[feature:redis-slowlog] Add Redis Slow Query Monitoring with Real-time Visualization

hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java

@@ -38,6 +38,7 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.hertzbeat.collector.collect.AbstractCollect;
 import org.apache.hertzbeat.collector.collect.common.cache.AbstractConnection;
@@ -71,6 +72,8 @@ public class RedisCommonCollectImpl extends AbstractCollect {
 
     private static final String UNIQUE_IDENTITY = "identity";
 
+    private static final String SLOW_LOG = "slowlog";
+
     private final ClientResources defaultClientResources;
     private final GlobalConnectionCache connectionCache = GlobalConnectionCache.getInstance();
 
@@ -90,7 +93,15 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException{
     @Override
     public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) {
         try {
-            if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) {
+            if (Objects.equals(metrics.getName(), SLOW_LOG)) {
+                if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) {
+                    List<Map<String, String>> redisSlowLogList = getClusterRedisSlowLog(metrics);
+                    doMetricsDataList(builder, redisSlowLogList, metrics);
+                } else {
+                    List<Map<String, String>> redisSlowLog = getSingleRedisSlowLog(metrics);
+                    doMetricsDataList(builder, redisSlowLog, metrics);
+                }
+            } else if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) {
                 List<Map<String, String>> redisInfoList = getClusterRedisInfo(metrics);
                 doMetricsDataList(builder, redisInfoList, metrics);
             } else {
@@ -371,4 +382,93 @@ public String supportProtocol() {
         return DispatchConstants.PROTOCOL_REDIS;
     }
 
+    /**
+     * Get slow log data from a single Redis instance
+     * @param metrics metrics config
+     * @return data
+     */
+    private List<Map<String, String>> getSingleRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException {
+        StatefulRedisConnection<String, String> connection = getSingleConnection(metrics.getRedis());
+        // Get the last 100 slow logs by default
+        List<Object> slowLogs = connection.sync().slowlogGet(100);
+
+        return parseSlowLogs(slowLogs);
+    }
+
+    /**
+     * Get slow log data from Redis cluster nodes
+     * @param metrics metrics config
+     * @return data
+     */
+    private List<Map<String, String>> getClusterRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException {
+        Map<String, StatefulRedisClusterConnection<String, String>> connectionMap = getConnectionList(metrics.getRedis());
+        List<Map<String, String>> allSlowLogs = new ArrayList<>();
+
+        connectionMap.forEach((identity, connection) -> {
+            // Get the last 100 slow logs by default
+            List<Object> slowLogs = connection.sync().slowlogGet(100);
+            List<Map<String, String>> parsedLogs = parseSlowLogs(slowLogs);
+
+            // Add node identity to each log entry
+            parsedLogs.forEach(log -> log.put(UNIQUE_IDENTITY, identity));
+
+            allSlowLogs.addAll(parsedLogs);
+        });
+
+        return allSlowLogs;
+    }
+
+    /**
+     * Parse Redis slow log response into a list of maps
+     * @param slowLogs raw slow log data from Redis
+     * @return parsed slow logs
+     */
+    private List<Map<String, String>> parseSlowLogs(List<Object> slowLogs) {
+        List<Map<String, String>> result = new ArrayList<>();
+
+        if (Objects.isNull(slowLogs) || slowLogs.isEmpty()) {
+            return result;
+        }
+
+        for (Object entry : slowLogs) {
+            if (entry instanceof List) {
+                List<?> logEntry = (List<?>) entry;
+                if (logEntry.size() >= 4) {
+                    Map<String, String> logMap = new HashMap<>();
+
+                    // ID
+                    logMap.put("id", String.valueOf(logEntry.get(0)));
+
+                    // Timestamp
+                    logMap.put("timestamp", String.valueOf(logEntry.get(1)));
+
+                    // Execution time in microseconds
+                    logMap.put("execution_time", String.valueOf(logEntry.get(2)));
+
+                    // Command and arguments
+                    if (logEntry.get(3) instanceof List) {
+                        List<?> cmdArgs = (List<?>) logEntry.get(3);
+                        String command = cmdArgs.stream()
+                                .map(String::valueOf)
+                                .collect(Collectors.joining(" "));
+                        logMap.put("command", command);
+                    }
+
+                    // Client IP and name (if available)
+                    if (logEntry.size() > 4 && logEntry.get(4) instanceof String) {
+                        logMap.put("client_ip", String.valueOf(logEntry.get(4)));
+                    }
+
+                    if (logEntry.size() > 5 && logEntry.get(5) instanceof String) {
+                        logMap.put("client_name", String.valueOf(logEntry.get(5)));
+                    }
+
+                    result.add(logMap);
+                }
+            }
+        }
+
+        return result;
+    }
+
 }

hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java

@@ -54,7 +54,7 @@ public class RedisCommonCollectE2eTest extends AbstractCollectE2eTest {
     private static final String HOST = "127.0.0.1";
     private static final int REDIS_PORT = 6379;
     private static final String REDIS_PATTERN = "1";
-    private static final List<String> ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace");
+    private static final List<String> ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace", "slowlog");
 
     private static GenericContainer<?> redisContainer;
 

hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java

@@ -110,7 +110,9 @@ protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, S
                 String.format("%s metrics values should not be empty, detail: %s", metricName, metricsData.getMsg()));
 
         for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) {
+            log.info("valueRow: {}", valueRow);
             for (int i = 0; i < valueRow.getColumnsCount(); i++) {
+                log.info("valueRow.getColumns(i): {}", valueRow.getColumns(i));
                 Assertions.assertFalse(valueRow.getColumns(i).isEmpty(),
                         String.format("%s metric column %d should not be empty", metricName, i));
                 if (!allowEmpty) {

hertzbeat-manager/src/main/resources/define/app-redis.yml

@@ -1656,3 +1656,61 @@ metrics:
         privateKey: ^_^sshPrivateKey^_^
         privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^
         shareConnection: ^_^sshShareConnection^_^
+
+  # metrics - slowlog
+  - name: slowlog
+    # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel
+    priority: 9
+    i18n:
+      zh-CN: 慢查询日志
+      en-US: Slow Query Log
+    # collect metrics content
+    fields:
+      - field: id
+        type: 0
+        i18n:
+          zh-CN: ID
+          en-US: ID
+      - field: timestamp
+        type: 0
+        i18n:
+          zh-CN: 时间戳
+          en-US: Timestamp
+      - field: execution_time
+        type: 0
+        unit: μs
+        i18n:
+          zh-CN: 执行时间
+          en-US: Execution Time
+      - field: command
+        type: 1
+        i18n:
+          zh-CN: 命令
+          en-US: Command
+      - field: client_ip
+        type: 1
+        i18n:
+          zh-CN: 客户端IP
+          en-US: Client IP
+      - field: client_name
+        type: 1
+        i18n:
+          zh-CN: 客户端名称
+          en-US: Client Name
+    protocol: redis
+    redis:
+      host: ^_^host^_^
+      port: ^_^port^_^
+      username: ^_^username^_^
+      password: ^_^password^_^
+      timeout: ^_^timeout^_^
+      sshTunnel:
+        enable: ^_^enableSshTunnel^_^
+        host: ^_^sshHost^_^
+        port: ^_^sshPort^_^
+        timeout: ^_^sshTimeout^_^
+        username: ^_^sshUsername^_^
+        password: ^_^sshPassword^_^
+        privateKey: ^_^sshPrivateKey^_^
+        privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^
+        shareConnection: ^_^sshShareConnection^_^

home/docs/help/redis.md

@@ -232,3 +232,20 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni
 | cmdstat_lpop  | none    | lpop command stat                                                                                                               |
 | cmdstat_rpop  | none    | rpop command stat                                                                                                               |
 | cmdstat_llen  | none    | llen command stat                                                                                                               |
+
+#### Metric set：slowlog
+
+|           Metric name            | Metric unit |                                            Metric help description                                             |
+|---------------------------|----------|-----------------------------------------------------------------------------------------------|
+| id                        | none     | Unique progressive identifier for every slow log entry                                                                            |
+| timestamp                 | none     | Unix timestamp at which the logged command was processed                                                                          |
+| execution_time            | μs       | The amount of time needed for its execution, in microseconds                                                                      |
+| command                   | none     | The command and its arguments                                                                                                     |
+| client_ip                 | none     | Client IP address (available for Redis 4.0 or higher)                                                                             |
+| client_name               | none     | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher)                                             |
+
+> Note: To configure Redis slow log settings, use the following Redis commands:
+>
+> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms)
+> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries
+> - The slow log metrics are displayed in real-time and updated automatically.

home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md

@@ -5,7 +5,7 @@ sidebar_label: Redis 数据库
 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控]
 ---
 
-> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。
+> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。
 
 ### 配置参数
 
@@ -237,3 +237,20 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控]
 | cmdstat_lpop  | 无    | lpop命令的统计信息                                                                                                               |
 | cmdstat_rpop  | 无    | rpop命令的统计信息                                                                                                               |
 | cmdstat_llen  | 无    | llen命令的统计信息                                                                                                               |
+
+#### 指标集合：slowlog
+
+|           指标名称            | 指标单位 |                                            指标帮助描述                                             |
+|---------------------------|----------|-----------------------------------------------------------------------------------------------|
+| id                        | 无       | 慢查询日志的唯一递增标识符                                                                             |
+| timestamp                 | 无       | 记录命令被处理时的Unix时间戳                                                                          |
+| execution_time            | μs       | 命令执行所需的时间（微秒）                                                                             |
+| command                   | 无       | 执行的命令及其参数                                                                                   |
+| client_ip                 | 无       | 客户端IP地址（Redis 4.0或更高版本可用）                                                                 |
+| client_name               | 无       | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用）                                        |
+
+> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令：
+>
+> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令
+> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目
+> - 慢查询日志指标会实时显示并自动更新。

home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md

@@ -5,7 +5,7 @@ sidebar_label: Redis 数据库
 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控]
 ---
 
-> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。
+> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。
 
 ### 配置参数
 
@@ -237,3 +237,20 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控]
 | cmdstat_lpop  | 无    | lpop命令的统计信息                                                                                                               |
 | cmdstat_rpop  | 无    | rpop命令的统计信息                                                                                                               |
 | cmdstat_llen  | 无    | llen命令的统计信息                                                                                                               |
+
+#### 指标集合：slowlog
+
+|           指标名称            | 指标单位 |                                            指标帮助描述                                             |
+|---------------------------|----------|-----------------------------------------------------------------------------------------------|
+| id                        | 无       | 慢查询日志的唯一递增标识符                                                                             |
+| timestamp                 | 无       | 记录命令被处理时的Unix时间戳                                                                          |
+| execution_time            | μs       | 命令执行所需的时间（微秒）                                                                             |
+| command                   | 无       | 执行的命令及其参数                                                                                   |
+| client_ip                 | 无       | 客户端IP地址（Redis 4.0或更高版本可用）                                                                 |
+| client_name               | 无       | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用）                                        |
+
+> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令：
+>
+> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令
+> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目
+> - 慢查询日志指标会实时显示并自动更新。

home/versioned_docs/version-v1.6.x/help/redis.md

@@ -5,7 +5,7 @@ sidebar_label: REDIS
 keywords: [ open source monitoring tool, open source Redis monitoring tool, monitoring Redis metrics ]
 ---
 
-> Collect and monitor the general performance Metrics of Redis database. Support REDIS1.0+.
+> Collect and monitor the general performance Metrics of Redis database including slow query logs. Support REDIS1.0+.
 
 ### Configuration parameter
 
@@ -232,3 +232,20 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni
 | cmdstat_lpop  | none    | lpop command stat                                                                                                               |
 | cmdstat_rpop  | none    | rpop command stat                                                                                                               |
 | cmdstat_llen  | none    | llen command stat                                                                                                               |
+
+#### Metric set：slowlog
+
+|           Metric name            | Metric unit |                                            Metric help description                                             |
+|---------------------------|----------|-----------------------------------------------------------------------------------------------|
+| id                        | none     | Unique progressive identifier for every slow log entry                                                                            |
+| timestamp                 | none     | Unix timestamp at which the logged command was processed                                                                          |
+| execution_time            | μs       | The amount of time needed for its execution, in microseconds                                                                      |
+| command                   | none     | The command and its arguments                                                                                                     |
+| client_ip                 | none     | Client IP address (available for Redis 4.0 or higher)                                                                             |
+| client_name               | none     | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher)                                             |
+
+> Note: To configure Redis slow log settings, use the following Redis commands:
+>
+> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms)
+> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries
+> - The slow log metrics are displayed in real-time and updated automatically.