diff --git a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java index 90b258545db..ffaf3563b00 100644 --- a/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java +++ b/hertzbeat-collector/hertzbeat-collector-basic/src/main/java/org/apache/hertzbeat/collector/collect/redis/RedisCommonCollectImpl.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.hertzbeat.collector.collect.AbstractCollect; import org.apache.hertzbeat.collector.collect.common.cache.AbstractConnection; @@ -71,6 +72,8 @@ public class RedisCommonCollectImpl extends AbstractCollect { private static final String UNIQUE_IDENTITY = "identity"; + private static final String SLOW_LOG = "slowlog"; + private final ClientResources defaultClientResources; private final GlobalConnectionCache connectionCache = GlobalConnectionCache.getInstance(); @@ -90,7 +93,15 @@ public void preCheck(Metrics metrics) throws IllegalArgumentException{ @Override public void collect(CollectRep.MetricsData.Builder builder, Metrics metrics) { try { - if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { + if (Objects.equals(metrics.getName(), SLOW_LOG)) { + if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { + List> redisSlowLogList = getClusterRedisSlowLog(metrics); + doMetricsDataList(builder, redisSlowLogList, metrics); + } else { + List> redisSlowLog = getSingleRedisSlowLog(metrics); + doMetricsDataList(builder, redisSlowLog, metrics); + } + } else if (Objects.nonNull(metrics.getRedis().getPattern()) && Objects.equals(metrics.getRedis().getPattern(), CLUSTER)) { List> redisInfoList = getClusterRedisInfo(metrics); doMetricsDataList(builder, redisInfoList, metrics); } else { @@ -371,4 +382,93 @@ public String supportProtocol() { return DispatchConstants.PROTOCOL_REDIS; } + /** + * Get slow log data from a single Redis instance + * @param metrics metrics config + * @return data + */ + private List> getSingleRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException { + StatefulRedisConnection connection = getSingleConnection(metrics.getRedis()); + // Get the last 100 slow logs by default + List slowLogs = connection.sync().slowlogGet(100); + + return parseSlowLogs(slowLogs); + } + + /** + * Get slow log data from Redis cluster nodes + * @param metrics metrics config + * @return data + */ + private List> getClusterRedisSlowLog(Metrics metrics) throws GeneralSecurityException, IOException { + Map> connectionMap = getConnectionList(metrics.getRedis()); + List> allSlowLogs = new ArrayList<>(); + + connectionMap.forEach((identity, connection) -> { + // Get the last 100 slow logs by default + List slowLogs = connection.sync().slowlogGet(100); + List> parsedLogs = parseSlowLogs(slowLogs); + + // Add node identity to each log entry + parsedLogs.forEach(log -> log.put(UNIQUE_IDENTITY, identity)); + + allSlowLogs.addAll(parsedLogs); + }); + + return allSlowLogs; + } + + /** + * Parse Redis slow log response into a list of maps + * @param slowLogs raw slow log data from Redis + * @return parsed slow logs + */ + private List> parseSlowLogs(List slowLogs) { + List> result = new ArrayList<>(); + + if (Objects.isNull(slowLogs) || slowLogs.isEmpty()) { + return result; + } + + for (Object entry : slowLogs) { + if (entry instanceof List) { + List logEntry = (List) entry; + if (logEntry.size() >= 4) { + Map logMap = new HashMap<>(); + + // ID + logMap.put("id", String.valueOf(logEntry.get(0))); + + // Timestamp + logMap.put("timestamp", String.valueOf(logEntry.get(1))); + + // Execution time in microseconds + logMap.put("execution_time", String.valueOf(logEntry.get(2))); + + // Command and arguments + if (logEntry.get(3) instanceof List) { + List cmdArgs = (List) logEntry.get(3); + String command = cmdArgs.stream() + .map(String::valueOf) + .collect(Collectors.joining(" ")); + logMap.put("command", command); + } + + // Client IP and name (if available) + if (logEntry.size() > 4 && logEntry.get(4) instanceof String) { + logMap.put("client_ip", String.valueOf(logEntry.get(4))); + } + + if (logEntry.size() > 5 && logEntry.get(5) instanceof String) { + logMap.put("client_name", String.valueOf(logEntry.get(5))); + } + + result.add(logMap); + } + } + } + + return result; + } + } diff --git a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java index 673cbb0d633..6aff9dd9208 100644 --- a/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-basic-e2e/src/test/java/org/apache/hertzbeat/collector/collect/basic/redis/RedisCommonCollectE2eTest.java @@ -54,7 +54,7 @@ public class RedisCommonCollectE2eTest extends AbstractCollectE2eTest { private static final String HOST = "127.0.0.1"; private static final int REDIS_PORT = 6379; private static final String REDIS_PATTERN = "1"; - private static final List ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace"); + private static final List ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace", "slowlog"); private static GenericContainer redisContainer; diff --git a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java index 7a7aa174203..7f1e42488c9 100644 --- a/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java +++ b/hertzbeat-e2e/hertzbeat-collector-common-e2e/src/test/java/org/apache/hertzbeat/collector/collect/AbstractCollectE2eTest.java @@ -110,7 +110,9 @@ protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, S String.format("%s metrics values should not be empty, detail: %s", metricName, metricsData.getMsg())); for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) { + log.info("valueRow: {}", valueRow); for (int i = 0; i < valueRow.getColumnsCount(); i++) { + log.info("valueRow.getColumns(i): {}", valueRow.getColumns(i)); Assertions.assertFalse(valueRow.getColumns(i).isEmpty(), String.format("%s metric column %d should not be empty", metricName, i)); if (!allowEmpty) { diff --git a/hertzbeat-manager/src/main/resources/define/app-redis.yml b/hertzbeat-manager/src/main/resources/define/app-redis.yml index a7d442cc470..9ebadcfb1be 100644 --- a/hertzbeat-manager/src/main/resources/define/app-redis.yml +++ b/hertzbeat-manager/src/main/resources/define/app-redis.yml @@ -1656,3 +1656,61 @@ metrics: privateKey: ^_^sshPrivateKey^_^ privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^ shareConnection: ^_^sshShareConnection^_^ + + # metrics - slowlog + - name: slowlog + # metrics scheduling priority(0->127)->(high->low), metrics with the same priority will be scheduled in parallel + priority: 9 + i18n: + zh-CN: 慢查询日志 + en-US: Slow Query Log + # collect metrics content + fields: + - field: id + type: 0 + i18n: + zh-CN: ID + en-US: ID + - field: timestamp + type: 0 + i18n: + zh-CN: 时间戳 + en-US: Timestamp + - field: execution_time + type: 0 + unit: μs + i18n: + zh-CN: 执行时间 + en-US: Execution Time + - field: command + type: 1 + i18n: + zh-CN: 命令 + en-US: Command + - field: client_ip + type: 1 + i18n: + zh-CN: 客户端IP + en-US: Client IP + - field: client_name + type: 1 + i18n: + zh-CN: 客户端名称 + en-US: Client Name + protocol: redis + redis: + host: ^_^host^_^ + port: ^_^port^_^ + username: ^_^username^_^ + password: ^_^password^_^ + timeout: ^_^timeout^_^ + sshTunnel: + enable: ^_^enableSshTunnel^_^ + host: ^_^sshHost^_^ + port: ^_^sshPort^_^ + timeout: ^_^sshTimeout^_^ + username: ^_^sshUsername^_^ + password: ^_^sshPassword^_^ + privateKey: ^_^sshPrivateKey^_^ + privateKeyPassphrase: ^_^sshPrivateKeyPassphrase^_^ + shareConnection: ^_^sshShareConnection^_^ diff --git a/home/docs/help/redis.md b/home/docs/help/redis.md index 2edbc47f482..e02f691292e 100644 --- a/home/docs/help/redis.md +++ b/home/docs/help/redis.md @@ -232,3 +232,20 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | cmdstat_lpop | none | lpop command stat | | cmdstat_rpop | none | rpop command stat | | cmdstat_llen | none | llen command stat | + +#### Metric set：slowlog + +| Metric name | Metric unit | Metric help description | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | none | Unique progressive identifier for every slow log entry | +| timestamp | none | Unix timestamp at which the logged command was processed | +| execution_time | μs | The amount of time needed for its execution, in microseconds | +| command | none | The command and its arguments | +| client_ip | none | Client IP address (available for Redis 4.0 or higher) | +| client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | + +> Note: To configure Redis slow log settings, use the following Redis commands: +> +> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) +> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries +> - The slow log metrics are displayed in real-time and updated automatically. diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md index d34ac3ca56c..ea9312eac35 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/current/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: Redis 数据库 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] --- -> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。 +> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。 ### 配置参数 @@ -237,3 +237,20 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | cmdstat_lpop | 无 | lpop命令的统计信息 | | cmdstat_rpop | 无 | rpop命令的统计信息 | | cmdstat_llen | 无 | llen命令的统计信息 | + +#### 指标集合：slowlog + +| 指标名称 | 指标单位 | 指标帮助描述 | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | 无 | 慢查询日志的唯一递增标识符 | +| timestamp | 无 | 记录命令被处理时的Unix时间戳 | +| execution_time | μs | 命令执行所需的时间（微秒） | +| command | 无 | 执行的命令及其参数 | +| client_ip | 无 | 客户端IP地址（Redis 4.0或更高版本可用） | +| client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | + +> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> +> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 +> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 +> - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md index d34ac3ca56c..ea9312eac35 100644 --- a/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md +++ b/home/i18n/zh-cn/docusaurus-plugin-content-docs/version-v1.6.x/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: Redis 数据库 keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] --- -> 对REDIS数据库的通用性能指标进行采集监控。支持REDIS1.0+。 +> 对REDIS数据库的通用性能指标（包括慢查询日志）进行采集监控。支持REDIS1.0+。 ### 配置参数 @@ -237,3 +237,20 @@ keywords: [开源监控系统, 开源数据库监控, Redis数据库监控] | cmdstat_lpop | 无 | lpop命令的统计信息 | | cmdstat_rpop | 无 | rpop命令的统计信息 | | cmdstat_llen | 无 | llen命令的统计信息 | + +#### 指标集合：slowlog + +| 指标名称 | 指标单位 | 指标帮助描述 | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | 无 | 慢查询日志的唯一递增标识符 | +| timestamp | 无 | 记录命令被处理时的Unix时间戳 | +| execution_time | μs | 命令执行所需的时间（微秒） | +| command | 无 | 执行的命令及其参数 | +| client_ip | 无 | 客户端IP地址（Redis 4.0或更高版本可用） | +| client_name | 无 | 客户端名称，如果通过CLIENT SETNAME命令设置（Redis 4.0或更高版本可用） | + +> 注意：要配置Redis慢查询日志设置，请使用以下Redis命令： +> +> - `CONFIG SET slowlog-log-slower-than 10000` - 记录执行时间超过10,000微秒（10毫秒）的命令 +> - `CONFIG SET slowlog-max-len 128` - 保留最近128条慢查询日志条目 +> - 慢查询日志指标会实时显示并自动更新。 diff --git a/home/versioned_docs/version-v1.6.x/help/redis.md b/home/versioned_docs/version-v1.6.x/help/redis.md index 2edbc47f482..64f1cd40aff 100644 --- a/home/versioned_docs/version-v1.6.x/help/redis.md +++ b/home/versioned_docs/version-v1.6.x/help/redis.md @@ -5,7 +5,7 @@ sidebar_label: REDIS keywords: [ open source monitoring tool, open source Redis monitoring tool, monitoring Redis metrics ] --- -> Collect and monitor the general performance Metrics of Redis database. Support REDIS1.0+. +> Collect and monitor the general performance Metrics of Redis database including slow query logs. Support REDIS1.0+. ### Configuration parameter @@ -232,3 +232,20 @@ keywords: [ open source monitoring tool, open source Redis monitoring tool, moni | cmdstat_lpop | none | lpop command stat | | cmdstat_rpop | none | rpop command stat | | cmdstat_llen | none | llen command stat | + +#### Metric set：slowlog + +| Metric name | Metric unit | Metric help description | +|---------------------------|----------|-----------------------------------------------------------------------------------------------| +| id | none | Unique progressive identifier for every slow log entry | +| timestamp | none | Unix timestamp at which the logged command was processed | +| execution_time | μs | The amount of time needed for its execution, in microseconds | +| command | none | The command and its arguments | +| client_ip | none | Client IP address (available for Redis 4.0 or higher) | +| client_name | none | Client name if set via the CLIENT SETNAME command (available for Redis 4.0 or higher) | + +> Note: To configure Redis slow log settings, use the following Redis commands: +> +> - `CONFIG SET slowlog-log-slower-than 10000` - Log commands that take longer than 10,000 microseconds (10ms) +> - `CONFIG SET slowlog-max-len 128` - Keep the last 128 slow log entries +> - The slow log metrics are displayed in real-time and updated automatically.