diff --git a/go.mod b/go.mod index fce668d5a3e..c2f5966a5d5 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ replace ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsekshyperpodreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/jmxreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260702195205-c728dfbd69c6 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver => github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 ) @@ -195,6 +196,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver v0.124.1 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.124.1 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver v0.124.1 @@ -619,7 +621,7 @@ require ( go.opentelemetry.io/collector/component/componentstatus v0.124.0 // indirect go.opentelemetry.io/collector/config/configcompression v1.30.0 go.opentelemetry.io/collector/config/configgrpc v0.124.0 // indirect - go.opentelemetry.io/collector/config/confignet v1.30.0 // indirect + go.opentelemetry.io/collector/config/confignet v1.30.0 go.opentelemetry.io/collector/config/configretry v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/httpprovider v1.30.0 // indirect go.opentelemetry.io/collector/confmap/provider/yamlprovider v1.30.0 // indirect diff --git a/go.sum b/go.sum index b535d7aebc8..33d106618b4 100644 --- a/go.sum +++ b/go.sum @@ -278,6 +278,8 @@ github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayr github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gNDpUc9eYHKy9YqGYK8toPIo3QZKdYQFLnaVQDA32A0= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9 h1:R4Ems+YMAXLtsEZwRXw8/cf1NkrzHQsc25WPdYbUaTM= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/jmxreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:wQV77sHJJdUMIvbJna8LjlNb0omEdiVSrwQnoVAlTMM= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260702195205-c728dfbd69c6 h1:TyjCXDOKL+uz9VBOxr9Zs08BgX624jPWUoU8xQ4sTrw= +github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/mysqlreceiver v0.0.0-20260702195205-c728dfbd69c6/go.mod h1:tnnmAnxuLxWKBG7NjOI3po8R+EAX/9cG1y7TWiQTMvA= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9 h1:ZLP3mZHMXzUjiXxXFVI9Z5LtU25qT13Qajz84mna/LY= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/postgresqlreceiver v0.0.0-20260617204523-70530ca095f9/go.mod h1:gSxi2fY+T8mNzlaU7qOEnaXTcKfhIpijDSyeCu5GOYo= github.com/amazon-contributing/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.0.0-20260617204523-70530ca095f9 h1:p0HB9BwhUdkaI0ALIb3hYxi7oCTDsGUuGadeM3R36AM= diff --git a/service/defaultcomponents/components.go b/service/defaultcomponents/components.go index 02731a24f26..efdabb5cd87 100644 --- a/service/defaultcomponents/components.go +++ b/service/defaultcomponents/components.go @@ -53,6 +53,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/journaldreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kafkareceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/kubeletstatsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/postgresqlreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver" @@ -116,6 +117,7 @@ func Factories() (otelcol.Factories, error) { nopreceiver.NewFactory(), otlpreceiver.NewFactory(), prometheusreceiver.NewFactory(), + mysqlreceiver.NewFactory(), postgresqlreceiver.NewFactory(), statsdreceiver.NewFactory(), systemmetricsreceiver.NewFactory(), diff --git a/service/defaultcomponents/components_test.go b/service/defaultcomponents/components_test.go index e979358847e..e14b67bf934 100644 --- a/service/defaultcomponents/components_test.go +++ b/service/defaultcomponents/components_test.go @@ -35,6 +35,7 @@ func TestComponents(t *testing.T) { "nop", "otlp", "prometheus", + "mysql", "postgresql", "statsd", "systemmetrics", diff --git a/translator/config/schema.json b/translator/config/schema.json index 3d52e38f145..200130c7914 100644 --- a/translator/config/schema.json +++ b/translator/config/schema.json @@ -1830,6 +1830,27 @@ "required": ["endpoint", "instance_name", "username", "password_file"], "additionalProperties": false } + }, + "mysql": { + "type": "array", + "items": { + "type": "object", + "properties": { + "endpoint": { "type": "string" }, + "instance_name": { "type": "string" }, + "username": { "type": "string" }, + "password_file": { "type": "string" }, + "logs": { + "type": "object", + "properties": { + "file_path": { "type": "string" } + }, + "additionalProperties": false + } + }, + "required": ["endpoint", "instance_name", "username", "password_file"], + "additionalProperties": false + } } }, "additionalProperties": false diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml index 7be83ef10a5..93032d2bce5 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_ec2_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -504,6 +504,14 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 1m0s + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s batch/opentelemetry_metrics: metadata_cardinality_limit: 1000 send_batch_max_size: 1000 @@ -514,14 +522,6 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - batch/opentelemetry_logs: - metadata_cardinality_limit: 1000 - metadata_keys: - - aws.log.group.name - - aws.log.stream.name - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 15s batch/xray: metadata_cardinality_limit: 1000 send_batch_max_size: 0 @@ -549,7 +549,7 @@ processors: middleware: agenthealth/statuscode refresh_tags_interval: 0s refresh_volumes_interval: 0s - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -837,7 +837,7 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -853,7 +853,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -864,7 +864,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -875,7 +875,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -891,7 +891,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1505,31 +1505,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -1564,13 +1564,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host: exporters: - awscloudwatch @@ -1578,13 +1578,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_procstat/1917393364 - telegraf_mem - - telegraf_netstat + - telegraf_procstat/1917393364 + - telegraf_processes - telegraf_cpu - telegraf_disk - telegraf_swap - - telegraf_processes + - telegraf_netstat metrics/host_metrics: exporters: - forward/opentelemetry @@ -1609,8 +1609,8 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_net - telegraf_diskio + - telegraf_net metrics/opentelemetry: exporters: - otlphttp/metrics diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml index 37b965b30b9..352c1b4f31b 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/combined_v1_v2_eks_config.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -551,6 +551,14 @@ processors: send_batch_max_size: 0 send_batch_size: 8192 timeout: 1m0s + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s batch/opentelemetry_metrics: metadata_cardinality_limit: 1000 send_batch_max_size: 1000 @@ -561,14 +569,6 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - batch/opentelemetry_logs: - metadata_cardinality_limit: 1000 - metadata_keys: - - aws.log.group.name - - aws.log.stream.name - send_batch_max_size: 10000 - send_batch_size: 10000 - timeout: 15s batch/xray: metadata_cardinality_limit: 1000 send_batch_max_size: 0 @@ -624,7 +624,7 @@ processors: metrics: metric: - IsMatch(name, "^(up|scrape_duration_seconds|scrape_samples_scraped|scrape_samples_post_metric_relabeling|scrape_series_added)$") - filter/dbi_exclude_monitor_0: + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -1370,7 +1370,7 @@ processors: - set(resource.attributes["k8s.workload.type"], "CronJob") where resource.attributes["k8s.workload.type"] == nil and resource.attributes["k8s.cronjob.name"] != nil - set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil - set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.workload.type"] == nil and resource.attributes["k8s.replicaset.name"] != nil - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -1386,7 +1386,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -1397,7 +1397,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -1408,7 +1408,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "my-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -1424,7 +1424,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "my-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -2347,31 +2347,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/emf_logs: @@ -2617,13 +2617,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host: exporters: - awscloudwatch @@ -2631,13 +2631,13 @@ service: - ec2tagger - awsentity/resource receivers: - - telegraf_procstat/1917393364 - - telegraf_cpu - telegraf_swap + - telegraf_processes - telegraf_mem + - telegraf_procstat/1917393364 - telegraf_netstat - - telegraf_processes - telegraf_disk + - telegraf_cpu metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.conf rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.conf diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json new file mode 100644 index 00000000000..5aca9f4f90f --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.json @@ -0,0 +1,22 @@ +{ + "agent": { + "region": "us-west-2" + }, + "opentelemetry": { + "collect": { + "database_insights": { + "mysql": [ + { + "endpoint": "localhost:3306", + "instance_name": "pentest-mysql", + "username": "cw_monitor", + "password_file": "sampleConfig/opentelemetry/testdata/.mysql_credentials", + "logs": { + "file_path": "/var/log/mysql/mysql.log" + } + } + ] + } + } + } +} diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml new file mode 100644 index 00000000000..3630425c1ca --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_mysql_config_linux.yaml @@ -0,0 +1,1581 @@ +connectors: + count/dbi_dbload_mysql: + datapoints: + metric.datapoint.count: + description: The number of data points observed. + logs: + mysql.active_sessions.by_db: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by database + mysql.active_sessions.by_host: + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by host + mysql.active_sessions.by_sql: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by query digest + mysql.active_sessions.by_sql_wait: + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by query and wait + mysql.active_sessions.by_user: + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by user + mysql.active_sessions.by_wait: + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Active sessions by wait event + mysql.active_sessions.count: + attributes: + - key: db.system.name + - key: db.namespace + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + description: Total active sessions + metrics: + metric.count: + description: The number of metrics observed. + spanevents: + trace.span.event.count: + description: The number of span events observed. + spans: + trace.span.count: + description: The number of spans observed. + forward/opentelemetry: {} + signaltometrics/dbi_topsql_mysql: + logs: + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + description: SQL execution count + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.count_star + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.count_star"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + description: Total execution time in picoseconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_timer_wait + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + description: Total lock time in seconds + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_lock_time + sum: + value: Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + description: Rows sent to client + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_sent + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + description: Rows examined + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_rows_examined + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + description: Total errors + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_errors + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + description: Rows sorted + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_rows + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + description: Temporary tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + description: Temporary disk tables created + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_created_tmp_disk_tables + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + description: Queries using no index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + description: Full joins without index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_full_join + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + description: Sorts done via full scan + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_sort_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + description: Queries using no good index + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_no_good_index_used + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"]) + unit: "" + - attributes: + - key: db.system.name + optional: false + - key: mysql.events_statements_summary_by_digest.digest + optional: false + - key: db.namespace + optional: false + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + description: Full table scans + include_resource_attributes: + - key: service.instance.id + optional: false + name: mysql.sum_select_scan + sum: + value: Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"]) + unit: "" +exporters: + otlphttp/logs: + auth: + authenticator: agenthealth/opentelemetry_logs + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: https://logs.us-west-2.amazonaws.com/v1/logs + max_idle_conns: 100 + metrics_endpoint: "" + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 + otlphttp/metrics: + auth: + authenticator: agenthealth/opentelemetry_metrics + compression: gzip + encoding: proto + idle_conn_timeout: 1m30s + logs_endpoint: "" + max_idle_conns: 100 + metrics_endpoint: https://monitoring.us-west-2.amazonaws.com/v1/metrics + retry_on_failure: + enabled: true + initial_interval: 5s + max_elapsed_time: 5m0s + max_interval: 30s + multiplier: 1.5 + randomization_factor: 0.5 + sending_queue: + block_on_overflow: false + blocking: false + enabled: true + num_consumers: 10 + queue_size: 1000 + sizer: {} + wait_for_result: false + timeout: 30s + traces_endpoint: "" + write_buffer_size: 524288 +extensions: + agenthealth/opentelemetry_logs: + additional_auth: headers_setter/logs + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + agenthealth/opentelemetry_metrics: + additional_auth: sigv4auth/monitoring + is_usage_data_enabled: true + stats: + operations: + - '*' + usage_flags: + mode: EC2 + region_type: ACJ + awscloudwatchlogsprovisioner: + additional_auth: sigv4auth/logs + imds_retries: 1 + logs_provision_failure_backoff: 30s + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 10 + headers_setter/logs: + additional_auth: awscloudwatchlogsprovisioner + headers: + - action: upsert + from_context: aws.log.group.name + key: x-aws-log-group + - action: upsert + from_context: aws.log.stream.name + key: x-aws-log-stream + sigv4auth/logs: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: logs + sigv4auth/monitoring: + assume_role: {} + imds_retries: 1 + max_retries: 2 + num_workers: 8 + region: us-west-2 + request_timeout_seconds: 30 + service: monitoring +processors: + attributestocontext/opentelemetry: + actions: + - from_resource_attribute: aws.log.group.name + key: aws.log.group.name + - from_resource_attribute: aws.log.stream.name + key: aws.log.stream.name + batch/opentelemetry_logs: + metadata_cardinality_limit: 1000 + metadata_keys: + - aws.log.group.name + - aws.log.stream.name + send_batch_max_size: 10000 + send_batch_size: 10000 + timeout: 15s + batch/opentelemetry_metrics: + metadata_cardinality_limit: 1000 + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 15s + filter/dbi_exclude_monitor_mysql_0: + error_mode: propagate + logs: + log_record: + - attributes["user.name"] == "cw_monitor" + metrics: {} + spans: {} + traces: {} + resourcedetection/opentelemetry: + aks: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + azure: + resource_attributes: + azure.resourcegroup.name: + enabled: true + azure.vm.name: + enabled: true + azure.vm.scaleset.name: + enabled: true + azure.vm.size: + enabled: true + cloud.account.id: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + tags: [] + consul: + address: "" + datacenter: "" + namespace: "" + resource_attributes: + cloud.region: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + token_file: "" + detectors: + - eks + - env + - ec2 + docker: + resource_attributes: + host.name: + enabled: true + os.type: + enabled: true + ec2: + fail_on_missing_metadata: false + max_attempts: 3 + max_backoff: 20s + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + host.id: + enabled: true + host.image.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + tags: + - ^kubernetes.io/cluster/.*$ + - ^aws:autoscaling:groupName + ecs: + resource_attributes: + aws.ecs.cluster.arn: + enabled: true + aws.ecs.launchtype: + enabled: true + aws.ecs.task.arn: + enabled: true + aws.ecs.task.family: + enabled: true + aws.ecs.task.id: + enabled: true + aws.ecs.task.revision: + enabled: true + aws.log.group.arns: + enabled: true + aws.log.group.names: + enabled: true + aws.log.stream.arns: + enabled: true + aws.log.stream.names: + enabled: true + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + eks: + resource_attributes: + cloud.account.id: + enabled: false + cloud.platform: + enabled: true + cloud.provider: + enabled: true + k8s.cluster.name: + enabled: false + elasticbeanstalk: + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + deployment.environment: + enabled: true + service.instance.id: + enabled: true + service.version: + enabled: true + gcp: + resource_attributes: + cloud.account.id: + enabled: true + cloud.availability_zone: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.id: + enabled: true + faas.instance: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + gcp.cloud_run.job.execution: + enabled: true + gcp.cloud_run.job.task_index: + enabled: true + gcp.gce.instance.hostname: + enabled: false + gcp.gce.instance.name: + enabled: false + gcp.gce.instance_group_manager.name: + enabled: true + gcp.gce.instance_group_manager.region: + enabled: true + gcp.gce.instance_group_manager.zone: + enabled: true + host.id: + enabled: true + host.name: + enabled: true + host.type: + enabled: true + k8s.cluster.name: + enabled: true + heroku: + resource_attributes: + cloud.provider: + enabled: true + heroku.app.id: + enabled: true + heroku.dyno.id: + enabled: true + heroku.release.commit: + enabled: true + heroku.release.creation_timestamp: + enabled: true + service.instance.id: + enabled: true + service.name: + enabled: true + service.version: + enabled: true + idle_conn_timeout: 1m30s + k8snode: + auth_type: serviceAccount + context: "" + kube_config_path: "" + node_from_env_var: "" + resource_attributes: + k8s.node.name: + enabled: true + k8s.node.uid: + enabled: true + kubeadm: + auth_type: serviceAccount + context: "" + kube_config_path: "" + resource_attributes: + k8s.cluster.name: + enabled: true + k8s.cluster.uid: + enabled: true + lambda: + resource_attributes: + aws.log.group.names: + enabled: true + aws.log.stream.names: + enabled: true + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + faas.instance: + enabled: true + faas.max_memory: + enabled: true + faas.name: + enabled: true + faas.version: + enabled: true + max_idle_conns: 100 + openshift: + address: "" + resource_attributes: + cloud.platform: + enabled: true + cloud.provider: + enabled: true + cloud.region: + enabled: true + k8s.cluster.name: + enabled: true + tls: {} + token: "" + override: true + system: + resource_attributes: + host.arch: + enabled: false + host.cpu.cache.l2.size: + enabled: false + host.cpu.family: + enabled: false + host.cpu.model.id: + enabled: false + host.cpu.model.name: + enabled: false + host.cpu.stepping: + enabled: false + host.cpu.vendor.id: + enabled: false + host.id: + enabled: false + host.ip: + enabled: false + host.mac: + enabled: false + host.name: + enabled: true + os.description: + enabled: false + os.type: + enabled: true + os.version: + enabled: false + timeout: 2s + transform/dbi_fix_start_time_mysql: + error_mode: ignore + flatten_data: false + log_statements: [] + metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") + trace_statements: [] + transform/dbi_logs_mysql_raw-events_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/raw-events") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_logs_mysql_server-logs_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/mysql/server-logs") + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-mysql"], "/")) + metric_statements: [] + trace_statements: [] + transform/dbi_resource_mysql_0: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "pentest-mysql") + metric_statements: + - context: resource + error_mode: propagate + statements: + - set(resource.attributes["db.system.name"], "mysql") + - set(resource.attributes["db.instance.name"], "pentest-mysql") + trace_statements: [] + transform/dbi_scope_mysql_0: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + trace_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-database-insights") + transform/host_metrics_scope: + error_mode: ignore + flatten_data: false + log_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + metric_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + trace_statements: + - context: scope + error_mode: ignore + statements: + - set(attributes["cloudwatch.source"], "cloudwatch-agent") + - set(attributes["cloudwatch.solution"], "otel-host-metrics") + transform/identity: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + metric_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + trace_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["service.name"], "unknown_service") where resource.attributes["service.name"] == nil + - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:ec2", resource.attributes["cloud.region"], resource.attributes["cloud.account.id"], Concat(["instance", resource.attributes["host.id"]], "/")], ":")) where resource.attributes["cloud.resource_id"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], resource.attributes["ec2.tag.aws:autoscaling:groupName"]], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["ec2.tag.aws:autoscaling:groupName"] != nil + - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], "default"], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil + transform/logs_cleanup: + error_mode: propagate + flatten_data: false + log_statements: + - context: resource + error_mode: propagate + statements: + - delete_key(resource.attributes, "aws.log.group.name") + - delete_key(resource.attributes, "aws.log.stream.name") + - delete_key(resource.attributes, "aws.log.source") + metric_statements: [] + trace_statements: [] + transform/logs_routing: + error_mode: ignore + flatten_data: false + log_statements: + - context: resource + error_mode: ignore + statements: + - set(resource.attributes["aws.log.group.name"], Concat(["/aws/cwagent", resource.attributes["aws.log.source"]], "/")) where resource.attributes["aws.log.group.name"] == nil and resource.attributes["aws.log.source"] != nil + - set(resource.attributes["aws.log.group.name"], "/aws/cwagent/default") where resource.attributes["aws.log.group.name"] == nil + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.namespace"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.namespace"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.name"], resource.attributes["service.name"]], "/")) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil and resource.attributes["service.name"] != nil and resource.attributes["service.name"] != "unknown_service" + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.id"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.id"] != nil + - set(resource.attributes["aws.log.stream.name"], resource.attributes["host.name"]) where resource.attributes["aws.log.stream.name"] == nil and resource.attributes["host.name"] != nil + - set(resource.attributes["aws.log.stream.name"], "default") where resource.attributes["aws.log.stream.name"] == nil + metric_statements: [] + trace_statements: [] +receivers: + filelog/mysql_0: + encoding: utf-8 + exclude_older_than: 0s + fingerprint_size: 1000 + force_flush_period: 500ms + id: file_input + include: + - /var/log/mysql/mysql.log + include_file_name: true + initial_buffer_size: 16384 + max_concurrent_files: 1024 + max_log_size: 1048576 + operators: [] + poll_interval: 200ms + retry_on_failure: + enabled: false + initial_interval: 1s + max_elapsed_time: 5m0s + max_interval: 30s + start_at: end + type: file_input + hostmetrics/opentelemetry: + collection_interval: 30s + initial_delay: 1s + scrapers: + cpu: + metrics: + system.cpu.frequency: + enabled: true + system.cpu.logical.count: + enabled: true + system.cpu.physical.count: + enabled: true + system.cpu.utilization: + enabled: true + disk: null + filesystem: + metrics: + system.filesystem.utilization: + enabled: true + load: null + memory: + metrics: + system.linux.memory.available: + enabled: true + system.linux.memory.dirty: + enabled: true + system.memory.limit: + enabled: true + system.memory.page_size: + enabled: true + system.memory.utilization: + enabled: true + network: null + process: + include: + match_type: regexp + names: + - mysqld.* + metrics: + process.cpu.utilization: + enabled: true + process.memory.utilization: + enabled: true + mute_process_all_errors: true + processes: null + timeout: 0s + mysql/events_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: false + mysql.connection.count: + enabled: false + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: false + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: false + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: false + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: false + mysql.replica.time_behind_source: + enabled: false + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: false + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor + mysql/metrics_0: + allow_native_passwords: true + collection_interval: 10s + endpoint: localhost:3306 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + initial_delay: 1s + metrics: + mysql.buffer_pool.data_pages: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.buffer_pool.limit: + enabled: true + mysql.buffer_pool.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.buffer_pool.page_flushes: + enabled: true + mysql.buffer_pool.pages: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.buffer_pool.usage: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.client.network.io: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.commands: + aggregation_strategy: sum + attributes: + - command + enabled: false + mysql.connection.count: + enabled: false + mysql.connection.errors: + aggregation_strategy: sum + attributes: + - error + enabled: false + mysql.double_writes: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.handlers: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.index.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.index.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + - index + enabled: true + mysql.joins: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.log_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.max_used_connections: + enabled: false + mysql.mysqlx_connections: + aggregation_strategy: sum + attributes: + - status + enabled: true + mysql.mysqlx_worker_threads: + aggregation_strategy: sum + attributes: + - kind + enabled: false + mysql.opened_resources: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.page_size: + enabled: false + mysql.prepared_statements: + aggregation_strategy: sum + attributes: + - command + enabled: true + mysql.query.client.count: + enabled: false + mysql.query.count: + enabled: false + mysql.query.slow.count: + enabled: false + mysql.replica.sql_delay: + enabled: false + mysql.replica.time_behind_source: + enabled: false + mysql.row_locks: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.row_operations: + aggregation_strategy: sum + attributes: + - operation + enabled: true + mysql.sessions: + enabled: false + mysql.sorts: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.statement_event.count: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + - kind + enabled: false + mysql.statement_event.wait.time: + aggregation_strategy: sum + attributes: + - schema + - digest + - digest_text + enabled: false + mysql.table.average_row_length: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.io.wait.count: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.io.wait.time: + aggregation_strategy: sum + attributes: + - operation + - table + - schema + enabled: true + mysql.table.lock_wait.read.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.read.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.count: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.lock_wait.write.time: + aggregation_strategy: sum + attributes: + - schema + - table + - kind + enabled: false + mysql.table.rows: + aggregation_strategy: sum + attributes: + - table + - schema + enabled: false + mysql.table.size: + aggregation_strategy: sum + attributes: + - table + - schema + - kind + enabled: false + mysql.table_open_cache: + aggregation_strategy: sum + attributes: + - status + enabled: false + mysql.threads: + aggregation_strategy: sum + attributes: + - kind + enabled: true + mysql.tmp_resources: + aggregation_strategy: sum + attributes: + - resource + enabled: true + mysql.uptime: + enabled: true + passfile: sampleConfig/opentelemetry/testdata/.mysql_credentials + query_sample_collection: + collection_interval: 1m0s + max_rows_per_query: 500 + resource_attributes: + mysql.instance.endpoint: + enabled: true + service.instance.id: + enabled: true + statement_events: + digest_text_limit: 120 + limit: 250 + time_limit: 24h0m0s + timeout: 0s + tls: + insecure: true + top_query_collection: + collection_interval: 1m0s + lookback_time: 60 + max_query_sample_count: 5000 + query_plan_cache_size: 1000 + query_plan_cache_ttl: 1h0m0s + top_query_count: 200 + transport: tcp + username: cw_monitor +service: + extensions: + - sigv4auth/monitoring + - agenthealth/opentelemetry_metrics + - sigv4auth/logs + - awscloudwatchlogsprovisioner + - headers_setter/logs + - agenthealth/opentelemetry_logs + pipelines: + logs/dbi_mysql_0: + exporters: + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + processors: + - filter/dbi_exclude_monitor_mysql_0 + receivers: + - mysql/metrics_0 + logs/dbi_mysql_rawevents_0: + exporters: + - forward/opentelemetry + processors: + - filter/dbi_exclude_monitor_mysql_0 + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_raw-events_0 + receivers: + - mysql/events_0 + logs/dbi_mysql_serverlogs_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - resourcedetection/opentelemetry + - transform/dbi_resource_mysql_0 + - transform/dbi_logs_mysql_server-logs_0 + receivers: + - filelog/mysql_0 + logs/opentelemetry: + exporters: + - otlphttp/logs + processors: + - resourcedetection/opentelemetry + - transform/identity + - transform/logs_routing + - attributestocontext/opentelemetry + - transform/logs_cleanup + - batch/opentelemetry_logs + receivers: + - forward/opentelemetry + metrics/dbi_mysql_0: + exporters: + - forward/opentelemetry + processors: + - transform/dbi_scope_mysql_0 + - transform/dbi_resource_mysql_0 + - transform/dbi_fix_start_time_mysql + receivers: + - mysql/metrics_0 + - count/dbi_dbload_mysql + - signaltometrics/dbi_topsql_mysql + metrics/host_metrics: + exporters: + - forward/opentelemetry + processors: + - transform/host_metrics_scope + receivers: + - hostmetrics/opentelemetry + metrics/opentelemetry: + exporters: + - otlphttp/metrics + processors: + - resourcedetection/opentelemetry + - transform/identity + - batch/opentelemetry_metrics + receivers: + - forward/opentelemetry + telemetry: + logs: + encoding: console + level: info + output_paths: + - /opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log + sampling: + enabled: true + initial: 2 + thereafter: 500 + tick: 10s + metrics: + level: None + traces: + level: None diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf new file mode 100644 index 00000000000..9d2168ce61a --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.conf @@ -0,0 +1,15 @@ +[agent] + collection_jitter = "0s" + debug = false + flush_interval = "1s" + flush_jitter = "0s" + hostname = "" + interval = "60s" + logfile = "/opt/aws/amazon-cloudwatch-agent/logs/amazon-cloudwatch-agent.log" + logtarget = "lumberjack" + metric_batch_size = 1000 + metric_buffer_limit = 10000 + omit_hostname = false + precision = "" + quiet = false + round_interval = false diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json similarity index 100% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.json rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.json diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml similarity index 97% rename from translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml rename to translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml index 33db4312e1d..a236a01e615 100644 --- a/translator/tocwconfig/sampleConfig/opentelemetry/dbi_config_linux.yaml +++ b/translator/tocwconfig/sampleConfig/opentelemetry/dbi_postgresql_config_linux.yaml @@ -1,5 +1,5 @@ connectors: - count/dbi_dbload: + count/dbi_dbload_postgresql: datapoints: metric.datapoint.count: description: The number of data points observed. @@ -79,7 +79,7 @@ connectors: trace.span.count: description: The number of spans observed. forward/opentelemetry: {} - signaltometrics/dbi_topsql: + signaltometrics/dbi_topsql_postgresql: logs: - attributes: - key: db.system.name @@ -339,11 +339,6 @@ processors: key: aws.log.group.name - from_resource_attribute: aws.log.stream.name key: aws.log.stream.name - batch/opentelemetry_metrics: - metadata_cardinality_limit: 1000 - send_batch_max_size: 1000 - send_batch_size: 1000 - timeout: 15s batch/opentelemetry_logs: metadata_cardinality_limit: 1000 metadata_keys: @@ -352,7 +347,12 @@ processors: send_batch_max_size: 10000 send_batch_size: 10000 timeout: 15s - filter/dbi_exclude_monitor_0: + batch/opentelemetry_metrics: + metadata_cardinality_limit: 1000 + send_batch_max_size: 1000 + send_batch_size: 1000 + timeout: 15s + filter/dbi_exclude_monitor_postgresql_0: error_mode: propagate logs: log_record: @@ -640,7 +640,7 @@ processors: os.version: enabled: false timeout: 2s - transform/dbi_fix_start_time: + transform/dbi_fix_start_time_postgresql: error_mode: ignore flatten_data: false log_statements: [] @@ -656,7 +656,7 @@ processors: - replace_match(datapoint.attributes["postgresql.query_id"], "", "unknown") - replace_match(datapoint.attributes["user.name"], "", "unknown") trace_statements: [] - transform/dbi_logs_raw-events_0: + transform/dbi_logs_postgresql_raw-events_0: error_mode: propagate flatten_data: false log_statements: @@ -667,7 +667,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_logs_server-logs_0: + transform/dbi_logs_postgresql_server-logs_0: error_mode: propagate flatten_data: false log_statements: @@ -678,7 +678,7 @@ processors: - set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "pentest-postgres"], "/")) metric_statements: [] trace_statements: [] - transform/dbi_resource_0: + transform/dbi_resource_postgresql_0: error_mode: propagate flatten_data: false log_statements: @@ -694,7 +694,7 @@ processors: - set(resource.attributes["db.system.name"], "postgresql") - set(resource.attributes["db.instance.name"], "pentest-postgres") trace_statements: [] - transform/dbi_scope_0: + transform/dbi_scope_postgresql_0: error_mode: ignore flatten_data: false log_statements: @@ -1098,31 +1098,31 @@ service: pipelines: logs/dbi_postgresql_0: exporters: - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql processors: - - filter/dbi_exclude_monitor_0 + - filter/dbi_exclude_monitor_postgresql_0 receivers: - postgresql/metrics_0 logs/dbi_postgresql_rawevents_0: exporters: - forward/opentelemetry processors: - - filter/dbi_exclude_monitor_0 - - transform/dbi_scope_0 + - filter/dbi_exclude_monitor_postgresql_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_raw-events_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_raw-events_0 receivers: - postgresql/events_0 logs/dbi_postgresql_serverlogs_0: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 + - transform/dbi_scope_postgresql_0 - resourcedetection/opentelemetry - - transform/dbi_resource_0 - - transform/dbi_logs_server-logs_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_logs_postgresql_server-logs_0 receivers: - filelog/postgresql_0 logs/opentelemetry: @@ -1141,13 +1141,13 @@ service: exporters: - forward/opentelemetry processors: - - transform/dbi_scope_0 - - transform/dbi_resource_0 - - transform/dbi_fix_start_time + - transform/dbi_scope_postgresql_0 + - transform/dbi_resource_postgresql_0 + - transform/dbi_fix_start_time_postgresql receivers: - postgresql/metrics_0 - - count/dbi_dbload - - signaltometrics/dbi_topsql + - count/dbi_dbload_postgresql + - signaltometrics/dbi_topsql_postgresql metrics/host_metrics: exporters: - forward/opentelemetry diff --git a/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials new file mode 100644 index 00000000000..63df2f15ccd --- /dev/null +++ b/translator/tocwconfig/sampleConfig/opentelemetry/testdata/.mysql_credentials @@ -0,0 +1,5 @@ +[client] +host=localhost +port=3306 +user=cw_monitor +password=test_password diff --git a/translator/tocwconfig/tocwconfig_unix_test.go b/translator/tocwconfig/tocwconfig_unix_test.go index ee5d7022c37..5b3a292482c 100644 --- a/translator/tocwconfig/tocwconfig_unix_test.go +++ b/translator/tocwconfig/tocwconfig_unix_test.go @@ -104,7 +104,7 @@ func TestDBIConfigLinux(t *testing.T) { context.CurrentContext().SetMode(config.ModeEC2) require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.pgpass", 0600)) expectedEnvVars := map[string]string{} - checkTranslation(t, "opentelemetry/dbi_config_linux", "linux", expectedEnvVars, "") + checkTranslation(t, "opentelemetry/dbi_postgresql_config_linux", "linux", expectedEnvVars, "") } func TestJournaldLogsUnits(t *testing.T) { @@ -183,3 +183,11 @@ func TestCombinedV1V2EKSConfig(t *testing.T) { }) assert.Empty(t, cmp.Diff(expected, actual, opt)) } + +func TestDBIMySQLConfigLinux(t *testing.T) { + resetContext(t) + context.CurrentContext().SetMode(config.ModeEC2) + require.NoError(t, os.Chmod("sampleConfig/opentelemetry/testdata/.mysql_credentials", 0600)) + expectedEnvVars := map[string]string{} + checkTranslation(t, "opentelemetry/dbi_mysql_config_linux", "linux", expectedEnvVars, "") +} diff --git a/translator/translate/otel/common/common.go b/translator/translate/otel/common/common.go index ad112c23c86..541844c1c92 100644 --- a/translator/translate/otel/common/common.go +++ b/translator/translate/otel/common/common.go @@ -140,8 +140,9 @@ const ( const ( DatabaseInsightsKey = "database_insights" PostgreSQLKey = "postgresql" + MySQLKey = "mysql" - // DBI component names + // DBI component name prefixes (shared across engines; index-suffixed per instance) DbiTransformFixStartTime = "dbi_fix_start_time" DbiTransformResource = "dbi_resource" DbiTransformLogs = "dbi_logs" @@ -159,6 +160,7 @@ const ( var ( DatabaseInsightsConfigKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey) DatabaseInsightsPostgresKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, PostgreSQLKey) + DatabaseInsightsMysqlKey = ConfigKey(OpenTelemetryKey, CollectKey, DatabaseInsightsKey, MySQLKey) ) const ( diff --git a/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml new file mode 100644 index 00000000000..0e0f381c1b4 --- /dev/null +++ b/translator/translate/otel/connector/count/dbi_dbload_mysql.yaml @@ -0,0 +1,56 @@ +logs: + mysql.active_sessions.by_wait: + description: "Active sessions by wait event" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_user: + description: "Active sessions by user" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: user.name + - key: db.namespace + mysql.active_sessions.by_db: + description: "Active sessions by database" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: db.namespace + mysql.active_sessions.by_sql: + description: "Active sessions by query digest" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: db.namespace + mysql.active_sessions.by_sql_wait: + description: "Active sessions by query and wait" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: mysql.events_statements_current.digest + - key: mysql.wait_type + - key: db.namespace + mysql.active_sessions.by_host: + description: "Active sessions by host" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: network.peer.address + - key: db.namespace + mysql.active_sessions.count: + description: "Total active sessions" + conditions: + - attributes["mysql.threads.processlist_command"] == "Query" + attributes: + - key: db.system.name + - key: db.namespace diff --git a/translator/translate/otel/connector/count/dbi_dbload.yaml b/translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/count/dbi_dbload.yaml rename to translator/translate/otel/connector/count/dbi_dbload_postgresql.yaml diff --git a/translator/translate/otel/connector/count/translator.go b/translator/translate/otel/connector/count/translator.go index ad803f43865..1ab0e54c0e4 100644 --- a/translator/translate/otel/connector/count/translator.go +++ b/translator/translate/otel/connector/count/translator.go @@ -15,20 +15,26 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_dbload.yaml +//go:embed dbi_dbload_postgresql.yaml var dbiDbloadConfig string +//go:embed dbi_dbload_mysql.yaml +var dbiDbloadMysqlConfig string + type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a count connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a count connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: countconnector.NewFactory(), } } @@ -40,9 +46,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*countconnector.Config) - if t.name == common.DbiConnectorDbload { + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiDbloadConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, dbiDbloadMysqlConfig) } - return nil, fmt.Errorf("unsupported count connector config: %s", t.name) + return nil, fmt.Errorf("unsupported count connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/count/translator_test.go b/translator/translate/otel/connector/count/translator_test.go index 0721d3de662..275af49dd61 100644 --- a/translator/translate/otel/connector/count/translator_test.go +++ b/translator/translate/otel/connector/count/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) - assert.Equal(t, "count/dbi_dbload", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "count/dbi_dbload_postgresql", tr.ID().String()) } func TestTranslateDbload(t *testing.T) { - tr := NewTranslator(common.DbiConnectorDbload) + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,8 +36,28 @@ func TestTranslateDbload(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_dbload_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported count connector config") + assert.Contains(t, err.Error(), "unsupported count connector engine") +} + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) + assert.Equal(t, "count/dbi_dbload_mysql", tr.ID().String()) +} + +func TestTranslateDbloadMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorDbload+"_"+common.MySQLKey, common.MySQLKey) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + countCfg := cfg.(*countconnector.Config) + assert.Len(t, countCfg.Logs, 7) + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_user") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_db") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_sql_wait") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.by_host") + assert.Contains(t, countCfg.Logs, "mysql.active_sessions.count") } diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml new file mode 100644 index 00000000000..bf3bb3131e0 --- /dev/null +++ b/translator/translate/otel/connector/signaltometrics/dbi_topsql_mysql.yaml @@ -0,0 +1,169 @@ +logs: + - name: mysql.count_star + include_resource_attributes: + - key: service.instance.id + description: "SQL execution count" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.count_star"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.count_star"])' + - name: mysql.sum_timer_wait + include_resource_attributes: + - key: service.instance.id + description: "Total execution time in picoseconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_timer_wait"])' + - name: mysql.sum_lock_time + include_resource_attributes: + - key: service.instance.id + description: "Total lock time in seconds" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_lock_time"] != nil + sum: + value: 'Double(attributes["mysql.events_statements_summary_by_digest.sum_lock_time"])' + - name: mysql.sum_rows_sent + include_resource_attributes: + - key: service.instance.id + description: "Rows sent to client" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_sent"])' + - name: mysql.sum_rows_examined + include_resource_attributes: + - key: service.instance.id + description: "Rows examined" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_rows_examined"])' + - name: mysql.sum_errors + include_resource_attributes: + - key: service.instance.id + description: "Total errors" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_errors"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_errors"])' + - name: mysql.sum_sort_rows + include_resource_attributes: + - key: service.instance.id + description: "Rows sorted" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_rows"])' + - name: mysql.sum_created_tmp_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_tables"])' + - name: mysql.sum_created_tmp_disk_tables + include_resource_attributes: + - key: service.instance.id + description: "Temporary disk tables created" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_created_tmp_disk_tables"])' + - name: mysql.sum_no_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_index_used"])' + - name: mysql.sum_select_full_join + include_resource_attributes: + - key: service.instance.id + description: "Full joins without index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_full_join"])' + - name: mysql.sum_sort_scan + include_resource_attributes: + - key: service.instance.id + description: "Sorts done via full scan" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_sort_scan"])' + - name: mysql.sum_no_good_index_used + include_resource_attributes: + - key: service.instance.id + description: "Queries using no good index" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_no_good_index_used"])' + - name: mysql.sum_select_scan + include_resource_attributes: + - key: service.instance.id + description: "Full table scans" + attributes: + - key: db.system.name + - key: mysql.events_statements_summary_by_digest.digest + - key: db.namespace + conditions: + - attributes["mysql.events_statements_summary_by_digest.sum_select_scan"] != nil + sum: + value: 'Int(attributes["mysql.events_statements_summary_by_digest.sum_select_scan"])' diff --git a/translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml b/translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml similarity index 100% rename from translator/translate/otel/connector/signaltometrics/dbi_topsql.yaml rename to translator/translate/otel/connector/signaltometrics/dbi_topsql_postgresql.yaml diff --git a/translator/translate/otel/connector/signaltometrics/translator.go b/translator/translate/otel/connector/signaltometrics/translator.go index 313b2f8f71e..a26fbe4870f 100644 --- a/translator/translate/otel/connector/signaltometrics/translator.go +++ b/translator/translate/otel/connector/signaltometrics/translator.go @@ -16,20 +16,26 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" ) -//go:embed dbi_topsql.yaml +//go:embed dbi_topsql_postgresql.yaml var dbiTopsqlConfig string +//go:embed dbi_topsql_mysql.yaml +var dbiTopsqlMysqlConfig string + type translator struct { name string + engine string factory connector.Factory } var _ common.ComponentTranslator = (*translator)(nil) -// NewTranslator creates a signaltometrics connector translator. The name determines which config to load. -func NewTranslator(name string) common.ComponentTranslator { +// NewTranslator creates a signaltometrics connector translator. The name sets the +// component ID; the engine ("postgresql" or "mysql") selects which config to load. +func NewTranslator(name string, engine string) common.ComponentTranslator { return &translator{ name: name, + engine: engine, factory: signaltometricsconnector.NewFactory(), } } @@ -41,9 +47,12 @@ func (t *translator) ID() component.ID { func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { cfg := t.factory.CreateDefaultConfig().(*signaltometricsconfig.Config) - if t.name == common.DbiConnectorTopsql { + switch t.engine { + case common.PostgreSQLKey: return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, dbiTopsqlMysqlConfig) } - return nil, fmt.Errorf("unsupported signaltometrics connector config: %s", t.name) + return nil, fmt.Errorf("unsupported signaltometrics connector engine: %s", t.engine) } diff --git a/translator/translate/otel/connector/signaltometrics/translator_test.go b/translator/translate/otel/connector/signaltometrics/translator_test.go index bcb37f40802..e7a8b68fe49 100644 --- a/translator/translate/otel/connector/signaltometrics/translator_test.go +++ b/translator/translate/otel/connector/signaltometrics/translator_test.go @@ -14,12 +14,12 @@ import ( ) func TestTranslatorID(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) - assert.Equal(t, "signaltometrics/dbi_topsql", tr.ID().String()) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) + assert.Equal(t, "signaltometrics/dbi_topsql_postgresql", tr.ID().String()) } func TestTranslateTopsql(t *testing.T) { - tr := NewTranslator(common.DbiConnectorTopsql) + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.PostgreSQLKey, common.PostgreSQLKey) cfg, err := tr.Translate(nil) require.NoError(t, err) require.NotNil(t, cfg) @@ -36,8 +36,35 @@ func TestTranslateTopsql(t *testing.T) { } func TestTranslateUnsupported(t *testing.T) { - tr := NewTranslator("unsupported") + tr := NewTranslator("dbi_topsql_unknown", "unsupported") _, err := tr.Translate(nil) require.Error(t, err) - assert.Contains(t, err.Error(), "unsupported signaltometrics connector config") + assert.Contains(t, err.Error(), "unsupported signaltometrics connector engine") +} + +func TestTranslatorID_MySQL(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) + assert.Equal(t, "signaltometrics/dbi_topsql_mysql", tr.ID().String()) +} + +func TestTranslateTopsqlMysql(t *testing.T) { + tr := NewTranslator(common.DbiConnectorTopsql+"_"+common.MySQLKey, common.MySQLKey) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + stmCfg := cfg.(*signaltometricsconfig.Config) + assert.Len(t, stmCfg.Logs, 14) + assert.Equal(t, "mysql.count_star", stmCfg.Logs[0].Name) + assert.Equal(t, "mysql.sum_timer_wait", stmCfg.Logs[1].Name) + assert.Equal(t, "mysql.sum_lock_time", stmCfg.Logs[2].Name) + assert.Equal(t, "mysql.sum_rows_sent", stmCfg.Logs[3].Name) + assert.Equal(t, "mysql.sum_rows_examined", stmCfg.Logs[4].Name) + assert.Equal(t, "mysql.sum_errors", stmCfg.Logs[5].Name) + assert.Equal(t, "mysql.sum_sort_rows", stmCfg.Logs[6].Name) + assert.Equal(t, "mysql.sum_created_tmp_tables", stmCfg.Logs[7].Name) + assert.Equal(t, "mysql.sum_created_tmp_disk_tables", stmCfg.Logs[8].Name) + assert.Equal(t, "mysql.sum_no_index_used", stmCfg.Logs[9].Name) + assert.Equal(t, "mysql.sum_select_full_join", stmCfg.Logs[10].Name) + assert.Equal(t, "mysql.sum_sort_scan", stmCfg.Logs[11].Name) + assert.Equal(t, "mysql.sum_no_good_index_used", stmCfg.Logs[12].Name) + assert.Equal(t, "mysql.sum_select_scan", stmCfg.Logs[13].Name) } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go index cfd42095321..f531ca9ca5e 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator.go @@ -20,6 +20,7 @@ import ( "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/resourcedetection" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/processor/transformprocessor" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/filelog" + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/mysql" "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/receiver/postgresql" ) @@ -32,6 +33,10 @@ const ( dbiServerLogs ) +// dbiTranslator generates DBI pipelines for a single database instance. The +// engine (carried on cfg) selects engine-specific receivers, connector configs, +// resource attributes, and log group paths. Component IDs are index-based so +// PostgreSQL and MySQL instances share one consistent naming scheme. type dbiTranslator struct { pipelineType dbiPipelineType instanceIndex int @@ -44,13 +49,13 @@ func (t *dbiTranslator) ID() pipeline.ID { idx := strconv.Itoa(t.instanceIndex) switch t.pipelineType { case dbiMetrics: - return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalMetrics, "dbi_"+t.cfg.engine+"_"+idx) case dbiLogToMetrics: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_"+idx) case dbiRawEvents: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_rawevents_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_rawevents_"+idx) case dbiServerLogs: - return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_postgresql_serverlogs_"+idx) + return pipeline.NewIDWithName(pipeline.SignalLogs, "dbi_"+t.cfg.engine+"_serverlogs_"+idx) } return pipeline.NewID(pipeline.SignalMetrics) } @@ -78,12 +83,12 @@ func (t *dbiTranslator) Translate(_ *confmap.Conf) (*common.ComponentTranslators func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) { idx := strconv.Itoa(t.instanceIndex) fwd := forward.NewTranslator(common.OpenTelemetryKey) - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("metrics"), countConn, s2mConn), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime)), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("metrics"), countConn, s2mConn), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+t.cfg.engine, transformprocessor.WithDbiFixStartTime(t.cfg.engine))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd, countConn, s2mConn), @@ -91,11 +96,11 @@ func (t *dbiTranslator) translateMetrics() (*common.ComponentTranslators, error) } func (t *dbiTranslator) translateLogToMetrics() (*common.ComponentTranslators, error) { - countConn := count.NewTranslator(common.DbiConnectorDbload) - s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql) + countConn := count.NewTranslator(common.DbiConnectorDbload+"_"+t.cfg.engine, t.cfg.engine) + s2mConn := signaltometrics.NewTranslator(common.DbiConnectorTopsql+"_"+t.cfg.engine, t.cfg.engine) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("metrics")), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("metrics")), Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter()), Exporters: common.NewTranslatorMap[component.Config, component.ID](countConn, s2mConn), Extensions: common.NewTranslatorMap[component.Config, component.ID](), @@ -108,8 +113,8 @@ func (t *dbiTranslator) translateRawEvents() (*common.ComponentTranslators, erro fwd := forward.NewTranslator(common.OpenTelemetryKey) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](t.pgReceiver("events", postgresql.WithQuerySampleInterval(60*time.Second))), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), + Receivers: common.NewTranslatorMap[component.Config, component.ID](t.receiver("events")), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.excludeMonitorFilter(), t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_"+t.cfg.engine+"_raw-events_"+idx, transformprocessor.WithLogStatements(t.logStatements("raw-events")))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), @@ -121,15 +126,27 @@ func (t *dbiTranslator) translateServerLogs() (*common.ComponentTranslators, err fwd := forward.NewTranslator(common.OpenTelemetryKey) return &common.ComponentTranslators{ - Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix("postgresql"), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), - Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), + Receivers: common.NewTranslatorMap[component.Config, component.ID](filelog.NewTranslator(filelog.WithNamePrefix(t.cfg.engine), filelog.WithIndex(t.instanceIndex), filelog.WithFilePath(t.cfg.logFilePath))), + Processors: common.NewTranslatorMap[component.Config, component.ID](t.scopeTransform(), resourcedetection.NewTranslator(resourcedetection.WithName(common.OpenTelemetryKey)), transformprocessor.NewTranslatorWithName(common.DbiTransformResource+"_"+t.cfg.engine+"_"+idx, transformprocessor.WithMetricStatements(t.resourceStatements()), transformprocessor.WithLogStatements(t.resourceStatements())), transformprocessor.NewTranslatorWithName(common.DbiTransformLogs+"_"+t.cfg.engine+"_server-logs_"+idx, transformprocessor.WithLogStatements(t.logStatements("server-logs")))), Exporters: common.NewTranslatorMap[component.Config, component.ID](fwd), Extensions: common.NewTranslatorMap[component.Config, component.ID](), Connectors: common.NewTranslatorMap[component.Config, component.ID](fwd), }, nil } -func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) common.ComponentTranslator { +// receiver builds the engine-specific receiver translator. name is "metrics" or +// "events"; for PostgreSQL events we override the query sample interval. +func (t *dbiTranslator) receiver(name string) common.ComponentTranslator { + if t.cfg.engine == common.MySQLKey { + return mysql.NewTranslator( + mysql.WithName(name), + mysql.WithIndex(t.instanceIndex), + mysql.WithEndpoint(t.cfg.endpoint), + mysql.WithUsername(t.cfg.username), + mysql.WithPassfile(t.cfg.passfile), + ) + } + opts := []postgresql.Option{ postgresql.WithName(name), postgresql.WithIndex(t.instanceIndex), @@ -139,19 +156,24 @@ func (t *dbiTranslator) pgReceiver(name string, extraOpts ...postgresql.Option) postgresql.WithCAFile(t.cfg.caFile), postgresql.WithIsLocalhost(t.cfg.isLocalhost), } - opts = append(opts, extraOpts...) + if name == "events" { + opts = append(opts, postgresql.WithQuerySampleInterval(60*time.Second)) + } return postgresql.NewTranslator(opts...) } func (t *dbiTranslator) excludeMonitorFilter() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - condition := fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) - return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+idx, condition) + condition := fmt.Sprintf(`attributes["user.name"] == "%s"`, t.cfg.username) + if t.cfg.engine == common.PostgreSQLKey { + condition = fmt.Sprintf(`attributes["user.name"] == "%s" or attributes["postgresql.rolname"] == "%s"`, t.cfg.username, t.cfg.username) + } + return filterprocessor.NewTranslatorWithLogCondition(common.DbiFilterExcludeMonitor+"_"+t.cfg.engine+"_"+idx, condition) } func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { idx := strconv.Itoa(t.instanceIndex) - return transformprocessor.NewTranslatorWithName("dbi_scope_"+idx, + return transformprocessor.NewTranslatorWithName("dbi_scope_"+t.cfg.engine+"_"+idx, transformprocessor.WithErrorMode("ignore"), transformprocessor.WithScopeStatements([]string{ `set(attributes["cloudwatch.source"], "cloudwatch-agent")`, @@ -162,14 +184,14 @@ func (t *dbiTranslator) scopeTransform() common.ComponentTranslator { func (t *dbiTranslator) resourceStatements() []string { return []string{ - `set(resource.attributes["db.system.name"], "postgresql")`, + fmt.Sprintf(`set(resource.attributes["db.system.name"], "%s")`, t.cfg.engine), fmt.Sprintf(`set(resource.attributes["db.instance.name"], "%s")`, t.cfg.instanceName), } } func (t *dbiTranslator) logStatements(destination string) []string { return []string{ - fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/postgresql/%s")`, destination), + fmt.Sprintf(`set(resource.attributes["aws.log.group.name"], "/aws/self-managed-database-insights/%s/%s")`, t.cfg.engine, destination), fmt.Sprintf(`set(resource.attributes["aws.log.stream.name"], Concat([resource.attributes["host.id"], "%s"], "/"))`, t.cfg.instanceName), } } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go index 04d913311fe..98ac0daa258 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translator_test.go @@ -16,20 +16,25 @@ import ( func TestDbiTranslatorID(t *testing.T) { tests := []struct { name string + engine string pipelineType dbiPipelineType index int want string }{ - {"Metrics_0", dbiMetrics, 0, "metrics/dbi_postgresql_0"}, - {"Metrics_1", dbiMetrics, 1, "metrics/dbi_postgresql_1"}, - {"LogToMetrics_0", dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, - {"RawEvents_0", dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, - {"ServerLogs_0", dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, - {"ServerLogs_2", dbiServerLogs, 2, "logs/dbi_postgresql_serverlogs_2"}, + {"PG_Metrics_0", common.PostgreSQLKey, dbiMetrics, 0, "metrics/dbi_postgresql_0"}, + {"PG_Metrics_1", common.PostgreSQLKey, dbiMetrics, 1, "metrics/dbi_postgresql_1"}, + {"PG_LogToMetrics_0", common.PostgreSQLKey, dbiLogToMetrics, 0, "logs/dbi_postgresql_0"}, + {"PG_RawEvents_0", common.PostgreSQLKey, dbiRawEvents, 0, "logs/dbi_postgresql_rawevents_0"}, + {"PG_ServerLogs_0", common.PostgreSQLKey, dbiServerLogs, 0, "logs/dbi_postgresql_serverlogs_0"}, + {"Mysql_Metrics_0", common.MySQLKey, dbiMetrics, 0, "metrics/dbi_mysql_0"}, + {"Mysql_Metrics_1", common.MySQLKey, dbiMetrics, 1, "metrics/dbi_mysql_1"}, + {"Mysql_LogToMetrics_0", common.MySQLKey, dbiLogToMetrics, 0, "logs/dbi_mysql_0"}, + {"Mysql_RawEvents_0", common.MySQLKey, dbiRawEvents, 0, "logs/dbi_mysql_rawevents_0"}, + {"Mysql_ServerLogs_0", common.MySQLKey, dbiServerLogs, 0, "logs/dbi_mysql_serverlogs_0"}, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index} + tr := &dbiTranslator{pipelineType: tc.pipelineType, instanceIndex: tc.index, cfg: dbiInstanceConfig{engine: tc.engine}} assert.Equal(t, tc.want, tr.ID().String()) }) } @@ -37,6 +42,7 @@ func TestDbiTranslatorID(t *testing.T) { func TestDbiTranslate(t *testing.T) { cfg := dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: "localhost:5432", username: "cw_monitor", passfile: "/etc/.pgpass", @@ -79,7 +85,7 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { tr := &dbiTranslator{ pipelineType: dbiMetrics, instanceIndex: 0, - cfg: dbiInstanceConfig{instanceName: "mydb"}, + cfg: dbiInstanceConfig{engine: common.PostgreSQLKey, instanceName: "mydb"}, } result, err := tr.Translate(nil) require.NoError(t, err) @@ -98,8 +104,78 @@ func TestDbiTranslateMetrics_ComponentIDs(t *testing.T) { connectors = append(connectors, c.ID().String()) }) - assert.ElementsMatch(t, []string{"postgresql/metrics_0", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, receivers) - assert.Equal(t, []string{"transform/dbi_scope_0", "transform/dbi_resource_0", "transform/dbi_fix_start_time"}, processors) + assert.ElementsMatch(t, []string{"postgresql/metrics_0", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_postgresql_0", "transform/dbi_resource_postgresql_0", "transform/dbi_fix_start_time_postgresql"}, processors) assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) - assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload", "signaltometrics/dbi_topsql"}, connectors) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_postgresql", "signaltometrics/dbi_topsql_postgresql"}, connectors) +} + +func TestDbiMysqlTranslate(t *testing.T) { + cfg := dbiInstanceConfig{ + engine: common.MySQLKey, + endpoint: "localhost:3306", + username: "cw_monitor", + passfile: "/etc/.mysql_credentials", + instanceName: "my-db", + logFilePath: "/var/log/mysql/mysql.log", + isLocalhost: true, + } + + tests := []struct { + name string + pipeline dbiPipelineType + expectedID string + nRecv int + nProc int + nExp int + nConn int + }{ + {"metrics", dbiMetrics, "metrics/dbi_mysql_0", 3, 3, 1, 3}, + {"log_to_metrics", dbiLogToMetrics, "logs/dbi_mysql_0", 1, 1, 2, 2}, + {"raw_events", dbiRawEvents, "logs/dbi_mysql_rawevents_0", 1, 5, 1, 1}, + {"server_logs", dbiServerLogs, "logs/dbi_mysql_serverlogs_0", 1, 4, 1, 1}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tr := &dbiTranslator{pipelineType: tc.pipeline, instanceIndex: 0, cfg: cfg} + assert.Equal(t, tc.expectedID, tr.ID().String()) + + result, err := tr.Translate(nil) + require.NoError(t, err) + assert.Equal(t, tc.nRecv, result.Receivers.Len()) + assert.Equal(t, tc.nProc, result.Processors.Len()) + assert.Equal(t, tc.nExp, result.Exporters.Len()) + assert.Equal(t, tc.nConn, result.Connectors.Len()) + }) + } +} + +func TestDbiMysqlTranslateMetrics_ComponentIDs(t *testing.T) { + tr := &dbiTranslator{ + pipelineType: dbiMetrics, + instanceIndex: 0, + cfg: dbiInstanceConfig{engine: common.MySQLKey, instanceName: "mydb"}, + } + result, err := tr.Translate(nil) + require.NoError(t, err) + + var receivers, processors, exporters, connectors []string + result.Receivers.Range(func(c common.Translator[component.Config, component.ID]) { + receivers = append(receivers, c.ID().String()) + }) + result.Processors.Range(func(c common.Translator[component.Config, component.ID]) { + processors = append(processors, c.ID().String()) + }) + result.Exporters.Range(func(c common.Translator[component.Config, component.ID]) { + exporters = append(exporters, c.ID().String()) + }) + result.Connectors.Range(func(c common.Translator[component.Config, component.ID]) { + connectors = append(connectors, c.ID().String()) + }) + + assert.ElementsMatch(t, []string{"mysql/metrics_0", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, receivers) + assert.Equal(t, []string{"transform/dbi_scope_mysql_0", "transform/dbi_resource_mysql_0", "transform/dbi_fix_start_time_mysql"}, processors) + assert.ElementsMatch(t, []string{"forward/opentelemetry"}, exporters) + assert.ElementsMatch(t, []string{"forward/opentelemetry", "count/dbi_dbload_mysql", "signaltometrics/dbi_topsql_mysql"}, connectors) } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go index bed6e035827..7d6c9a66a60 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators.go @@ -20,6 +20,7 @@ import ( var ottlSafeRegex = regexp.MustCompile(`^[a-zA-Z0-9._@/:\-]+$`) type dbiInstanceConfig struct { + engine string endpoint string username string passfile string @@ -35,15 +36,22 @@ func NewTranslators(conf *confmap.Conf) common.PipelineTranslatorMap { return translators } - instances := parseDbiPostgresqlInstances(conf) - for i, cfg := range instances { - translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) - translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) - if cfg.logFilePath != "" { - translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + // Each engine is indexed independently; the engine name is part of every + // component ID, so PostgreSQL and MySQL instances never collide. + addInstances := func(instances []dbiInstanceConfig) { + for i, cfg := range instances { + translators.Set(&dbiTranslator{pipelineType: dbiMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiLogToMetrics, instanceIndex: i, cfg: cfg}) + translators.Set(&dbiTranslator{pipelineType: dbiRawEvents, instanceIndex: i, cfg: cfg}) + if cfg.logFilePath != "" { + translators.Set(&dbiTranslator{pipelineType: dbiServerLogs, instanceIndex: i, cfg: cfg}) + } } } + + addInstances(parseDbiPostgresqlInstances(conf)) + addInstances(parseDbiMysqlInstances(conf)) + return translators } @@ -67,6 +75,7 @@ func parseDbiPostgresqlInstances(conf *confmap.Conf) []dbiInstanceConfig { instances := make([]dbiInstanceConfig, 0, len(raw)) for _, r := range raw { instances = append(instances, dbiInstanceConfig{ + engine: common.PostgreSQLKey, endpoint: r.Endpoint, username: r.Username, passfile: r.PasswordFile, @@ -89,8 +98,40 @@ func validateOttlSafe(field, value string) error { return nil } +type mysqlRawInstance struct { + Endpoint string `mapstructure:"endpoint"` + Username string `mapstructure:"username"` + PasswordFile string `mapstructure:"password_file"` + InstanceName string `mapstructure:"instance_name"` + Logs struct { + FilePath string `mapstructure:"file_path"` + } `mapstructure:"logs"` +} + +func parseDbiMysqlInstances(conf *confmap.Conf) []dbiInstanceConfig { + arr, _ := conf.Get(common.DatabaseInsightsMysqlKey).([]any) + var raw []mysqlRawInstance + if err := mapstructure.Decode(arr, &raw); err != nil { + return nil + } + instances := make([]dbiInstanceConfig, 0, len(raw)) + for _, r := range raw { + instances = append(instances, dbiInstanceConfig{ + engine: common.MySQLKey, + endpoint: r.Endpoint, + username: r.Username, + passfile: r.PasswordFile, + instanceName: r.InstanceName, + logFilePath: r.Logs.FilePath, + isLocalhost: isLocalhostEndpoint(r.Endpoint), + }) + } + return instances +} + func isLocalhostEndpoint(endpoint string) bool { return strings.HasPrefix(endpoint, "localhost") || strings.HasPrefix(endpoint, "127.0.0.1") || + strings.HasPrefix(endpoint, "[::1]") || strings.HasPrefix(endpoint, "::1") } diff --git a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go index 0595f19c3dd..92a3488313c 100644 --- a/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/databaseinsights/translators_test.go @@ -108,6 +108,84 @@ func TestNewTranslators_MultiInstance(t *testing.T) { func TestIsLocalhostEndpoint(t *testing.T) { assert.True(t, isLocalhostEndpoint("localhost:5432")) assert.True(t, isLocalhostEndpoint("127.0.0.1:5432")) + assert.True(t, isLocalhostEndpoint("[::1]:3306")) + assert.True(t, isLocalhostEndpoint("::1:3306")) assert.False(t, isLocalhostEndpoint("db.example.com:5432")) assert.False(t, isLocalhostEndpoint("")) } + +func TestNewTranslators_MySQL_WithLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql.log"}, + }, + }, + }, + }, + }, + }) + assert.Equal(t, 4, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_NoLogs(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "my-db", + }, + }, + }, + }, + }, + }) + assert.Equal(t, 3, NewTranslators(cfg).Len()) +} + +func TestNewTranslators_MySQL_MultiInstance(t *testing.T) { + cfg := confmap.NewFromStringMap(map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", "username": "cw_monitor", + "password_file": "/etc/.mysql_credentials", "instance_name": "db1", + "logs": map[string]interface{}{"file_path": "/var/log/mysql/mysql1.log"}, + }, + map[string]interface{}{ + "endpoint": "db2.example.com:3306", "username": "monitor", + "password_file": "/etc/.mysql_credentials2", "instance_name": "db2", + }, + }, + }, + }, + }, + }) + translators := NewTranslators(cfg) + assert.Equal(t, 7, translators.Len()) // 4 + 3 + + var ids []string + for _, k := range translators.Keys() { + ids = append(ids, k.String()) + } + expected := []string{ + "metrics/dbi_mysql_0", + "logs/dbi_mysql_0", + "logs/dbi_mysql_rawevents_0", + "logs/dbi_mysql_serverlogs_0", + "metrics/dbi_mysql_1", + "logs/dbi_mysql_1", + "logs/dbi_mysql_rawevents_1", + } + assert.ElementsMatch(t, expected, ids) +} diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go index 272cf00d231..9862de47205 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator.go @@ -38,10 +38,17 @@ func (t *hostMetricsTranslator) Translate(conf *confmap.Conf) (*common.Component var opts []hostmetrics.Option opts = append(opts, hostmetrics.WithName(common.OpenTelemetryKey)) if conf.IsSet(common.DatabaseInsightsConfigKey) { + var processNames []string + if conf.IsSet(common.DatabaseInsightsPostgresKey) { + processNames = append(processNames, "post(gres|master).*") + } + if conf.IsSet(common.DatabaseInsightsMysqlKey) { + processNames = append(processNames, "mysqld.*") + } opts = append(opts, hostmetrics.WithProcessScraper(map[string]any{ "include": map[string]any{ "match_type": "regexp", - "names": []string{"post(gres|master).*"}, + "names": processNames, }, "mute_process_all_errors": true, "metrics": map[string]any{ diff --git a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go index 4783823d18f..f0f23a2a161 100644 --- a/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/hostmetrics/translator_test.go @@ -23,9 +23,10 @@ func TestHostMetricsTranslator(t *testing.T) { assert.EqualValues(t, "metrics/host_metrics", tt.ID().String()) testCases := map[string]struct { - input map[string]interface{} - wantErr error - expectProcess bool + input map[string]interface{} + wantErr error + expectProcess bool + wantProcessNames []string }{ "WithNilConf": { input: nil, @@ -56,7 +57,41 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, + }, + "WithMysqlDatabaseInsightsKey": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"mysqld.*"}, + }, + "WithBothEngines": { + input: map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:5432", "username": "cwagent"}, + }, + "mysql": []interface{}{ + map[string]interface{}{"endpoint": "localhost:3306", "username": "cwagent"}, + }, + }, + }, + }, + }, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*", "mysqld.*"}, }, "WithBothKeys": { input: map[string]interface{}{ @@ -71,7 +106,8 @@ func TestHostMetricsTranslator(t *testing.T) { }, }, }, - expectProcess: true, + expectProcess: true, + wantProcessNames: []string{"post(gres|master).*"}, }, } for name, tc := range testCases { @@ -109,7 +145,7 @@ func TestHostMetricsTranslator(t *testing.T) { assert.True(t, exists, "expected process scraper") include := processCfg["include"].(map[string]any) assert.Equal(t, "regexp", include["match_type"]) - assert.Equal(t, []string{"postgres.*"}, include["names"]) + assert.Equal(t, tc.wantProcessNames, include["names"]) } } }) diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go index 94f645ee651..369c00932a0 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule.go @@ -10,6 +10,8 @@ import ( func init() { otelRule := mergeJsonUtil.NewSectionMergeRule("opentelemetry", "/") collectRule := mergeJsonUtil.NewSectionMergeRule("collect", otelRule.Path) + dbiRule := mergeJsonUtil.NewSectionMergeRule("database_insights", collectRule.Path) + collectRule.MergeMap[dbiRule.SectionKey] = dbiRule otelRule.MergeMap[collectRule.SectionKey] = collectRule mergeJsonUtil.MergeRuleMap[otelRule.SectionKey] = otelRule } diff --git a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go index f5918d7b181..2d6e120cf95 100644 --- a/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go +++ b/translator/translate/otel/pipeline/opentelemetry/merge_rule_test.go @@ -32,3 +32,46 @@ func TestMergeRule(t *testing.T) { assert.Contains(t, otel, "collect") assert.Contains(t, otel["collect"].(map[string]interface{}), "host_metrics") } + +func TestMergeRule_DatabaseInsights(t *testing.T) { + // Verify that postgresql and mysql from separate configs merge correctly + // under database_insights (simulates separate SSM parameters) + source1 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "postgresql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:5432", + "instance_name": "pg-test", + }, + }, + }, + }, + }, + } + source2 := map[string]interface{}{ + "opentelemetry": map[string]interface{}{ + "collect": map[string]interface{}{ + "database_insights": map[string]interface{}{ + "mysql": []interface{}{ + map[string]interface{}{ + "endpoint": "localhost:3306", + "instance_name": "mysql-test", + }, + }, + }, + }, + }, + } + + // Merge source1 into source2 (source2 acts as result) + mergeJsonUtil.MergeRuleMap["opentelemetry"].Merge(source1, source2) + + otel := source2["opentelemetry"].(map[string]interface{}) + collect := otel["collect"].(map[string]interface{}) + dbi := collect["database_insights"].(map[string]interface{}) + + assert.Contains(t, dbi, "postgresql", "postgresql should be preserved after merge") + assert.Contains(t, dbi, "mysql", "mysql should be preserved after merge") +} diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml new file mode 100644 index 00000000000..6c528ef72e5 --- /dev/null +++ b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_mysql.yaml @@ -0,0 +1,10 @@ +error_mode: ignore +metric_statements: + - context: datapoint + error_mode: ignore + statements: + - set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0 + - replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU") + - replace_match(datapoint.attributes["network.peer.address"], "", "unknown") + - replace_match(datapoint.attributes["mysql.events_statements_current.digest"], "", "unknown") + - replace_match(datapoint.attributes["user.name"], "", "unknown") diff --git a/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml b/translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml similarity index 100% rename from translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time.yaml rename to translator/translate/otel/processor/transformprocessor/transform_dbi_fix_start_time_postgresql.yaml diff --git a/translator/translate/otel/processor/transformprocessor/translate.go b/translator/translate/otel/processor/transformprocessor/translate.go index b354979ab27..14102fc8e11 100644 --- a/translator/translate/otel/processor/transformprocessor/translate.go +++ b/translator/translate/otel/processor/transformprocessor/translate.go @@ -26,9 +26,12 @@ var transformJmxDropConfig string //go:embed transform_efa_config.yaml var transformEfaConfig string -//go:embed transform_dbi_fix_start_time.yaml +//go:embed transform_dbi_fix_start_time_postgresql.yaml var transformDbiFixStartTimeConfig string +//go:embed transform_dbi_fix_start_time_mysql.yaml +var transformDbiFixStartTimeMysqlConfig string + //go:embed transform_identity_host.yaml var transformIdentityHostConfig string @@ -71,13 +74,22 @@ func WithScopeStatements(statements []string) Option { } } +// WithDbiFixStartTime selects the DBI fix-start-time config for the given engine +// ("postgresql" or "mysql"). The component is named by the caller (index-based). +func WithDbiFixStartTime(engine string) Option { + return func(t *translator) { + t.dbiFixStartTimeEngine = engine + } +} + type translator struct { - name string - factory processor.Factory - logStatements []string - metricStatements []string - scopeStatements []string - errorMode string + name string + factory processor.Factory + logStatements []string + metricStatements []string + scopeStatements []string + errorMode string + dbiFixStartTimeEngine string } var _ common.ComponentTranslator = (*translator)(nil) @@ -134,8 +146,15 @@ func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { if strings.HasPrefix(t.name, common.PipelineNameHostDeltaMetrics) { return common.GetYamlFileToYamlConfig(cfg, transformEfaConfig) } - if t.name == common.DbiTransformFixStartTime { - return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + // DBI fix-start-time: content is engine-specific, component name is index-based. + if t.dbiFixStartTimeEngine != "" { + switch t.dbiFixStartTimeEngine { + case common.PostgreSQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeConfig) + case common.MySQLKey: + return common.GetYamlFileToYamlConfig(cfg, transformDbiFixStartTimeMysqlConfig) + } + return nil, fmt.Errorf("unsupported dbi fix-start-time engine: %s", t.dbiFixStartTimeEngine) } if t.name == common.Identity { if context.CurrentContext().KubernetesMode() != "" { diff --git a/translator/translate/otel/processor/transformprocessor/translator_test.go b/translator/translate/otel/processor/transformprocessor/translator_test.go index ffbb4adaf1e..639060c2eba 100644 --- a/translator/translate/otel/processor/transformprocessor/translator_test.go +++ b/translator/translate/otel/processor/transformprocessor/translator_test.go @@ -76,8 +76,8 @@ func TestJmxTranslate(t *testing.T) { } func TestDbiFixStartTimeTranslate(t *testing.T) { - transl := NewTranslatorWithName(common.DbiTransformFixStartTime) - assert.Equal(t, "transform/dbi_fix_start_time", transl.ID().String()) + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.PostgreSQLKey, WithDbiFixStartTime(common.PostgreSQLKey)) + assert.Equal(t, "transform/dbi_fix_start_time_postgresql", transl.ID().String()) cfg, err := transl.Translate(nil) require.NoError(t, err) @@ -89,6 +89,21 @@ func TestDbiFixStartTimeTranslate(t *testing.T) { assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[6]) } +func TestDbiFixStartTimeMysqlTranslate(t *testing.T) { + transl := NewTranslatorWithName(common.DbiTransformFixStartTime+"_"+common.MySQLKey, WithDbiFixStartTime(common.MySQLKey)) + assert.Equal(t, "transform/dbi_fix_start_time_mysql", transl.ID().String()) + + cfg, err := transl.Translate(nil) + require.NoError(t, err) + actualCfg := cfg.(*transformprocessor.Config) + require.Len(t, actualCfg.MetricStatements, 1) + assert.Equal(t, "datapoint", string(actualCfg.MetricStatements[0].Context)) + require.Len(t, actualCfg.MetricStatements[0].Statements, 5) + assert.Equal(t, "set(datapoint.start_time_unix_nano, datapoint.time_unix_nano) where datapoint.start_time_unix_nano == 0", actualCfg.MetricStatements[0].Statements[0]) + assert.Equal(t, `replace_match(datapoint.attributes["mysql.wait_type"], "", "CPU")`, actualCfg.MetricStatements[0].Statements[1]) + assert.Equal(t, `replace_match(datapoint.attributes["user.name"], "", "unknown")`, actualCfg.MetricStatements[0].Statements[4]) +} + func TestDbiResourceTranslate(t *testing.T) { stmts := []string{ `set(resource.attributes["db.system.name"], "postgresql")`, diff --git a/translator/translate/otel/receiver/mysql/translator.go b/translator/translate/otel/receiver/mysql/translator.go new file mode 100644 index 00000000000..7d514df384a --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator.go @@ -0,0 +1,83 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "strconv" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/confmap" + "go.opentelemetry.io/collector/receiver" + + "github.com/aws/amazon-cloudwatch-agent/translator/translate/otel/common" +) + +type Option func(*translator) + +type translator struct { + factory receiver.Factory + name string + endpoint string + username string + passfile string + index int + topQueryInterval time.Duration +} + +func WithName(name string) Option { return func(t *translator) { t.name = name } } +func WithEndpoint(ep string) Option { return func(t *translator) { t.endpoint = ep } } +func WithUsername(u string) Option { return func(t *translator) { t.username = u } } +func WithPassfile(p string) Option { return func(t *translator) { t.passfile = p } } +func WithIndex(i int) Option { return func(t *translator) { t.index = i } } +func WithTopQueryInterval(d time.Duration) Option { + return func(t *translator) { t.topQueryInterval = d } +} + +func NewTranslator(opts ...Option) common.ComponentTranslator { + t := &translator{ + factory: mysqlreceiver.NewFactory(), + name: "metrics", + topQueryInterval: 60 * time.Second, + } + for _, opt := range opts { + opt(t) + } + return t +} + +func (t *translator) ID() component.ID { + return component.MustNewIDWithName("mysql", t.name+"_"+strconv.Itoa(t.index)) +} + +func (t *translator) Translate(_ *confmap.Conf) (component.Config, error) { + cfg := t.factory.CreateDefaultConfig().(*mysqlreceiver.Config) + + cfg.Endpoint = t.endpoint + cfg.Username = t.username + cfg.Passfile = t.passfile + cfg.Transport = confignet.TransportTypeTCP + + // DBI only supports local MySQL instances, so the connection does not use + // TLS. Remote (TLS) connections are intentionally not supported. + cfg.TLS = configtls.ClientConfig{ + Insecure: true, + } + + cfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled = true + cfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled = true + + cfg.QuerySampleCollection.MaxRowsPerQuery = 500 + + cfg.TopQueryCollection.CollectionInterval = t.topQueryInterval + cfg.TopQueryCollection.TopQueryCount = 200 + cfg.TopQueryCollection.MaxQuerySampleCount = 5000 + cfg.TopQueryCollection.QueryPlanCacheSize = 1000 + cfg.TopQueryCollection.QueryPlanCacheTTL = time.Hour + + return cfg, nil +} diff --git a/translator/translate/otel/receiver/mysql/translator_test.go b/translator/translate/otel/receiver/mysql/translator_test.go new file mode 100644 index 00000000000..587b13f02d3 --- /dev/null +++ b/translator/translate/otel/receiver/mysql/translator_test.go @@ -0,0 +1,58 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +package mysql + +import ( + "testing" + "time" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/mysqlreceiver" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslator_ID(t *testing.T) { + tr := NewTranslator(WithName("metrics"), WithIndex(0)) + assert.Equal(t, "mysql/metrics_0", tr.ID().String()) + + tr = NewTranslator(WithName("events"), WithIndex(1)) + assert.Equal(t, "mysql/events_1", tr.ID().String()) +} + +func TestTranslator_Translate_Localhost(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithIndex(0), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + mysqlCfg := cfg.(*mysqlreceiver.Config) + + assert.Equal(t, "localhost:3306", mysqlCfg.Endpoint) + assert.Equal(t, "cw_monitor", mysqlCfg.Username) + assert.Equal(t, "/etc/.mysql_credentials", mysqlCfg.Passfile) + assert.True(t, mysqlCfg.TLS.Insecure) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerQuerySample.Enabled) + assert.True(t, mysqlCfg.LogsBuilderConfig.Events.DbServerTopQuery.Enabled) + assert.Equal(t, uint64(500), mysqlCfg.QuerySampleCollection.MaxRowsPerQuery) + assert.Equal(t, uint64(200), mysqlCfg.TopQueryCollection.TopQueryCount) + assert.Equal(t, uint64(5000), mysqlCfg.TopQueryCollection.MaxQuerySampleCount) + assert.Equal(t, 1000, mysqlCfg.TopQueryCollection.QueryPlanCacheSize) + assert.Equal(t, time.Hour, mysqlCfg.TopQueryCollection.QueryPlanCacheTTL) + assert.Equal(t, 60*time.Second, mysqlCfg.TopQueryCollection.CollectionInterval) +} + +func TestTranslator_Translate_CustomInterval(t *testing.T) { + tr := NewTranslator( + WithEndpoint("localhost:3306"), + WithUsername("cw_monitor"), + WithPassfile("/etc/.mysql_credentials"), + WithTopQueryInterval(30*time.Second), + ) + cfg, err := tr.Translate(nil) + require.NoError(t, err) + assert.Equal(t, 30*time.Second, cfg.(*mysqlreceiver.Config).TopQueryCollection.CollectionInterval) +}