diff --git a/src/sources/topsql/upstream/tidb/parser.rs b/src/sources/topsql/upstream/tidb/parser.rs index ec610981..96a7b7b4 100644 --- a/src/sources/topsql/upstream/tidb/parser.rs +++ b/src/sources/topsql/upstream/tidb/parser.rs @@ -405,7 +405,10 @@ impl TopSqlSubResponseParser { &[ (LABEL_NAME, METRIC_NAME_SQL_META.to_owned()), (LABEL_SQL_DIGEST, hex::encode_upper(sql_meta.sql_digest)), - (LABEL_NORMALIZED_SQL, sql_meta.normalized_sql), + ( + LABEL_NORMALIZED_SQL, + truncate_label_value(sql_meta.normalized_sql), + ), (LABEL_IS_INTERNAL_SQL, sql_meta.is_internal_sql.to_string()), ], &[Utc::now()], @@ -444,10 +447,13 @@ impl TopSqlSubResponseParser { &[ (LABEL_NAME, METRIC_NAME_PLAN_META.to_owned()), (LABEL_PLAN_DIGEST, hex::encode_upper(plan_meta.plan_digest)), - (LABEL_NORMALIZED_PLAN, plan_meta.normalized_plan), + ( + LABEL_NORMALIZED_PLAN, + truncate_label_value(plan_meta.normalized_plan), + ), ( LABEL_ENCODED_NORMALIZED_PLAN, - plan_meta.encoded_normalized_plan, + truncate_label_value(plan_meta.encoded_normalized_plan), ), ], &[Utc::now()], diff --git a/src/sources/topsql_v2/upstream/parser.rs b/src/sources/topsql_v2/upstream/parser.rs index 5db4f974..7fa10593 100644 --- a/src/sources/topsql_v2/upstream/parser.rs +++ b/src/sources/topsql_v2/upstream/parser.rs @@ -2,6 +2,21 @@ use std::sync::Arc; use vector_lib::event::LogEvent; use crate::sources::topsql_v2::schema_cache::SchemaCache; + +pub fn truncate_label_value(s: String) -> String { + // Truncate label value if it's too long, the default limit is 16KB in vminsert. + const MAX_LABEL_LEN: usize = 16384; + if s.len() > MAX_LABEL_LEN { + let mut idx = MAX_LABEL_LEN; + while idx != 0 && !s.is_char_boundary(idx) { + idx -= 1; + } + s[..idx].to_string() + } else { + s + } +} + pub trait UpstreamEventParser { type UpstreamEvent; diff --git a/src/sources/topsql_v2/upstream/tidb/parser.rs b/src/sources/topsql_v2/upstream/tidb/parser.rs index 50d1abac..b69df344 100644 --- a/src/sources/topsql_v2/upstream/tidb/parser.rs +++ b/src/sources/topsql_v2/upstream/tidb/parser.rs @@ -14,7 +14,7 @@ use crate::sources::topsql_v2::upstream::consts::{ METRIC_NAME_TOTAL_RU, METRIC_NAME_EXEC_COUNT, METRIC_NAME_EXEC_DURATION, SOURCE_TABLE_TIDB_TOPSQL, SOURCE_TABLE_TOPSQL_PLAN_META, SOURCE_TABLE_TOPSQL_SQL_META, SOURCE_TABLE_TOPRU, }; -use crate::sources::topsql_v2::upstream::parser::UpstreamEventParser; +use crate::sources::topsql_v2::upstream::parser::{truncate_label_value, UpstreamEventParser}; use crate::sources::topsql_v2::upstream::tidb::proto::top_sql_sub_response::RespOneof; use crate::sources::topsql_v2::upstream::tidb::proto::{ PlanMeta, SqlMeta, TopSqlRecord, TopSqlRecordItem, TopSqlSubResponse, @@ -287,7 +287,10 @@ impl TopSqlSubResponseParser { log.insert(LABEL_SOURCE_TABLE, SOURCE_TABLE_TOPSQL_SQL_META); log.insert(LABEL_SQL_DIGEST, sql_digest); - log.insert(LABEL_NORMALIZED_SQL, sql_meta.normalized_sql); + log.insert( + LABEL_NORMALIZED_SQL, + truncate_label_value(sql_meta.normalized_sql.clone()), + ); let now = Utc::now(); log.insert(LABEL_TIMESTAMPS, LogValue::from(now.timestamp())); let date_str = now.format("%Y-%m-%d").to_string(); @@ -299,15 +302,19 @@ impl TopSqlSubResponseParser { fn parse_tidb_plan_meta(plan_meta: PlanMeta) -> Vec { let mut events = vec![]; let plan_digest = hex::encode_upper(plan_meta.plan_digest); - let encoded_normalized_plan = - hex::encode_upper(plan_meta.encoded_normalized_plan); + let encoded_normalized_plan = truncate_label_value(hex::encode_upper( + plan_meta.encoded_normalized_plan, + )); let mut event = Event::Log(LogEvent::default()); let log = event.as_mut_log(); // Add metadata with Vector prefix (ensure all fields have values) log.insert(LABEL_SOURCE_TABLE, SOURCE_TABLE_TOPSQL_PLAN_META); log.insert(LABEL_PLAN_DIGEST, plan_digest); - log.insert(LABEL_NORMALIZED_PLAN, plan_meta.normalized_plan); + log.insert( + LABEL_NORMALIZED_PLAN, + truncate_label_value(plan_meta.normalized_plan.clone()), + ); log.insert( LABEL_ENCODED_NORMALIZED_PLAN, encoded_normalized_plan,