diff --git a/Cargo.lock b/Cargo.lock index d23731697fa21..6ef537db506c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2419,6 +2419,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "criterion", "csv-core", "derivative", "derive_more 2.1.1", @@ -12567,6 +12568,7 @@ dependencies = [ "rand 0.9.2", "rand_distr", "regex", + "rust_decimal", "ryu", "schannel", "security-framework 3.5.1", diff --git a/changelog.d/add_decimal_type.feature.md b/changelog.d/add_decimal_type.feature.md new file mode 100644 index 0000000000000..da8bb944d558b --- /dev/null +++ b/changelog.d/add_decimal_type.feature.md @@ -0,0 +1,7 @@ +Added a new `parse_float` option to the JSON decoder. When set to `"decimal"`, floating-point +numbers are parsed as exact decimal values (up to 28-29 significant digits) instead of IEEE 754 +`f64`, which can lose precision for very large or high-precision numbers. + +Note: When using Vector-to-Vector communication (`vector` source/sink with the native codec), +all receiving instances must be upgraded to this version before enabling `parse_float: "decimal"` +on any sender. Older Vector versions will silently drop events containing decimal values. diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 90a5fd2780109..764ef66e7cd07 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -57,6 +57,7 @@ vector-vrl-functions.workspace = true toml = { version = "0.9.8", optional = true } [dev-dependencies] +criterion.workspace = true futures.workspace = true indoc.workspace = true tokio = { workspace = true, features = ["test-util"] } @@ -68,6 +69,10 @@ tracing-test = "0.2.6" uuid.workspace = true vrl.workspace = true +[[bench]] +name = "json_decimal_bench" +harness = false + [features] arrow = ["dep:arrow"] opentelemetry = ["dep:opentelemetry-proto"] diff --git a/lib/codecs/benches/json_decimal_bench.rs b/lib/codecs/benches/json_decimal_bench.rs new file mode 100644 index 0000000000000..ca51cc200bb0b --- /dev/null +++ b/lib/codecs/benches/json_decimal_bench.rs @@ -0,0 +1,133 @@ +//! Benchmark for JSON deserialization with and without decimal precision preservation. +//! +//! Run with: +//! cargo bench -p codecs --bench json_decimal_bench + +use bytes::Bytes; +use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use vector_core::config::LogNamespace; + +use codecs::decoding::format::{Deserializer, JsonDeserializer}; + +// Sample JSON payloads +const SIMPLE_JSON: &str = r#"{"message": "hello world", "level": "info", "count": 42}"#; + +const NUMERIC_JSON: &str = r#"{ + "id": 12345, + "temperature": 23.5, + "pressure": 1013.25, + "humidity": 65.0, + "wind_speed": 12.3 +}"#; + +const HIGH_PRECISION_JSON: &str = r#"{ + "measurement": 12345678901234567890.12345678901234, + "tolerance": 0.00000001234567890123, + "total_count": 999999999999999999.99 +}"#; + +const MIXED_JSON: &str = r#"{ + "user_id": "abc123", + "amount": 1234.56, + "high_precision_amount": 12345678901234567890.12, + "items": [1, 2, 3, 4, 5], + "metadata": {"key": "value"} +}"#; + +fn bench_json_deserialize(c: &mut Criterion) { + let mut group = c.benchmark_group("json_deserialize"); + + // Test cases: (name, json_payload) + let test_cases = [ + ("simple", SIMPLE_JSON), + ("numeric", NUMERIC_JSON), + ("high_precision", HIGH_PRECISION_JSON), + ("mixed", MIXED_JSON), + ]; + + for (name, json) in test_cases { + let input = Bytes::from(json); + group.throughput(Throughput::Bytes(input.len() as u64)); + + // Baseline: without decimal precision + let deserializer = + JsonDeserializer::new(false, codecs::decoding::format::ParseFloat::Float); + group.bench_with_input(BenchmarkId::new("baseline", name), &input, |b, input| { + b.iter(|| { + deserializer + .parse(input.clone(), LogNamespace::Vector) + .expect("baseline deserialization should not fail") + }) + }); + + // With decimal precision enabled + { + let deserializer_precision = + JsonDeserializer::new(false, codecs::decoding::format::ParseFloat::Decimal); + group.bench_with_input( + BenchmarkId::new("decimal_precision", name), + &input, + |b, input| { + b.iter(|| { + deserializer_precision + .parse(input.clone(), LogNamespace::Vector) + .expect("decimal precision deserialization should not fail") + }) + }, + ); + } + } + + group.finish(); +} + +fn bench_json_deserialize_batch(c: &mut Criterion) { + let mut group = c.benchmark_group("json_deserialize_batch"); + + for size in [100, 1_000, 10_000, 100_000] { + let batch: Vec = (0..size) + .map(|i| { + Bytes::from(format!( + r#"{{"id": {}, "value": 123.456, "big": 12345678901234567890.12}}"#, + i + )) + }) + .collect(); + + let total_bytes: usize = batch.iter().map(|b| b.len()).sum(); + group.throughput(Throughput::Bytes(total_bytes as u64)); + + let deserializer = + JsonDeserializer::new(false, codecs::decoding::format::ParseFloat::Float); + group.bench_function(format!("baseline_{size}"), |b| { + b.iter(|| { + for input in &batch { + let _ = deserializer + .parse(input.clone(), LogNamespace::Vector) + .expect("baseline batch deserialization should not fail"); + } + }) + }); + + let deserializer_precision = + JsonDeserializer::new(false, codecs::decoding::format::ParseFloat::Decimal); + group.bench_function(format!("decimal_precision_{size}"), |b| { + b.iter(|| { + for input in &batch { + let _ = deserializer_precision + .parse(input.clone(), LogNamespace::Vector) + .expect("decimal precision batch deserialization should not fail"); + } + }) + }); + } + + group.finish(); +} + +criterion_group!( + benches, + bench_json_deserialize, + bench_json_deserialize_batch +); +criterion_main!(benches); diff --git a/lib/codecs/src/decoding/format/json.rs b/lib/codecs/src/decoding/format/json.rs index a2f9b0c4502ff..1581cc6f69c6d 100644 --- a/lib/codecs/src/decoding/format/json.rs +++ b/lib/codecs/src/decoding/format/json.rs @@ -1,17 +1,38 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; +use serde_json::value::RawValue; use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::{ config::{DataType, LogNamespace, log_schema}, - event::Event, + event::{Event, LogEvent}, schema, }; +use vrl::core::Value; use vrl::value::Kind; use super::{Deserializer, default_lossy}; +/// Controls how JSON floating-point numbers are parsed. +#[configurable_component] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[serde(rename_all = "lowercase")] +pub enum ParseFloat { + /// Parse floating-point numbers as IEEE 754 `f64` values (default). + /// + /// This may lose precision for very large integers or decimals with many significant digits. + #[default] + Float, + + /// Parse floating-point numbers as exact decimal values using `rust_decimal::Decimal`. + /// + /// This preserves the exact string representation from the JSON source, + /// supporting up to 28-29 significant digits without rounding errors. + #[configurable(metadata(status = "beta"))] + Decimal, +} + /// Config used to build a `JsonDeserializer`. #[configurable_component] #[derive(Debug, Clone, Default)] @@ -78,6 +99,19 @@ pub struct JsonDeserializerOptions { )] #[derivative(Default(value = "default_lossy()"))] pub lossy: bool, + + /// Controls how JSON floating-point numbers are parsed. + /// + /// Accepted values: `"float"` (default) or `"decimal"`. + /// + /// When set to `"decimal"`, non-integer numbers are parsed as exact decimal values + /// (using `rust_decimal::Decimal`), preserving precision for values like + /// `12345678901234567890.123`. + /// + /// When set to `"float"` (default), numbers are parsed as i64 or f64, which may lose + /// precision for very large integers or decimals with many significant digits. + #[serde(default, skip_serializing_if = "vector_core::serde::is_default")] + pub parse_float: ParseFloat, } /// Deserializer that builds `Event`s from a byte frame containing JSON. @@ -86,12 +120,28 @@ pub struct JsonDeserializerOptions { pub struct JsonDeserializer { #[derivative(Default(value = "default_lossy()"))] lossy: bool, + + parse_float: ParseFloat, } impl JsonDeserializer { /// Creates a new `JsonDeserializer`. - pub fn new(lossy: bool) -> Self { - Self { lossy } + pub fn new(lossy: bool, parse_float: ParseFloat) -> Self { + Self { lossy, parse_float } + } + + /// Parse bytes as JSON, handling lossy UTF-8 conversion if configured. + fn parse_json_bytes( + &self, + bytes: &[u8], + ) -> vector_common::Result { + if self.lossy { + let s = String::from_utf8_lossy(bytes); + serde_json::from_str(&s) + } else { + serde_json::from_slice(bytes) + } + .map_err(|e| format!("Error parsing JSON: {e:?}").into()) } } @@ -107,26 +157,35 @@ impl Deserializer for JsonDeserializer { return Ok(smallvec![]); } - let json: serde_json::Value = match self.lossy { - true => serde_json::from_str(&String::from_utf8_lossy(&bytes)), - false => serde_json::from_slice(&bytes), - } - .map_err(|error| format!("Error parsing JSON: {error:?}"))?; + let value = if self.parse_float == ParseFloat::Decimal { + let raw_json: Box = self.parse_json_bytes(&bytes)?; + Value::try_from(raw_json.as_ref()) + .map_err(|error| format!("Error parsing JSON: {error:?}"))? + } else { + let json: serde_json::Value = self.parse_json_bytes(&bytes)?; + Value::from(json) + }; - // If the root is an Array, split it into multiple events - let mut events = match json { - serde_json::Value::Array(values) => values - .into_iter() - .map(|json| Event::from_json_value(json, log_namespace)) - .collect::, _>>()?, - _ => smallvec![Event::from_json_value(json, log_namespace)?], + let values = match value { + Value::Array(values) => values, + other => vec![other], }; let events = match log_namespace { - LogNamespace::Vector => events, + LogNamespace::Vector => values + .into_iter() + .map(|value| Event::Log(LogEvent::from(value))) + .collect(), LogNamespace::Legacy => { - let timestamp = Utc::now(); + let mut events = values + .into_iter() + .map(|value| match value { + Value::Object(fields) => Ok(Event::Log(LogEvent::from(fields))), + _ => Err("Attempted to convert non-Object JSON into an Event.".into()), + }) + .collect::>>()?; + let timestamp = Utc::now(); if let Some(timestamp_key) = log_schema().timestamp_key_target_path() { for event in &mut events { let log = event.as_mut_log(); @@ -148,6 +207,7 @@ impl From<&JsonDeserializerConfig> for JsonDeserializer { fn from(config: &JsonDeserializerConfig) -> Self { Self { lossy: config.json.lossy, + parse_float: config.json.parse_float, } } } @@ -155,7 +215,6 @@ impl From<&JsonDeserializerConfig> for JsonDeserializer { #[cfg(test)] mod tests { use vector_core::config::log_schema; - use vrl::core::Value; use super::*; @@ -263,7 +322,7 @@ mod tests { #[test] fn deserialize_lossy_replace_invalid_utf8() { let input = Bytes::from(b"{ \"foo\": \"Hello \xF0\x90\x80World\" }".as_slice()); - let deserializer = JsonDeserializer::new(true); + let deserializer = JsonDeserializer::new(true, ParseFloat::Float); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { let events = deserializer.parse(input.clone(), namespace).unwrap(); @@ -290,10 +349,56 @@ mod tests { #[test] fn deserialize_non_lossy_error_invalid_utf8() { let input = Bytes::from(b"{ \"foo\": \"Hello \xF0\x90\x80World\" }".as_slice()); - let deserializer = JsonDeserializer::new(false); + let deserializer = JsonDeserializer::new(false, ParseFloat::Float); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { assert!(deserializer.parse(input.clone(), namespace).is_err()); } } } + +#[cfg(test)] +mod decimal_precision_tests { + use super::*; + + #[test] + fn preserves_high_precision_decimal() { + let input = Bytes::from(r#"{"val": 12345678901234567890.123}"#); + let deser = JsonDeserializer::new(false, ParseFloat::Decimal); + let events = deser.parse(input, LogNamespace::Vector).unwrap(); + let log = events[0].as_log(); + let val = log.get("val").unwrap(); + assert!(val.is_decimal(), "Expected Decimal, got {:?}", val); + } + + #[test] + fn integers_become_integer() { + let input = Bytes::from(r#"{"int": 42}"#); + let deser = JsonDeserializer::new(false, ParseFloat::Decimal); + let events = deser.parse(input, LogNamespace::Vector).unwrap(); + let log = events[0].as_log(); + assert!(matches!(log.get("int"), Some(Value::Integer(42)))); + } + + #[test] + fn decimals_become_decimal() { + let input = Bytes::from(r#"{"float": 3.14}"#); + let deser = JsonDeserializer::new(false, ParseFloat::Decimal); + let events = deser.parse(input, LogNamespace::Vector).unwrap(); + let log = events[0].as_log(); + let val = log.get("float").unwrap(); + assert!(val.is_decimal(), "Expected Decimal, got {:?}", val); + } + + #[test] + fn float_mode_uses_standard_conversion() { + let input = Bytes::from(r#"{"int": 42, "float": 3.14}"#); + let deser = JsonDeserializer::new(false, ParseFloat::Float); + let events = deser.parse(input, LogNamespace::Vector).unwrap(); + let log = events[0].as_log(); + let int_val = log.get("int").unwrap(); + let float_val = log.get("float").unwrap(); + assert!(int_val.is_integer(), "Expected Integer, got {:?}", int_val); + assert!(float_val.is_float(), "Expected Float, got {:?}", float_val); + } +} diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 701d11c8403d5..47b3de45ab448 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -22,7 +22,7 @@ pub use avro::{AvroDeserializer, AvroDeserializerConfig, AvroDeserializerOptions use dyn_clone::DynClone; pub use gelf::{GelfDeserializer, GelfDeserializerConfig, GelfDeserializerOptions}; pub use influxdb::{InfluxdbDeserializer, InfluxdbDeserializerConfig}; -pub use json::{JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions}; +pub use json::{JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions, ParseFloat}; pub use native::{NativeDeserializer, NativeDeserializerConfig}; pub use native_json::{ NativeJsonDeserializer, NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, diff --git a/lib/codecs/src/decoding/mod.rs b/lib/codecs/src/decoding/mod.rs index c87337856454a..54eb42540ffb8 100644 --- a/lib/codecs/src/decoding/mod.rs +++ b/lib/codecs/src/decoding/mod.rs @@ -18,7 +18,7 @@ pub use format::{ GelfDeserializerConfig, GelfDeserializerOptions, InfluxdbDeserializer, InfluxdbDeserializerConfig, JsonDeserializer, JsonDeserializerConfig, JsonDeserializerOptions, NativeDeserializer, NativeDeserializerConfig, NativeJsonDeserializer, - NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, ProtobufDeserializer, + NativeJsonDeserializerConfig, NativeJsonDeserializerOptions, ParseFloat, ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions, }; #[cfg(feature = "opentelemetry")] diff --git a/lib/codecs/src/encoding/format/cef.rs b/lib/codecs/src/encoding/format/cef.rs index 9ecc95821a492..7cc0280d413ab 100644 --- a/lib/codecs/src/encoding/format/cef.rs +++ b/lib/codecs/src/encoding/format/cef.rs @@ -352,6 +352,7 @@ fn get_log_event_value(log: &LogEvent, field: &ConfigTargetPath) -> String { Some(Value::Float(float)) => float.to_string(), Some(Value::Boolean(bool)) => bool.to_string(), Some(Value::Timestamp(timestamp)) => timestamp.to_rfc3339_opts(SecondsFormat::AutoSi, true), + Some(Value::Decimal(decimal)) => decimal.to_string(), Some(Value::Null) => String::from(""), // Other value types: Array, Regex, Object are not supported by the CEF format. Some(_) => String::from(""), diff --git a/lib/codecs/src/encoding/format/csv.rs b/lib/codecs/src/encoding/format/csv.rs index d967b0d33a567..928c047612a8f 100644 --- a/lib/codecs/src/encoding/format/csv.rs +++ b/lib/codecs/src/encoding/format/csv.rs @@ -254,6 +254,7 @@ impl Encoder for CsvSerializer { Some(Value::Timestamp(timestamp)) => { timestamp.to_rfc3339_opts(SecondsFormat::AutoSi, true) } + Some(Value::Decimal(decimal)) => decimal.to_string(), Some(Value::Null) => String::new(), // Other value types: Array, Regex, Object are not supported by the CSV format. Some(_) => String::new(), diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index a7e33377bc09c..aea19a9fa76d9 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -42,6 +42,7 @@ prost-types.workspace = true prost.workspace = true quanta = { version = "0.12.6", default-features = false } regex = { version = "1.12.3", default-features = false, features = ["std", "perf"] } +rust_decimal.workspace = true ryu = { version = "1", default-features = false } serde.workspace = true serde_json.workspace = true diff --git a/lib/vector-core/proto/event.proto b/lib/vector-core/proto/event.proto index 303aa6c2fe168..46a416b146766 100644 --- a/lib/vector-core/proto/event.proto +++ b/lib/vector-core/proto/event.proto @@ -68,6 +68,7 @@ message Value { ValueMap map = 7; ValueArray array = 8; ValueNull null = 9; + string decimal = 10; } } diff --git a/lib/vector-core/src/event/discriminant.rs b/lib/vector-core/src/event/discriminant.rs index b84342a8f6b85..2a473abe4211b 100644 --- a/lib/vector-core/src/event/discriminant.rs +++ b/lib/vector-core/src/event/discriminant.rs @@ -61,6 +61,7 @@ fn value_eq(this: &Value, other: &Value) -> bool { (Value::Bytes(this), Value::Bytes(other)) => this.eq(other), (Value::Boolean(this), Value::Boolean(other)) => this.eq(other), (Value::Integer(this), Value::Integer(other)) => this.eq(other), + (Value::Decimal(this), Value::Decimal(other)) => this.eq(other), (Value::Timestamp(this), Value::Timestamp(other)) => this.eq(other), (Value::Null, Value::Null) => true, // Non-trivial. @@ -136,6 +137,7 @@ fn hash_value(hasher: &mut H, value: &Value) { Value::Array(val) => hash_array(hasher, val), Value::Object(val) => hash_map(hasher, val), Value::Null => hash_null(hasher), + Value::Decimal(val) => val.hash(hasher), } } diff --git a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs index 832dc70cfd343..9739914570b9a 100644 --- a/lib/vector-core/src/event/estimated_json_encoded_size_of.rs +++ b/lib/vector-core/src/event/estimated_json_encoded_size_of.rs @@ -79,6 +79,7 @@ impl EstimatedJsonEncodedSizeOf for Value { Value::Float(v) => v.estimated_json_encoded_size_of(), Value::Boolean(v) => v.estimated_json_encoded_size_of(), Value::Null => NULL_SIZE, + Value::Decimal(v) => v.estimated_json_encoded_size_of(), } } } @@ -424,6 +425,12 @@ impl EstimatedJsonEncodedSizeOf for i64 { } } +impl EstimatedJsonEncodedSizeOf for rust_decimal::Decimal { + fn estimated_json_encoded_size_of(&self) -> JsonSize { + JsonSize::new(self.to_string().len()) + } +} + impl EstimatedJsonEncodedSizeOf for usize { fn estimated_json_encoded_size_of(&self) -> JsonSize { (*self as u64).estimated_json_encoded_size_of() diff --git a/lib/vector-core/src/event/proto.rs b/lib/vector-core/src/event/proto.rs index 21311267e1aa5..4e219b363e917 100644 --- a/lib/vector-core/src/event/proto.rs +++ b/lib/vector-core/src/event/proto.rs @@ -2,6 +2,7 @@ use std::{collections::BTreeMap, sync::Arc}; use chrono::TimeZone; use ordered_float::NotNan; +use rust_decimal::Decimal; use uuid::Uuid; use super::{MetricTags, WithMetadata}; @@ -711,6 +712,11 @@ fn decode_value(input: Value) -> Option { Some(value::Kind::Map(map)) => decode_map(map.fields), Some(value::Kind::Array(array)) => decode_array(array.items), Some(value::Kind::Null(_)) => Some(super::Value::Null), + Some(value::Kind::Decimal(s)) => s + .parse::() + .map(super::Value::Decimal) + .map_err(|error| error!(%error, decimal = %s, "Failed to parse decimal value.")) + .ok(), None => { error!("Encoded event contains unknown value kind."); None @@ -749,6 +755,7 @@ fn encode_value(value: super::Value) -> Value { super::Value::Object(fields) => Some(value::Kind::Map(encode_map(fields))), super::Value::Array(items) => Some(value::Kind::Array(encode_array(items))), super::Value::Null => Some(value::Kind::Null(ValueNull::NullValue as i32)), + super::Value::Decimal(d) => Some(value::Kind::Decimal(d.to_string())), }, } } diff --git a/src/transforms/dedupe/transform.rs b/src/transforms/dedupe/transform.rs index 8c4fd1d731dca..83e019cf75261 100644 --- a/src/transforms/dedupe/transform.rs +++ b/src/transforms/dedupe/transform.rs @@ -62,6 +62,7 @@ const fn type_id_for_value(val: &Value) -> TypeId { Value::Array(_) => 6, Value::Null => 7, Value::Regex(_) => 8, + Value::Decimal(_) => 9, } } diff --git a/src/transforms/reduce/merge_strategy.rs b/src/transforms/reduce/merge_strategy.rs index 0e673c9788982..4d06ce7a1f350 100644 --- a/src/transforms/reduce/merge_strategy.rs +++ b/src/transforms/reduce/merge_strategy.rs @@ -588,6 +588,7 @@ impl From for Box { Value::Bytes(_) => Box::new(DiscardMerger::new(v)), Value::Regex(_) => Box::new(DiscardMerger::new(v)), Value::Array(_) => Box::new(DiscardMerger::new(v)), + Value::Decimal(_) => Box::new(DiscardMerger::new(v)), } } }