diff --git a/Cargo.lock b/Cargo.lock index dd3f9ce0..c68617d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if 1.0.1", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -696,6 +696,18 @@ dependencies = [ "serde", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block" version = "0.1.6" @@ -755,6 +767,29 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bson" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969a9ba84b0ff843813e7249eed1678d9b6607ce5a3b8f0a47af3fcf7978e6e" +dependencies = [ + "ahash", + "base64 0.22.1", + "bitvec", + "getrandom 0.2.16", + "getrandom 0.3.3", + "hex", + "indexmap 2.10.0", + "js-sys", + "once_cell", + "rand 0.9.1", + "serde", + "serde_bytes", + "serde_json", + "time", + "uuid", +] + [[package]] name = "bstr" version = "1.12.0" @@ -1014,6 +1049,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if 1.0.1", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chromiumoxide" version = "0.7.0" @@ -1292,6 +1338,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -1313,6 +1379,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -1432,6 +1507,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-assembler-x64" version = "0.117.2" @@ -1618,6 +1702,12 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam" version = "0.8.4" @@ -1757,6 +1847,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.104", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.104", +] + [[package]] name = "dashmap" version = "5.5.3" @@ -1970,6 +2095,28 @@ dependencies = [ "serde", ] +[[package]] +name = "derive-syn-parse" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "derive-where" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_arbitrary" version = "1.4.2" @@ -2000,7 +2147,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" dependencies = [ - "derive_more-impl", + "derive_more-impl 1.0.0", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl 2.1.1", ] [[package]] @@ -2015,6 +2171,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case 0.10.0", + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "syn 2.0.104", + "unicode-xid", +] + [[package]] name = "digest" version = "0.10.7" @@ -2573,6 +2743,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.1.31" @@ -2752,11 +2928,23 @@ dependencies = [ "cfg-if 1.0.1", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasi 0.14.2+wasi-0.2.4", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if 1.0.1", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + [[package]] name = "gif" version = "0.14.1" @@ -3088,6 +3276,76 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hickory-net" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2295ed2f9c31e471e1428a8f88a3f0e1f4b27c15049592138d1eebe9c35b183" +dependencies = [ + "async-trait", + "cfg-if 1.0.1", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.1", + "thiserror 2.0.12", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bab31817bfb44672a252e97fe81cd0c18d1b2cf892108922f6818820df8c643" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.1", + "ring 0.17.14", + "thiserror 2.0.12", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d58d28879ceecde6607729660c2667a081ccdc082e082675042793960f178c" +dependencies = [ + "cfg-if 1.0.1", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot 0.12.4", + "rand 0.10.1", + "resolv-conf", + "smallvec 1.15.1", + "system-configuration 0.7.0", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "hipstr" version = "0.6.0" @@ -3653,11 +3911,27 @@ dependencies = [ "libc", ] +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2 0.6.4", + "widestring", + "windows-registry", + "windows-result 0.4.1", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +dependencies = [ + "serde", +] [[package]] name = "iri-string" @@ -3775,12 +4049,61 @@ dependencies = [ "syn 2.0.104", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if 1.0.1", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.12", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "simd_cesu8", + "syn 2.0.104", +] + [[package]] name = "jni-sys" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.104", +] + [[package]] name = "jobserver" version = "0.1.33" @@ -4101,6 +4424,54 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_magic" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc33f9f0351468d26fbc53d9ce00a096c8522ecb42f19b50f34f2c422f76d21d" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "macro_magic_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1687dc887e42f352865a393acae7cf79d98fab6351cde1f58e9e057da89bf150" +dependencies = [ + "const-random", + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "macro_magic_macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.104", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -4385,12 +4756,105 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch 0.9.18", + "crossbeam-utils 0.8.21", + "equivalent", + "parking_lot 0.12.4", + "portable-atomic", + "smallvec 1.15.1", + "tagptr", + "uuid", +] + [[package]] name = "monch" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b52c1b33ff98142aecea13138bd399b68aa7ab5d9546c300988c345004001eea" +[[package]] +name = "mongocrypt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da0cd419a51a5fb44819e290fbdb0665a54f21dead8923446a799c7f4d26ad9" +dependencies = [ + "bson", + "mongocrypt-sys", + "once_cell", + "serde", +] + +[[package]] +name = "mongocrypt-sys" +version = "0.1.5+1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224484c5d09285a7b8cb0a0c117e847ebd14cb6e4470ecf68cdb89c503b0edb9" + +[[package]] +name = "mongodb" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "276ba0cd571553d1f6936c6f180964776ece6ab7507dc8765f8a9c9c49d8cd00" +dependencies = [ + "base64 0.22.1", + "bitflags 2.9.1", + "bson", + "derive-where", + "derive_more 2.1.1", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hickory-net", + "hickory-proto", + "hickory-resolver", + "hmac", + "macro_magic", + "md-5", + "mongocrypt", + "mongodb-internal-macros", + "pbkdf2 0.12.2", + "percent-encoding", + "rand 0.9.1", + "rustc_version_runtime", + "rustls 0.23.28", + "serde", + "serde_bytes", + "serde_with 3.14.1", + "sha1", + "sha2", + "socket2 0.6.4", + "stringprep", + "strsim", + "take_mut", + "thiserror 2.0.12", + "tokio", + "tokio-rustls 0.26.2", + "tokio-util", + "typed-builder", + "uuid", + "webpki-roots 1.0.1", +] + +[[package]] +name = "mongodb-internal-macros" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ceb1a9a1018e470077ec94cf3a8c2d0e6da542b2c05ea95a59a0a627147375" +dependencies = [ + "macro_magic", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "moxcms" version = "0.8.1" @@ -4512,13 +4976,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "ndk-sys" version = "0.5.0+25.2.9519653" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" dependencies = [ - "jni-sys", + "jni-sys 0.3.0", ] [[package]] @@ -4867,6 +5337,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -5092,6 +5566,15 @@ dependencies = [ "sha2", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -5379,6 +5862,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "presser" version = "0.3.1" @@ -5717,6 +6211,18 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.5" @@ -5738,6 +6244,17 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -5776,6 +6293,12 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "range-alloc" version = "0.1.5" @@ -6274,6 +6797,7 @@ dependencies = [ "reflow_actor", "reflow_actor_macro", "reflow_graph", + "reflow_tracing", "reflow_tracing_protocol", "reqwest 0.12.22", "rusty_pool", @@ -6281,7 +6805,7 @@ dependencies = [ "serde", "serde-wasm-bindgen 0.6.5", "serde_json", - "serde_with", + "serde_with 2.3.3", "serde_yaml", "sha2", "shared_memory", @@ -6500,6 +7024,7 @@ dependencies = [ "reflow_components", "reflow_pack_loader", "reflow_rt", + "reflow_tracing_protocol", "serde", "serde_json", "tokio", @@ -6594,6 +7119,7 @@ dependencies = [ "anyhow", "async-trait", "brotli", + "bson", "chrono", "dashmap 6.1.0", "derive_more 1.0.0", @@ -6603,11 +7129,13 @@ dependencies = [ "hostname", "lazy_static", "lz4_flex", + "mongodb", "num_cpus", "once_cell", "parking_lot 0.12.4", "prometheus", "reflow_tracing_protocol", + "reqwest 0.12.22", "semver 1.0.26", "serde", "serde_json", @@ -6634,14 +7162,17 @@ dependencies = [ "chrono", "console_error_panic_hook", "derive_more 0.99.20", + "flume", "futures-util", "gloo-events", "gloo-utils 0.2.0", + "hex", "js-sys", "lz4_flex", "serde", "serde-wasm-bindgen 0.6.5", "serde_json", + "sha2", "thiserror 2.0.12", "tokio", "tokio-tungstenite 0.27.0", @@ -6653,6 +7184,7 @@ dependencies = [ "wasm-bindgen-test", "web-sys", "web-time", + "zstd 0.13.3", ] [[package]] @@ -6792,7 +7324,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tokio-util", @@ -6846,6 +7378,12 @@ dependencies = [ "webpki-roots 1.0.1", ] +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "rgb" version = "0.8.53" @@ -7034,6 +7572,16 @@ dependencies = [ "semver 1.0.26", ] +[[package]] +name = "rustc_version_runtime" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dd18cd2bae1820af0b6ad5e54f4a51d0f3fcc53b05f845675074efcc7af071d" +dependencies = [ + "rustc_version 0.4.1", + "semver 1.0.26", +] + [[package]] name = "rustfft" version = "6.4.1" @@ -7425,6 +7973,29 @@ dependencies = [ "time", ] +[[package]] +name = "serde_with" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" +dependencies = [ + "serde", + "serde_derive", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "3.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -7445,7 +8016,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ "cfg-if 1.0.1", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -7456,7 +8027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if 1.0.1", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -7473,7 +8044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if 1.0.1", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -7574,6 +8145,22 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version 0.4.1", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "0.3.11" @@ -7650,6 +8237,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "spade" version = "2.14.0" @@ -8339,7 +8936,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation 0.9.4", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.9.1", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", ] [[package]] @@ -8352,6 +8960,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys 0.8.7", + "libc", +] + [[package]] name = "system-interface" version = "0.27.3" @@ -8368,6 +8986,24 @@ dependencies = [ "winx", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "take_mut" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.12.16" @@ -8482,6 +9118,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tiny-skia" version = "0.11.4" @@ -8834,7 +9479,10 @@ checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes 1.10.1", "futures-core", + "futures-io", "futures-sink", + "futures-util", + "hashbrown 0.15.4", "pin-project-lite", "tokio", ] @@ -9188,6 +9836,26 @@ version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" +[[package]] +name = "typed-builder" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398a3a3c918c96de527dc11e6e846cd549d4508030b8a33e1da12789c856b81a" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e48cea23f68d1f78eb7bc092881b6bb88d3d6b5b7e6234f6f9c911da1ffb221" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "typenum" version = "1.18.0" @@ -10134,6 +10802,12 @@ dependencies = [ "safe_arch", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "wiggle" version = "30.0.2" @@ -10407,6 +11081,17 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -10840,6 +11525,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xml-rs" version = "0.8.28" @@ -10964,7 +11658,7 @@ dependencies = [ "crossbeam-utils 0.8.21", "flate2", "hmac", - "pbkdf2", + "pbkdf2 0.11.0", "sha1", "time", "zstd 0.11.2+zstd.1.5.2", diff --git a/crates/reflow_actor/src/message.rs b/crates/reflow_actor/src/message.rs index e215971b..cd2d668e 100644 --- a/crates/reflow_actor/src/message.rs +++ b/crates/reflow_actor/src/message.rs @@ -690,7 +690,8 @@ impl Message { .map_err(|e| MessageError::Compression(e.to_string())) } - fn type_name(&self) -> &'static str { + /// Stable variant name (e.g. `"Integer"`), used for trace message types. + pub fn type_name(&self) -> &'static str { match self { Message::Flow => "Flow", Message::Event(_) => "Event", diff --git a/crates/reflow_network/Cargo.toml b/crates/reflow_network/Cargo.toml index 9c1df19c..8685924d 100644 --- a/crates/reflow_network/Cargo.toml +++ b/crates/reflow_network/Cargo.toml @@ -135,6 +135,9 @@ wasm = [] [dev-dependencies] wasm-bindgen-test = "0.3.50" tracing-subscriber = "0.3" +# Embed the tracing collector in distributed e2e tests to assert that a flow +# spanning two processes is stitched into one trace on a shared collector. +reflow_tracing = { path = "../reflow_tracing" } [[example]] name = "distributed_example" diff --git a/crates/reflow_network/src/connector.rs b/crates/reflow_network/src/connector.rs index 51f53022..10bfe165 100644 --- a/crates/reflow_network/src/connector.rs +++ b/crates/reflow_network/src/connector.rs @@ -139,15 +139,19 @@ impl Connector { }); from_actor_load_count.dec(); - // Send tracing event if tracing is enabled + // Send tracing event if tracing is enabled. The message itself + // is the content; the integration computes the checksum/size + // (and optionally captures content) per the configured knobs. if let Some(ref tracing) = tracing_integration { - let message_size = std::mem::size_of_val(&msg); + use reflow_tracing_protocol::PerformanceMetrics; + let message_type = msg.type_name(); let _ = tracing .trace_message_sent( from_actor_id.clone(), _from_port.clone(), - format!("{:?}", std::mem::discriminant(&msg)), - message_size, + message_type, + &msg, + PerformanceMetrics::default(), ) .await; @@ -158,8 +162,9 @@ impl Connector { _from_port.clone(), to_actor_id.clone(), to_port.clone(), - format!("{:?}", std::mem::discriminant(&msg)), - message_size, + message_type, + &msg, + PerformanceMetrics::default(), ) .await; } @@ -227,8 +232,9 @@ impl Connector { continue; }; - let message_size = std::mem::size_of_val(&msg); - let msg_discriminant = format!("{:?}", std::mem::discriminant(&msg)); + // Capture the message for tracing before it is moved into the + // downstream channel (only when tracing is active). + let traced_msg = tracing_integration.as_ref().map(|_| msg.clone()); let encodable = if let Message::Bytes(_) = &msg { crate::message::EncodableValue::from(serde_json::Value::String( @@ -259,13 +265,16 @@ impl Connector { ) }); - if let Some(ref tracing) = tracing_integration { + if let (Some(tracing), Some(content)) = (&tracing_integration, &traced_msg) { + use reflow_tracing_protocol::PerformanceMetrics; + let message_type = content.type_name(); let _ = tracing .trace_message_sent( from_actor_id.clone(), from_port.clone(), - msg_discriminant.clone(), - message_size, + message_type, + content, + PerformanceMetrics::default(), ) .await; let _ = tracing @@ -274,8 +283,9 @@ impl Connector { from_port.clone(), to_actor_id.clone(), to_port.clone(), - msg_discriminant, - message_size, + message_type, + content, + PerformanceMetrics::default(), ) .await; } @@ -333,9 +343,9 @@ impl Connector { continue; }; - // Capture tracing info from &msg before moving - let message_size = std::mem::size_of_val(&msg); - let msg_discriminant = format!("{:?}", std::mem::discriminant(&msg)); + // Capture the message for tracing before it is moved + // into the downstream channel (only when tracing is active). + let traced_msg = tracing_integration.as_ref().map(|_| msg.clone()); // Emit MessageSent event — skip expensive serialization for binary blobs let encodable = if let Message::Bytes(_) = &msg { @@ -376,13 +386,18 @@ impl Connector { }); // Send tracing event if tracing is enabled - if let Some(ref tracing) = tracing_integration { + if let (Some(tracing), Some(content)) = + (&tracing_integration, &traced_msg) + { + use reflow_tracing_protocol::PerformanceMetrics; + let message_type = content.type_name(); let _ = tracing .trace_message_sent( from_actor_id.clone(), from_port.clone(), - msg_discriminant.clone(), - message_size, + message_type, + content, + PerformanceMetrics::default(), ) .await; @@ -392,8 +407,9 @@ impl Connector { from_port.clone(), to_actor_id.clone(), to_port.clone(), - msg_discriminant, - message_size, + message_type, + content, + PerformanceMetrics::default(), ) .await; } diff --git a/crates/reflow_network/src/integration_tests.rs b/crates/reflow_network/src/integration_tests.rs index a367427c..900ebd55 100644 --- a/crates/reflow_network/src/integration_tests.rs +++ b/crates/reflow_network/src/integration_tests.rs @@ -120,6 +120,7 @@ mod tests { target_port: "input".to_string(), payload: crate::message::Message::String(std::sync::Arc::new("hello".to_string())), timestamp: chrono::Utc::now(), + trace_context: None, }; let result = router.handle_incoming_message(msg).await; assert!(result.is_err()); @@ -137,6 +138,45 @@ mod tests { let actors = router.get_local_actor_list(); assert!(actors.is_empty()); } + + #[test] + fn remote_message_trace_context_is_wire_backward_compatible() { + use crate::router::{RemoteMessage, TraceContext}; + use reflow_tracing_protocol::TraceId; + + // An older peer sends no `trace_context`; it must deserialize to None + // rather than failing. + let legacy = r#"{ + "message_id":"m1","source_network":"a","source_actor":"s", + "target_network":"b","target_actor":"t","target_port":"in", + "payload":{"type":"Flow"},"timestamp":"2020-01-01T00:00:00Z" + }"#; + let msg: RemoteMessage = + serde_json::from_str(legacy).expect("deserialize without trace_context"); + assert!(msg.trace_context.is_none()); + + // A context round-trips and preserves the propagated trace id. + let trace_id = TraceId::new(); + let with_ctx = RemoteMessage { + message_id: "m2".into(), + source_network: "a".into(), + source_actor: "s".into(), + target_network: "b".into(), + target_actor: "t".into(), + target_port: "in".into(), + payload: crate::message::Message::Flow, + timestamp: chrono::Utc::now(), + trace_context: Some(TraceContext { + trace_id: trace_id.clone(), + flow_id: None, + parent_span_id: "span-1".into(), + parent_event_id: None, + }), + }; + let encoded = serde_json::to_string(&with_ctx).unwrap(); + let decoded: RemoteMessage = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded.trace_context.unwrap().trace_id, trace_id); + } } // === 5.3: Subgraph composition tests === diff --git a/crates/reflow_network/src/network.rs b/crates/reflow_network/src/network.rs index 17213cda..26f03d60 100644 --- a/crates/reflow_network/src/network.rs +++ b/crates/reflow_network/src/network.rs @@ -46,6 +46,10 @@ use std::sync::{Arc, Mutex}; #[cfg_attr(target_arch = "wasm32", derive(Tsify))] #[cfg_attr(target_arch = "wasm32", tsify(into_wasm_abi))] #[cfg_attr(target_arch = "wasm32", tsify(from_wasm_abi))] +// Container-level default so callers can supply a partial config (e.g. just +// `tracing`) and have the rest fall back to defaults — important for SDKs that +// enable tracing through a config object. +#[serde(default)] pub struct NetworkConfig { pub compression: CompressionConfig, pub tracing: TracingConfig, @@ -156,6 +160,12 @@ pub struct Network { event_handle: Vec, compression_config: CompressionConfig, pub(crate) tracing_integration: Option, + /// Local tap for trace events, so SDKs can consume traces without a + /// collector. Fed by `tracing_integration` when tracing is enabled. + trace_event_emitter: ( + flume::Sender, + flume::Receiver, + ), } unsafe impl Send for Network {} @@ -366,12 +376,26 @@ impl Network { } } +/// Capacity of the per-network local trace-event tap. Bounds memory when a +/// consumer lags or never drains; excess events are dropped best-effort. +const TRACE_TAP_CAPACITY: usize = 4096; + impl Network { pub fn new(config: NetworkConfig) -> Self { - // Initialize tracing if enabled + // Local trace-event tap. Bounded so it can never grow without bound if + // nothing drains it; the tap is installed lazily (see + // `get_trace_receiver`) so when no local consumer exists, no events are + // buffered at all. `try_send` drops on a full buffer — best-effort, + // never blocks the data plane. + let trace_event_emitter = flume::bounded(TRACE_TAP_CAPACITY); + + // Initialize tracing if enabled. The local tap is NOT wired here — it is + // attached on first `get_trace_receiver()` call so that enabling tracing + // purely for a remote collector costs no local buffering. let tracing_integration = if config.tracing.enabled { let client = TracingClient::new(config.tracing.clone()); - Some(TracingIntegration::new(client)) + let integration = TracingIntegration::new(client); + Some(integration) } else { None }; @@ -390,6 +414,7 @@ impl Network { event_handle: Vec::new(), compression_config: config.compression, tracing_integration, + trace_event_emitter, } } @@ -810,18 +835,25 @@ impl Network { timestamp, }); - // Trace the message being sent + // Trace the message being sent. The message is the content; the + // integration computes checksum/size per the configured knobs. if let Some(ref tracing) = self.tracing_integration { - let message_type = format!("{:?}", std::mem::discriminant(&data)); - let size_bytes = serde_json::to_string(&data).unwrap_or_default().len(); + use reflow_tracing_protocol::PerformanceMetrics; let tracing_clone = tracing.clone(); let id_clone = id.to_string(); let port_clone = port.to_string(); + let content = data.clone(); #[cfg(not(target_arch = "wasm32"))] { tokio::runtime::Handle::current().spawn(async move { let _ = tracing_clone - .trace_message_sent(&id_clone, &port_clone, &message_type, size_bytes) + .trace_message_sent( + &id_clone, + &port_clone, + content.type_name(), + &content, + PerformanceMetrics::default(), + ) .await; }); } @@ -829,7 +861,13 @@ impl Network { { spawn_local(async move { let _ = tracing_clone - .trace_message_sent(&id_clone, &port_clone, &message_type, size_bytes) + .trace_message_sent( + &id_clone, + &port_clone, + content.type_name(), + &content, + PerformanceMetrics::default(), + ) .await; }); } @@ -1367,11 +1405,15 @@ impl Network { { let tracing_integration = self.tracing_integration.clone(); tokio::runtime::Handle::current().spawn(async move { - // Shutdown tracing first to flush any pending events - if let Some(ref tracing) = tracing_integration - && let Err(e) = tracing.client().shutdown().await - { - tracing::warn!("Failed to shutdown tracing client: {}", e); + if let Some(ref tracing) = tracing_integration { + // Finalize the session trace so it's persisted as Completed, + // then flush and close the client. + let _ = tracing + .end_flow_trace(reflow_tracing_protocol::ExecutionStatus::Completed) + .await; + if let Err(e) = tracing.client().shutdown().await { + tracing::warn!("Failed to shutdown tracing client: {}", e); + } } }); } @@ -1498,6 +1540,22 @@ impl Network { pub fn get_event_receiver(&self) -> flume::Receiver { self.network_event_emitter.1.clone() } + + /// Subscribe to this network's live trace events locally — no tracing + /// collector required. Each `TraceEvent` recorded while tracing is enabled + /// is delivered here in addition to being shipped to the configured server. + /// Returns an empty (never-fed) receiver when tracing is disabled. + /// + /// Installs the local tap on first call (idempotent), so enabling tracing + /// solely for a collector buffers nothing locally. Call before `start()` + /// for full coverage. The tap is bounded ([`TRACE_TAP_CAPACITY`]); if a + /// consumer stops draining, new events are dropped rather than retained. + pub fn get_trace_receiver(&self) -> flume::Receiver { + if let Some(ref tracing) = self.tracing_integration { + tracing.set_local_tap(self.trace_event_emitter.0.clone()); + } + self.trace_event_emitter.1.clone() + } } /// GraphNetwork diff --git a/crates/reflow_network/src/router.rs b/crates/reflow_network/src/router.rs index f02fe5b3..bb9849b5 100644 --- a/crates/reflow_network/src/router.rs +++ b/crates/reflow_network/src/router.rs @@ -1,6 +1,7 @@ use crate::{bridge::RemoteConnection, message::Message, network::Network}; use anyhow::Result; use parking_lot::RwLock; +use reflow_tracing_protocol::{EventId, FlowId, TraceId}; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, sync::Arc}; @@ -32,6 +33,32 @@ pub struct RemoteMessage { pub target_port: String, pub payload: Message, pub timestamp: chrono::DateTime, + /// Trace context propagated across the process boundary. `Option` + + /// `serde(default)` keeps the wire backward-compatible: older peers that + /// don't send this field simply deserialize to `None`. + #[serde(default)] + pub trace_context: Option, +} + +/// Trace context carried on a [`RemoteMessage`] so a flow that spans multiple +/// processes aggregates into one end-to-end trace on a shared collector. +/// +/// The originating network's session `trace_id` propagates unchanged; the +/// receiving network records the cross-process hop under that same id, so — +/// with every network pointed at one tracing server — both processes' events +/// land in the same `FlowTrace`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TraceContext { + /// The shared trace id the whole flow is recorded under. + pub trace_id: TraceId, + /// The originating flow id, when available. + #[serde(default)] + pub flow_id: Option, + /// Span id of the sending hop, for linking the inbound event to its parent. + pub parent_span_id: String, + /// Event id that caused this hop, when available. + #[serde(default)] + pub parent_event_id: Option, } impl Default for MessageRouter { @@ -86,7 +113,9 @@ impl MessageRouter { port ); - // Create remote message + // Create remote message, attaching the local network's session trace + // context so the receiving process can record this hop under the same + // trace id (unified, shared-collector tracing). let remote_message = RemoteMessage { message_id: uuid::Uuid::new_v4().to_string(), source_network, @@ -96,6 +125,7 @@ impl MessageRouter { target_port: port.to_string(), payload: message, timestamp: chrono::Utc::now(), + trace_context: self.local_trace_context(), }; // Find connection for target network @@ -130,6 +160,65 @@ impl MessageRouter { } } + /// Build the trace context to attach to outbound remote messages from the + /// local network's current session trace. `None` when tracing is disabled + /// or no flow trace has been started. + fn local_trace_context(&self) -> Option { + let guard = self.local_network.read(); + let network = guard.as_ref()?.read(); + let tracing = network.tracing_integration.as_ref()?; + let trace_id = tracing.current_trace_id()?; + Some(TraceContext { + trace_id, + flow_id: None, + parent_span_id: uuid::Uuid::new_v4().to_string(), + parent_event_id: None, + }) + } + + /// Prepare the cross-process hop event for an inbound message, attributed to + /// the propagated trace id so it aggregates with the origin's events on a + /// shared collector. Returns the client + trace id + event to record, or + /// `None` if the message carries no context or tracing is disabled. + fn build_inbound_hop( + &self, + message: &RemoteMessage, + ) -> Option<( + Arc, + TraceId, + reflow_tracing_protocol::TraceEvent, + )> { + use reflow_tracing_protocol::{MessageSnapshot, PerformanceMetrics, TraceEvent}; + + let ctx = message.trace_context.as_ref()?; + let guard = self.local_network.read(); + let network = guard.as_ref()?.read(); + let tracing = network.tracing_integration.as_ref()?; + let client = tracing.client(); + + let snapshot = MessageSnapshot::capture( + message.payload.type_name(), + &message.payload, + client.capture_checksum(), + client.capture_content(), + ); + // The cross-process delivery, modeled as a data-flow from the remote + // source actor into the local target actor. + let mut event = TraceEvent::data_flow( + message.source_actor.clone(), + message.target_port.clone(), + message.target_actor.clone(), + message.target_port.clone(), + snapshot, + PerformanceMetrics::default(), + ); + // Link this hop to the sending span from the originating process. + event.causality.span_id = ctx.parent_span_id.clone(); + event.causality.parent_event_id = ctx.parent_event_id.clone(); + + Some((client, ctx.trace_id.clone(), event)) + } + pub async fn handle_incoming_message( &self, message: RemoteMessage, @@ -142,43 +231,55 @@ impl MessageRouter { message.target_port ); - // Send message to local network - let local_network_guard = self.local_network.read(); - if let Some(ref local_network_arc) = *local_network_guard { - let network = local_network_arc.read(); + // Build the cross-process hop event (and grab the tracing client) before + // the payload is delivered, so we can attribute it to the propagated + // trace id once delivery succeeds. + let hop = self.build_inbound_hop(&message); - tracing::info!( - "🔍 ROUTER: Sending to local network, available actors: {:?}", - network.actors.keys().collect::>() - ); - tracing::info!( - "🔍 ROUTER: Available nodes: {:?}", - network.nodes.keys().collect::>() - ); + // Send message to local network. Scope the (non-async) network guard so + // it is never held across the await below. + let deliver = { + let local_network_guard = self.local_network.read(); + if let Some(ref local_network_arc) = *local_network_guard { + let network = local_network_arc.read(); + tracing::info!( + "🔍 ROUTER: Sending to local network, available actors: {:?}", + network.actors.keys().collect::>() + ); + network.send_to_actor( + &message.target_actor, + &message.target_port, + message.payload, + ) + } else { + tracing::error!("❌ ROUTER: No local network configured"); + return Err(anyhow::anyhow!("No local network configured")); + } + }; - match network.send_to_actor( - &message.target_actor, - &message.target_port, - message.payload, - ) { - Ok(_) => { - tracing::info!( - "✅ ROUTER: Successfully routed message to local actor {}", - message.target_actor - ); - } - Err(e) => { - tracing::error!( - "❌ ROUTER: Failed to route message to local actor {}: {}", - message.target_actor, - e - ); - return Err(e); - } + match deliver { + Ok(_) => { + tracing::info!( + "✅ ROUTER: Successfully routed message to local actor {}", + message.target_actor + ); } - } else { - tracing::error!("❌ ROUTER: No local network configured"); - return Err(anyhow::anyhow!("No local network configured")); + Err(e) => { + tracing::error!( + "❌ ROUTER: Failed to route message to local actor {}: {}", + message.target_actor, + e + ); + return Err(e); + } + } + + // Attribute the inbound hop to the propagated trace (fire-and-forget so + // tracing never blocks the data plane). + if let Some((client, trace_id, event)) = hop { + tokio::spawn(async move { + let _ = client.record_event(trace_id, event).await; + }); } Ok(()) diff --git a/crates/reflow_network/tests/distributed_e2e_test.rs b/crates/reflow_network/tests/distributed_e2e_test.rs index 811422f0..46403883 100644 --- a/crates/reflow_network/tests/distributed_e2e_test.rs +++ b/crates/reflow_network/tests/distributed_e2e_test.rs @@ -108,6 +108,38 @@ fn make_config(network_id: &str, instance_id: &str, port: u16) -> DistributedCon make_config_with_auth(network_id, instance_id, port, None) } +/// Like [`make_config`] but with the local network's tracing pointed at a +/// shared collector, so cross-process trace stitching can be asserted. +fn make_config_with_tracing( + network_id: &str, + instance_id: &str, + port: u16, + server_url: &str, +) -> DistributedConfig { + use reflow_tracing_protocol::client::TracingConfig; + let mut cfg = make_config(network_id, instance_id, port); + cfg.local_network_config.tracing = TracingConfig { + server_url: server_url.to_string(), + enabled: true, + ..TracingConfig::default() + }; + cfg +} + +/// Boot an in-process tracing collector on an ephemeral port; return its ws URL. +async fn start_collector() -> String { + use tokio::net::TcpListener; + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = reflow_tracing::server::TraceServer::new(reflow_tracing::config::Config::default()) + .await + .unwrap(); + tokio::spawn(async move { + let _ = server.run(listener).await; + }); + format!("ws://{}", addr) +} + fn make_config_with_auth( network_id: &str, instance_id: &str, @@ -236,6 +268,113 @@ async fn message_via_remote_actor_proxy() { client.shutdown().await.ok(); } +/// A flow that crosses the process boundary is stitched into one trace on a +/// shared collector: the client's session `trace_id` propagates on the wire, +/// and the server process records the cross-process hop under that same id. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn cross_process_flow_is_one_unified_trace() { + use reflow_tracing_protocol::client::{TracingClient, TracingConfig}; + use reflow_tracing_protocol::{TraceEventType, TraceQuery}; + + let collector = start_collector().await; + let (sink_tx, _sink_rx) = flume::unbounded::>(); + + let server = DistributedNetwork::new(make_config_with_tracing( + "server-net", + "server-tr", + 9120, + &collector, + )) + .await + .expect("server::new"); + server + .register_local_actor("recorder", RecorderActor::new(sink_tx), None) + .expect("register recorder"); + let mut server = server; + server.start().await.expect("server::start"); + + let mut client = DistributedNetwork::new(make_config_with_tracing( + "client-net", + "client-tr", + 9121, + &collector, + )) + .await + .expect("client::new"); + client.start().await.expect("client::start"); + + client + .connect_to_network("127.0.0.1:9120") + .await + .expect("connect_to_network"); + tokio::time::sleep(Duration::from_millis(200)).await; + + client + .register_remote_actor("recorder", "server-net") + .await + .expect("register_remote_actor"); + + // A querying client against the same collector. + let q = TracingClient::new(TracingConfig { + server_url: collector.clone(), + ..TracingConfig::default() + }); + q.connect().await.expect("query client connect"); + tokio::time::sleep(Duration::from_millis(150)).await; + + let query_all = || TraceQuery { + flow_id: None, + execution_id: None, + time_range: None, + status: None, + actor_filter: None, + limit: Some(200), + offset: None, + }; + + // Wait until both networks' session traces are live on the collector, so the + // source side has a trace id to propagate when we route the message. + for _ in 0..40 { + if q.query_traces(query_all()).await.unwrap_or_default().len() >= 2 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + + // Push through the proxy so the message crosses the bridge carrying context. + let payload = Message::String(Arc::new("trace-me".to_string())); + { + let local = client.get_local_network(); + let net = local.read(); + net.send_to_actor("recorder@server-net", "in", payload.clone()) + .expect("send_to_actor via proxy"); + } + + // Query the shared collector for a trace containing the cross-process hop: + // a DataFlow whose destination is the remote `recorder`, recorded by the + // server process under the client's propagated trace id. + let mut found = false; + for _ in 0..40 { + let traces = q.query_traces(query_all()).await.unwrap_or_default(); + found = traces.iter().any(|t| { + t.events.iter().any(|e| { + matches!(&e.event_type, TraceEventType::DataFlow { to_actor, .. } if to_actor == "recorder") + }) + }); + if found { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!( + found, + "expected a unified trace containing the cross-process hop to `recorder`" + ); + + server.shutdown().await.ok(); + client.shutdown().await.ok(); +} + /// When server and client share the same auth_token, federation /// works exactly as without auth. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/crates/reflow_rt_capi/Cargo.toml b/crates/reflow_rt_capi/Cargo.toml index c0fc443f..b9d574e6 100644 --- a/crates/reflow_rt_capi/Cargo.toml +++ b/crates/reflow_rt_capi/Cargo.toml @@ -19,6 +19,7 @@ crate-type = ["cdylib", "staticlib", "rlib"] [dependencies] reflow_rt = { version = "0.2.1", path = "../reflow_rt" } +reflow_tracing_protocol = { version = "0.2.1", path = "../reflow_tracing_protocol" } reflow_pack_loader = { version = "0.2.1", path = "../reflow_pack_loader" } reflow_components = { version = "0.2.1", path = "../reflow_components", default-features = false, optional = true } anyhow = { workspace = true } diff --git a/crates/reflow_rt_capi/include/reflow_rt.h b/crates/reflow_rt_capi/include/reflow_rt.h index a353ba95..c1a5352d 100644 --- a/crates/reflow_rt_capi/include/reflow_rt.h +++ b/crates/reflow_rt_capi/include/reflow_rt.h @@ -122,6 +122,17 @@ typedef struct rfl_stream_recv rfl_stream_recv; */ typedef struct rfl_subgraph_builder rfl_subgraph_builder; +/** + * Opaque handle to a tracing-collector client. + */ +typedef struct rfl_trace_client rfl_trace_client; + +/** + * Opaque handle to a subscriber on a network's local trace-event stream. + * One subscriber per handle. + */ +typedef struct rfl_traces rfl_traces; + /** * Function pointer: the body of a callback actor. * @@ -243,6 +254,61 @@ void rfl_events_free(struct rfl_events *e); */ char *rfl_version(void); +/** + * Subscribe to a network's live trace events locally — no collector required. + * Call **before** `rfl_network_start` for full coverage. Returns NULL on a + * null argument. + */ +struct rfl_traces *rfl_network_traces(struct rfl_network *n); + +/** + * Poll for the next trace event, blocking up to `timeout_ms` milliseconds. + * On success writes a newly allocated JSON `TraceEvent` to `*out_json` (free + * with `rfl_string_free`). Returns `Ok`, `InvalidState` on timeout, or + * `Runtime` if the channel is closed. + */ +enum rfl_status rfl_traces_recv(struct rfl_traces *t, uint32_t timeout_ms, char **out_json); + +/** + * Free a traces handle. Safe on NULL. + */ +void rfl_traces_free(struct rfl_traces *t); + +/** + * Connect to a tracing collector at `server_url` (e.g. "ws://127.0.0.1:8080"). + * Returns NULL on failure (see `rfl_last_error_message`). + */ +struct rfl_trace_client *rfl_trace_client_connect(const char *server_url); + +/** + * Query historical traces. `query_json` is a JSON `TraceQuery`, or NULL for + * "all". Writes a JSON array of `FlowTrace` to `*out_json` (free with + * `rfl_string_free`). + */ +enum rfl_status rfl_trace_client_query(struct rfl_trace_client *c, + const char *query_json, + char **out_json); + +/** + * Subscribe to live trace events from the collector. `filters_json` is a JSON + * `SubscriptionFilters`, or NULL for no filtering. After this returns `Ok`, + * poll events with `rfl_trace_client_recv`. + */ +enum rfl_status rfl_trace_client_subscribe(struct rfl_trace_client *c, const char *filters_json); + +/** + * Poll for the next live trace event after `rfl_trace_client_subscribe`, + * blocking up to `timeout_ms`. Writes a JSON `TraceEvent` to `*out_json`. + */ +enum rfl_status rfl_trace_client_recv(struct rfl_trace_client *c, + uint32_t timeout_ms, + char **out_json); + +/** + * Free a trace-client handle. Safe on NULL. + */ +void rfl_trace_client_free(struct rfl_trace_client *c); + /** * Add a node. * `metadata_json` may be NULL or a JSON object string (`{"key": ...}`). diff --git a/crates/reflow_rt_capi/src/lib.rs b/crates/reflow_rt_capi/src/lib.rs index ba157bb4..aee17e66 100644 --- a/crates/reflow_rt_capi/src/lib.rs +++ b/crates/reflow_rt_capi/src/lib.rs @@ -418,6 +418,254 @@ pub extern "C" fn rfl_version() -> *mut c_char { CString::new(env!("CARGO_PKG_VERSION")).unwrap().into_raw() } +// ─── tracing ─────────────────────────────────────────────────────────────── +// +// Two consumer surfaces, mirroring the `rfl_events` poll pattern: +// • Local tap — `rfl_network_traces` polls a network's own trace events with +// no collector required (the ergonomic default for local monitoring). +// • Collector client — `rfl_trace_client_*` connect/query/subscribe against a +// shared tracing server for historical and distributed monitoring. +// +// Enabling tracing is done via `NetworkConfig.tracing` in the config JSON passed +// to `rfl_network_new_with_config` (server_url + enabled + capture knobs). + +use reflow_tracing_protocol::client::{TracingClient, TracingConfig}; +use reflow_tracing_protocol::{SubscriptionFilters, TraceEvent, TraceQuery}; + +/// Serialize a value to a newly allocated JSON C string in `*out`. +unsafe fn write_json(v: &T, out: *mut *mut c_char) -> rfl_status { + match serde_json::to_string(v) { + Ok(json) => match CString::new(json) { + Ok(cstr) => { + unsafe { *out = cstr.into_raw() }; + rfl_status::Ok + } + Err(_) => { + set_last_error("json contains a NUL byte"); + rfl_status::InvalidJson + } + }, + Err(e) => to_status_runtime(e), + } +} + +/// Opaque handle to a subscriber on a network's local trace-event stream. +/// One subscriber per handle. +pub struct rfl_traces { + rx: flume::Receiver, +} + +/// Subscribe to a network's live trace events locally — no collector required. +/// Call **before** `rfl_network_start` for full coverage. Returns NULL on a +/// null argument. +#[no_mangle] +pub unsafe extern "C" fn rfl_network_traces(n: *mut rfl_network) -> *mut rfl_traces { + clear_last_error(); + if n.is_null() { + set_last_error("network pointer is null"); + return std::ptr::null_mut(); + } + let handle = unsafe { &*n }; + let rx = handle.net.lock().unwrap().get_trace_receiver(); + Box::into_raw(Box::new(rfl_traces { rx })) +} + +/// Poll for the next trace event, blocking up to `timeout_ms` milliseconds. +/// On success writes a newly allocated JSON `TraceEvent` to `*out_json` (free +/// with `rfl_string_free`). Returns `Ok`, `InvalidState` on timeout, or +/// `Runtime` if the channel is closed. +#[no_mangle] +pub unsafe extern "C" fn rfl_traces_recv( + t: *mut rfl_traces, + timeout_ms: u32, + out_json: *mut *mut c_char, +) -> rfl_status { + clear_last_error(); + if t.is_null() || out_json.is_null() { + return rfl_status::NullArg; + } + let handle = unsafe { &*t }; + let deadline = std::time::Duration::from_millis(timeout_ms as u64); + match handle.rx.recv_timeout(deadline) { + Ok(evt) => unsafe { write_json(&evt, out_json) }, + Err(flume::RecvTimeoutError::Timeout) => rfl_status::InvalidState, + Err(flume::RecvTimeoutError::Disconnected) => { + set_last_error("trace channel disconnected"); + rfl_status::Runtime + } + } +} + +/// Free a traces handle. Safe on NULL. +#[no_mangle] +pub unsafe extern "C" fn rfl_traces_free(t: *mut rfl_traces) { + if !t.is_null() { + drop(unsafe { Box::from_raw(t) }); + } +} + +/// Opaque handle to a tracing-collector client. +pub struct rfl_trace_client { + client: TracingClient, + sub_rx: Mutex>>, +} + +/// Connect to a tracing collector at `server_url` (e.g. "ws://127.0.0.1:8080"). +/// Returns NULL on failure (see `rfl_last_error_message`). +#[no_mangle] +pub unsafe extern "C" fn rfl_trace_client_connect( + server_url: *const c_char, +) -> *mut rfl_trace_client { + clear_last_error(); + let url = match unsafe { cstr_to_str(server_url, "server_url") } { + Ok(s) => s.to_string(), + Err(_) => return std::ptr::null_mut(), + }; + let client = TracingClient::new(TracingConfig { + server_url: url, + ..TracingConfig::default() + }); + let rt = runtime(); + if let Err(e) = rt.block_on(client.connect()) { + set_last_error(format!("trace client connect: {e}")); + return std::ptr::null_mut(); + } + Box::into_raw(Box::new(rfl_trace_client { + client, + sub_rx: Mutex::new(None), + })) +} + +/// Query historical traces. `query_json` is a JSON `TraceQuery`, or NULL for +/// "all". Writes a JSON array of `FlowTrace` to `*out_json` (free with +/// `rfl_string_free`). +#[no_mangle] +pub unsafe extern "C" fn rfl_trace_client_query( + c: *mut rfl_trace_client, + query_json: *const c_char, + out_json: *mut *mut c_char, +) -> rfl_status { + clear_last_error(); + if c.is_null() || out_json.is_null() { + return rfl_status::NullArg; + } + let handle = unsafe { &*c }; + let query = if query_json.is_null() { + empty_trace_query() + } else { + let s = match unsafe { cstr_to_str(query_json, "query_json") } { + Ok(s) => s, + Err(e) => return e, + }; + match serde_json::from_str::(s) { + Ok(q) => q, + Err(e) => { + set_last_error(format!("query_json: {e}")); + return rfl_status::InvalidJson; + } + } + }; + let rt = runtime(); + match rt.block_on(handle.client.query_traces(query)) { + Ok(traces) => unsafe { write_json(&traces, out_json) }, + Err(e) => to_status_runtime(e), + } +} + +/// Subscribe to live trace events from the collector. `filters_json` is a JSON +/// `SubscriptionFilters`, or NULL for no filtering. After this returns `Ok`, +/// poll events with `rfl_trace_client_recv`. +#[no_mangle] +pub unsafe extern "C" fn rfl_trace_client_subscribe( + c: *mut rfl_trace_client, + filters_json: *const c_char, +) -> rfl_status { + clear_last_error(); + if c.is_null() { + return rfl_status::NullArg; + } + let handle = unsafe { &*c }; + let filters = if filters_json.is_null() { + SubscriptionFilters { + flow_ids: None, + actor_ids: None, + event_types: None, + status_filter: None, + } + } else { + let s = match unsafe { cstr_to_str(filters_json, "filters_json") } { + Ok(s) => s, + Err(e) => return e, + }; + match serde_json::from_str::(s) { + Ok(f) => f, + Err(e) => { + set_last_error(format!("filters_json: {e}")); + return rfl_status::InvalidJson; + } + } + }; + // Bounded so a consumer that stops polling can't grow this without bound; + // the notification tap drops on a full buffer (best-effort). + let (tx, rx) = flume::bounded(4096); + handle.client.set_notification_tap(tx); + *handle.sub_rx.lock().unwrap() = Some(rx); + let rt = runtime(); + match rt.block_on(handle.client.subscribe(filters)) { + Ok(_) => rfl_status::Ok, + Err(e) => to_status_runtime(e), + } +} + +/// Poll for the next live trace event after `rfl_trace_client_subscribe`, +/// blocking up to `timeout_ms`. Writes a JSON `TraceEvent` to `*out_json`. +#[no_mangle] +pub unsafe extern "C" fn rfl_trace_client_recv( + c: *mut rfl_trace_client, + timeout_ms: u32, + out_json: *mut *mut c_char, +) -> rfl_status { + clear_last_error(); + if c.is_null() || out_json.is_null() { + return rfl_status::NullArg; + } + let handle = unsafe { &*c }; + let rx = { handle.sub_rx.lock().unwrap().clone() }; + let Some(rx) = rx else { + set_last_error("not subscribed; call rfl_trace_client_subscribe first"); + return rfl_status::InvalidState; + }; + let deadline = std::time::Duration::from_millis(timeout_ms as u64); + match rx.recv_timeout(deadline) { + Ok(evt) => unsafe { write_json(&evt, out_json) }, + Err(flume::RecvTimeoutError::Timeout) => rfl_status::InvalidState, + Err(flume::RecvTimeoutError::Disconnected) => { + set_last_error("subscription channel disconnected"); + rfl_status::Runtime + } + } +} + +/// Free a trace-client handle. Safe on NULL. +#[no_mangle] +pub unsafe extern "C" fn rfl_trace_client_free(c: *mut rfl_trace_client) { + if !c.is_null() { + drop(unsafe { Box::from_raw(c) }); + } +} + +fn empty_trace_query() -> TraceQuery { + TraceQuery { + flow_id: None, + execution_id: None, + time_range: None, + status: None, + actor_filter: None, + limit: None, + offset: None, + } +} + // ─── shared helpers ──────────────────────────────────────────────────────── unsafe fn cstr_to_str<'a>(p: *const c_char, name: &str) -> Result<&'a str, rfl_status> { diff --git a/crates/reflow_tracing/Cargo.toml b/crates/reflow_tracing/Cargo.toml index 9fb4271d..319375cd 100644 --- a/crates/reflow_tracing/Cargo.toml +++ b/crates/reflow_tracing/Cargo.toml @@ -34,9 +34,18 @@ derive_more = { version = "1.0.0", features = ["display"] } tokio-tungstenite = "0.27.0" tungstenite = "0.27.0" -# Database (SQLite) - required for persistent tracing storage +# Database (SQLite always; PostgreSQL via the `postgres` feature) for persistent +# tracing storage. The Postgres driver is added by the feature, below. sqlx = { version = "0.8.2", features = ["runtime-tokio-rustls", "sqlite", "chrono", "uuid"] } +# MongoDB backend (via the `mongodb` feature), optional. +mongodb = { version = "3.1", optional = true } +bson = { version = "2.13", optional = true } + +# OTLP export bridge (via the `otlp` feature) — ships finalized traces to any +# OpenTelemetry backend (Monoscope, Jaeger, Tempo, …) over OTLP/HTTP+JSON. +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"], optional = true } + # Compression zstd = "0.13.2" lz4_flex = "0.11.3" @@ -58,6 +67,15 @@ tokio-test = "0.4" tempfile = "3.8" [features] -default = ["server", "client"] +default = ["server", "client", "storage"] server = [] client = [] +# Durable storage backends. `storage` enables the SQLite backend (always +# available). `postgres` and `mongodb` add the respective drivers. +storage = [] +postgres = ["storage", "sqlx/postgres"] +mongodb = ["storage", "dep:mongodb", "dep:bson"] +# Convenience: every durable backend at once. +all-backends = ["postgres", "mongodb"] +# Export finalized traces to an OpenTelemetry collector (Monoscope, Jaeger, …). +otlp = ["dep:reqwest"] diff --git a/crates/reflow_tracing/src/config.rs b/crates/reflow_tracing/src/config.rs index dd6bf49e..ded84137 100644 --- a/crates/reflow_tracing/src/config.rs +++ b/crates/reflow_tracing/src/config.rs @@ -10,6 +10,27 @@ pub struct Config { pub storage: StorageConfig, pub compression: CompressionConfig, pub metrics: MetricsConfig, + /// Optional OpenTelemetry export. When enabled, each finalized trace is also + /// shipped to an OTLP collector (Monoscope, Jaeger, Tempo, …). + #[serde(default)] + pub otlp: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OtlpConfig { + /// Turn export on. Defaults off even when a config block is present. + #[serde(default)] + pub enabled: bool, + /// Base OTLP/HTTP endpoint, e.g. "http://localhost:4318" (Monoscope, or any + /// OpenTelemetry collector). `/v1/traces` is appended automatically. + pub endpoint: String, + /// `service.name` resource attribute reported to the backend. + #[serde(default = "default_service_name")] + pub service_name: String, +} + +fn default_service_name() -> String { + "reflow".to_string() } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -108,6 +129,7 @@ impl Default for Config { port: 9090, endpoint: "/metrics".to_string(), }, + otlp: None, } } } diff --git a/crates/reflow_tracing/src/lib.rs b/crates/reflow_tracing/src/lib.rs new file mode 100644 index 00000000..319dbe54 --- /dev/null +++ b/crates/reflow_tracing/src/lib.rs @@ -0,0 +1,11 @@ +//! # Reflow Tracing Server (library surface) +//! +//! Exposes the tracing server's building blocks so it can be embedded — most +//! importantly in integration tests and SDK collectors — in addition to being +//! run as the `reflow_tracing` binary. + +pub mod config; +pub mod otlp; +pub mod protocol; +pub mod server; +pub mod storage; diff --git a/crates/reflow_tracing/src/main.rs b/crates/reflow_tracing/src/main.rs index 5c36f716..dbc7a207 100644 --- a/crates/reflow_tracing/src/main.rs +++ b/crates/reflow_tracing/src/main.rs @@ -8,13 +8,8 @@ use std::net::SocketAddr; use tokio::net::TcpListener; use tracing::{info, warn}; -mod config; -mod protocol; -mod server; -mod storage; - -use config::Config; -use server::TraceServer; +use reflow_tracing::config::Config; +use reflow_tracing::server::TraceServer; #[tokio::main] async fn main() -> Result<()> { diff --git a/crates/reflow_tracing/src/otlp.rs b/crates/reflow_tracing/src/otlp.rs new file mode 100644 index 00000000..4b41552a --- /dev/null +++ b/crates/reflow_tracing/src/otlp.rs @@ -0,0 +1,328 @@ +//! OTLP export bridge. +//! +//! Converts a finalized [`FlowTrace`] into OpenTelemetry spans and ships them to +//! any OTLP/HTTP collector (Monoscope, Jaeger, Tempo, Grafana, Honeycomb, …) via +//! `POST {endpoint}/v1/traces`. We hand-roll the OTLP/JSON payload rather than +//! pulling the full OpenTelemetry SDK: the wire format is documented and stable, +//! and reflow's trace model is already span-shaped, so the mapping is direct. +//! +//! Mapping: +//! - `FlowTrace` → one OTel **trace** (the reflow `trace_id`, a 128-bit UUID, is +//! used directly as the 16-byte OTel trace id). +//! - a synthetic **root span** spans the flow (`flow:{flow_id}`). +//! - each `TraceEvent` → a **child span**, parented to its +//! `causality.parent_event_id` when set, else the root. `ActorFailed` sets an +//! error status; `DataFlow`/message attributes (incl. the content checksum) +//! become span attributes. +//! +//! Per the OTLP/JSON encoding, trace/span ids are **hex** strings and unix-nano +//! timestamps are decimal **strings**. + +use crate::config::OtlpConfig; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use reflow_tracing_protocol::{FlowTrace, TraceEvent, TraceEventType}; +use serde_json::{json, Value}; + +fn to_hex(bytes: &[u8]) -> String { + let mut s = String::with_capacity(bytes.len() * 2); + for b in bytes { + s.push_str(&format!("{:02x}", b)); + } + s +} + +fn unix_nanos(t: DateTime) -> String { + t.timestamp_nanos_opt().unwrap_or(0).max(0).to_string() +} + +/// `{"key": k, "value": {"stringValue": v}}` +fn attr(key: &str, value: impl Into) -> Value { + json!({ "key": key, "value": { "stringValue": value.into() } }) +} + +fn event_type_name(t: &TraceEventType) -> &'static str { + match t { + TraceEventType::ActorCreated => "ActorCreated", + TraceEventType::ActorStarted => "ActorStarted", + TraceEventType::ActorCompleted => "ActorCompleted", + TraceEventType::ActorFailed => "ActorFailed", + TraceEventType::MessageSent => "MessageSent", + TraceEventType::MessageReceived => "MessageReceived", + TraceEventType::StateChanged => "StateChanged", + TraceEventType::PortConnected => "PortConnected", + TraceEventType::PortDisconnected => "PortDisconnected", + TraceEventType::DataFlow { .. } => "DataFlow", + TraceEventType::NetworkEvent => "NetworkEvent", + } +} + +fn span_name(e: &TraceEvent) -> String { + match &e.event_type { + TraceEventType::DataFlow { to_actor, .. } => { + format!("dataflow:{}->{}", e.actor_id, to_actor) + } + other => format!("{}:{}", event_type_name(other), e.actor_id), + } +} + +fn build_event_span(trace_id_hex: &str, root_span_id: &str, e: &TraceEvent) -> Value { + let span_id = to_hex(&e.event_id.0.as_bytes()[..8]); + let parent = e + .causality + .parent_event_id + .as_ref() + .map(|p| to_hex(&p.0.as_bytes()[..8])) + .unwrap_or_else(|| root_span_id.to_string()); + + let start = e.timestamp.timestamp_nanos_opt().unwrap_or(0).max(0); + let dur = e.data.performance_metrics.execution_time_ns.unwrap_or(0) as i64; + let end = start + dur; + + let mut attrs = vec![ + attr("reflow.actor_id", e.actor_id.clone()), + attr("reflow.event_type", event_type_name(&e.event_type)), + ]; + if let Some(port) = &e.data.port { + attrs.push(attr("reflow.port", port.clone())); + } + if let TraceEventType::DataFlow { to_actor, to_port } = &e.event_type { + attrs.push(attr("reflow.to_actor", to_actor.clone())); + attrs.push(attr("reflow.to_port", to_port.clone())); + } + if let Some(m) = &e.data.message { + attrs.push(attr("reflow.message.type", m.message_type.clone())); + attrs.push(attr("reflow.message.size_bytes", m.size_bytes.to_string())); + if !m.checksum.is_empty() { + attrs.push(attr("reflow.message.checksum", m.checksum.clone())); + } + } + + let mut span = json!({ + "traceId": trace_id_hex, + "spanId": span_id, + "parentSpanId": parent, + "name": span_name(e), + "kind": 1, // SPAN_KIND_INTERNAL + "startTimeUnixNano": start.to_string(), + "endTimeUnixNano": end.to_string(), + "attributes": attrs, + }); + if matches!(e.event_type, TraceEventType::ActorFailed) { + span["status"] = json!({ + "code": 2, // STATUS_CODE_ERROR + "message": e.data.error.clone().unwrap_or_default(), + }); + } + span +} + +/// Convert a finalized `FlowTrace` to an OTLP/JSON `ExportTraceServiceRequest`. +pub fn flow_trace_to_otlp_json(trace: &FlowTrace, service_name: &str) -> Value { + let trace_id_hex = to_hex(trace.trace_id.0.as_bytes()); + let root_span_id = to_hex(&trace.execution_id.0.as_bytes()[..8]); + + let start = trace.start_time.timestamp_nanos_opt().unwrap_or(0).max(0); + let end = trace + .end_time + .map(|t| t.timestamp_nanos_opt().unwrap_or(0).max(0)) + .unwrap_or_else(|| { + trace + .events + .iter() + .filter_map(|e| e.timestamp.timestamp_nanos_opt()) + .max() + .unwrap_or(start) + }); + + let mut spans = Vec::with_capacity(trace.events.len() + 1); + // Root span for the flow. + spans.push(json!({ + "traceId": trace_id_hex, + "spanId": root_span_id, + "name": format!("flow:{}", trace.flow_id.0), + "kind": 1, + "startTimeUnixNano": start.to_string(), + "endTimeUnixNano": end.to_string(), + "attributes": [ + attr("reflow.flow_id", trace.flow_id.0.clone()), + attr("reflow.execution_id", trace.execution_id.0.to_string()), + attr("reflow.status", format!("{:?}", trace.status)), + ], + })); + // One span per event. + for e in &trace.events { + spans.push(build_event_span(&trace_id_hex, &root_span_id, e)); + } + + json!({ + "resourceSpans": [{ + "resource": { "attributes": [ attr("service.name", service_name) ] }, + "scopeSpans": [{ + "scope": { "name": "reflow_tracing", "version": env!("CARGO_PKG_VERSION") }, + "spans": spans, + }], + }], + }) +} + +/// Exports finalized traces to an OTLP/HTTP collector. +#[cfg(feature = "otlp")] +pub struct OtlpExporter { + traces_url: String, + service_name: String, + client: reqwest::Client, +} + +#[cfg(feature = "otlp")] +impl OtlpExporter { + pub fn new(config: OtlpConfig) -> Result { + let base = config.endpoint.trim_end_matches('/'); + Ok(Self { + traces_url: format!("{base}/v1/traces"), + service_name: config.service_name, + client: reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .build()?, + }) + } + + /// Best-effort export of one finalized trace. Errors are returned for the + /// caller to log; export should never block the data plane. + pub async fn export_trace(&self, trace: &FlowTrace) -> Result<()> { + let body = flow_trace_to_otlp_json(trace, &self.service_name); + let resp = self.client.post(&self.traces_url).json(&body).send().await?; + if !resp.status().is_success() { + let code = resp.status(); + let text = resp.text().await.unwrap_or_default(); + anyhow::bail!("OTLP export rejected ({code}): {text}"); + } + Ok(()) + } +} + +// Stub when the feature is disabled, so the server can hold an +// `Option` unconditionally. +#[cfg(not(feature = "otlp"))] +pub struct OtlpExporter; + +#[cfg(not(feature = "otlp"))] +impl OtlpExporter { + pub fn new(_config: OtlpConfig) -> Result { + anyhow::bail!("OTLP export not compiled in — build reflow_tracing with --features otlp") + } + pub async fn export_trace(&self, _trace: &FlowTrace) -> Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + use reflow_tracing_protocol::{ + ExecutionId, ExecutionStatus, FlowId, FlowVersion, MessageSnapshot, PerformanceMetrics, + TraceEvent, TraceId, TraceMetadata, + }; + use std::collections::HashMap; + + fn trace() -> FlowTrace { + let mut df = TraceEvent::data_flow( + "reader".into(), + "out".into(), + "writer".into(), + "in".into(), + MessageSnapshot::capture("String", &serde_json::json!("hi"), true, false), + PerformanceMetrics::cheap(Some(1_500_000), Some(2)), + ); + df.causality.parent_event_id = None; + FlowTrace { + trace_id: TraceId::new(), + flow_id: FlowId::new("flow_x"), + execution_id: ExecutionId::new(), + version: FlowVersion { + major: 1, + minor: 0, + patch: 0, + git_hash: None, + timestamp: Utc::now(), + }, + start_time: Utc::now(), + end_time: Some(Utc::now()), + status: ExecutionStatus::Completed, + events: vec![TraceEvent::actor_created("reader".into()), df], + metadata: TraceMetadata { + user_id: None, + session_id: None, + environment: "test".into(), + hostname: "h".into(), + process_id: 1, + thread_id: "t".into(), + tags: HashMap::new(), + }, + } + } + + #[test] + fn maps_flow_trace_to_valid_otlp_json() { + let t = trace(); + let v = flow_trace_to_otlp_json(&t, "reflow-test"); + + // trace id is the 128-bit reflow uuid as 32 lowercase hex chars. + let spans = &v["resourceSpans"][0]["scopeSpans"][0]["spans"]; + let trace_id = spans[0]["traceId"].as_str().unwrap(); + assert_eq!(trace_id.len(), 32); + assert_eq!(trace_id, &t.trace_id.0.simple().to_string()); + + // root span + one span per event. + assert_eq!(spans.as_array().unwrap().len(), t.events.len() + 1); + // root has no parent; every span id is 16 hex chars. + assert!(spans[0].get("parentSpanId").is_none()); + for s in spans.as_array().unwrap() { + assert_eq!(s["spanId"].as_str().unwrap().len(), 16); + assert_eq!(s["traceId"].as_str().unwrap(), trace_id); + } + + // service.name resource attribute is set. + let rattrs = &v["resourceSpans"][0]["resource"]["attributes"]; + assert_eq!(rattrs[0]["key"], "service.name"); + assert_eq!(rattrs[0]["value"]["stringValue"], "reflow-test"); + + // the data-flow span carries the content checksum + destination. + let df = spans + .as_array() + .unwrap() + .iter() + .find(|s| s["name"].as_str().unwrap_or("").starts_with("dataflow:")) + .expect("data-flow span"); + let attrs = df["attributes"].as_array().unwrap(); + let has = |k: &str| attrs.iter().any(|a| a["key"] == k); + assert!(has("reflow.to_actor")); + assert!(has("reflow.message.checksum")); + // execution_time_ns (1.5ms) makes end > start. + assert!( + df["endTimeUnixNano"].as_str().unwrap() > df["startTimeUnixNano"].as_str().unwrap() + ); + } + + /// Live POST to a real OTLP/HTTP collector (Monoscope, an OTel collector on + /// :4318, …). Runs only with `--features otlp` and `REFLOW_TEST_OTLP_ENDPOINT`. + #[cfg(feature = "otlp")] + #[tokio::test] + async fn live_export_to_collector() { + let endpoint = match std::env::var("REFLOW_TEST_OTLP_ENDPOINT") { + Ok(e) if !e.is_empty() => e, + _ => { + eprintln!("skipping OTLP live test — set REFLOW_TEST_OTLP_ENDPOINT (e.g. http://localhost:4318)"); + return; + } + }; + let exporter = OtlpExporter::new(OtlpConfig { + enabled: true, + endpoint, + service_name: "reflow-otlp-test".into(), + }) + .unwrap(); + exporter.export_trace(&trace()).await.expect("export to collector"); + } +} diff --git a/crates/reflow_tracing/src/server.rs b/crates/reflow_tracing/src/server.rs index d9ee08fa..65e93802 100644 --- a/crates/reflow_tracing/src/server.rs +++ b/crates/reflow_tracing/src/server.rs @@ -1,4 +1,6 @@ use anyhow::Result; +use chrono::Utc; +use dashmap::DashMap; use futures::{SinkExt, StreamExt}; use std::collections::HashMap; use std::sync::Arc; @@ -183,6 +185,17 @@ pub struct TraceServer { storage: Box, metrics: Arc>, subscription_manager: SubscriptionManager, + /// In-flight traces being aggregated from a live stream of `RecordEvent`s. + /// A trace lives here from `StartTrace` (or first event) until it is + /// finalized (`EndTrace` or client disconnect), at which point it is + /// written through to `storage`. + active_traces: Arc>, + /// Trace ids started by each connection, so a disconnect can finalize any + /// traces the client left open. + connection_traces: Arc>>, + /// Optional OTLP export bridge: finalized traces are also shipped to an + /// OpenTelemetry collector (Monoscope, Jaeger, …). + otlp: Option>, } #[derive(Debug, Default)] @@ -203,11 +216,29 @@ impl TraceServer { let storage = StorageBackend::create(&config.storage).await?; info!("Initialized storage backend: {}", config.storage.backend); + // Build the OTLP exporter if configured and enabled. + let otlp = match config.otlp.as_ref() { + Some(otlp_cfg) if otlp_cfg.enabled => match crate::otlp::OtlpExporter::new(otlp_cfg.clone()) { + Ok(exp) => { + info!("OTLP export enabled → {}", otlp_cfg.endpoint); + Some(Arc::new(exp)) + } + Err(e) => { + warn!("OTLP export disabled: {}", e); + None + } + }, + _ => None, + }; + Ok(Self { config, storage, metrics: Arc::new(Mutex::new(ServerMetrics::default())), subscription_manager: SubscriptionManager::new(), + active_traces: Arc::new(DashMap::new()), + connection_traces: Arc::new(DashMap::new()), + otlp, }) } @@ -299,38 +330,83 @@ impl TraceServer { ) -> Result { match request { TracingRequest::StartTrace { + trace_id, flow_id, - version: _, + version, } => { - let trace_id = TraceId::new(); - // For now, just return the trace ID. In a real implementation, - // you'd create a new FlowTrace and store it + // Create the in-flight trace under the client-owned id and + // remember it against this connection so a disconnect can + // finalize it if no explicit EndTrace arrives. + let trace = Self::new_flow_trace(trace_id.clone(), flow_id.clone(), version); + self.active_traces.insert(trace_id.clone(), trace); + self.connection_traces + .entry(client_id) + .or_default() + .push(trace_id.clone()); info!("Started trace {} for flow {}", trace_id, flow_id); Ok(TracingResponse::TraceStarted { trace_id }) } TracingRequest::RecordEvent { trace_id, event } => { - // Notify all subscribers about this event + // Append to the in-flight trace (lazily create if an event + // arrives before StartTrace), then broadcast to subscribers. + { + let mut entry = self + .active_traces + .entry(trace_id.clone()) + .or_insert_with(|| { + Self::new_flow_trace( + trace_id.clone(), + FlowId::new("unknown"), + Self::default_version(), + ) + }); + entry.events.push(event.clone()); + } self.subscription_manager .notify_event(trace_id, event) .await; - Ok(TracingResponse::EventRecorded { success: true, error: None, }) } - TracingRequest::GetTrace { trace_id } => { - match self.storage.get_trace(trace_id).await? { - Some(trace) => Ok(TracingResponse::TraceData { trace: Some(trace) }), - None => Ok(TracingResponse::TraceData { trace: None }), - } + TracingRequest::EndTrace { trace_id, status } => { + self.finalize_trace(&trace_id, status).await?; + Ok(TracingResponse::EventRecorded { + success: true, + error: None, + }) } - TracingRequest::QueryTraces { query } => { - let traces = self.storage.query_traces(query).await?; - let traces_len = traces.len(); + TracingRequest::GetTrace { + request_id, + trace_id, + } => { + // In-flight traces take precedence over finalized storage. + let trace = match self.active_traces.get(&trace_id) { + Some(t) => Some(t.clone()), + None => self.storage.get_trace(trace_id).await?, + }; + Ok(TracingResponse::TraceData { request_id, trace }) + } + TracingRequest::QueryTraces { request_id, query } => { + let mut traces = self.storage.query_traces(query.clone()).await?; + // Merge in matching in-flight traces not yet in storage. + for entry in self.active_traces.iter() { + let t = entry.value(); + if Self::query_matches(&query, t) + && !traces.iter().any(|s| s.trace_id == t.trace_id) + { + traces.push(t.clone()); + } + } + if let Some(limit) = query.limit { + traces.truncate(limit); + } + let total_count = traces.len(); Ok(TracingResponse::QueryResults { + request_id, traces, - total_count: traces_len, + total_count, }) } TracingRequest::GetFlowVersions { flow_id: _ } => { @@ -368,6 +444,114 @@ impl TraceServer { } } + /// Build a fresh in-flight `FlowTrace` with `Running` status. + fn new_flow_trace(trace_id: TraceId, flow_id: FlowId, version: FlowVersion) -> FlowTrace { + FlowTrace { + trace_id, + flow_id, + execution_id: ExecutionId::new(), + version, + start_time: Utc::now(), + end_time: None, + status: ExecutionStatus::Running, + events: Vec::new(), + metadata: Self::default_metadata(), + } + } + + fn default_version() -> FlowVersion { + FlowVersion { + major: 0, + minor: 0, + patch: 0, + git_hash: None, + timestamp: Utc::now(), + } + } + + fn default_metadata() -> TraceMetadata { + TraceMetadata { + user_id: None, + session_id: None, + environment: "unknown".to_string(), + hostname: "unknown".to_string(), + process_id: std::process::id(), + thread_id: format!("{:?}", std::thread::current().id()), + tags: HashMap::new(), + } + } + + /// Move a trace out of the active map, stamp its terminal status, and + /// write it through to storage. No-op if the trace is already finalized. + async fn finalize_trace(&self, trace_id: &TraceId, status: ExecutionStatus) -> Result<()> { + if let Some((_, mut trace)) = self.active_traces.remove(trace_id) { + trace.status = status; + trace.end_time = Some(Utc::now()); + + // Best-effort OTLP export of the finalized trace (fire-and-forget so + // it never blocks finalize or the data plane). Clone only when an + // exporter is configured. + if let Some(otlp) = &self.otlp { + let otlp = Arc::clone(otlp); + let trace_for_otlp = trace.clone(); + tokio::spawn(async move { + if let Err(e) = otlp.export_trace(&trace_for_otlp).await { + tracing::debug!("OTLP export failed: {}", e); + } + }); + } + + self.storage.store_trace(trace).await?; + let mut metrics = self.metrics.lock().await; + metrics.traces_stored += 1; + } + Ok(()) + } + + /// Finalize every still-open trace a disconnected client started. + async fn finalize_connection_traces(&self, client_id: &str) { + if let Some((_, trace_ids)) = self.connection_traces.remove(client_id) { + for trace_id in trace_ids { + if let Err(e) = self + .finalize_trace(&trace_id, ExecutionStatus::Completed) + .await + { + warn!("Failed to finalize trace {} on disconnect: {}", trace_id, e); + } + } + } + } + + /// Minimal filter for including in-flight traces in query results. + fn query_matches(query: &TraceQuery, trace: &FlowTrace) -> bool { + if let Some(flow_id) = &query.flow_id + && &trace.flow_id != flow_id + { + return false; + } + if let Some(execution_id) = &query.execution_id + && &trace.execution_id != execution_id + { + return false; + } + if let Some(status) = &query.status + && &trace.status != status + { + return false; + } + if let Some((start, end)) = &query.time_range + && (trace.start_time < *start || trace.start_time > *end) + { + return false; + } + if let Some(actor) = &query.actor_filter + && !trace.events.iter().any(|e| &e.actor_id == actor) + { + return false; + } + true + } + async fn send_trace_response( &self, sender: Arc< @@ -787,6 +971,9 @@ impl TraceServer { } } + // Persist any traces the client left open so they remain queryable. + server.finalize_connection_traces(&addr.to_string()).await; + Ok(()) } } diff --git a/crates/reflow_tracing/src/storage/mod.rs b/crates/reflow_tracing/src/storage/mod.rs index 81836c86..0631b48f 100644 --- a/crates/reflow_tracing/src/storage/mod.rs +++ b/crates/reflow_tracing/src/storage/mod.rs @@ -5,6 +5,8 @@ use crate::config::StorageConfig; use reflow_tracing_protocol::{FlowTrace, TraceId, TraceQuery}; pub mod memory; +pub mod mongo; +pub mod postgres; #[allow(dead_code)] pub mod sqlite; @@ -48,9 +50,35 @@ impl StorageBackend { let storage = sqlite::SqliteStorage::new(sqlite_config.clone()).await?; Ok(Box::new(storage)) } - _ => Err(anyhow::anyhow!( - "Unsupported storage backend: {}", - config.backend + "postgres" | "postgresql" => { + let pg_config = config + .postgres + .as_ref() + .ok_or_else(|| anyhow::anyhow!("PostgreSQL storage config missing"))?; + let storage = postgres::PostgresStorage::new(pg_config.clone()).await?; + Ok(Box::new(storage)) + } + "timescale" | "timescaledb" => { + // TimescaleDB speaks the Postgres protocol; it reuses the + // `storage.postgres` connection config and adds a hypertable. + let pg_config = config + .postgres + .as_ref() + .ok_or_else(|| anyhow::anyhow!("TimescaleDB uses the [storage.postgres] config; it is missing"))?; + let storage = postgres::PostgresStorage::new_timescale(pg_config.clone()).await?; + Ok(Box::new(storage)) + } + "mongodb" | "mongo" => { + let mongo_config = config + .mongodb + .as_ref() + .ok_or_else(|| anyhow::anyhow!("MongoDB storage config missing"))?; + let storage = mongo::MongoStorage::new(mongo_config.clone()).await?; + Ok(Box::new(storage)) + } + other => Err(anyhow::anyhow!( + "Unsupported storage backend: '{other}'. Available: memory, sqlite, \ + postgres / timescale (--features postgres), mongodb (--features mongodb)." )), } } diff --git a/crates/reflow_tracing/src/storage/mongo.rs b/crates/reflow_tracing/src/storage/mongo.rs new file mode 100644 index 00000000..0bca9c15 --- /dev/null +++ b/crates/reflow_tracing/src/storage/mongo.rs @@ -0,0 +1,210 @@ +//! MongoDB trace storage (feature `mongodb`). +//! +//! Each `FlowTrace` is stored as one document keyed by its `trace_id` (`_id`), +//! with denormalized top-level fields (`flow_id`, `execution_id`, `status`, +//! `start_time`, `end_time`) for indexed querying and the full trace nested +//! under `trace`. Document storage maps naturally onto the JSON-shaped trace. + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use chrono::DateTime; + +use super::{StorageStats, TraceStorage}; +use crate::config::MongoDbConfig; +use reflow_tracing_protocol::{FlowTrace, TraceId, TraceQuery}; + +#[cfg(feature = "mongodb")] +use { + bson::{doc, Document}, + futures::TryStreamExt, + mongodb::{Client, Collection}, + serde::{Deserialize, Serialize}, +}; + +#[cfg(feature = "mongodb")] +#[derive(Serialize, Deserialize)] +struct TraceDoc { + #[serde(rename = "_id")] + id: String, + flow_id: String, + execution_id: String, + status: String, + start_time: i64, + end_time: Option, + event_count: i64, + trace: FlowTrace, +} + +#[cfg(feature = "mongodb")] +impl TraceDoc { + fn from_trace(trace: FlowTrace) -> Self { + Self { + id: trace.trace_id.0.to_string(), + flow_id: trace.flow_id.0.clone(), + execution_id: trace.execution_id.0.to_string(), + status: serde_json::to_string(&trace.status).unwrap_or_default(), + start_time: trace.start_time.timestamp(), + end_time: trace.end_time.map(|t| t.timestamp()), + event_count: trace.events.len() as i64, + trace, + } + } +} + +#[cfg(feature = "mongodb")] +pub struct MongoStorage { + collection: Collection, +} + +#[cfg(feature = "mongodb")] +impl MongoStorage { + pub async fn new(config: MongoDbConfig) -> Result { + let client = Client::with_uri_str(&config.connection_url) + .await + .map_err(|e| anyhow!("Failed to connect to MongoDB: {e}"))?; + let collection = client + .database(&config.database_name) + .collection::(&config.collection_name); + // Best-effort indexes for the query columns. + let db = client.database(&config.database_name); + for key in ["flow_id", "execution_id", "status", "start_time"] { + let _ = db + .run_command(doc! { + "createIndexes": &config.collection_name, + "indexes": [ { "key": { key: 1 }, "name": format!("idx_{key}") } ], + }) + .await; + } + Ok(Self { collection }) + } +} + +#[cfg(feature = "mongodb")] +#[async_trait] +impl TraceStorage for MongoStorage { + async fn store_trace(&self, trace: FlowTrace) -> Result { + let trace_id = trace.trace_id.clone(); + let doc = TraceDoc::from_trace(trace); + self.collection + .replace_one(doc! { "_id": &doc.id }, &doc) + .upsert(true) + .await + .map_err(|e| anyhow!("store trace: {e}"))?; + Ok(trace_id) + } + + async fn get_trace(&self, trace_id: TraceId) -> Result> { + let found = self + .collection + .find_one(doc! { "_id": trace_id.0.to_string() }) + .await + .map_err(|e| anyhow!("get trace: {e}"))?; + Ok(found.map(|d| d.trace)) + } + + async fn query_traces(&self, query: TraceQuery) -> Result> { + let mut filter = Document::new(); + if let Some(ref f) = query.flow_id { + filter.insert("flow_id", &f.0); + } + if let Some(ref e) = query.execution_id { + filter.insert("execution_id", e.0.to_string()); + } + if let Some(ref s) = query.status { + filter.insert("status", serde_json::to_string(s).unwrap_or_default()); + } + if let Some((start, end)) = &query.time_range { + filter.insert( + "start_time", + doc! { "$gte": start.timestamp(), "$lte": end.timestamp() }, + ); + } + + let mut find = self.collection.find(filter).sort(doc! { "start_time": -1 }); + if let Some(limit) = query.limit { + find = find.limit(limit as i64); + } + if let Some(offset) = query.offset { + find = find.skip(offset as u64); + } + let cursor = find.await.map_err(|e| anyhow!("query traces: {e}"))?; + let docs: Vec = cursor + .try_collect() + .await + .map_err(|e| anyhow!("collect traces: {e}"))?; + Ok(docs.into_iter().map(|d| d.trace).collect()) + } + + async fn delete_trace(&self, trace_id: TraceId) -> Result { + let res = self + .collection + .delete_one(doc! { "_id": trace_id.0.to_string() }) + .await + .map_err(|e| anyhow!("delete trace: {e}"))?; + Ok(res.deleted_count > 0) + } + + async fn get_stats(&self) -> Result { + let total_traces = self + .collection + .count_documents(doc! {}) + .await + .map_err(|e| anyhow!("count traces: {e}"))? as usize; + + // Aggregate event totals and the time window. + let mut total_events = 0usize; + let mut oldest: Option = None; + let mut newest: Option = None; + let mut cursor = self + .collection + .find(doc! {}) + .await + .map_err(|e| anyhow!("scan traces: {e}"))?; + while let Some(d) = cursor.try_next().await.map_err(|e| anyhow!("scan: {e}"))? { + total_events += d.event_count.max(0) as usize; + oldest = Some(oldest.map_or(d.start_time, |o| o.min(d.start_time))); + newest = Some(newest.map_or(d.start_time, |n| n.max(d.start_time))); + } + + Ok(StorageStats { + total_traces, + total_events, + storage_size_bytes: 0, // not cheaply available per-collection + oldest_trace_timestamp: oldest.and_then(|ts| DateTime::from_timestamp(ts, 0)), + newest_trace_timestamp: newest.and_then(|ts| DateTime::from_timestamp(ts, 0)), + }) + } +} + +// Fallback stub when the feature is disabled. +#[cfg(not(feature = "mongodb"))] +pub struct MongoStorage; + +#[cfg(not(feature = "mongodb"))] +impl MongoStorage { + pub async fn new(_config: MongoDbConfig) -> Result { + Err(anyhow!( + "MongoDB backend not compiled in — build reflow_tracing with --features mongodb" + )) + } +} + +#[cfg(not(feature = "mongodb"))] +#[async_trait] +impl TraceStorage for MongoStorage { + async fn store_trace(&self, _trace: FlowTrace) -> Result { + Err(anyhow!("MongoDB backend not compiled in")) + } + async fn get_trace(&self, _trace_id: TraceId) -> Result> { + Err(anyhow!("MongoDB backend not compiled in")) + } + async fn query_traces(&self, _query: TraceQuery) -> Result> { + Err(anyhow!("MongoDB backend not compiled in")) + } + async fn delete_trace(&self, _trace_id: TraceId) -> Result { + Err(anyhow!("MongoDB backend not compiled in")) + } + async fn get_stats(&self) -> Result { + Err(anyhow!("MongoDB backend not compiled in")) + } +} diff --git a/crates/reflow_tracing/src/storage/postgres.rs b/crates/reflow_tracing/src/storage/postgres.rs new file mode 100644 index 00000000..659dfc3b --- /dev/null +++ b/crates/reflow_tracing/src/storage/postgres.rs @@ -0,0 +1,410 @@ +//! PostgreSQL trace storage (feature `postgres`). +//! +//! Mirrors the SQLite backend's model: each `FlowTrace` is stored as a single +//! (optionally zstd-compressed) JSON blob in a `traces` table, alongside +//! denormalized columns (`flow_id`, `execution_id`, `status`, `start_time`, +//! `end_time`, `event_count`) for indexed querying. This keeps the storage +//! schema-agnostic to the evolving `FlowTrace` shape while staying queryable. + +use anyhow::{anyhow, Result}; +use async_trait::async_trait; +use chrono::DateTime; + +use super::{StorageStats, TraceStorage}; +use crate::config::PostgresConfig; +use reflow_tracing_protocol::{FlowTrace, TraceId, TraceQuery}; + +#[cfg(feature = "postgres")] +use sqlx::{postgres::PgPoolOptions, PgPool, Row}; +#[cfg(feature = "postgres")] +use std::time::Duration; + +/// Compress with zstd above a threshold; returns (bytes, compressed?). +#[cfg(feature = "postgres")] +fn maybe_compress(serialized: Vec, threshold: usize) -> Result<(Vec, bool)> { + if serialized.len() > threshold { + let c = zstd::bulk::compress(&serialized, 3) + .map_err(|e| anyhow!("zstd compress failed: {e}"))?; + Ok((c, true)) + } else { + Ok((serialized, false)) + } +} + +#[cfg(feature = "postgres")] +fn maybe_decompress(data: Vec, compressed: bool) -> Result> { + if compressed { + zstd::bulk::decompress(&data, 64 * 1024 * 1024) + .map_err(|e| anyhow!("zstd decompress failed: {e}")) + } else { + Ok(data) + } +} + +/// Chunk width for the TimescaleDB hypertable, in seconds (start_time is unix +/// seconds). 7 days is a reasonable default for trace volumes. +#[cfg(feature = "postgres")] +const TIMESCALE_CHUNK_SECONDS: i64 = 7 * 24 * 60 * 60; + +#[cfg(feature = "postgres")] +pub struct PostgresStorage { + pool: PgPool, + compress_threshold: usize, + /// When true, the trace table is a TimescaleDB hypertable partitioned on + /// `start_time` and upserts key on `(trace_id, start_time)`. + timescale: bool, +} + +#[cfg(feature = "postgres")] +impl PostgresStorage { + /// Plain PostgreSQL. + pub async fn new(config: PostgresConfig) -> Result { + Self::connect(config, false).await + } + + /// TimescaleDB: same as Postgres, but converts `traces` into a hypertable + /// partitioned on `start_time`. Degrades gracefully to a plain table if the + /// `timescaledb` extension isn't available. + pub async fn new_timescale(config: PostgresConfig) -> Result { + Self::connect(config, true).await + } + + async fn connect(config: PostgresConfig, timescale: bool) -> Result { + let pool = PgPoolOptions::new() + .max_connections(config.max_connections.max(1)) + .min_connections(config.min_connections) + .acquire_timeout(Duration::from_secs(config.acquire_timeout_secs.max(1))) + .connect(&config.connection_url) + .await + .map_err(|e| anyhow!("Failed to connect to PostgreSQL: {e}"))?; + + Self::create_tables(&pool, timescale).await?; + Ok(Self { + pool, + compress_threshold: 1024, + timescale, + }) + } + + async fn create_tables(pool: &PgPool, timescale: bool) -> Result<()> { + // A hypertable's unique/primary key must include the partitioning column + // (`start_time`), so the timescale schema keys on `(trace_id, start_time)`. + let create = if timescale { + r#" + CREATE TABLE IF NOT EXISTS traces ( + trace_id TEXT NOT NULL, + flow_id TEXT NOT NULL, + execution_id TEXT NOT NULL, + status TEXT NOT NULL, + start_time BIGINT NOT NULL, + end_time BIGINT, + event_count BIGINT NOT NULL DEFAULT 0, + data BYTEA NOT NULL, + compressed BOOLEAN NOT NULL DEFAULT FALSE, + size_bytes BIGINT NOT NULL, + created_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT, + PRIMARY KEY (trace_id, start_time) + ) + "# + } else { + r#" + CREATE TABLE IF NOT EXISTS traces ( + trace_id TEXT PRIMARY KEY, + flow_id TEXT NOT NULL, + execution_id TEXT NOT NULL, + status TEXT NOT NULL, + start_time BIGINT NOT NULL, + end_time BIGINT, + event_count BIGINT NOT NULL DEFAULT 0, + data BYTEA NOT NULL, + compressed BOOLEAN NOT NULL DEFAULT FALSE, + size_bytes BIGINT NOT NULL, + created_at BIGINT NOT NULL DEFAULT EXTRACT(EPOCH FROM NOW())::BIGINT + ) + "# + }; + sqlx::query(create) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create traces table: {e}"))?; + + for idx in [ + "CREATE INDEX IF NOT EXISTS idx_traces_flow_id ON traces(flow_id)", + "CREATE INDEX IF NOT EXISTS idx_traces_execution_id ON traces(execution_id)", + "CREATE INDEX IF NOT EXISTS idx_traces_status ON traces(status)", + "CREATE INDEX IF NOT EXISTS idx_traces_start_time ON traces(start_time)", + ] { + sqlx::query(idx) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create index: {e}"))?; + } + + if timescale { + // Best-effort: enable the extension and convert to a hypertable. If + // the extension isn't installed these fail and we keep the plain + // (composite-key) table — still correct, just not time-partitioned. + let _ = sqlx::query("CREATE EXTENSION IF NOT EXISTS timescaledb") + .execute(pool) + .await; + // Point lookups by trace_id alone (get/delete) need a non-unique + // index since the PK now leads with both columns. + let _ = sqlx::query("CREATE INDEX IF NOT EXISTS idx_traces_trace_id ON traces(trace_id)") + .execute(pool) + .await; + let hyper = format!( + "SELECT create_hypertable('traces', 'start_time', \ + chunk_time_interval => {TIMESCALE_CHUNK_SECONDS}, \ + if_not_exists => TRUE, migrate_data => TRUE)" + ); + if let Err(e) = sqlx::query(&hyper).execute(pool).await { + tracing::warn!( + "TimescaleDB hypertable not created (continuing with a plain table): {e}" + ); + } + } + Ok(()) + } +} + +#[cfg(feature = "postgres")] +#[async_trait] +impl TraceStorage for PostgresStorage { + async fn store_trace(&self, trace: FlowTrace) -> Result { + let trace_id = trace.trace_id.clone(); + let serialized = + serde_json::to_vec(&trace).map_err(|e| anyhow!("serialize trace: {e}"))?; + let (data, compressed) = maybe_compress(serialized, self.compress_threshold)?; + + // The hypertable keys on (trace_id, start_time) and can't UPDATE the + // partition column; plain Postgres keys on trace_id alone. `start_time` + // is set once at trace creation, so the composite key is stable. + let sql = if self.timescale { + r#" + INSERT INTO traces + (trace_id, flow_id, execution_id, status, start_time, end_time, + event_count, data, compressed, size_bytes) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (trace_id, start_time) DO UPDATE SET + flow_id = EXCLUDED.flow_id, + execution_id = EXCLUDED.execution_id, + status = EXCLUDED.status, + end_time = EXCLUDED.end_time, + event_count = EXCLUDED.event_count, + data = EXCLUDED.data, + compressed = EXCLUDED.compressed, + size_bytes = EXCLUDED.size_bytes + "# + } else { + r#" + INSERT INTO traces + (trace_id, flow_id, execution_id, status, start_time, end_time, + event_count, data, compressed, size_bytes) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (trace_id) DO UPDATE SET + flow_id = EXCLUDED.flow_id, + execution_id = EXCLUDED.execution_id, + status = EXCLUDED.status, + start_time = EXCLUDED.start_time, + end_time = EXCLUDED.end_time, + event_count = EXCLUDED.event_count, + data = EXCLUDED.data, + compressed = EXCLUDED.compressed, + size_bytes = EXCLUDED.size_bytes + "# + }; + + sqlx::query(sql) + .bind(trace.trace_id.0.to_string()) + .bind(&trace.flow_id.0) + .bind(trace.execution_id.0.to_string()) + .bind(serde_json::to_string(&trace.status).unwrap_or_default()) + .bind(trace.start_time.timestamp()) + .bind(trace.end_time.map(|t| t.timestamp())) + .bind(trace.events.len() as i64) + .bind(&data) + .bind(compressed) + .bind(data.len() as i64) + .execute(&self.pool) + .await + .map_err(|e| anyhow!("insert trace: {e}"))?; + + Ok(trace_id) + } + + async fn get_trace(&self, trace_id: TraceId) -> Result> { + let row = sqlx::query("SELECT data, compressed FROM traces WHERE trace_id = $1") + .bind(trace_id.0.to_string()) + .fetch_optional(&self.pool) + .await + .map_err(|e| anyhow!("query trace: {e}"))?; + + match row { + Some(row) => { + let data: Vec = row.get("data"); + let compressed: bool = row.get("compressed"); + let bytes = maybe_decompress(data, compressed)?; + let trace: FlowTrace = + serde_json::from_slice(&bytes).map_err(|e| anyhow!("deserialize trace: {e}"))?; + Ok(Some(trace)) + } + None => Ok(None), + } + } + + async fn query_traces(&self, query: TraceQuery) -> Result> { + // Build a parameterized query with $-placeholders. + let mut sql = String::from("SELECT data, compressed FROM traces WHERE 1=1"); + let mut n = 0; + // Collected bind values as strings/ints in order; bind by matching type below. + let mut flow_id: Option = None; + let mut execution_id: Option = None; + let mut status: Option = None; + let mut time_range: Option<(i64, i64)> = None; + + if let Some(ref f) = query.flow_id { + n += 1; + sql.push_str(&format!(" AND flow_id = ${n}")); + flow_id = Some(f.0.clone()); + } + if let Some(ref e) = query.execution_id { + n += 1; + sql.push_str(&format!(" AND execution_id = ${n}")); + execution_id = Some(e.0.to_string()); + } + if let Some(ref s) = query.status { + n += 1; + sql.push_str(&format!(" AND status = ${n}")); + status = Some(serde_json::to_string(s).unwrap_or_default()); + } + if let Some((start, end)) = &query.time_range { + let a = n + 1; + let b = n + 2; + n += 2; + sql.push_str(&format!(" AND start_time BETWEEN ${a} AND ${b}")); + time_range = Some((start.timestamp(), end.timestamp())); + } + sql.push_str(" ORDER BY start_time DESC"); + if let Some(limit) = query.limit { + sql.push_str(&format!(" LIMIT {}", limit as i64)); + } + if let Some(offset) = query.offset { + sql.push_str(&format!(" OFFSET {}", offset as i64)); + } + + let mut q = sqlx::query(&sql); + if let Some(v) = flow_id { + q = q.bind(v); + } + if let Some(v) = execution_id { + q = q.bind(v); + } + if let Some(v) = status { + q = q.bind(v); + } + if let Some((a, b)) = time_range { + q = q.bind(a).bind(b); + } + + let rows = q + .fetch_all(&self.pool) + .await + .map_err(|e| anyhow!("query traces: {e}"))?; + + let mut traces = Vec::with_capacity(rows.len()); + for row in rows { + let data: Vec = row.get("data"); + let compressed: bool = row.get("compressed"); + let bytes = maybe_decompress(data, compressed)?; + let trace: FlowTrace = + serde_json::from_slice(&bytes).map_err(|e| anyhow!("deserialize trace: {e}"))?; + traces.push(trace); + } + Ok(traces) + } + + async fn delete_trace(&self, trace_id: TraceId) -> Result { + let affected = sqlx::query("DELETE FROM traces WHERE trace_id = $1") + .bind(trace_id.0.to_string()) + .execute(&self.pool) + .await + .map_err(|e| anyhow!("delete trace: {e}"))? + .rows_affected(); + Ok(affected > 0) + } + + async fn get_stats(&self) -> Result { + let total_traces: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM traces") + .fetch_one(&self.pool) + .await + .map_err(|e| anyhow!("count traces: {e}"))?; + let total_events: Option = + sqlx::query_scalar("SELECT COALESCE(SUM(event_count), 0) FROM traces") + .fetch_one(&self.pool) + .await + .map_err(|e| anyhow!("sum events: {e}"))?; + let storage_size_bytes: Option = + sqlx::query_scalar("SELECT COALESCE(SUM(size_bytes), 0) FROM traces") + .fetch_one(&self.pool) + .await + .map_err(|e| anyhow!("sum size: {e}"))?; + let oldest: Option = sqlx::query_scalar("SELECT MIN(start_time) FROM traces") + .fetch_one(&self.pool) + .await + .ok() + .flatten(); + let newest: Option = sqlx::query_scalar("SELECT MAX(start_time) FROM traces") + .fetch_one(&self.pool) + .await + .ok() + .flatten(); + + Ok(StorageStats { + total_traces: total_traces as usize, + total_events: total_events.unwrap_or(0) as usize, + storage_size_bytes: storage_size_bytes.unwrap_or(0) as usize, + oldest_trace_timestamp: oldest.and_then(|ts| DateTime::from_timestamp(ts, 0)), + newest_trace_timestamp: newest.and_then(|ts| DateTime::from_timestamp(ts, 0)), + }) + } +} + +// Fallback stub when the feature is disabled, so `StorageBackend::create` can +// reference the type unconditionally and return a clear error. +#[cfg(not(feature = "postgres"))] +pub struct PostgresStorage; + +#[cfg(not(feature = "postgres"))] +impl PostgresStorage { + pub async fn new(_config: PostgresConfig) -> Result { + Err(anyhow!( + "PostgreSQL backend not compiled in — build reflow_tracing with --features postgres" + )) + } + + pub async fn new_timescale(_config: PostgresConfig) -> Result { + Err(anyhow!( + "TimescaleDB backend not compiled in — build reflow_tracing with --features postgres" + )) + } +} + +#[cfg(not(feature = "postgres"))] +#[async_trait] +impl TraceStorage for PostgresStorage { + async fn store_trace(&self, _trace: FlowTrace) -> Result { + Err(anyhow!("PostgreSQL backend not compiled in")) + } + async fn get_trace(&self, _trace_id: TraceId) -> Result> { + Err(anyhow!("PostgreSQL backend not compiled in")) + } + async fn query_traces(&self, _query: TraceQuery) -> Result> { + Err(anyhow!("PostgreSQL backend not compiled in")) + } + async fn delete_trace(&self, _trace_id: TraceId) -> Result { + Err(anyhow!("PostgreSQL backend not compiled in")) + } + async fn get_stats(&self) -> Result { + Err(anyhow!("PostgreSQL backend not compiled in")) + } +} diff --git a/crates/reflow_tracing/src/storage/sqlite.rs b/crates/reflow_tracing/src/storage/sqlite.rs index 684fc6b3..0ba59a51 100644 --- a/crates/reflow_tracing/src/storage/sqlite.rs +++ b/crates/reflow_tracing/src/storage/sqlite.rs @@ -163,10 +163,12 @@ pub struct StorageMetrics { #[cfg(feature = "storage")] impl SqliteStorage { pub async fn new(config: SqliteConfig) -> Result { + // `mode=rwc` = read-write-create, so a fresh database file is created + // if it doesn't exist yet (plain `sqlite:` errors on a missing file). let database_url = if config.database_path == ":memory:" { "sqlite::memory:".to_string() } else { - format!("sqlite:{}", config.database_path) + format!("sqlite:{}?mode=rwc", config.database_path) }; let storage_config = SqliteStorageConfig::from(config); @@ -464,17 +466,16 @@ impl TraceStorage for SqliteStorage { async fn store_trace(&self, trace: FlowTrace) -> Result { let trace_id = trace.trace_id.clone(); - // Add to write buffer for batch processing - let mut buffer = self.write_buffer.write().await; - buffer.traces.push_back(trace); - buffer.pending_size += 1; - - // Force flush if buffer is full - if buffer.pending_size >= self.config.write_buffer_size { - drop(buffer); // Release lock before flush - Self::flush_write_buffer(&self.write_buffer, &self.pool, &self.compression_engine) - .await?; - } + // Write through directly to the database. A tracing collector stores + // one trace per finalized flow — modest enough that write-batching isn't + // worth the read-after-write and delete races a buffer introduces. This + // matches the Postgres backend's synchronous semantics. + Self::batch_store_traces( + &self.pool, + std::slice::from_ref(&trace), + &self.compression_engine, + ) + .await?; Ok(trace_id) } @@ -490,6 +491,16 @@ impl TraceStorage for SqliteStorage { return Ok(Some(trace)); } + // Read-after-write: the trace may still be in the write buffer, not yet + // flushed to the database. Serve the buffered copy so a store immediately + // followed by a get/query is consistent. + { + let buffer = self.write_buffer.read().await; + if let Some(trace) = buffer.traces.iter().rev().find(|t| t.trace_id == trace_id) { + return Ok(Some(trace.clone())); + } + } + // Load from database let row = sqlx::query("SELECT data, compressed FROM traces WHERE trace_id = ?") .bind(&trace_id.0.to_string()) @@ -596,12 +607,33 @@ impl TraceStorage for SqliteStorage { .collect(); let results = futures::future::join_all(futures).await; - let traces: Result, _> = results + // A trace id can vanish between the SELECT and the load (concurrent + // delete); skip rather than fail the whole query. + let mut traces: Vec = results .into_iter() - .map(|result| result?.ok_or_else(|| anyhow::anyhow!("Trace not found"))) + .filter_map(|result| result.ok().flatten()) .collect(); - Ok(traces?) + // Merge in not-yet-flushed buffered traces that match the query, so a + // store immediately followed by a query is consistent. + { + let buffer = self.write_buffer.read().await; + for t in buffer.traces.iter() { + if trace_matches_query(&query, t) + && !traces.iter().any(|x| x.trace_id == t.trace_id) + { + traces.push(t.clone()); + } + } + } + + // Newest first, then apply the limit across the merged set. + traces.sort_by(|a, b| b.start_time.cmp(&a.start_time)); + if let Some(limit) = query.limit { + traces.truncate(limit); + } + + Ok(traces) } async fn delete_trace(&self, trace_id: TraceId) -> Result { @@ -612,7 +644,21 @@ impl TraceStorage for SqliteStorage { .map_err(|e| anyhow::anyhow!("Failed to delete trace: {}", e))? .rows_affected(); - Ok(rows_affected > 0) + // Purge every other copy: the not-yet-flushed write buffer and the read + // cache. Otherwise a delete could be undone by a later flush, or a stale + // copy served from cache/buffer. + let removed_from_buffer = { + let mut buffer = self.write_buffer.write().await; + let before = buffer.traces.len(); + buffer.traces.retain(|t| t.trace_id != trace_id); + buffer.traces.len() != before + }; + { + let mut cache = self.cache.write().await; + cache.remove(&trace_id); + } + + Ok(rows_affected > 0 || removed_from_buffer) } async fn get_stats(&self) -> Result { @@ -813,6 +859,13 @@ impl LruCache { } } + fn remove(&mut self, trace_id: &TraceId) { + if let Some(entry) = self.cache.remove(trace_id) { + self.size_bytes = self.size_bytes.saturating_sub(entry.size_bytes); + self.usage_order.retain(|id| id != trace_id); + } + } + fn insert(&mut self, trace_id: TraceId, trace: FlowTrace) { let entry_size = estimate_trace_size(&trace); @@ -965,6 +1018,32 @@ impl Default for StorageMetrics { } } +/// Does a trace satisfy a query's column filters? Used to merge not-yet-flushed +/// buffered traces into query results (matches the SQL WHERE clause). +fn trace_matches_query(query: &TraceQuery, trace: &FlowTrace) -> bool { + if let Some(ref f) = query.flow_id + && &trace.flow_id != f + { + return false; + } + if let Some(ref e) = query.execution_id + && &trace.execution_id != e + { + return false; + } + if let Some(ref s) = query.status + && &trace.status != s + { + return false; + } + if let Some((start, end)) = &query.time_range + && (trace.start_time < *start || trace.start_time > *end) + { + return false; + } + true +} + fn estimate_trace_size(trace: &FlowTrace) -> usize { // Rough estimation of trace size in memory let base_size = std::mem::size_of::(); diff --git a/crates/reflow_tracing/tests/aggregation_e2e.rs b/crates/reflow_tracing/tests/aggregation_e2e.rs new file mode 100644 index 00000000..77990282 --- /dev/null +++ b/crates/reflow_tracing/tests/aggregation_e2e.rs @@ -0,0 +1,177 @@ +//! Phase 1 end-to-end: a stream of `RecordEvent`s sent under one client-owned +//! `trace_id` must be aggregated by the server into a single `FlowTrace` that +//! `get_trace` and `query_traces` return — i.e. correlation + persistence + +//! request/response correlation all work over the real WebSocket client. + +use std::time::Duration; + +use reflow_tracing::config::Config; +use reflow_tracing::server::TraceServer; +use reflow_tracing_protocol::client::{TracingClient, TracingConfig}; +use reflow_tracing_protocol::{ + ExecutionStatus, FlowId, FlowTrace, FlowVersion, MessageSnapshot, PerformanceMetrics, + TraceEvent, TraceId, TraceQuery, +}; +use tokio::net::TcpListener; + +/// Bind an in-process tracing server on an ephemeral port and return its ws URL. +async fn start_server() -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = TraceServer::new(Config::default()).await.unwrap(); + tokio::spawn(async move { + let _ = server.run(listener).await; + }); + format!("ws://{}", addr) +} + +fn version() -> FlowVersion { + FlowVersion { + major: 1, + minor: 0, + patch: 0, + git_hash: None, + timestamp: chrono::Utc::now(), + } +} + +async fn poll_get(client: &TracingClient, trace_id: &TraceId) -> Option { + for _ in 0..30 { + if let Ok(Some(t)) = client.get_trace(trace_id.clone()).await { + return Some(t); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + None +} + +#[tokio::test] +async fn records_and_queries_a_correlated_trace() { + let url = start_server().await; + + let client = TracingClient::new(TracingConfig { + server_url: url, + ..TracingConfig::default() + }); + client.connect().await.expect("connect to tracing server"); + // Let the connection's writer task install its message sender. + tokio::time::sleep(Duration::from_millis(150)).await; + + // Start a flow trace — the client owns the id. + let flow_id = FlowId::new("phase1_test_flow"); + let trace_id = client + .start_trace(flow_id.clone(), version()) + .await + .expect("start_trace"); + + // Three correlated events under the same trace id. + client + .record_event(trace_id.clone(), TraceEvent::actor_created("reader".into())) + .await + .unwrap(); + client + .record_event( + trace_id.clone(), + TraceEvent::data_flow( + "reader".into(), + "out".into(), + "writer".into(), + "in".into(), + MessageSnapshot::capture("String", &serde_json::json!("hello"), true, false), + PerformanceMetrics::default(), + ), + ) + .await + .unwrap(); + client + .record_event( + trace_id.clone(), + TraceEvent::actor_completed("writer".into()), + ) + .await + .unwrap(); + + // Let the events flush, then finalize the trace. + tokio::time::sleep(Duration::from_millis(200)).await; + client + .end_trace(trace_id.clone(), ExecutionStatus::Completed) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + + // get_trace returns the finalized trace with all three events correlated. + let trace = poll_get(&client, &trace_id).await.expect("trace persisted"); + assert_eq!(trace.trace_id, trace_id); + assert_eq!(trace.flow_id, flow_id); + assert_eq!( + trace.events.len(), + 3, + "all events correlated into one trace, got {:?}", + trace.events.iter().map(|e| &e.event_type).collect::>() + ); + assert!(matches!(trace.status, ExecutionStatus::Completed)); + + // The data-flow event carries a populated content checksum, and by default + // (capture_content off) retains no content bytes. + let df = trace + .events + .iter() + .find(|e| matches!(e.event_type, reflow_tracing_protocol::TraceEventType::DataFlow { .. })) + .expect("data flow event present"); + let snap = df.data.message.as_ref().expect("data flow has a message snapshot"); + assert!( + snap.checksum.starts_with("sha256:") && snap.checksum.len() == "sha256:".len() + 64, + "content checksum populated: {:?}", + snap.checksum + ); + assert!( + snap.serialized_data.is_empty(), + "content must not be captured by default" + ); + + // query_traces by flow_id also returns it. + let results = client + .query_traces(TraceQuery { + flow_id: Some(flow_id.clone()), + execution_id: None, + time_range: None, + status: None, + actor_filter: None, + limit: Some(10), + offset: None, + }) + .await + .expect("query_traces"); + assert!( + results.iter().any(|t| t.trace_id == trace_id), + "trace must be findable by flow_id query" + ); +} + +#[tokio::test] +async fn in_flight_trace_is_visible_before_finalize() { + let url = start_server().await; + + let client = TracingClient::new(TracingConfig { + server_url: url, + ..TracingConfig::default() + }); + client.connect().await.expect("connect"); + tokio::time::sleep(Duration::from_millis(150)).await; + + let flow_id = FlowId::new("phase1_inflight_flow"); + let trace_id = client + .start_trace(flow_id, version()) + .await + .expect("start_trace"); + client + .record_event(trace_id.clone(), TraceEvent::actor_created("a".into())) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(200)).await; + + // No EndTrace yet — the active (Running) trace must still be queryable. + let trace = poll_get(&client, &trace_id).await.expect("in-flight trace"); + assert!(matches!(trace.status, ExecutionStatus::Running)); + assert_eq!(trace.events.len(), 1); +} diff --git a/crates/reflow_tracing/tests/storage_backends.rs b/crates/reflow_tracing/tests/storage_backends.rs new file mode 100644 index 00000000..66a951a4 --- /dev/null +++ b/crates/reflow_tracing/tests/storage_backends.rs @@ -0,0 +1,233 @@ +//! Storage-backend integration tests. +//! +//! SQLite runs against a temp file (no external server). PostgreSQL runs only +//! when `REFLOW_TEST_POSTGRES_URL` is set to a reachable instance, and is +//! skipped otherwise. + +use chrono::Utc; +use reflow_tracing::storage::TraceStorage; +use reflow_tracing_protocol::{ + ExecutionStatus, ExecutionId, FlowId, FlowTrace, FlowVersion, TraceEvent, TraceId, + TraceMetadata, TraceQuery, +}; +use std::collections::HashMap; + +fn sample_trace(flow_id: &str) -> FlowTrace { + FlowTrace { + trace_id: TraceId::new(), + flow_id: FlowId::new(flow_id), + execution_id: ExecutionId::new(), + version: FlowVersion { + major: 1, + minor: 0, + patch: 0, + git_hash: None, + timestamp: Utc::now(), + }, + start_time: Utc::now(), + end_time: Some(Utc::now()), + status: ExecutionStatus::Completed, + events: vec![ + TraceEvent::actor_created("reader".into()), + TraceEvent::actor_completed("reader".into()), + ], + metadata: TraceMetadata { + user_id: None, + session_id: None, + environment: "test".into(), + hostname: "localhost".into(), + process_id: std::process::id(), + thread_id: "t".into(), + tags: HashMap::new(), + }, + } +} + +fn empty_query(flow_id: Option<&str>) -> TraceQuery { + TraceQuery { + flow_id: flow_id.map(FlowId::new), + execution_id: None, + time_range: None, + status: None, + actor_filter: None, + limit: Some(50), + offset: None, + } +} + +#[tokio::test] +async fn sqlite_store_query_delete_roundtrip() { + use reflow_tracing::config::SqliteConfig; + use reflow_tracing::storage::sqlite::SqliteStorage; + + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("traces.db"); + let storage = SqliteStorage::new(SqliteConfig { + database_path: path.to_string_lossy().into_owned(), + wal_mode: true, + journal_mode: "WAL".into(), + synchronous: "NORMAL".into(), + cache_size: -2000, + }) + .await + .expect("open sqlite storage"); + + let trace = sample_trace("sqlite_flow"); + let id = storage.store_trace(trace.clone()).await.expect("store"); + + // store_trace buffers; the background flush runs ~every second. + let mut got = None; + for _ in 0..40 { + if let Some(t) = storage.get_trace(id.clone()).await.expect("get") { + got = Some(t); + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + let got = got.expect("trace persisted to sqlite"); + assert_eq!(got.trace_id, id); + assert_eq!(got.flow_id, trace.flow_id); + assert_eq!(got.events.len(), 2); + + let results = storage + .query_traces(empty_query(Some("sqlite_flow"))) + .await + .expect("query"); + assert!(results.iter().any(|t| t.trace_id == id)); + + assert!(storage.delete_trace(id.clone()).await.expect("delete")); + assert!(storage.get_trace(id).await.expect("get after delete").is_none()); +} + +#[tokio::test] +async fn mongodb_store_query_delete_roundtrip() { + let url = match std::env::var("REFLOW_TEST_MONGODB_URL") { + Ok(u) if !u.is_empty() => u, + _ => { + eprintln!("skipping mongodb test — set REFLOW_TEST_MONGODB_URL to run"); + return; + } + }; + + use reflow_tracing::config::MongoDbConfig; + use reflow_tracing::storage::mongo::MongoStorage; + + let storage = MongoStorage::new(MongoDbConfig { + connection_url: url, + database_name: "reflow_tracing_test".into(), + collection_name: "traces".into(), + }) + .await + .expect("connect mongodb"); + + let trace = sample_trace("mongo_flow"); + let id = storage.store_trace(trace.clone()).await.expect("store"); + + let got = storage + .get_trace(id.clone()) + .await + .expect("get") + .expect("trace present"); + assert_eq!(got.trace_id, id); + assert_eq!(got.events.len(), 2); + + let results = storage + .query_traces(empty_query(Some("mongo_flow"))) + .await + .expect("query"); + assert!(results.iter().any(|t| t.trace_id == id)); + + assert!(storage.delete_trace(id.clone()).await.expect("delete")); + assert!(storage.get_trace(id).await.expect("get after delete").is_none()); +} + +#[tokio::test] +async fn postgres_store_query_delete_roundtrip() { + let url = match std::env::var("REFLOW_TEST_POSTGRES_URL") { + Ok(u) if !u.is_empty() => u, + _ => { + eprintln!("skipping postgres test — set REFLOW_TEST_POSTGRES_URL to run"); + return; + } + }; + + use reflow_tracing::config::PostgresConfig; + use reflow_tracing::storage::postgres::PostgresStorage; + + let storage = PostgresStorage::new(PostgresConfig { + connection_url: url, + max_connections: 4, + min_connections: 1, + acquire_timeout_secs: 5, + }) + .await + .expect("connect postgres"); + + let trace = sample_trace("pg_flow"); + let id = storage.store_trace(trace.clone()).await.expect("store"); + + // Postgres writes are synchronous — no flush wait needed. + let got = storage + .get_trace(id.clone()) + .await + .expect("get") + .expect("trace present"); + assert_eq!(got.trace_id, id); + assert_eq!(got.events.len(), 2); + + let results = storage + .query_traces(empty_query(Some("pg_flow"))) + .await + .expect("query"); + assert!(results.iter().any(|t| t.trace_id == id)); + + let stats = storage.get_stats().await.expect("stats"); + assert!(stats.total_traces >= 1); + + assert!(storage.delete_trace(id.clone()).await.expect("delete")); + assert!(storage.get_trace(id).await.expect("get after delete").is_none()); +} + +#[tokio::test] +async fn timescaledb_store_query_delete_roundtrip() { + let url = match std::env::var("REFLOW_TEST_TIMESCALE_URL") { + Ok(u) if !u.is_empty() => u, + _ => { + eprintln!("skipping timescaledb test — set REFLOW_TEST_TIMESCALE_URL to run"); + return; + } + }; + + use reflow_tracing::config::PostgresConfig; + use reflow_tracing::storage::postgres::PostgresStorage; + + // TimescaleDB reuses the Postgres connection config; the hypertable is set + // up on connect (degrades to a plain table if the extension is absent). + let storage = PostgresStorage::new_timescale(PostgresConfig { + connection_url: url, + max_connections: 4, + min_connections: 1, + acquire_timeout_secs: 5, + }) + .await + .expect("connect timescaledb"); + + let trace = sample_trace("timescale_flow"); + let id = storage.store_trace(trace.clone()).await.expect("store"); + + let got = storage + .get_trace(id.clone()) + .await + .expect("get") + .expect("trace present"); + assert_eq!(got.trace_id, id); + + let results = storage + .query_traces(empty_query(Some("timescale_flow"))) + .await + .expect("query"); + assert!(results.iter().any(|t| t.trace_id == id)); + + assert!(storage.delete_trace(id.clone()).await.expect("delete")); + assert!(storage.get_trace(id).await.expect("get after delete").is_none()); +} diff --git a/crates/reflow_tracing_protocol/Cargo.toml b/crates/reflow_tracing_protocol/Cargo.toml index 34ec7987..20d432cf 100644 --- a/crates/reflow_tracing_protocol/Cargo.toml +++ b/crates/reflow_tracing_protocol/Cargo.toml @@ -16,9 +16,14 @@ categories = ["development-tools", "development-tools::debugging"] [dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +# Content-digest (checksum) of traced messages. SHA-256 over canonical JSON. +sha2 = "0.10" +hex = "0.4" chrono = { version = "0.4", features = ["serde", "wasmbind"] } web-time = "1.1" +# Local trace-event tap so SDKs can consume traces without a collector. +flume = "0.11" derive_more = "0.99" tracing = { workspace = true } futures-util = "0.3.31" @@ -29,6 +34,9 @@ thiserror = { workspace = true } tokio = { version = "1", features = ["full"] } tokio-tungstenite = "0.27.0" uuid = { version = "1.0", features = ["v4", "serde"] } +# The tracing server compresses large frames (zstd by default); the native +# client must be able to read them back on the query/response path. +zstd = "0.13" [target.'cfg(target_arch = "wasm32")'.dependencies] uuid = { version = "1.0", features = ["v4", "serde", "js"] } diff --git a/crates/reflow_tracing_protocol/src/checksum.rs b/crates/reflow_tracing_protocol/src/checksum.rs new file mode 100644 index 00000000..350582b4 --- /dev/null +++ b/crates/reflow_tracing_protocol/src/checksum.rs @@ -0,0 +1,160 @@ +//! # Message content checksums +//! +//! A [`MessageSnapshot`](crate::MessageSnapshot) carries a `checksum`: a +//! deterministic, content-only digest of a single traced message. It exists +//! for content integrity, dedup, and as a stable reference downstream +//! consumers can rely on. One message, one digest — this is **not** an +//! event-chain or tamper-evident log, and checksums are never chained across +//! events. +//! +//! ## Algorithm and encoding +//! +//! SHA-256, stored algorithm-prefixed as `"sha256:<64 lowercase hex>"`. The +//! prefix makes the scheme self-describing for any verifier and leaves room to +//! migrate later without breaking the `String` wire type. +//! +//! ## Hash domain — content only +//! +//! The digest is computed over the **message value and nothing else**. Event +//! metadata — `event_id`, `timestamp`, `span_id`, `actor_id`, port names, +//! sizes — is explicitly excluded. Two identical payloads sent at different +//! times by different actors therefore produce the same checksum, which is +//! what makes dedup and content-identity work. +//! +//! ## Canonical form (so the digest is reproducible everywhere) +//! +//! Determinism is the load-bearing requirement: the checksum must be identical +//! across processes, hosts, CPU architectures, and SDK languages. That requires +//! a documented *canonical serialization*, not the in-memory layout. We use +//! **RFC 8785-style canonical JSON (JCS)**, realized as follows — these rules +//! are written as prose so any independent verifier can re-derive the exact +//! bytes that get hashed: +//! +//! 1. Serialize the message value to JSON via its serde representation. For +//! Reflow's `Message` that is the externally-tagged form +//! `{"type": , "data": }` (the `data` key is absent for +//! unit variants such as `Flow`). +//! 2. **Object keys are sorted** in ascending order. (Implementation note: we +//! sort by Rust `str` ordering, i.e. Unicode scalar / UTF-8 byte order. This +//! matches RFC 8785's UTF-16 code-unit ordering for every character in the +//! Basic Multilingual Plane; the two differ only for supplementary-plane +//! characters (≥ U+10000) appearing *in object keys*, which Reflow message +//! field names never use.) +//! 3. **No insignificant whitespace** — the compact JSON form. +//! 4. **Strings** use standard JSON escaping (RFC 8259): `"` and `\` are +//! escaped, C0 control characters are emitted as `\u00xx`. +//! 5. **Numbers**: integers are emitted exactly; finite floating-point values +//! use the shortest round-tripping decimal, with exponential notation in the +//! ECMAScript style (`1e+21`, `1e-7`). Non-finite floats (NaN/±Inf) are not +//! representable in JSON and serialize to `null`. +//! 6. The hash input is the UTF-8 bytes of the resulting string. +//! +//! The digest is always computed over the **decompressed** content, so toggling +//! transport/storage compression never changes the checksum. +//! +//! ## Edge cases +//! +//! Signal-only and empty payloads (`Message::Flow`, `Optional(None)`, +//! zero-byte) hash their canonical typed representation and so yield a stable, +//! well-defined value — never an empty string. Callers should reserve the +//! empty string strictly for "not computed". + +use serde::Serialize; +use sha2::{Digest, Sha256}; + +/// The canonical-form prefix; bump only if the algorithm changes. +pub const CHECKSUM_PREFIX: &str = "sha256:"; + +/// Canonical JSON (JCS-style) bytes for a serializable value — the exact bytes +/// the checksum is computed over. Routing through `serde_json::Value` is what +/// gives us sorted object keys (its map is a `BTreeMap`); the compact +/// serializer gives us no insignificant whitespace. +pub fn canonical_json(content: &T) -> Result { + let value = serde_json::to_value(content)?; + serde_json::to_string(&value) +} + +/// Content checksum of a serializable message value, encoded +/// `"sha256:<64 lowercase hex>"`. +/// +/// Infallible: a value that cannot be represented as JSON (which, for Reflow +/// `Message`s, does not occur — non-finite floats serialize to `null`) falls +/// back to hashing a stable typed marker rather than producing an empty or +/// non-deterministic result. +pub fn content_checksum(content: &T) -> String { + match canonical_json(content) { + Ok(canonical) => checksum_of_canonical(&canonical), + Err(_) => digest(b"reflow:uncanonicalizable"), + } +} + +/// Checksum of an already-canonical JSON string. Use this when you have already +/// computed the canonical form (e.g. to also measure its size or capture it as +/// content) to avoid canonicalizing twice. +pub fn checksum_of_canonical(canonical: &str) -> String { + digest(canonical.as_bytes()) +} + +/// `"sha256:"` digest of raw bytes. +fn digest(bytes: &[u8]) -> String { + let mut hasher = Sha256::new(); + hasher.update(bytes); + format!("{}{}", CHECKSUM_PREFIX, hex::encode(hasher.finalize())) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn checksum_is_prefixed_64_hex() { + let c = content_checksum(&json!({"a": 1})); + assert!(c.starts_with("sha256:")); + assert_eq!(c.len(), "sha256:".len() + 64); + assert!(c["sha256:".len()..] + .chars() + .all(|ch| ch.is_ascii_hexdigit() && !ch.is_ascii_uppercase())); + } + + #[test] + fn checksum_is_content_only_and_order_independent() { + // Same content, different key insertion order -> same digest. + let a = content_checksum(&json!({"x": 1, "y": 2})); + let b = content_checksum(&json!({"y": 2, "x": 1})); + assert_eq!(a, b, "object key order must not affect the checksum"); + } + + #[test] + fn distinct_content_distinct_checksum() { + assert_ne!( + content_checksum(&json!({"x": 1})), + content_checksum(&json!({"x": 2})) + ); + } + + #[test] + fn empty_and_signal_values_have_stable_nonempty_digests() { + let flow = content_checksum(&json!(null)); + assert!(flow.starts_with("sha256:")); + assert_ne!(flow, ""); + // Stable across calls. + assert_eq!(flow, content_checksum(&json!(null))); + } + + #[test] + fn canonical_json_sorts_nested_keys_compactly() { + let canonical = canonical_json(&json!({"b": 1, "a": {"d": 1, "c": 2}})).unwrap(); + assert_eq!(canonical, r#"{"a":{"c":2,"d":1},"b":1}"#); + } + + #[test] + fn known_vector_sha256_of_empty_object() { + // Canonical form of {} is the two bytes "{}"; pin the digest so any + // SDK re-implementation can check itself against this vector. + assert_eq!( + content_checksum(&json!({})), + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + ); + } +} diff --git a/crates/reflow_tracing_protocol/src/client.rs b/crates/reflow_tracing_protocol/src/client.rs index 70a5ab39..21f75536 100644 --- a/crates/reflow_tracing_protocol/src/client.rs +++ b/crates/reflow_tracing_protocol/src/client.rs @@ -28,8 +28,61 @@ use wasm_bindgen::JsCast; #[cfg(target_arch = "wasm32")] use web_sys::{CloseEvent, ErrorEvent, MessageEvent, WebSocket}; -/// Configuration for the tracing client +/// Route a server reply to the caller waiting on its `request_id`. +/// Replies without a correlation id (notifications, acks) are ignored here. +#[cfg(not(target_arch = "wasm32"))] +fn route_tracing_response(state: &Arc>, text: &str) { + let response = match serde_json::from_str::(text) { + Ok(r) => r, + Err(_) => { + debug!("Ignoring unparseable tracing response"); + return; + } + }; + match response { + // Real-time subscription push → forward to the notification tap. + TracingResponse::EventNotification { event, .. } => { + let tap = { state.lock().unwrap().notification_tap.clone() }; + if let Some(sender) = tap { + let _ = sender.try_send(event); + } + } + // Correlated reply → wake the waiting caller. + other => { + let request_id = match &other { + TracingResponse::QueryResults { request_id, .. } => Some(request_id.to_string()), + TracingResponse::TraceData { request_id, .. } => Some(request_id.to_string()), + _ => None, + }; + if let Some(key) = request_id { + let mut guard = state.lock().unwrap(); + if let Some(sender) = guard.pending_requests.remove(&key) { + let _ = sender.send(other); + } + } + } + } +} + +/// Decode a binary WebSocket frame to its JSON text. Frames may be +/// zstd-compressed (the server's default) or raw UTF-8 JSON. +#[cfg(not(target_arch = "wasm32"))] +fn decode_frame(data: &[u8]) -> Option { + if let Ok(decompressed) = zstd::decode_all(data) { + if let Ok(text) = String::from_utf8(decompressed) { + return Some(text); + } + } + String::from_utf8(data.to_vec()).ok() +} + +/// Configuration for the tracing client. +/// +/// `#[serde(default)]` lets callers (and every SDK) enable tracing with just +/// the fields they care about, e.g. `{ "server_url": "...", "enabled": true }`; +/// all other fields fall back to [`TracingConfig::default`]. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] pub struct TracingConfig { /// WebSocket server URL (e.g., "ws://localhost:8080") pub server_url: String, @@ -43,6 +96,22 @@ pub struct TracingConfig { pub retry_config: RetryConfig, /// Whether tracing is enabled pub enabled: bool, + /// Compute a content checksum for each traced message. Cheap; default on. + /// See [`crate::checksum`]. + #[serde(default = "default_true")] + pub capture_checksum: bool, + /// Retain full message content in `serialized_data`. Heavy and potentially + /// sensitive; default off. Independent of `capture_checksum`. + #[serde(default)] + pub capture_content: bool, + /// Sample the heavy `PerformanceMetrics` fields (memory/cpu/throughput). + /// Default off — cheap timing/queue-depth are always measured. + #[serde(default)] + pub enable_perf_sampling: bool, +} + +fn default_true() -> bool { + true } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -74,12 +143,17 @@ type Result = std::result::Result; #[derive(Debug)] struct ClientState { connection_status: ConnectionStatus, - event_buffer: VecDeque, + /// Buffered events paired with the trace they belong to, so the trace id + /// survives the batch path instead of being replaced by a fresh random id. + event_buffer: VecDeque<(TraceId, TraceEvent)>, #[cfg(not(target_arch = "wasm32"))] pending_requests: StdHashMap>, last_batch_time: Instant, reconnect_attempts: usize, current_trace_id: Option, + /// When subscribed, real-time `EventNotification`s from the server are + /// forwarded here so a consumer (e.g. an SDK monitor) can poll them. + notification_tap: Option>, #[cfg(not(target_arch = "wasm32"))] message_sender: Option>, #[cfg(target_arch = "wasm32")] @@ -128,6 +202,9 @@ impl Default for TracingConfig { backoff_multiplier: 2.0, }, enabled: true, + capture_checksum: true, + capture_content: false, + enable_perf_sampling: false, } } } @@ -143,6 +220,7 @@ impl TracingClient { last_batch_time: Instant::now(), reconnect_attempts: 0, current_trace_id: None, + notification_tap: None, #[cfg(not(target_arch = "wasm32"))] message_sender: None, #[cfg(target_arch = "wasm32")] @@ -256,37 +334,14 @@ impl TracingClient { while let Some(msg) = ws_receiver.next().await { match msg { Ok(WsMessage::Text(text)) => { - debug!("Received WebSocket message: {}", text); - // Handle different response types - if let Ok(response) = serde_json::from_str::(&text) { - let mut state_guard = state_for_receiver.lock().unwrap(); - - // Convert TraceResponse to TracingResponse for compatibility - let converted_response = match response { - TraceResponse::TraceStored { trace_id } => { - TracingResponse::TraceStarted { trace_id } - } - TraceResponse::TracesFound { traces } => { - TracingResponse::QueryResults { - traces, - total_count: 0, - } - } - TraceResponse::TraceData { trace } => { - TracingResponse::TraceData { trace: Some(trace) } - } - TraceResponse::Error { message } => TracingResponse::Error { - message, - code: ErrorCode::InternalError, - }, - TraceResponse::Metrics { data: _ } => TracingResponse::Pong, - }; - - // Notify any pending requests - #[cfg(not(target_arch = "wasm32"))] - if let Some((_, sender)) = state_guard.pending_requests.drain().next() { - let _ = sender.send(converted_response); - }; + // The server replies with `TracingResponse`. Route replies + // that carry a `request_id` to their waiting caller. + route_tracing_response(&state_for_receiver, &text); + } + Ok(WsMessage::Binary(data)) => { + // Large responses are compressed (zstd by default). + if let Some(text) = decode_frame(&data) { + route_tracing_response(&state_for_receiver, &text); } } Ok(WsMessage::Close(_)) => { @@ -339,12 +394,10 @@ impl TracingClient { info!("Sending batch of {} trace events", events_to_send.len()); if let Some(sender) = sender_ref { - for event in events_to_send { - // Send individual events as TracingRequest::RecordEvent - let request = TracingRequest::RecordEvent { - trace_id: TraceId::new(), - event, - }; + for (trace_id, event) in events_to_send { + // Send individual events as TracingRequest::RecordEvent, + // preserving the trace id the event was recorded under. + let request = TracingRequest::RecordEvent { trace_id, event }; if let Ok(message) = serde_json::to_string(&request) { if let Err(e) = sender.send(message) { @@ -420,8 +473,13 @@ impl TracingClient { let trace_id = TraceId::new(); - // Send StartTrace request - let request = TracingRequest::StartTrace { flow_id, version }; + // Send StartTrace request. The client owns the trace id; the server + // creates/stores the trace under this exact id. + let request = TracingRequest::StartTrace { + trace_id: trace_id.clone(), + flow_id, + version, + }; let message = serde_json::to_string(&request)?; // Send the message immediately @@ -436,8 +494,8 @@ impl TracingClient { Ok(trace_id) } - /// Record a trace event - pub async fn record_event(&self, _trace_id: TraceId, event: TraceEvent) -> Result<()> { + /// Record a trace event under the given trace. + pub async fn record_event(&self, trace_id: TraceId, event: TraceEvent) -> Result<()> { if !self.config.enabled { return Ok(()); } @@ -447,7 +505,9 @@ impl TracingClient { // Check if we should send immediately let should_send_immediately = { let mut state = self.state.lock().unwrap(); - state.event_buffer.push_back(event.clone()); + state + .event_buffer + .push_back((trace_id.clone(), event.clone())); // Check if we should send immediately if self.config.batch_size == 1 { @@ -463,10 +523,7 @@ impl TracingClient { // Send immediately if needed (lock is already dropped) if should_send_immediately { - let request = TracingRequest::RecordEvent { - trace_id: TraceId::new(), - event, - }; + let request = TracingRequest::RecordEvent { trace_id, event }; if let Ok(message) = serde_json::to_string(&request) { self.send_message(message).await?; @@ -476,6 +533,18 @@ impl TracingClient { Ok(()) } + /// Finalize a trace, marking its terminal status on the server. + pub async fn end_trace(&self, trace_id: TraceId, status: ExecutionStatus) -> Result<()> { + if !self.config.enabled { + return Ok(()); + } + + let request = TracingRequest::EndTrace { trace_id, status }; + let message = serde_json::to_string(&request)?; + self.send_message(message).await?; + Ok(()) + } + /// Send a ping to the server pub async fn ping(&self) -> Result<()> { if !self.config.enabled { @@ -488,32 +557,100 @@ impl TracingClient { Ok(()) } - /// Query traces from the server + /// Query traces from the server. pub async fn query_traces(&self, query: TraceQuery) -> Result> { if !self.config.enabled { return Ok(Vec::new()); } - let request = TracingRequest::QueryTraces { query }; - let message = serde_json::to_string(&request)?; - self.send_message(message).await?; + #[cfg(not(target_arch = "wasm32"))] + { + let request_id = RequestId::new(); + let rx = self.register_request(&request_id); - // For now, return empty - would need proper response handling - Ok(Vec::new()) + let request = TracingRequest::QueryTraces { + request_id: request_id.clone(), + query, + }; + self.send_message(serde_json::to_string(&request)?).await?; + + match self.await_response(&request_id, rx).await? { + TracingResponse::QueryResults { traces, .. } => Ok(traces), + TracingResponse::Error { message, .. } => { + Err(TracingError::WebSocketError(message)) + } + _ => Ok(Vec::new()), + } + } + + #[cfg(target_arch = "wasm32")] + { + let _ = query; + Ok(Vec::new()) + } } - /// Get a specific trace by ID + /// Get a specific trace by ID. pub async fn get_trace(&self, trace_id: TraceId) -> Result> { if !self.config.enabled { return Ok(None); } - let request = TracingRequest::GetTrace { trace_id }; - let message = serde_json::to_string(&request)?; - self.send_message(message).await?; + #[cfg(not(target_arch = "wasm32"))] + { + let request_id = RequestId::new(); + let rx = self.register_request(&request_id); + + let request = TracingRequest::GetTrace { + request_id: request_id.clone(), + trace_id, + }; + self.send_message(serde_json::to_string(&request)?).await?; + + match self.await_response(&request_id, rx).await? { + TracingResponse::TraceData { trace, .. } => Ok(trace), + TracingResponse::Error { message, .. } => { + Err(TracingError::WebSocketError(message)) + } + _ => Ok(None), + } + } + + #[cfg(target_arch = "wasm32")] + { + let _ = trace_id; + Ok(None) + } + } + + /// Register a pending request and return the receiver to await its reply. + #[cfg(not(target_arch = "wasm32"))] + fn register_request( + &self, + request_id: &RequestId, + ) -> tokio::sync::oneshot::Receiver { + let (tx, rx) = tokio::sync::oneshot::channel(); + let mut state = self.state.lock().unwrap(); + state.pending_requests.insert(request_id.to_string(), tx); + rx + } - // For now, return None - would need proper response handling - Ok(None) + /// Await a correlated reply, cleaning up the pending entry on timeout. + #[cfg(not(target_arch = "wasm32"))] + async fn await_response( + &self, + request_id: &RequestId, + rx: tokio::sync::oneshot::Receiver, + ) -> Result { + match tokio::time::timeout(Duration::from_secs(10), rx).await { + Ok(Ok(response)) => Ok(response), + Ok(Err(_)) => Err(TracingError::Disconnected), + Err(_) => { + let mut state = self.state.lock().unwrap(); + state.pending_requests.remove(&request_id.to_string()); + Err(TracingError::Timeout) + } + } } /// Subscribe to real-time trace events @@ -534,6 +671,27 @@ impl TracingClient { state.connection_status == ConnectionStatus::Connected } + /// Whether to compute a content checksum for each traced message (cheap). + pub fn capture_checksum(&self) -> bool { + self.config.capture_checksum + } + + /// Whether to retain full message content in snapshots (heavy, opt-in). + pub fn capture_content(&self) -> bool { + self.config.capture_content + } + + /// Whether the heavy `PerformanceMetrics` fields should be sampled. + pub fn enable_perf_sampling(&self) -> bool { + self.config.enable_perf_sampling + } + + /// Forward real-time `EventNotification`s (from a `subscribe`) to `sender`, + /// so a consumer can poll live trace events from the collector. + pub fn set_notification_tap(&self, sender: flume::Sender) { + self.state.lock().unwrap().notification_tap = Some(sender); + } + /// Get the current connection status pub fn connection_status(&self) -> String { let state = self.state.lock().unwrap(); @@ -558,11 +716,8 @@ impl TracingClient { if !events_to_send.is_empty() { info!("Flushing {} pending trace events", events_to_send.len()); - for event in events_to_send { - let request = TracingRequest::RecordEvent { - trace_id: TraceId::new(), - event, - }; + for (trace_id, event) in events_to_send { + let request = TracingRequest::RecordEvent { trace_id, event }; if let Ok(message) = serde_json::to_string(&request) { let _ = self.send_message(message).await; @@ -593,11 +748,8 @@ impl TracingClient { info!("Flushing {} trace events", events_to_send.len()); - for event in events_to_send { - let request = TracingRequest::RecordEvent { - trace_id: TraceId::new(), - event, - }; + for (trace_id, event) in events_to_send { + let request = TracingRequest::RecordEvent { trace_id, event }; if let Ok(message) = serde_json::to_string(&request) { self.send_message(message).await?; @@ -660,6 +812,9 @@ impl TracingClient { pub struct TracingIntegration { client: Arc, current_trace_id: Arc>>, + /// Optional local tap: every recorded event is also fanned here, so an SDK + /// can consume live trace events without standing up a collector. + local_tap: Arc>>>, } unsafe impl Send for TracingIntegration {} @@ -671,9 +826,18 @@ impl TracingIntegration { Self { client: Arc::new(client), current_trace_id: Arc::new(Mutex::new(None)), + local_tap: Arc::new(Mutex::new(None)), } } + /// Install a local tap. Every subsequently recorded `TraceEvent` is cloned + /// into `sender` in addition to being shipped to the collector, so an SDK + /// can observe live traces without a server. A disconnected receiver simply + /// drops events (the send is best-effort and never blocks tracing). + pub fn set_local_tap(&self, sender: flume::Sender) { + *self.local_tap.lock().unwrap() = Some(sender); + } + /// Start tracing for a flow pub async fn start_flow_trace(&self, flow_id: impl Into) -> Result { let flow_id = FlowId::new(flow_id); @@ -695,49 +859,28 @@ impl TracingIntegration { /// Record an actor creation event pub async fn trace_actor_created(&self, actor_id: impl Into) -> Result<()> { - let event = TraceEvent::actor_created(actor_id.into()); - let trace_id = self.current_trace_id.lock().unwrap().clone(); - if let Some(trace_id) = trace_id { - self.client.record_event(trace_id, event).await - } else { - // Use a default trace ID if none is set - self.client.record_event(TraceId::new(), event).await - } + self.record(TraceEvent::actor_created(actor_id.into())).await } /// Record an actor process completion event pub async fn trace_actor_completed(&self, actor_id: impl Into) -> Result<()> { - let event = TraceEvent::actor_completed(actor_id.into()); - let trace_id = self.current_trace_id.lock().unwrap().clone(); - if let Some(trace_id) = trace_id { - self.client.record_event(trace_id, event).await - } else { - // Use a default trace ID if none is set - self.client.record_event(TraceId::new(), event).await - } + self.record(TraceEvent::actor_completed(actor_id.into())) + .await } - /// Record a message sent event - pub async fn trace_message_sent( + /// Record a message-sent event, capturing a snapshot of `content` per the + /// configured `capture_checksum` / `capture_content` knobs. + pub async fn trace_message_sent( &self, actor_id: impl Into, port: impl Into, message_type: impl Into, - size_bytes: usize, + content: &T, + metrics: PerformanceMetrics, ) -> Result<()> { - let event = TraceEvent::message_sent( - actor_id.into(), - port.into(), - message_type.into(), - size_bytes, - ); - let trace_id = self.current_trace_id.lock().unwrap().clone(); - if let Some(trace_id) = trace_id { - self.client.record_event(trace_id, event).await - } else { - // Use a default trace ID if none is set - self.client.record_event(TraceId::new(), event).await - } + let snapshot = self.snapshot(message_type, content); + let event = TraceEvent::message_sent(actor_id.into(), port.into(), snapshot, metrics); + self.record(event).await } /// Record an actor failure event @@ -746,44 +889,78 @@ impl TracingIntegration { actor_id: impl Into, error: impl Into, ) -> Result<()> { - let event = TraceEvent::actor_failed(actor_id.into(), error.into()); - let trace_id = self.current_trace_id.lock().unwrap().clone(); - if let Some(trace_id) = trace_id { - self.client.record_event(trace_id, event).await - } else { - // Use a default trace ID if none is set - self.client.record_event(TraceId::new(), event).await - } + self.record(TraceEvent::actor_failed(actor_id.into(), error.into())) + .await } - /// Record a data flow event between two actors - pub async fn trace_data_flow( + /// Record a data-flow event between two actors, capturing a snapshot of + /// `content` per the configured capture knobs. + pub async fn trace_data_flow( &self, from_actor: impl Into, from_port: impl Into, to_actor: impl Into, to_port: impl Into, message_type: impl Into, - size_bytes: usize, + content: &T, + metrics: PerformanceMetrics, ) -> Result<()> { + let snapshot = self.snapshot(message_type, content); let event = TraceEvent::data_flow( from_actor.into(), from_port.into(), to_actor.into(), to_port.into(), - message_type.into(), - size_bytes, + snapshot, + metrics, ); + self.record(event).await + } + + /// Build a message snapshot honoring the client's capture configuration. + fn snapshot( + &self, + message_type: impl Into, + content: &T, + ) -> MessageSnapshot { + MessageSnapshot::capture( + message_type, + content, + self.client.capture_checksum(), + self.client.capture_content(), + ) + } + + /// Record an event under the active flow trace (or a fresh id if none), + /// fanning a copy to the local tap first (best-effort, never blocks). + async fn record(&self, event: TraceEvent) -> Result<()> { + let tap = self.local_tap.lock().unwrap().clone(); + if let Some(sender) = tap { + let _ = sender.try_send(event.clone()); + } + let trace_id = self.current_trace_id.lock().unwrap().clone(); + match trace_id { + Some(trace_id) => self.client.record_event(trace_id, event).await, + None => self.client.record_event(TraceId::new(), event).await, + } + } + /// Finalize the current flow trace with a terminal status. + pub async fn end_flow_trace(&self, status: ExecutionStatus) -> Result<()> { let trace_id = self.current_trace_id.lock().unwrap().clone(); if let Some(trace_id) = trace_id { - self.client.record_event(trace_id, event).await + self.client.end_trace(trace_id, status).await } else { - // Use a default trace ID if none is set - self.client.record_event(TraceId::new(), event).await + Ok(()) } } + /// The trace id of the active flow, if one has been started. Used by the + /// distributed router to propagate trace context across process boundaries. + pub fn current_trace_id(&self) -> Option { + self.current_trace_id.lock().unwrap().clone() + } + /// Get access to the underlying client pub fn client(&self) -> Arc { Arc::clone(&self.client) @@ -818,10 +995,10 @@ macro_rules! trace_actor_event { let _ = tracing.trace_actor_created($actor_id).await; } }; - (message_sent, $actor_id:expr, $port:expr, $msg_type:expr, $size:expr) => { + (message_sent, $actor_id:expr, $port:expr, $msg_type:expr, $content:expr, $metrics:expr) => { if let Some(tracing) = $crate::tracing::global_tracing() { let _ = tracing - .trace_message_sent($actor_id, $port, $msg_type, $size) + .trace_message_sent($actor_id, $port, $msg_type, $content, $metrics) .await; } }; @@ -869,8 +1046,52 @@ mod tests { assert!(actor_result.is_ok()); let message_result = integration - .trace_message_sent("test_actor", "output", "TestMessage", 128) + .trace_message_sent( + "test_actor", + "output", + "TestMessage", + &serde_json::json!({ "k": 1 }), + PerformanceMetrics::default(), + ) .await; assert!(message_result.is_ok()); } + + #[tokio::test] + async fn local_tap_receives_recorded_events() { + // Tracing disabled (no server), but a local tap still observes events. + let client = TracingClient::new(TracingConfig { + enabled: false, + ..Default::default() + }); + let integration = TracingIntegration::new(client); + let (tx, rx) = flume::unbounded(); + integration.set_local_tap(tx); + + integration.trace_actor_created("actor-1").await.unwrap(); + integration + .trace_message_sent( + "actor-1", + "out", + "String", + &serde_json::json!("hi"), + PerformanceMetrics::default(), + ) + .await + .unwrap(); + + let e1 = rx.try_recv().expect("actor_created reaches the tap"); + assert!(matches!(e1.event_type, TraceEventType::ActorCreated)); + let e2 = rx.try_recv().expect("message_sent reaches the tap"); + assert!(matches!(e2.event_type, TraceEventType::MessageSent)); + // The snapshot built on the tapped event carries a content checksum. + assert!( + e2.data + .message + .as_ref() + .unwrap() + .checksum + .starts_with("sha256:") + ); + } } diff --git a/crates/reflow_tracing_protocol/src/lib.rs b/crates/reflow_tracing_protocol/src/lib.rs index dfecc007..4ea8c68a 100644 --- a/crates/reflow_tracing_protocol/src/lib.rs +++ b/crates/reflow_tracing_protocol/src/lib.rs @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use uuid::Uuid; +pub mod checksum; pub mod client; /// Unique identifier for a trace @@ -27,6 +28,11 @@ pub struct ExecutionId(pub Uuid); #[derive(Debug, Display, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] pub struct EventId(pub Uuid); +/// Correlation identifier for matching a request/response pair over the +/// (otherwise fire-and-forget) WebSocket channel. +#[derive(Debug, Display, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub struct RequestId(pub Uuid); + /// Flow version for versioning support #[derive(Debug, Clone, Serialize, Deserialize)] pub struct FlowVersion { @@ -41,8 +47,12 @@ pub struct FlowVersion { #[derive(Debug, Clone, Serialize, Deserialize)] #[allow(clippy::large_enum_variant)] pub enum TracingRequest { - /// Start a new flow trace + /// Start a new flow trace. The client owns the `trace_id` so that the + /// same id can be propagated peer-to-peer in distributed flows before any + /// server round-trip; the server creates/stores the trace under this id. StartTrace { + #[serde(default)] + trace_id: TraceId, flow_id: FlowId, version: FlowVersion, }, @@ -51,10 +61,23 @@ pub enum TracingRequest { trace_id: TraceId, event: TraceEvent, }, + /// Finalize a trace, marking its terminal status. + EndTrace { + trace_id: TraceId, + status: ExecutionStatus, + }, /// Get a specific trace by ID - GetTrace { trace_id: TraceId }, + GetTrace { + #[serde(default)] + request_id: RequestId, + trace_id: TraceId, + }, /// Query traces with filters - QueryTraces { query: TraceQuery }, + QueryTraces { + #[serde(default)] + request_id: RequestId, + query: TraceQuery, + }, /// Get all versions of a flow GetFlowVersions { flow_id: FlowId }, /// Health check @@ -75,9 +98,15 @@ pub enum TracingResponse { error: Option, }, /// Response to GetTrace - TraceData { trace: Option }, + TraceData { + #[serde(default)] + request_id: RequestId, + trace: Option, + }, /// Response to QueryTraces QueryResults { + #[serde(default)] + request_id: RequestId, traces: Vec, total_count: usize, }, @@ -175,13 +204,96 @@ pub struct TraceEventData { pub custom_attributes: HashMap, } -/// Snapshot of message data for replay +/// Current version of the `serialized_data` content format. Bump when the +/// canonical content encoding or codec semantics change. +pub const CONTENT_FORMAT_VERSION: u32 = 1; + +/// Snapshot of a single traced message. +/// +/// Two independent capture decisions populate this (see [`MessageSnapshot::capture`]): +/// - `checksum` — cheap, content-only identity (default on). See [`crate::checksum`]. +/// - `serialized_data` — heavy, opt-in full content (default off). +/// +/// Invariant when content is captured: `checksum == sha256(canonical(decompress(serialized_data)))`, +/// and `serialized_data` round-trips to a message equal to the original. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MessageSnapshot { + /// Discriminant/type name of the message (e.g. `"Integer"`). pub message_type: String, + /// Pre-compression content size in bytes — the same bytes the `checksum` + /// covers (length of the canonical JSON form). Stays meaningful whether or + /// not content was captured and regardless of compression. pub size_bytes: usize, + /// `"sha256:<64 lowercase hex>"`, or `""` if not computed. Never chained. pub checksum: String, - pub serialized_data: Vec, // Compressed message data + /// Optional captured content (default empty). Decoded per `content_codec`. + /// Empty strictly means "not captured", distinct from a captured typed-empty. + #[serde(default)] + pub serialized_data: Vec, + /// Codec for `serialized_data`. `"none"` = uncompressed canonical JSON. + #[serde(default = "default_content_codec")] + pub content_codec: String, + /// Format/version tag so a reader in another SDK can decode safely. + #[serde(default)] + pub content_format_version: u32, + /// Compressed/stored footprint of `serialized_data`, when captured. + /// `None` when content is not captured. Do not overload `size_bytes` for this. + #[serde(default)] + pub stored_bytes: Option, +} + +fn default_content_codec() -> String { + "none".to_string() +} + +impl MessageSnapshot { + /// Capture a snapshot of `content` honoring the two orthogonal toggles. + /// + /// `capture_checksum` (cheap, recommended default-on) computes the + /// content-only SHA-256. `capture_content` (heavy, default-off) additionally + /// retains the canonical content bytes in `serialized_data` (codec `"none"`, + /// i.e. uncompressed canonical JSON), preserving the checksum invariant. + pub fn capture( + message_type: impl Into, + content: &T, + capture_checksum: bool, + capture_content: bool, + ) -> Self { + // Canonicalize once if either toggle needs it. + let canonical = if capture_checksum || capture_content { + crate::checksum::canonical_json(content).ok() + } else { + None + }; + let size_bytes = canonical.as_ref().map(|c| c.len()).unwrap_or(0); + let checksum = if capture_checksum { + canonical + .as_ref() + .map(|c| crate::checksum::checksum_of_canonical(c)) + .unwrap_or_default() + } else { + String::new() + }; + let (serialized_data, stored_bytes) = if capture_content { + let bytes = canonical + .as_ref() + .map(|c| c.as_bytes().to_vec()) + .unwrap_or_default(); + let len = bytes.len(); + (bytes, Some(len)) + } else { + (Vec::new(), None) + }; + Self { + message_type: message_type.into(), + size_bytes, + checksum, + serialized_data, + content_codec: default_content_codec(), + content_format_version: CONTENT_FORMAT_VERSION, + stored_bytes, + } + } } /// State differences for time travel debugging @@ -199,14 +311,50 @@ pub enum StateDiffType { MemoryOnly, } -/// Performance metrics for observability +/// Per-event performance metrics. +/// +/// Each field is `Option`: `None` means **unmeasured**, `Some(0)` means +/// **measured and zero**. The distinction is load-bearing — downstream +/// aggregation (averages, percentiles, alerts) must exclude unmeasured samples +/// rather than averaging zeros into them. Units are pinned in each field's doc +/// and must not drift across SDKs. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PerformanceMetrics { - pub execution_time_ns: u64, - pub memory_usage_bytes: usize, - pub cpu_usage_percent: f32, - pub queue_depth: usize, - pub throughput_msgs_per_sec: f64, + /// Wall-clock duration of the traced operation, in **nanoseconds** + /// (monotonic clock). Cheap — measured on the always-on path. + #[serde(default)] + pub execution_time_ns: Option, + /// Memory attributable to the operation as a **delta** (allocation + /// high-water during the tick), in **bytes** — not process RSS. Heavy: + /// gated behind `enable_perf_sampling`; `None` when off. + #[serde(default)] + pub memory_usage_bytes: Option, + /// CPU usage averaged over the operation window, **0–100 percent**. Heavy + /// and interval-shaped: gated; `None` when off. + #[serde(default)] + pub cpu_usage_percent: Option, + /// Instantaneous inbound mailbox depth at capture time, a **count**. Cheap — + /// measured on the always-on path. + #[serde(default)] + pub queue_depth: Option, + /// Throughput over a documented rolling window, in **messages per second**. + /// Needs a window denominator: gated; `None` when off. + #[serde(default)] + pub throughput_msgs_per_sec: Option, +} + +impl PerformanceMetrics { + /// Cheap-path metrics: the always-affordable fields measured (or `None` if + /// not available here), the heavy/gated fields left unmeasured. + pub fn cheap(execution_time_ns: Option, queue_depth: Option) -> Self { + Self { + execution_time_ns, + memory_usage_bytes: None, + cpu_usage_percent: None, + queue_depth, + throughput_msgs_per_sec: None, + } + } } /// Causality information for dependency tracking @@ -315,6 +463,18 @@ impl FlowId { } } +impl RequestId { + pub fn new() -> Self { + Self(Uuid::new_v4()) + } +} + +impl Default for RequestId { + fn default() -> Self { + Self::new() + } +} + impl Default for TraceId { fn default() -> Self { Self::new() @@ -334,13 +494,14 @@ impl Default for EventId { } impl Default for PerformanceMetrics { + /// All fields unmeasured. fn default() -> Self { Self { - execution_time_ns: 0, - memory_usage_bytes: 0, - cpu_usage_percent: 0.0, - queue_depth: 0, - throughput_msgs_per_sec: 0.0, + execution_time_ns: None, + memory_usage_bytes: None, + cpu_usage_percent: None, + queue_depth: None, + throughput_msgs_per_sec: None, } } } @@ -375,8 +536,8 @@ impl TraceEvent { from_port: String, to_actor: String, to_port: String, - message_type: String, - size_bytes: usize, + snapshot: MessageSnapshot, + metrics: PerformanceMetrics, ) -> Self { Self { event_id: EventId::new(), @@ -385,15 +546,10 @@ impl TraceEvent { actor_id: from_actor, data: TraceEventData { port: Some(from_port), - message: Some(MessageSnapshot { - message_type, - size_bytes, - checksum: String::new(), // TODO: Calculate actual checksum - serialized_data: Vec::new(), // TODO: Add optional data capture - }), + message: Some(snapshot), state_diff: None, error: None, - performance_metrics: PerformanceMetrics::default(), + performance_metrics: metrics, custom_attributes: HashMap::new(), }, causality: CausalityInfo { @@ -408,8 +564,8 @@ impl TraceEvent { pub fn message_sent( actor_id: String, port: String, - message_type: String, - size_bytes: usize, + snapshot: MessageSnapshot, + metrics: PerformanceMetrics, ) -> Self { Self { event_id: EventId::new(), @@ -418,15 +574,10 @@ impl TraceEvent { actor_id, data: TraceEventData { port: Some(port), - message: Some(MessageSnapshot { - message_type, - size_bytes, - checksum: String::new(), // TODO: Calculate actual checksum - serialized_data: Vec::new(), // TODO: Add optional data capture - }), + message: Some(snapshot), state_diff: None, error: None, - performance_metrics: PerformanceMetrics::default(), + performance_metrics: metrics, custom_attributes: HashMap::new(), }, causality: CausalityInfo { diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index f48f7d8d..bd4a9be0 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -56,6 +56,11 @@ - [Data Flow Tracing](observability/data-flow-tracing.md) - [Configuration](observability/configuration.md) - [Storage Backends](observability/storage-backends.md) + - [SQLite](observability/storage/sqlite.md) + - [PostgreSQL](observability/storage/postgres.md) + - [TimescaleDB](observability/storage/timescaledb.md) + - [MongoDB](observability/storage/mongodb.md) +- [OTLP Export (Monoscope)](observability/otlp-export.md) - [Production Deployment](observability/deployment.md) --- diff --git a/docs/observability/configuration.md b/docs/observability/configuration.md index 882e2921..a5a4f275 100644 --- a/docs/observability/configuration.md +++ b/docs/observability/configuration.md @@ -6,22 +6,16 @@ Reflow's observability framework provides flexible configuration options to suit ### TracingConfig Structure +`TracingConfig` derives `Default` and is `#[serde(default)]`, so you only need to +set the fields you care about — the rest fall back to their defaults: + ```rust use reflow_network::tracing::TracingConfig; -use std::time::Duration; let config = TracingConfig { server_url: "ws://localhost:8080".to_string(), - batch_size: 50, - batch_timeout: Duration::from_millis(1000), - enable_compression: true, enabled: true, - retry_config: RetryConfig { - max_retries: 3, - initial_delay: Duration::from_millis(500), - max_delay: Duration::from_secs(5), - backoff_multiplier: 2.0, - }, + ..TracingConfig::default() }; ``` @@ -30,12 +24,37 @@ let config = TracingConfig { | Parameter | Type | Default | Description | |-----------|------|---------|-------------| | `server_url` | String | `"ws://localhost:8080"` | WebSocket URL of the tracing server | -| `batch_size` | usize | `50` | Number of events to batch before sending | -| `batch_timeout` | Duration | `1000ms` | Maximum time to wait before sending incomplete batch | -| `enable_compression` | bool | `true` | Enable gzip compression for network transmission | | `enabled` | bool | `true` | Global enable/disable switch for tracing | +| `batch_size` | usize | `1` | Events to batch before sending (1 = send immediately) | +| `batch_timeout` | Duration | `100ms` | Maximum time to wait before sending an incomplete batch | +| `enable_compression` | bool | `true` | Enable compression for large payloads | +| `capture_checksum` | bool | `true` | Compute a SHA-256 content checksum for each traced message (cheap) | +| `capture_content` | bool | `false` | Retain full message content in `serialized_data` (heavy, opt-in) | +| `enable_perf_sampling` | bool | `false` | Sample the heavy `PerformanceMetrics` fields (memory/cpu/throughput) | | `retry_config` | RetryConfig | See below | Configuration for retry logic | +### Capture controls + +Content fidelity is governed by two independent toggles (see +[Data Flow Tracing](data-flow-tracing.md)): + +- **`capture_checksum`** (default **on**) — a cheap, content-only + `"sha256:"` digest of each message, for content identity, dedup, and + integrity. Computed over a canonical form, so it's identical across processes, + hosts, and SDK languages. +- **`capture_content`** (default **off**) — additionally retains the message + bytes in `serialized_data`. Heavier, and may contain sensitive payloads — opt + in deliberately. + +`enable_perf_sampling` (default off) gates the expensive `PerformanceMetrics` +fields; cheap timing/queue-depth are always recorded, the rest report +*unmeasured* (`None`) when off. + +> **Note:** `TracingConfig` is a plain serde struct. There is no built-in +> `from_env`/`from_file` loader — load it yourself with `serde_json`/`toml` (or +> set `NetworkConfig.tracing` directly). The environment-variable and TOML +> examples below are illustrative patterns, not built-in functions. + ### Retry Configuration ```rust @@ -67,10 +86,18 @@ export REFLOW_TRACING_MAX_DELAY_MS=30000 ### Configuration from Environment +`TracingConfig` has no built-in env loader; read the variables yourself and +build the struct (defaults fill the rest): + ```rust use reflow_network::tracing::TracingConfig; -let config = TracingConfig::from_env().unwrap_or_default(); +let config = TracingConfig { + enabled: std::env::var("REFLOW_TRACING_ENABLED").map(|v| v == "true").unwrap_or(true), + server_url: std::env::var("REFLOW_TRACING_SERVER_URL") + .unwrap_or_else(|_| "ws://localhost:8080".to_string()), + ..TracingConfig::default() +}; ``` ## File-Based Configuration @@ -101,10 +128,14 @@ event_types = ["ActorCreated", "DataFlow", "ActorFailed"] ### Loading from File +Deserialize with any serde format — `TracingConfig` is `Deserialize` and +`#[serde(default)]`, so partial files work: + ```rust use reflow_network::tracing::TracingConfig; -let config = TracingConfig::from_file("tracing.toml")?; +let text = std::fs::read_to_string("tracing.toml")?; +let config: TracingConfig = toml::from_str(&text)?; ``` ## Performance Tuning diff --git a/docs/observability/data-flow-tracing.md b/docs/observability/data-flow-tracing.md index 43b1e607..9d452b67 100644 --- a/docs/observability/data-flow-tracing.md +++ b/docs/observability/data-flow-tracing.md @@ -18,23 +18,17 @@ Traditional actor monitoring focuses on individual actor behavior - creation, co Data Flow Tracing operates at the **connector level**, intercepting messages as they flow between actors: ```rust -// Automatic tracing in connector implementation -impl Connector { - pub async fn send_message(&self, message: Message) -> Result<()> { - // Send the actual message - self.channel.send(message.clone()).await?; - - // Automatically record data flow event - if let Some(tracing) = global_tracing() { - tracing.trace_data_flow( - &self.from_actor, &self.from_port, - &self.to_actor, &self.to_port, - message.type_name(), message.size_bytes() - ).await?; - } - - Ok(()) - } +// Automatic tracing at the connector level (simplified). The message itself is +// passed as content; the integration computes the snapshot (checksum, size, and +// — if capture_content is on — the bytes) per the configured capture knobs. +if let Some(tracing) = &network.tracing_integration { + tracing.trace_data_flow( + from_actor_id, from_port, + to_actor_id, to_port, + msg.type_name(), // message type label + &msg, // content (Message: Serialize) + reflow_tracing_protocol::PerformanceMetrics::default(), + ).await?; } ``` @@ -107,37 +101,59 @@ graph LR style E fill:#f3e5f5 ``` -Query for complete data lineage: +Query for complete data lineage. `TraceQuery` has these fields: `flow_id`, +`execution_id`, `time_range`, `status`, `actor_filter`, `limit`, `offset`. +There is no `event_types`/`custom_filter` field — filter the returned events +client-side (e.g. by `event_type` or `data.message.checksum`): ```rust -// Find all data flow for a specific message +use reflow_tracing_protocol::{TraceEventType, TraceQuery}; + let query = TraceQuery { - event_types: Some(vec![TraceEventType::DataFlow { - to_actor: "*".to_string(), - to_port: "*".to_string() - }]), - custom_filter: Some("message_id = 'msg_12345'"), - ..Default::default() + flow_id: None, + execution_id: None, + time_range: None, + status: None, + actor_filter: Some("data_processor".to_string()), + limit: Some(200), + offset: None, }; -let lineage = tracing_client.query_traces(query).await?; +let traces = tracing_client.query_traces(query).await?; +// Narrow to data-flow events for a specific payload by its content checksum: +let lineage: Vec<_> = traces.iter() + .flat_map(|t| &t.events) + .filter(|e| matches!(e.event_type, TraceEventType::DataFlow { .. })) + .filter(|e| e.data.message.as_ref() + .map(|m| m.checksum == "sha256:9f86d081…") + .unwrap_or(false)) + .collect(); ``` ### 2. Performance Analysis -Identify bottlenecks in your data processing pipeline: +Identify bottlenecks. Query by time range, then filter on the (optional) +performance metrics client-side. Note `execution_time_ns` is `Option` +(`None` = unmeasured), and the heavy fields require `enable_perf_sampling`: ```rust -// Query for slow data transfers -let slow_transfers = TraceQuery { - event_types: Some(vec![TraceEventType::DataFlow { - to_actor: "*".to_string(), - to_port: "*".to_string() - }]), - performance_filter: Some("execution_time_ns > 10000000"), // > 10ms +let recent = TraceQuery { + flow_id: None, + execution_id: None, time_range: Some((Utc::now() - Duration::hours(1), Utc::now())), - ..Default::default() + status: None, + actor_filter: None, + limit: Some(500), + offset: None, }; +let traces = tracing_client.query_traces(recent).await?; +let slow: Vec<_> = traces.iter() + .flat_map(|t| &t.events) + .filter(|e| matches!(e.event_type, TraceEventType::DataFlow { .. })) + .filter(|e| e.data.performance_metrics.execution_time_ns + .map(|ns| ns > 10_000_000) // > 10ms; ignores unmeasured (None) + .unwrap_or(false)) + .collect(); ``` ### 3. System Dependency Mapping @@ -195,6 +211,12 @@ let tracing_config = TracingConfig { ### Selective Tracing +> The built-in controls are the `capture_checksum` / `capture_content` config +> toggles (cheap identity always on; heavy content opt-in). The +> `SelectiveConnector` / `DataFlowSampler` / `should_trace_message` code below is +> **illustrative user-authored** sampling — there is no such built-in type. Note +> the real `trace_data_flow` signature takes `(…, message_type, &content, metrics)`. + For high-throughput systems, you might want to selectively trace certain data flows: ```rust @@ -257,84 +279,56 @@ impl DataFlowSampler { } ``` -## Advanced Features - -### Message Content Capture +## Content fidelity (checksums & capture) -For debugging purposes, you can optionally capture message content: +Every data-flow event carries a `MessageSnapshot`. Fidelity is governed by two +config toggles, not hand-rolled helpers: ```rust -let event = TraceEvent::data_flow_with_content( - from_actor, from_port, - to_actor, to_port, - message_type, size_bytes, - Some(message.serialize()?) // Optional content capture -); +let tracing_config = TracingConfig { + server_url: "ws://localhost:8080".to_string(), + enabled: true, + capture_checksum: true, // default ON — cheap content digest + capture_content: false, // default OFF — retain raw bytes (heavy/sensitive) + ..TracingConfig::default() +}; ``` -⚠️ **Security Warning**: Be careful when capturing message content in production. Ensure no sensitive data is included. - -### Custom Metadata +### Checksum (always-on, cheap) -Add custom metadata to data flow events: +With `capture_checksum` on (the default), each snapshot gets a content-only +`"sha256:<64 lowercase hex>"` digest over a canonical form of the message — +identical across processes, hosts, CPU architectures, and SDK languages. Use it +for content identity, dedup, and integrity: ```rust -// Enhanced data flow tracing with custom metadata -pub async fn trace_enhanced_data_flow( - tracing: &TracingIntegration, - from_actor: &str, from_port: &str, - to_actor: &str, to_port: &str, - message: &Message, - custom_metadata: HashMap -) -> Result<()> { - let mut event = TraceEvent::data_flow( - from_actor.to_string(), from_port.to_string(), - to_actor.to_string(), to_port.to_string(), - message.type_name(), message.size_bytes() - ); - - // Add custom metadata - event.data.custom_attributes.extend(custom_metadata); - - // Add message-specific metadata - event.data.custom_attributes.insert( - "message_priority".to_string(), - json!(message.priority()) - ); - event.data.custom_attributes.insert( - "message_correlation_id".to_string(), - json!(message.correlation_id()) - ); - - tracing.record_event(event).await +if let Some(msg) = &event.data.message { + println!("type={} size={}B checksum={}", msg.message_type, msg.size_bytes, msg.checksum); } ``` -### Causality Tracking +`size_bytes` is the pre-compression content size (the same bytes the checksum +covers). The digest is computed over the *decompressed* content, so toggling +compression never changes it. -Link data flow events to their triggering events: +### Content capture (opt-in, heavy) -```rust -pub async fn trace_causally_linked_data_flow( - tracing: &TracingIntegration, - triggering_event_id: EventId, - from_actor: &str, from_port: &str, - to_actor: &str, to_port: &str, - message: &Message -) -> Result<()> { - let mut event = TraceEvent::data_flow( - from_actor.to_string(), from_port.to_string(), - to_actor.to_string(), to_port.to_string(), - message.type_name(), message.size_bytes() - ); - - // Link to triggering event - event.causality.parent_event_id = Some(triggering_event_id); - event.causality.dependency_chain.push(triggering_event_id); - - tracing.record_event(event).await -} -``` +`capture_content` additionally retains the message bytes in +`serialized_data` (self-describing via `content_codec` + `content_format_version`, +with `stored_bytes` for the stored footprint). Invariant: +`checksum == sha256(canonical(decompress(serialized_data)))`. + +⚠️ **Security**: captured content may contain sensitive payloads. Keep +`capture_content` off unless you need full replay, and scope it narrowly. + +### Causality fields + +Each event has a `causality` block (`parent_event_id`, `root_cause_event_id`, +`dependency_chain`, `span_id`). Across process boundaries the inbound hop is +linked to the sending span (see the distributed-tracing section in the +[overview](overview.md)). +Fine-grained per-message causal chaining within a process is a documented +follow-up — the fields exist but are not yet auto-populated for in-process hops. ## Performance Considerations @@ -357,37 +351,34 @@ Data Flow Tracing introduces minimal overhead: ### Monitoring Performance Impact -Monitor the tracing system's own performance: - -```rust -// Monitor tracing overhead -let tracing_metrics = global_tracing() - .unwrap() - .get_performance_metrics() - .await?; - -println!("Events per second: {}", tracing_metrics.events_per_second); -println!("Average latency: {}ms", tracing_metrics.avg_latency_ms); -println!("Memory usage: {}MB", tracing_metrics.memory_usage_mb); -``` +The server tracks basic counters (connections, messages, traces stored/queried), +queryable via the legacy `TraceMessage::GetMetrics` request. There is no +`global_tracing().get_performance_metrics()` API. The cheap path +(`capture_checksum` on, `capture_content` off, `enable_perf_sampling` off) keeps +per-event overhead to a checksum; turn `capture_content` / `enable_perf_sampling` +on only when you need the extra data. ## Visualization and Analysis ### Data Flow Diagrams -Generate visual representations of your data flow: +There is no built-in `DataFlowGraph`/renderer. Query the data-flow events and +build a graph from them in your tool of choice — each `DataFlow` event gives you +the `actor_id` (source), `event_type.DataFlow { to_actor, to_port }` +(destination), and `data.message` (type/size/checksum): ```rust -// Generate data flow graph for the last hour -let flow_data = tracing_client.query_data_flows( - TraceQuery { - time_range: Some((Utc::now() - Duration::hours(1), Utc::now())), - ..Default::default() - } -).await?; +let traces = tracing_client.query_traces(TraceQuery { + flow_id: None, execution_id: None, + time_range: Some((Utc::now() - Duration::hours(1), Utc::now())), + status: None, actor_filter: None, limit: Some(1000), offset: None, +}).await?; -let graph = DataFlowGraph::from_events(&flow_data); -graph.render_to_file("data_flow_diagram.svg")?; +for e in traces.iter().flat_map(|t| &t.events) { + if let TraceEventType::DataFlow { to_actor, to_port } = &e.event_type { + // edge: e.actor_id -> to_actor (port to_port), size e.data.message… + } +} ``` ### Real-time Dashboard diff --git a/docs/observability/deployment.md b/docs/observability/deployment.md index 692bd7b4..fd7a6829 100644 --- a/docs/observability/deployment.md +++ b/docs/observability/deployment.md @@ -2,6 +2,15 @@ This guide covers deploying Reflow's observability framework in production environments, including scalability considerations, security best practices, and operational procedures. +> **Status:** the storage backends shown here are real — build the +> `reflow_tracing` server with `--features postgres` (or `mongodb`, or +> `all-backends`) and set `storage.backend` accordingly; `memory`/`sqlite` ship +> by default (see [Storage Backends](storage-backends.md)). The PostgreSQL +> **replication/partitioning** tuning below is an operational target, not +> auto-configured by the server. The distributed model is a **shared +> collector**: point every network's `TracingConfig.server_url` at one +> `reflow_tracing` server and a cross-process flow is stitched into one trace. + ## Architecture Overview ### Production Architecture diff --git a/docs/observability/getting-started.md b/docs/observability/getting-started.md index fa21f840..ea8d0020 100644 --- a/docs/observability/getting-started.md +++ b/docs/observability/getting-started.md @@ -106,19 +106,12 @@ async fn main() -> Result<()> { println!("🚀 Starting traced actor network"); - // Step 1: Configure tracing + // Step 1: Configure tracing. TracingConfig is #[serde(default)] and derives + // Default, so set only what you need. let tracing_config = TracingConfig { server_url: "ws://127.0.0.1:8080".to_string(), - batch_size: 10, - batch_timeout: Duration::from_millis(500), - enable_compression: false, enabled: true, - retry_config: reflow_network::tracing::RetryConfig { - max_retries: 3, - initial_delay: Duration::from_millis(100), - max_delay: Duration::from_secs(5), - backoff_multiplier: 2.0, - }, + ..TracingConfig::default() }; // Step 2: Initialize global tracing @@ -179,10 +172,11 @@ async fn main() -> Result<()> { tracing.trace_actor_created("manual_actor").await?; tracing.trace_message_sent( - "manual_actor", - "output", - "ManualMessage", - 256 + "manual_actor", + "output", + "ManualMessage", + &"hello", // content (anything Serialize) + reflow_tracing_protocol::PerformanceMetrics::default(), ).await?; println!("✅ Manual events recorded"); @@ -244,6 +238,27 @@ You'll see real-time trace events: ... ``` +## Consuming Traces in Your App (no separate client needed) + +You don't have to run the monitoring client — every SDK can subscribe to the +network's **local trace tap** directly. Enable tracing in the config and call +`traces()`: + +```python +# Python — same for Node (net.traces()), Go (net.Traces()), C++/JVM (traces()) +net = Network({"tracing": {"server_url": "ws://127.0.0.1:8080", "enabled": True}}) +stream = net.traces() +net.start() +while True: + evt = stream.recv(timeout_ms=500) + if evt: print(evt["event_type"], evt.get("actor_id")) +``` + +The local tap delivers events even if the collector at `server_url` is +unreachable. To query historical traces or subscribe to a shared collector +across processes, use the collector client (e.g. C ABI +`rfl_trace_client_connect` / `_query` / `_subscribe`). + ## What Just Happened? Your simple actor network generated several types of trace events: @@ -319,6 +334,7 @@ telnet 127.0.0.1 8080 For production systems, consider: - Increasing `batch_size` to reduce network overhead - Enabling compression with `enable_compression: true` -- Using PostgreSQL backend for better concurrent performance +- Leaving `capture_content` off (default) so only the cheap checksum is computed +- Using the SQLite backend for durable storage (the `memory` backend is for tests) Get help in our [troubleshooting guide](../reference/troubleshooting-guide.md) or check the [architecture documentation](architecture.md) for deeper understanding. diff --git a/docs/observability/otlp-export.md b/docs/observability/otlp-export.md new file mode 100644 index 00000000..a05d823b --- /dev/null +++ b/docs/observability/otlp-export.md @@ -0,0 +1,69 @@ +# OpenTelemetry (OTLP) Export — Monoscope & friends + +The `reflow_tracing` collector can **export finalized traces to any OpenTelemetry +backend** over OTLP/HTTP — [Monoscope](https://github.com/monoscope-tech/monoscope), +Jaeger, Grafana Tempo, Honeycomb, Uptrace, or a vanilla OpenTelemetry Collector. +This is the bridge from reflow's own trace store to an operator-facing monitoring +dashboard. + +It's complementary, not either/or: + +- the **bespoke collector** stays the reflow-native system of record (FlowTrace + replay, content checksums, causality, the distributed unified-trace model); +- the **OTLP export** feeds a dashboard layer for live waterfalls, service maps, + alerting, and correlation with the surrounding app's logs/metrics. + +Because the integration is the **OTLP wire protocol** (not linking any backend's +code), it works with every OTel system and carries no library/license coupling. + +## Enable it + +Build the server with the `otlp` feature and add an `[otlp]` block: + +```bash +cargo build -p reflow_tracing --features otlp +``` + +```toml +[otlp] +enabled = true +# Base OTLP/HTTP endpoint; "/v1/traces" is appended automatically. +# Monoscope / an OTel Collector listens for OTLP/HTTP on :4318. +endpoint = "http://localhost:4318" +service_name = "reflow" +``` + +Each trace is exported once, when it **finalizes** (on `EndTrace` or client +disconnect), as a best-effort fire-and-forget POST — export never blocks the +collector or the data plane. A failed export is logged at `debug` and dropped. + +## How traces map to OpenTelemetry + +reflow's trace model is already span-shaped, so the mapping is direct: + +| reflow | OpenTelemetry span(s) | +|---|---| +| `FlowTrace.trace_id` (128-bit UUID) | the OTel **trace id** (16 bytes), used directly | +| the flow as a whole | a synthetic **root span** `flow:{flow_id}` over `start_time`..`end_time` | +| each `TraceEvent` | a **child span**, parented to `causality.parent_event_id` (else the root) | +| `DataFlow { to_actor, to_port }` | span `dataflow:{from}->{to}` + `reflow.to_actor` / `reflow.to_port` attrs | +| `PerformanceMetrics.execution_time_ns` | span **duration** (`end = start + ns`) | +| `MessageSnapshot.checksum` | attribute `reflow.message.checksum` | +| `ActorFailed` (+ `error`) | span **status = ERROR** with the message | +| cross-process `TraceContext` (shared collector) | one **distributed trace** in the dashboard | + +Per the OTLP/JSON encoding, trace/span ids are hex strings and timestamps are +decimal unix-nanos strings; the payload is a standard +`ExportTraceServiceRequest` (`resourceSpans → scopeSpans → spans`). + +## Monoscope specifically + +Monoscope is OpenTelemetry-native (logs + traces + metrics, 750+ integrations, +self-hostable via Docker Compose). Point `endpoint` at its OTLP/HTTP listener and +reflow flows show up alongside everything else it ingests — including the +**cross-process flows** stitched by the [shared-collector model](overview.md), +which render as single distributed traces. + +> Don't write traces into Monoscope's internal store directly (its TimeFusion DB +> is Postgres-compatible, so it's tempting): OTLP is the supported ingestion +> path and keeps you decoupled from its schema. diff --git a/docs/observability/overview.md b/docs/observability/overview.md index bb996e0e..c3c8f0b0 100644 --- a/docs/observability/overview.md +++ b/docs/observability/overview.md @@ -7,8 +7,8 @@ Reflow provides a comprehensive observability framework that enables deep intros ### 🔍 **Comprehensive Event Tracing** - **Actor Lifecycle**: Track creation, startup, execution, completion, and failures - **Message Flow**: Monitor all message passing between actors with detailed metadata -- **Data Flow Tracing**: NEW - Automatic tracing of data flow between connected actors -- **State Changes**: Capture state transitions with diff support for time-travel debugging +- **Data Flow Tracing**: Automatic tracing of data flow between connected actors, with content checksums +- **State Changes**: _Planned_ — the `StateChanged` event and `StateDiff` type exist, but state diffs are not yet captured automatically (time-travel debugging is a follow-up) - **Network Events**: Monitor distributed network operations and health ### 📊 **Real-time Monitoring** @@ -22,10 +22,20 @@ Reflow provides a comprehensive observability framework that enables deep intros - **PostgreSQL**: Production-ready backend with ACID guarantees and concurrent access - **Memory**: High-performance in-memory storage for testing and temporary analysis -### 🌐 **Distributed Tracing** -- **Cross-Network Visibility**: Trace execution across multiple network instances -- **Causality Tracking**: Maintain event dependency chains across distributed components -- **Span Integration**: Compatible with OpenTelemetry and Jaeger for unified observability +### 🌐 **Distributed Tracing (shared collector)** +- **Cross-Process Visibility**: A flow that spans multiple network instances is + stitched into **one** end-to-end `FlowTrace`. The originating network's + `trace_id` propagates across the bridge (on `RemoteMessage`), and each + receiving network records its cross-process hop under that same id. +- **Unified, Jaeger-style model**: point every participating network's + `TracingConfig.server_url` at one shared `reflow_tracing` server; it aggregates + every process's events into the same trace. +- **Content checksums**: each traced message carries a deterministic + `"sha256:"` content digest, identical across processes and SDK languages. + +> The stack is bespoke (WebSocket protocol + server + SQLite/memory storage + +> replay), not OpenTelemetry/OTLP. An OTLP export adapter is a possible future +> addition, not a current feature. ## Architecture Overview @@ -113,10 +123,12 @@ let network = Network::new(network_config); ``` ### Manual Event Recording -For custom events and detailed control: +For custom events and detailed control. `trace_data_flow`/`trace_message_sent` +take the **message itself** as content — the integration computes the snapshot +(checksum, size, optional content) per the configured capture knobs: ```rust -use reflow_tracing_protocol::{TraceEvent, TracingIntegration}; +use reflow_tracing_protocol::PerformanceMetrics; // Record custom events if let Some(tracing) = global_tracing() { @@ -124,11 +136,38 @@ if let Some(tracing) = global_tracing() { tracing.trace_data_flow( "source_actor", "output", "target_actor", "input", - "CustomMessage", 1024 + "CustomMessage", // message type label + &message, // the content (anything Serialize) + PerformanceMetrics::default(), ).await?; } ``` +### Consuming traces from an SDK (no collector required) +Tracing is first-class in every SDK. Enable it in the network config and +subscribe to the **local tap** — live trace events with no server needed: + +```python +# Python +net = Network({"tracing": {"server_url": "ws://localhost:8080", "enabled": True}}) +traces = net.traces() +net.start() +evt = traces.recv(timeout_ms=500) # dict: event_type, actor_id, data.message.checksum, … +``` + +```javascript +// Node +const net = new Network({ tracing: { server_url: "ws://localhost:8080", enabled: true } }); +const traces = net.traces(); +net.start(); +const evt = await traces.recv(); // { event_type, actor_id, data: { message: { checksum } } } +``` + +Equivalent surfaces exist in Go (`net.Traces()`), C++ (`net.traces()`), and the +JVM (`network.traces()`), plus a C ABI (`rfl_network_traces`) and a +collector-client (`rfl_trace_client_connect/_query/_subscribe`) for historical +and distributed monitoring. + ## Data Flow Tracing The latest enhancement to the observability framework provides automatic data flow tracing: @@ -186,5 +225,6 @@ DataFlow { - Learn about [event types and their uses](event-types.md) - Explore [storage backend options](storage-backends.md) +- Ship traces to a dashboard via [OTLP export (Monoscope, Jaeger, Tempo, …)](otlp-export.md) - Set up [production monitoring](deployment.md) - Integrate with [existing monitoring systems](../tutorials/advanced-tracing-setup.md) diff --git a/docs/observability/storage-backends.md b/docs/observability/storage-backends.md index 3f9ba165..645d4c13 100644 --- a/docs/observability/storage-backends.md +++ b/docs/observability/storage-backends.md @@ -4,13 +4,46 @@ Reflow's observability framework supports multiple storage backends to accommoda ## Overview -The tracing system provides a pluggable storage architecture that allows you to choose the most appropriate backend for your needs: - -- **Memory Storage**: Fast, ephemeral storage for development and testing -- **SQLite Storage**: Lightweight, embedded database for small to medium deployments -- **PostgreSQL Storage**: Robust, scalable database for production environments -- **ClickHouse Storage**: High-performance analytical database for massive scale -- **Custom Storage**: Implement your own storage adapter +> **Implementation status & build features.** The backend is selected by +> `storage.backend` in the server config. `memory` and `sqlite` are always +> available; `postgres` and `mongodb` are compiled in via cargo features: +> +> | `storage.backend` | Status | Build | +> |---|---|---| +> | `memory` | available | default | +> | `sqlite` | available | default (the `storage` feature) | +> | `postgres` / `postgresql` | available | `--features postgres` | +> | `timescale` / `timescaledb` | available | `--features postgres` | +> | `mongodb` / `mongo` | available | `--features mongodb` | +> +> `--features all-backends` enables Postgres + MongoDB together. Selecting a +> backend that wasn't compiled in returns a clear error. + +The tracing system provides a pluggable storage architecture (the `TraceStorage` +trait — `store_trace`/`get_trace`/`query_traces`/`delete_trace`/`get_stats`) so +backends are interchangeable. Postgres and MongoDB store each `FlowTrace` as one +row/document keyed by `trace_id`, with denormalized `flow_id`/`execution_id`/ +`status`/`start_time` columns/fields for indexed querying. + +- **Memory Storage** _(available)_: Fast, ephemeral storage for development and testing +- **SQLite Storage** _(available)_: Lightweight, embedded database for small to medium deployments; writes are synchronous (write-through) +- **PostgreSQL Storage** _(available, `--features postgres`)_: Robust, scalable database for production environments +- **TimescaleDB Storage** _(available, `--features postgres`)_: PostgreSQL + a time-series hypertable on `start_time` — a natural fit for traces +- **MongoDB Storage** _(available, `--features mongodb`)_: Document store; the JSON-shaped trace maps naturally to a document +- **Custom Storage**: Implement the `TraceStorage` trait yourself + +### Integration guides + +Step-by-step setup for each database — build feature, run the DB, the exact +config block, the auto-created schema, verification, and operations: + +- **[SQLite](storage/sqlite.md)** — embedded, default, zero-ops +- **[PostgreSQL](storage/postgres.md)** — production, multi-instance, high-concurrency +- **[TimescaleDB](storage/timescaledb.md)** — Postgres + time-series hypertable (retention/compression) +- **[MongoDB](storage/mongodb.md)** — document store + +This page below is the conceptual overview (model, comparison, selection); the +guides above are the practical walkthroughs. ## Memory Storage @@ -123,7 +156,28 @@ let read_config = SqliteConfig { - **File size**: Large databases can become unwieldy - **Network access**: No remote access without additional tools -## PostgreSQL Storage +## PostgreSQL Storage _(available — build with `--features postgres`)_ + +Selected via the server config (`storage.backend = "postgres"` + a `storage.postgres` +section). The schema (`traces` table + indexes) is created automatically on +startup; you do **not** need to run the DDL by hand. Configured fields: +`connection_url`, `max_connections`, `min_connections`, `acquire_timeout_secs`. + +```toml +[storage] +backend = "postgres" + +[storage.postgres] +connection_url = "postgresql://user:pass@localhost/traces" +max_connections = 20 +min_connections = 5 +acquire_timeout_secs = 5 +``` + +> The `PostgresStorage::*` / `PostgresConfig { schema_name, enable_partitioning, … }` +> and manual schema/partitioning snippets below are **design notes** for advanced +> tuning, not the current API — the implemented config is the four fields above +> and the schema is auto-managed. ### When to Use @@ -234,119 +288,62 @@ ALTER SYSTEM SET default_statistics_target = 100; SELECT pg_reload_conf(); ``` -## ClickHouse Storage +## TimescaleDB Storage _(available — build with `--features postgres`)_ -### When to Use +TimescaleDB is PostgreSQL with a time-series extension — same wire protocol, same +driver. So the `postgres` backend already connects to a TimescaleDB instance +unchanged. The dedicated `timescale` backend additionally converts the `traces` +table into a **hypertable** partitioned on `start_time`, which is a natural fit +for inherently time-series trace data: time-chunked storage, fast time-range +queries, and (operator-configured) native compression and retention policies. -- Very high-volume trace data (millions of events per second) -- Analytical workloads and reporting -- Time-series analysis -- Long-term data retention with compression -- Real-time dashboards and monitoring +It reuses the **`[storage.postgres]`** connection config — just set the backend: -### Configuration - -```rust -use reflow_tracing::storage::ClickHouseStorage; +```toml +[storage] +backend = "timescale" # or "timescaledb" -let storage = ClickHouseStorage::new("http://localhost:8123").await?; +[storage.postgres] +connection_url = "postgresql://user:pass@localhost/traces" +max_connections = 20 +min_connections = 5 +acquire_timeout_secs = 5 ``` -### Features - -- **Columnar storage**: Excellent compression and analytical performance -- **Distributed architecture**: Horizontal scaling -- **Real-time ingestion**: Handle massive write loads -- **Advanced analytics**: Built-in analytical functions -- **Time-series optimized**: Purpose-built for time-ordered data - -```rust -use reflow_tracing::storage::{ClickHouseStorage, ClickHouseConfig}; - -let config = ClickHouseConfig { - url: "http://clickhouse:8123".to_string(), - database: "tracing".to_string(), - cluster: Some("cluster".to_string()), - username: Some("default".to_string()), - password: None, - compression: CompressionMethod::LZ4, - batch_size: 10000, - flush_interval_ms: 5000, - max_memory_usage: 1_000_000_000, // 1GB - max_execution_time_ms: 300_000, // 5 minutes -}; - -let storage = ClickHouseStorage::with_config(config).await?; -``` - -### Schema Design - -```sql --- Optimized ClickHouse schema -CREATE TABLE tracing.events_local ON CLUSTER cluster ( - timestamp DateTime64(3), - trace_id UUID, - event_id UUID, - flow_id String, - execution_id UUID, - event_type LowCardinality(String), - actor_id String, - port String, - message_type String, - message_size UInt32, - execution_time_ns UInt64, - memory_usage UInt64, - cpu_usage Float32, - data String -- JSON as string for flexibility -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/events', '{replica}') -PARTITION BY toYYYYMM(timestamp) -ORDER BY (timestamp, trace_id, event_id) -SETTINGS index_granularity = 8192; - --- Distributed table -CREATE TABLE tracing.events ON CLUSTER cluster AS tracing.events_local -ENGINE = Distributed(cluster, tracing, events_local, rand()); - --- Materialized views for aggregations -CREATE MATERIALIZED VIEW tracing.event_metrics -ENGINE = SummingMergeTree() -PARTITION BY toYYYYMM(timestamp) -ORDER BY (timestamp, actor_id, event_type) -AS SELECT - toStartOfMinute(timestamp) as timestamp, - actor_id, - event_type, - count() as event_count, - avg(execution_time_ns) as avg_execution_time, - max(execution_time_ns) as max_execution_time, - sum(message_size) as total_bytes -FROM tracing.events_local -GROUP BY timestamp, actor_id, event_type; +On startup the server runs `CREATE EXTENSION IF NOT EXISTS timescaledb` and +`create_hypertable('traces', 'start_time', …)` (7-day chunks by default, +`if_not_exists`/`migrate_data` so it's idempotent). If the extension isn't +installed it logs a warning and continues with a plain table — still correct, +just not time-partitioned. The schema keys on `(trace_id, start_time)` (a +hypertable's unique key must include the partition column). + +> Native compression and retention are left to the operator as policies, e.g. +> `SELECT add_retention_policy('traces', INTERVAL '30 days')` and +> `ALTER TABLE traces SET (timescaledb.compress)` + +> `add_compression_policy('traces', INTERVAL '7 days')`. + +## MongoDB Storage _(available — build with `--features mongodb`)_ + +A document store: each `FlowTrace` is one document keyed by `trace_id` (`_id`), +with denormalized `flow_id` / `execution_id` / `status` / `start_time` fields +(indexed on startup) and the full trace nested under `trace`. Selected via the +server config: + +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_url = "mongodb://localhost:27017" +database_name = "reflow_tracing" +collection_name = "traces" ``` -### Performance Tuning +### When to Use -```xml - - - - - 10000000000 - 1 - random - - - - - - default - - ::/0 - - - - -``` +- You already run MongoDB and want traces alongside other documents +- Flexible, schema-less retention of the evolving trace shape +- Horizontal scale via sharding on `flow_id` / `_id` ## Custom Storage Implementation @@ -423,15 +420,15 @@ impl StorageBackend for RedisStorage { ### Decision Matrix -| Feature | Memory | SQLite | PostgreSQL | ClickHouse | Custom | -|---------|--------|---------|------------|------------|---------| -| **Persistence** | ❌ | ✅ | ✅ | ✅ | Depends | -| **Concurrency** | Medium | Low | High | Very High | Depends | -| **Scale** | Small | Medium | Large | Massive | Depends | -| **Setup Complexity** | None | Low | Medium | High | Varies | -| **Query Flexibility** | Limited | High | Very High | High | Depends | -| **Analytics** | Basic | Good | Excellent | Outstanding | Depends | -| **Operational Overhead** | None | Low | Medium | High | Varies | +| Feature | Memory | SQLite | PostgreSQL | TimescaleDB | MongoDB | Custom | +|---------|--------|---------|------------|-------------|---------|---------| +| **Persistence** | ❌ | ✅ | ✅ | ✅ | ✅ | Depends | +| **Concurrency** | Medium | Low | High | High | High | Depends | +| **Scale** | Small | Medium | Large | Large | Large | Depends | +| **Setup Complexity** | None | Low | Medium | Medium | Medium | Varies | +| **Query Flexibility** | Limited | High | Very High | Very High | High (document) | Depends | +| **Time-series / retention** | ❌ | ❌ | manual | ✅ hypertable + TTL | manual | Depends | +| **Operational Overhead** | None | Low | Medium | Medium | Medium | Varies | ### Recommendations @@ -453,10 +450,13 @@ let storage = SqliteStorage::new("traces.db").await?; let storage = PostgresStorage::new("postgresql://...").await?; ``` -**Large Scale/Analytics**: -```rust -// ClickHouse for high-volume scenarios -let storage = ClickHouseStorage::new("http://clickhouse:8123").await?; +**Time-series / retention at scale**: +```toml +# TimescaleDB: PostgreSQL + a hypertable on start_time, with native retention +[storage] +backend = "timescale" +[storage.postgres] +connection_url = "postgresql://user:pass@localhost/traces" ``` ## Migration Between Backends diff --git a/docs/observability/storage/mongodb.md b/docs/observability/storage/mongodb.md new file mode 100644 index 00000000..1e6d3e92 --- /dev/null +++ b/docs/observability/storage/mongodb.md @@ -0,0 +1,91 @@ +# MongoDB Integration Guide + +MongoDB is a **document-store** backend — the JSON-shaped `FlowTrace` maps +naturally onto a BSON document. A good fit if you already operate MongoDB, want +schema-less retention of the evolving trace shape, or scale horizontally via +sharding. + +## Build + +Behind a cargo feature (adds the `mongodb` + `bson` drivers): + +```bash +cargo build -p reflow_tracing --features mongodb +# or every durable backend at once: +cargo build -p reflow_tracing --features all-backends +``` + +## Run a database + +```bash +docker run -d --name reflow-mongo -p 27017:27017 mongo:7 +``` + +## Configure + +Set `storage.backend` to `mongodb` (or `mongo`) and a `[storage.mongodb]` block. +All fields: + +```toml +[storage] +backend = "mongodb" + +[storage.mongodb] +connection_url = "mongodb://localhost:27017" +database_name = "reflow_tracing" +collection_name = "traces" +``` + +## Document model + +Each `FlowTrace` is one document keyed by its `trace_id` (`_id`), with +denormalized top-level fields for querying and the full trace nested under +`trace`: + +```json +{ + "_id": "", + "flow_id": "...", "execution_id": "...", "status": "...", + "start_time": 1718000000, "end_time": 1718000005, "event_count": 12, + "trace": { /* the full FlowTrace */ } +} +``` + +Stores use an upsert (`replace_one … upsert`), and the server creates indexes on +`flow_id`, `execution_id`, `status`, and `start_time` on startup. + +## Verify + +```bash +reflow_tracing # logs: "Initialized storage backend: mongodb" + +mongosh "mongodb://localhost:27017/reflow_tracing" \ + --eval 'db.traces.countDocuments()' +mongosh "mongodb://localhost:27017/reflow_tracing" \ + --eval 'db.traces.findOne({}, {flow_id:1, status:1, event_count:1})' +``` + +Live integration test: + +```bash +REFLOW_TEST_MONGODB_URL="mongodb://localhost:27017" \ + cargo test -p reflow_tracing --features mongodb --test storage_backends \ + mongodb_store_query_delete_roundtrip +``` + +## Operations + +- **Retention (TTL)**: a TTL index expires old traces automatically — e.g. add a + BSON `Date` field and + `db.traces.createIndex({ expires_at: 1 }, { expireAfterSeconds: 0 })`, or run a + periodic `deleteMany({ start_time: { $lt: cutoff } })`. +- **Sharding**: shard on `_id` (trace_id) or `flow_id` for horizontal scale. +- **Backups**: `mongodump` / replica sets. + +## Troubleshooting + +- **`MongoDB backend not compiled in`** — rebuild with `--features mongodb`. +- **`MongoDB storage config missing`** — `backend = "mongodb"` but no + `[storage.mongodb]` block. +- **Connection errors** — check `connection_url`, that the server is up, and auth + (`mongodb://user:pass@host:27017`). diff --git a/docs/observability/storage/postgres.md b/docs/observability/storage/postgres.md new file mode 100644 index 00000000..81048d69 --- /dev/null +++ b/docs/observability/storage/postgres.md @@ -0,0 +1,94 @@ +# PostgreSQL Integration Guide + +PostgreSQL is the recommended backend for **production, multi-instance, and +high-concurrency** deployments — a real server with connection pooling and strong +durability. For time-series-heavy workloads, see the +[TimescaleDB guide](timescaledb.md), which reuses this same config. + +## Build + +PostgreSQL is behind a cargo feature (adds the sqlx Postgres driver): + +```bash +cargo build -p reflow_tracing --features postgres +# or every durable backend at once: +cargo build -p reflow_tracing --features all-backends +``` + +## Run a database + +```bash +docker run -d --name reflow-pg \ + -e POSTGRES_DB=traces -e POSTGRES_USER=reflow -e POSTGRES_PASSWORD=secret \ + -p 5432:5432 postgres:16 +``` + +## Configure + +Set `storage.backend` to `postgres` (or `postgresql`) and a `[storage.postgres]` +block. All fields: + +```toml +[storage] +backend = "postgres" + +[storage.postgres] +connection_url = "postgresql://reflow:secret@localhost:5432/traces" +max_connections = 20 +min_connections = 5 +acquire_timeout_secs = 5 +``` + +## Schema + +Created automatically on startup (the `traces` table + indexes) — no manual DDL. +Each `FlowTrace` is one row: the full trace as a (zstd-compressed when large) +JSON `BYTEA` blob, plus denormalized columns: + +| column | type | purpose | +|---|---|---| +| `trace_id` (PK) | `TEXT` | identity | +| `flow_id`, `execution_id`, `status` | `TEXT` | query filters | +| `start_time`, `end_time` | `BIGINT` | time-range queries | +| `event_count` | `BIGINT` | stats without loading the blob | +| `data`, `compressed`, `size_bytes` | `BYTEA`/`BOOL`/`BIGINT` | payload | + +Stores are **synchronous** with an `INSERT … ON CONFLICT (trace_id) DO UPDATE` +upsert (idempotent re-finalize), and indexes cover `flow_id`, `execution_id`, +`status`, `start_time`. + +## Verify + +```bash +reflow_tracing # logs: "Initialized storage backend: postgres" + +psql "$CONN" -c '\dt' # traces +psql "$CONN" -c 'SELECT count(*) FROM traces;' +``` + +Run the bundled integration test against a live instance: + +```bash +REFLOW_TEST_POSTGRES_URL="postgresql://reflow:secret@localhost:5432/traces" \ + cargo test -p reflow_tracing --features postgres --test storage_backends \ + postgres_store_query_delete_roundtrip +``` + +## Operations + +- **Pooling**: size `max_connections` to your collector concurrency; `sqlx` + manages the pool with `acquire_timeout_secs`. +- **Retention**: schedule + `DELETE FROM traces WHERE start_time < EXTRACT(EPOCH FROM now() - INTERVAL '30 days')`, + or use [TimescaleDB](timescaledb.md) for native retention. +- **Backups**: standard `pg_dump` / streaming replication / PITR. +- **Scale**: read replicas, partitioning. If your queries are dominated by + time ranges, TimescaleDB's hypertable is a better fit. + +## Troubleshooting + +- **`PostgreSQL backend not compiled in`** — rebuild with `--features postgres`. +- **`PostgreSQL storage config missing`** — `backend = "postgres"` but no + `[storage.postgres]` block. +- **Connection refused / auth failed** — check `connection_url`, that the server + is reachable, and credentials. diff --git a/docs/observability/storage/sqlite.md b/docs/observability/storage/sqlite.md new file mode 100644 index 00000000..8507ee04 --- /dev/null +++ b/docs/observability/storage/sqlite.md @@ -0,0 +1,83 @@ +# SQLite Integration Guide + +SQLite is the **default durable backend** — a single embedded file, no server to +run. Ideal for development, single-node deployments, and anywhere you want +persistence without operating a database. + +## Build + +SQLite ships in the default build (the `storage` feature is on by default): + +```bash +cargo build -p reflow_tracing # SQLite included +``` + +## Configure + +Set `storage.backend = "sqlite"` and a `[storage.sqlite]` block. All fields: + +```toml +[storage] +backend = "sqlite" + +[storage.sqlite] +database_path = "traces.db" # file path, or ":memory:" for an ephemeral DB +wal_mode = true # Write-Ahead Logging (recommended) +journal_mode = "WAL" +synchronous = "NORMAL" # NORMAL is a good durability/speed balance +cache_size = -2000 # KB when negative (here ~2 MB), pages when positive +``` + +The database file is **created automatically** if it doesn't exist (the server +opens it with `mode=rwc`); the parent directory must already exist. + +## Schema + +Created on startup — you never run DDL by hand. Each `FlowTrace` is stored as one +row in `traces`: the full trace is a (zstd-compressed when large) JSON blob in +`data`, alongside denormalized columns for querying: + +| column | purpose | +|---|---| +| `trace_id` (PK) | trace identity | +| `flow_id`, `execution_id`, `status` | query filters | +| `start_time`, `end_time` | time-range queries | +| `data` (BLOB), `compressed`, `size_bytes` | the trace payload | + +Indexes cover `flow_id`, `execution_id`, `status`, `start_time`. Writes are +**synchronous (write-through)**, so a trace is queryable the instant it's stored. + +## Verify + +```bash +# point the server at a sqlite config and start it +reflow_tracing # logs: "Initialized storage backend: sqlite" + +# after some traces have flowed, inspect the file directly +sqlite3 traces.db '.tables' # traces, trace_events +sqlite3 traces.db 'SELECT count(*) FROM traces;' +``` + +Or query through the protocol (`get_trace` / `query_traces`) from any SDK / the +monitoring client. + +## Operations + +- **Backups**: it's a single file — copy it (use the SQLite backup API or copy + while WAL-checkpointed). The WAL (`traces.db-wal`) must travel with the file. +- **Concurrency**: SQLite is single-writer. Fine for one collector; for high + concurrent write volume move to [PostgreSQL](postgres.md) / + [TimescaleDB](timescaledb.md). +- **Retention**: SQLite has no automatic TTL. Prune with + `DELETE FROM traces WHERE start_time < strftime('%s','now','-30 days')` on a + schedule, or use the in-process delete API. +- **Cache**: bump `cache_size` (negative = KB) for read-heavy workloads. + +## Troubleshooting + +- **`unable to open database file`** — the parent directory doesn't exist, or the + process can't write there. Create the directory / fix permissions. +- **No traces appear** — confirm `backend = "sqlite"` (not `memory`) and that the + network's `TracingConfig.enabled = true` points at this server. +- **`Storage feature is not enabled`** — you built with `--no-default-features`; + re-enable the `storage` feature. diff --git a/docs/observability/storage/timescaledb.md b/docs/observability/storage/timescaledb.md new file mode 100644 index 00000000..480b7472 --- /dev/null +++ b/docs/observability/storage/timescaledb.md @@ -0,0 +1,106 @@ +# TimescaleDB Integration Guide + +TimescaleDB is **PostgreSQL plus a time-series extension** — same protocol, same +driver. Traces are inherently time-series (every trace has a `start_time`), so +the `timescale` backend converts the `traces` table into a **hypertable** +partitioned on `start_time`: time-chunked storage, fast time-range scans, and +native compression/retention policies. + +Because it speaks the Postgres protocol, it **reuses the +[`[storage.postgres]`](postgres.md) connection config** — only the backend name +differs. + +## Build + +Uses the same feature as PostgreSQL: + +```bash +cargo build -p reflow_tracing --features postgres +``` + +## Run a database + +```bash +docker run -d --name reflow-ts \ + -e POSTGRES_DB=traces -e POSTGRES_USER=reflow -e POSTGRES_PASSWORD=secret \ + -p 5432:5432 timescale/timescaledb:latest-pg16 +``` + +## Configure + +Set the backend to `timescale` (or `timescaledb`); the connection config lives +under `[storage.postgres]`: + +```toml +[storage] +backend = "timescale" + +[storage.postgres] +connection_url = "postgresql://reflow:secret@localhost:5432/traces" +max_connections = 20 +min_connections = 5 +acquire_timeout_secs = 5 +``` + +## What the server sets up + +On startup it creates the `traces` table, then best-effort runs: + +```sql +CREATE EXTENSION IF NOT EXISTS timescaledb; +SELECT create_hypertable('traces', 'start_time', + chunk_time_interval => 604800, -- 7-day chunks (start_time is unix seconds) + if_not_exists => TRUE, migrate_data => TRUE); +``` + +Key differences from plain Postgres: + +- the schema keys on **`(trace_id, start_time)`** — a hypertable's unique key must + include the partition column. `start_time` is set once per trace, so the key is + stable and the `ON CONFLICT (trace_id, start_time)` upsert is idempotent. +- a non-unique index on `trace_id` keeps point lookups (`get`/`delete`) fast. +- **graceful degradation**: if the `timescaledb` extension isn't installed, the + server logs a warning and continues with a plain (composite-key) table — still + correct, just not time-partitioned. + +## Verify + +```bash +reflow_tracing # logs: "Initialized storage backend: timescale" + +# confirm the hypertable exists +psql "$CONN" -c "SELECT hypertable_name FROM timescaledb_information.hypertables;" +``` + +Live integration test: + +```bash +REFLOW_TEST_TIMESCALE_URL="postgresql://reflow:secret@localhost:5432/traces" \ + cargo test -p reflow_tracing --features postgres --test storage_backends \ + timescaledb_store_query_delete_roundtrip +``` + +## Operations — the payoff + +These are the reasons to pick TimescaleDB; configure them as policies on the +hypertable: + +```sql +-- Native retention: drop chunks older than 30 days, automatically. +SELECT add_retention_policy('traces', INTERVAL '30 days'); + +-- Native compression of older chunks. +ALTER TABLE traces SET (timescaledb.compress); +SELECT add_compression_policy('traces', INTERVAL '7 days'); +``` + +Time-range queries (`query_traces` with a `time_range`) hit only the relevant +chunks. Tune `chunk_time_interval` for your trace volume if 7 days isn't ideal. + +## Troubleshooting + +- **Hypertable warning in logs** — the extension isn't installed; either install + it (`timescale/timescaledb` image) or accept the plain-table fallback. +- **`TimescaleDB uses the [storage.postgres] config; it is missing`** — add the + `[storage.postgres]` block. +- Everything else mirrors the [PostgreSQL guide](postgres.md). diff --git a/sdk/cpp/include/reflow/reflow.hpp b/sdk/cpp/include/reflow/reflow.hpp index 8f6e2402..7a9a04ca 100644 --- a/sdk/cpp/include/reflow/reflow.hpp +++ b/sdk/cpp/include/reflow/reflow.hpp @@ -859,6 +859,35 @@ class EventStream { UniqueHandle ptr_; }; +// ─── TraceStream ─────────────────────────────────────────────────────────── + +/// A network's local trace-event stream — no collector required. Requires +/// tracing to be enabled in the network config +/// (`{"tracing":{"server_url":"ws://...","enabled":true}}`). +class TraceStream { +public: + explicit TraceStream(rfl_traces* t) : ptr_(t) { + if (!ptr_) detail::throw_status(rfl_status_Runtime, "TraceStream"); + } + + /// Block for up to `timeout_ms` for the next trace-event JSON. Returns + /// nullopt on timeout or when the channel has closed. + std::optional recv(uint32_t timeout_ms) { + char* out = nullptr; + rfl_status s = rfl_traces_recv(ptr_.get(), timeout_ms, &out); + if (s == rfl_status_Ok) { + if (out == nullptr) return std::nullopt; + return detail::take_c_string(out); + } + if (s == rfl_status_InvalidState) return std::nullopt; // timeout + if (s == rfl_status_Runtime) return std::nullopt; // channel closed + detail::throw_status(s, "TraceStream::recv"); + } + +private: + UniqueHandle ptr_; +}; + // ─── Network ─────────────────────────────────────────────────────────────── class Network { @@ -905,6 +934,14 @@ class Network { return EventStream(e); } + /// Subscribe to this network's local trace events (tracing must be enabled + /// in the config). The returned stream owns the underlying `rfl_traces*`. + TraceStream traces() { + rfl_traces* t = rfl_network_traces(ptr_.get()); + if (t == nullptr) detail::throw_status(rfl_status_Runtime, "Network::traces"); + return TraceStream(t); + } + /// Register an actor under a template id. Consumes the actor handle. void register_actor(std::string_view template_id, Actor&& actor) { detail::check(rfl_network_register_actor(ptr_.get(), diff --git a/sdk/cpp/include/reflow/reflow_rt.h b/sdk/cpp/include/reflow/reflow_rt.h index 2e15187c..c1a5352d 100644 --- a/sdk/cpp/include/reflow/reflow_rt.h +++ b/sdk/cpp/include/reflow/reflow_rt.h @@ -122,6 +122,17 @@ typedef struct rfl_stream_recv rfl_stream_recv; */ typedef struct rfl_subgraph_builder rfl_subgraph_builder; +/** + * Opaque handle to a tracing-collector client. + */ +typedef struct rfl_trace_client rfl_trace_client; + +/** + * Opaque handle to a subscriber on a network's local trace-event stream. + * One subscriber per handle. + */ +typedef struct rfl_traces rfl_traces; + /** * Function pointer: the body of a callback actor. * @@ -243,6 +254,61 @@ void rfl_events_free(struct rfl_events *e); */ char *rfl_version(void); +/** + * Subscribe to a network's live trace events locally — no collector required. + * Call **before** `rfl_network_start` for full coverage. Returns NULL on a + * null argument. + */ +struct rfl_traces *rfl_network_traces(struct rfl_network *n); + +/** + * Poll for the next trace event, blocking up to `timeout_ms` milliseconds. + * On success writes a newly allocated JSON `TraceEvent` to `*out_json` (free + * with `rfl_string_free`). Returns `Ok`, `InvalidState` on timeout, or + * `Runtime` if the channel is closed. + */ +enum rfl_status rfl_traces_recv(struct rfl_traces *t, uint32_t timeout_ms, char **out_json); + +/** + * Free a traces handle. Safe on NULL. + */ +void rfl_traces_free(struct rfl_traces *t); + +/** + * Connect to a tracing collector at `server_url` (e.g. "ws://127.0.0.1:8080"). + * Returns NULL on failure (see `rfl_last_error_message`). + */ +struct rfl_trace_client *rfl_trace_client_connect(const char *server_url); + +/** + * Query historical traces. `query_json` is a JSON `TraceQuery`, or NULL for + * "all". Writes a JSON array of `FlowTrace` to `*out_json` (free with + * `rfl_string_free`). + */ +enum rfl_status rfl_trace_client_query(struct rfl_trace_client *c, + const char *query_json, + char **out_json); + +/** + * Subscribe to live trace events from the collector. `filters_json` is a JSON + * `SubscriptionFilters`, or NULL for no filtering. After this returns `Ok`, + * poll events with `rfl_trace_client_recv`. + */ +enum rfl_status rfl_trace_client_subscribe(struct rfl_trace_client *c, const char *filters_json); + +/** + * Poll for the next live trace event after `rfl_trace_client_subscribe`, + * blocking up to `timeout_ms`. Writes a JSON `TraceEvent` to `*out_json`. + */ +enum rfl_status rfl_trace_client_recv(struct rfl_trace_client *c, + uint32_t timeout_ms, + char **out_json); + +/** + * Free a trace-client handle. Safe on NULL. + */ +void rfl_trace_client_free(struct rfl_trace_client *c); + /** * Add a node. * `metadata_json` may be NULL or a JSON object string (`{"key": ...}`). @@ -551,13 +617,8 @@ enum rfl_status rfl_ctx_state_set(struct rfl_actor_ctx *ctx, const char *value_json); /** - * Per-actor pools — named `{id: value}` maps living under reserved - * `_pool:` state keys. The canonical pattern for variable - * fan-in: multiple upstream connections upsert with stable per-upstream - * ids, the consumer reads the whole pool on each tick. - * - * All five operations require `MemoryState` (the default backend); - * custom states yield `InvalidState`. + * Upsert `value_json` into pool `pool_name` under `id`. Creates the + * pool entry if it doesn't exist yet. */ enum rfl_status rfl_ctx_pool_upsert(struct rfl_actor_ctx *ctx, const char *pool_name, @@ -575,7 +636,8 @@ enum rfl_status rfl_ctx_pool_remove(struct rfl_actor_ctx *ctx, /** * Read the entire pool as a JSON object `{id: value, ...}`. Returns * `"{}"` if the pool is empty or absent. Caller frees via - * `rfl_string_free`. Returns NULL only on argument errors. + * `rfl_string_free`. Returns NULL only on argument errors; absence + * of the pool is encoded as the empty object. */ char *rfl_ctx_pool_get_json(struct rfl_actor_ctx *ctx, const char *pool_name); @@ -587,18 +649,20 @@ uintptr_t rfl_ctx_pool_count(struct rfl_actor_ctx *ctx, const char *pool_name); /** * Drop the entire pool. Idempotent. */ -enum rfl_status rfl_ctx_pool_clear(struct rfl_actor_ctx *ctx, - const char *pool_name); +enum rfl_status rfl_ctx_pool_clear(struct rfl_actor_ctx *ctx, const char *pool_name); /** * Emit a typed message on `port`. Transfers ownership of the message — * do **not** call `rfl_message_free` afterwards. Prefer this over the * JSON variant for hot-path emits. * - * Per-tick semantics: emits accumulate in a HashMap that drains when - * the callback returns. Multiple emits to the *same* port collapse - * to the last write. For sources that publish a stream of values - * from inside a single `run`, use `rfl_ctx_send_message`. + * **Per-tick semantics**: `rfl_ctx_emit_message` accumulates outputs + * in a HashMap that drains when the callback returns. Multiple emits + * to the *same* port within one callback collapse to the last + * write. For source actors that need to publish a *stream* of + * values from inside a single `run`, use `rfl_ctx_send_message`, + * which writes straight to the outport channel and publishes one + * packet per call. */ enum rfl_status rfl_ctx_emit_message(struct rfl_actor_ctx *ctx, const char *port, @@ -607,9 +671,15 @@ enum rfl_status rfl_ctx_emit_message(struct rfl_actor_ctx *ctx, /** * Mid-tick flush: send a typed message straight to the outport * channel, bypassing the per-callback `outputs` HashMap. Use this - * when the same callback publishes multiple values on the same port - * — `rfl_ctx_emit_message` would overwrite. Transfers ownership of - * the message; do **not** free it afterwards. + * when the same callback needs to publish multiple values on the + * same port — `rfl_ctx_emit_message` would overwrite. Transfers + * ownership of the message; do **not** free it afterwards. + * + * Mirrors Python's `ctx.send` and JVM's `ctx.send`. The message is + * queued on the source actor's outport channel and reaches every + * connected downstream actor through the normal connector + * mechanics — consumers don't need to know whether the producer + * used `emit` or `send`. */ enum rfl_status rfl_ctx_send_message(struct rfl_actor_ctx *ctx, const char *port, diff --git a/sdk/go/network.go b/sdk/go/network.go index e415285f..65d50dad 100644 --- a/sdk/go/network.go +++ b/sdk/go/network.go @@ -220,3 +220,63 @@ func (e *EventStream) Close() { e.ptr = nil runtime.SetFinalizer(e, nil) } + +// Traces subscribes to the network's local trace-event stream. Requires +// tracing to be enabled in the config (e.g. NewNetworkWithConfig with +// {"tracing": {"server_url": "ws://...", "enabled": true}}). Events stream +// locally with no collector required. +func (n *Network) Traces() *TraceStream { + p := C.rfl_network_traces(n.ptr) + if p == nil { + return nil + } + t := &TraceStream{ptr: p} + runtime.SetFinalizer(t, (*TraceStream).Close) + return t +} + +// TraceStream delivers TraceEvent objects as JSON. +type TraceStream struct { + ptr *C.rfl_traces +} + +// Recv blocks up to `timeout` for the next trace event. On timeout returns +// (nil, nil). On channel close, returns an error. +func (t *TraceStream) Recv(timeout time.Duration) (map[string]any, error) { + if t == nil || t.ptr == nil { + return nil, fmt.Errorf("reflow.TraceStream.Recv: reader is closed") + } + ms := timeout.Milliseconds() + if ms < 0 { + ms = 0 + } + var out *C.char + st := C.rfl_traces_recv(t.ptr, C.uint32_t(ms), &out) + switch st { + case C.rfl_status_Ok: + if out == nil { + return nil, nil + } + defer C.rfl_string_free(out) + raw := []byte(C.GoString(out)) + m := map[string]any{} + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("decode trace event: %w", err) + } + return m, nil + case C.rfl_status_InvalidState: + return nil, nil // timeout + default: + return nil, statusToError(int32(st), "TraceStream.Recv") + } +} + +// Close releases the subscription. +func (t *TraceStream) Close() { + if t == nil || t.ptr == nil { + return + } + C.rfl_traces_free(t.ptr) + t.ptr = nil + runtime.SetFinalizer(t, nil) +} diff --git a/sdk/go/tracing_test.go b/sdk/go/tracing_test.go new file mode 100644 index 00000000..fcf8f942 --- /dev/null +++ b/sdk/go/tracing_test.go @@ -0,0 +1,93 @@ +package reflow + +import ( + "strings" + "testing" + "time" +) + +// First-class tracing in the Go SDK: enable via config, consume locally via +// Network.Traces() (no collector required), and observe correlated events with +// content checksums. Reuses newDoubler/newCollect from reflow_test.go. +func TestNetworkTracing(t *testing.T) { + net, err := NewNetworkWithConfig(map[string]any{ + "tracing": map[string]any{ + "server_url": "ws://127.0.0.1:8080", + "enabled": true, + }, + }) + if err != nil { + t.Fatalf("NewNetworkWithConfig: %v", err) + } + defer net.Close() + + d := newDoubler() + c := newCollect() + if err := net.RegisterGoActor("tpl_doubler", d); err != nil { + t.Fatalf("register doubler: %v", err) + } + if err := net.RegisterGoActor("tpl_collect", c); err != nil { + t.Fatalf("register collect: %v", err) + } + if err := net.AddNode("a", "tpl_doubler", nil); err != nil { + t.Fatalf("AddNode a: %v", err) + } + if err := net.AddNode("b", "tpl_collect", nil); err != nil { + t.Fatalf("AddNode b: %v", err) + } + if err := net.AddConnection("a", "out", "b", "in"); err != nil { + t.Fatalf("AddConnection: %v", err) + } + if err := net.AddInitial("a", "in", map[string]any{"type": "Integer", "data": 21}); err != nil { + t.Fatalf("AddInitial: %v", err) + } + + traces := net.Traces() + if traces == nil { + t.Fatal("Traces() returned nil — tracing not enabled?") + } + defer traces.Close() + + if err := net.Start(); err != nil { + t.Fatalf("Start: %v", err) + } + + seen := map[string]bool{} + sawChecksum := false + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + evt, err := traces.Recv(200 * time.Millisecond) + if err != nil { + break + } + if evt == nil { + continue // timeout + } + switch et := evt["event_type"].(type) { + case string: + seen[et] = true + case map[string]any: + for k := range et { + seen[k] = true + } + } + if data, ok := evt["data"].(map[string]any); ok { + if msg, ok := data["message"].(map[string]any); ok { + if cs, ok := msg["checksum"].(string); ok && strings.HasPrefix(cs, "sha256:") { + sawChecksum = true + } + } + } + if seen["ActorCreated"] && sawChecksum { + break + } + } + _ = net.Shutdown() + + if !seen["ActorCreated"] { + t.Errorf("expected ActorCreated trace event; seen=%v", seen) + } + if !sawChecksum { + t.Error("expected a data-flow snapshot carrying a sha256 checksum") + } +} diff --git a/sdk/jvm/src/main/java/ai/offbit/reflow/Network.java b/sdk/jvm/src/main/java/ai/offbit/reflow/Network.java index a092ed69..1b71df0f 100644 --- a/sdk/jvm/src/main/java/ai/offbit/reflow/Network.java +++ b/sdk/jvm/src/main/java/ai/offbit/reflow/Network.java @@ -53,6 +53,15 @@ public EventStream events() { return new EventStream(nativeEvents(nativePtr)); } + /** + * Subscribe to this network's live trace events (JSON). Requires tracing to + * be enabled in the config, e.g. {@code new Network("{\"tracing\":{\"server_url\":\"ws://localhost:8080\",\"enabled\":true}}")}. + * Events stream locally with no collector required. + */ + public TraceStream traces() { + return new TraceStream(nativeTraces(nativePtr)); + } + @Override public void close() { if (nativePtr != 0) { @@ -76,5 +85,6 @@ protected void finalize() { private static native void nativeStart(long ptr); private static native void nativeShutdown(long ptr); private static native long nativeEvents(long ptr); + private static native long nativeTraces(long ptr); private static native void nativeFree(long ptr); } diff --git a/sdk/jvm/src/main/java/ai/offbit/reflow/TraceStream.java b/sdk/jvm/src/main/java/ai/offbit/reflow/TraceStream.java new file mode 100644 index 00000000..a468a492 --- /dev/null +++ b/sdk/jvm/src/main/java/ai/offbit/reflow/TraceStream.java @@ -0,0 +1,32 @@ +package ai.offbit.reflow; + +/** Subscription on a Network's local trace-event stream. Events are JSON strings. */ +public final class TraceStream implements AutoCloseable { + static { Reflow.ensureLoaded(); } + + private long nativePtr; + + TraceStream(long ptr) { this.nativePtr = ptr; } + + /** Block up to {@code timeoutMs}. Returns the trace-event JSON or null on timeout/close. */ + public String recv(int timeoutMs) { + return nativeRecv(nativePtr, timeoutMs); + } + + @Override + public void close() { + if (nativePtr != 0) { + nativeFree(nativePtr); + nativePtr = 0; + } + } + + @Override + @SuppressWarnings("deprecation") + protected void finalize() { + close(); + } + + private static native String nativeRecv(long ptr, int timeoutMs); + private static native void nativeFree(long ptr); +} diff --git a/sdk/jvm/src/native/Cargo.lock b/sdk/jvm/src/native/Cargo.lock index 4f03d203..1bef5c2b 100644 --- a/sdk/jvm/src/native/Cargo.lock +++ b/sdk/jvm/src/native/Cargo.lock @@ -2482,6 +2482,7 @@ dependencies = [ "parking_lot", "reflow_components", "reflow_rt", + "reflow_tracing_protocol", "serde", "serde_json", "tokio", @@ -2511,14 +2512,17 @@ dependencies = [ "chrono", "console_error_panic_hook", "derive_more", + "flume", "futures-util", "gloo-events", "gloo-utils 0.2.0", + "hex", "js-sys", "lz4_flex", "serde", "serde-wasm-bindgen 0.6.5", "serde_json", + "sha2", "thiserror 2.0.18", "tokio", "tokio-tungstenite", @@ -2530,6 +2534,7 @@ dependencies = [ "wasm-bindgen-test", "web-sys", "web-time", + "zstd", ] [[package]] diff --git a/sdk/jvm/src/native/Cargo.toml b/sdk/jvm/src/native/Cargo.toml index 8e5a5627..6de03085 100644 --- a/sdk/jvm/src/native/Cargo.toml +++ b/sdk/jvm/src/native/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] [dependencies] reflow_rt = { version = "0.2.0", path = "../../../../crates/reflow_rt", default-features = false, features = ["components-core"] } reflow_components = { version = "0.2.0", path = "../../../../crates/reflow_components", default-features = false, features = ["av-core"] } +reflow_tracing_protocol = { version = "0.2.1", path = "../../../../crates/reflow_tracing_protocol" } jni = "0.21" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/sdk/jvm/src/native/src/lib.rs b/sdk/jvm/src/native/src/lib.rs index d09b3e2e..c95cad11 100644 --- a/sdk/jvm/src/native/src/lib.rs +++ b/sdk/jvm/src/native/src/lib.rs @@ -2361,6 +2361,57 @@ pub extern "system" fn Java_ai_offbit_reflow_EventStream_nativeFree( unsafe { let _ = box_from_ptr::(ptr); } } +// ─── Trace stream ────────────────────────────────────────────────────────── + +#[no_mangle] +pub extern "system" fn Java_ai_offbit_reflow_Network_nativeTraces( + _env: JNIEnv, + _class: JClass, + ptr: jlong, +) -> jlong { + let h = match unsafe { as_ref::(ptr) } { + Some(h) => h, + None => return 0, + }; + let rx = h.inner.lock().unwrap().get_trace_receiver(); + Box::into_raw(Box::new(TraceStreamHandle { rx })) as jlong +} + +pub struct TraceStreamHandle { + rx: flume::Receiver, +} + +#[no_mangle] +pub extern "system" fn Java_ai_offbit_reflow_TraceStream_nativeRecv<'local>( + mut env: JNIEnv<'local>, + _class: JClass<'local>, + ptr: jlong, + timeout_ms: jint, +) -> jobject { + let h = match unsafe { as_ref::(ptr) } { + Some(h) => h, + None => return std::ptr::null_mut(), + }; + let d = std::time::Duration::from_millis(timeout_ms.max(0) as u64); + match h.rx.recv_timeout(d) { + Ok(evt) => { + let s = serde_json::to_string(&evt).unwrap_or_default(); + env.new_string(&s).map(|j| j.into_raw()).unwrap_or(std::ptr::null_mut()) + } + Err(flume::RecvTimeoutError::Timeout) => std::ptr::null_mut(), + Err(flume::RecvTimeoutError::Disconnected) => std::ptr::null_mut(), + } +} + +#[no_mangle] +pub extern "system" fn Java_ai_offbit_reflow_TraceStream_nativeFree( + _env: JNIEnv, + _class: JClass, + ptr: jlong, +) { + unsafe { let _ = box_from_ptr::(ptr); } +} + // ─── Subgraph builder ────────────────────────────────────────────────────── pub struct SubgraphBuilderHandle { diff --git a/sdk/node/Cargo.lock b/sdk/node/Cargo.lock index 454cd53d..6be02694 100644 --- a/sdk/node/Cargo.lock +++ b/sdk/node/Cargo.lock @@ -2513,6 +2513,7 @@ dependencies = [ "parking_lot", "reflow_components", "reflow_rt", + "reflow_tracing_protocol", "serde", "serde_json", "tokio", @@ -2542,14 +2543,17 @@ dependencies = [ "chrono", "console_error_panic_hook", "derive_more", + "flume", "futures-util", "gloo-events", "gloo-utils 0.2.0", + "hex", "js-sys", "lz4_flex", "serde", "serde-wasm-bindgen 0.6.5", "serde_json", + "sha2", "thiserror 2.0.18", "tokio", "tokio-tungstenite", @@ -2561,6 +2565,7 @@ dependencies = [ "wasm-bindgen-test", "web-sys", "web-time", + "zstd", ] [[package]] diff --git a/sdk/node/Cargo.toml b/sdk/node/Cargo.toml index a07fccf0..1ba5bc1c 100644 --- a/sdk/node/Cargo.toml +++ b/sdk/node/Cargo.toml @@ -13,6 +13,7 @@ crate-type = ["cdylib"] [dependencies] reflow_rt = { version = "0.2.0", path = "../../crates/reflow_rt", default-features = false, features = ["components-core"] } reflow_components = { version = "0.2.0", path = "../../crates/reflow_components", default-features = false, features = ["av-core"] } +reflow_tracing_protocol = { version = "0.2.1", path = "../../crates/reflow_tracing_protocol" } napi = { version = "2", default-features = false, features = ["napi9", "async", "serde-json", "tokio_rt"] } napi-derive = "2" serde = { version = "1", features = ["derive"] } diff --git a/sdk/node/src/lib.rs b/sdk/node/src/lib.rs index d5451203..ee6d0b33 100644 --- a/sdk/node/src/lib.rs +++ b/sdk/node/src/lib.rs @@ -1557,6 +1557,17 @@ impl ReflowNetwork { let rx = self.inner.lock().unwrap().get_event_receiver(); EventStream { rx } } + + /// Subscribe to this network's live trace events. Returns a `TraceStream` + /// whose `recv` awaits the next event. Requires tracing to be enabled in + /// the config, e.g. + /// `new Network({ tracing: { server_url: "ws://localhost:8080", enabled: true } })`. + /// Events stream locally with no collector required. + #[napi] + pub fn traces(&self) -> TraceStream { + let rx = self.inner.lock().unwrap().get_trace_receiver(); + TraceStream { rx } + } } // ─── Event stream ────────────────────────────────────────────────────────── @@ -1579,3 +1590,25 @@ impl EventStream { } } } + +// ─── Trace stream ────────────────────────────────────────────────────────── + +#[napi] +pub struct TraceStream { + rx: flume::Receiver, +} + +#[napi] +impl TraceStream { + /// Await the next trace event (as a plain object). Resolves `null` if the + /// stream is closed. + #[napi] + pub async fn recv(&self) -> Result> { + match self.rx.recv_async().await { + Ok(evt) => serde_json::to_value(&evt) + .map(Some) + .map_err(|e| Error::from_reason(format!("serialize trace event: {e}"))), + Err(_) => Ok(None), + } + } +} diff --git a/sdk/node/test/09_tracing.mjs b/sdk/node/test/09_tracing.mjs new file mode 100644 index 00000000..a2be612f --- /dev/null +++ b/sdk/node/test/09_tracing.mjs @@ -0,0 +1,79 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { Actor, Message, Network } from "../reflow.mjs"; + +// First-class tracing in the Node SDK: enable via config, consume locally via +// Network.traces() (no collector required), and observe correlated events with +// content checksums. + +class Doubler extends Actor { + static component = "doubler"; + static inports = ["in"]; + static outports = ["out"]; + run(ctx) { + const n = (ctx.inputs?.in && (ctx.inputs.in.data ?? 0)) || 0; + ctx.done({ out: Message.integer(Number(n) * 2) }); + } +} + +class Collect extends Actor { + static component = "collect"; + static inports = ["in"]; + static outports = []; + constructor(sink) { + super(); + this.sink = sink; + } + run(ctx) { + if (ctx.inputs?.in) this.sink.push(ctx.inputs.in); + ctx.done(); + } +} + +function eventTypeNames(evt) { + const et = evt.event_type; + if (typeof et === "string") return [et]; + if (et && typeof et === "object") return Object.keys(et); + return []; +} + +test("Network.traces() yields correlated trace events with checksums", async () => { + const sink = []; + // Tracing enabled. The collector URL need not be reachable — events still + // flow to the local tap. + const net = new Network({ + tracing: { server_url: "ws://127.0.0.1:8080", enabled: true }, + }); + net.registerActor("tpl_doubler", new Doubler()); + net.registerActor("tpl_collect", new Collect(sink)); + net.addNode("d", "tpl_doubler"); + net.addNode("c", "tpl_collect"); + net.addConnection("d", "out", "c", "in"); + net.addInitial("d", "in", { type: "Integer", data: 21 }); + + const traces = net.traces(); + net.start(); + + const seen = new Set(); + let sawChecksum = false; + const deadline = Date.now() + 3000; + while (Date.now() < deadline) { + // recv() resolves null on close; race a timeout so we never hang. + const evt = await Promise.race([ + traces.recv(), + new Promise((r) => setTimeout(() => r(undefined), 250)), + ]); + if (!evt) continue; + for (const name of eventTypeNames(evt)) seen.add(name); + const msg = evt.data && evt.data.message; + if (msg && typeof msg.checksum === "string" && msg.checksum.startsWith("sha256:")) { + sawChecksum = true; + } + if (seen.has("ActorCreated") && sawChecksum) break; + } + + net.shutdown(); + + assert.ok(seen.has("ActorCreated"), `seen event types: ${[...seen].join(", ")}`); + assert.ok(sawChecksum, "expected a data-flow snapshot carrying a sha256 checksum"); +}); diff --git a/sdk/python/Cargo.lock b/sdk/python/Cargo.lock index 19942eaf..bb52c4fe 100644 --- a/sdk/python/Cargo.lock +++ b/sdk/python/Cargo.lock @@ -2531,6 +2531,7 @@ dependencies = [ "pythonize", "reflow_components", "reflow_rt", + "reflow_tracing_protocol", "serde", "serde_json", "tokio", @@ -2560,14 +2561,17 @@ dependencies = [ "chrono", "console_error_panic_hook", "derive_more", + "flume", "futures-util", "gloo-events", "gloo-utils 0.2.0", + "hex", "js-sys", "lz4_flex", "serde", "serde-wasm-bindgen 0.6.5", "serde_json", + "sha2", "thiserror 2.0.18", "tokio", "tokio-tungstenite", @@ -2579,6 +2583,7 @@ dependencies = [ "wasm-bindgen-test", "web-sys", "web-time", + "zstd", ] [[package]] diff --git a/sdk/python/Cargo.toml b/sdk/python/Cargo.toml index 327b9b89..e55ad275 100644 --- a/sdk/python/Cargo.toml +++ b/sdk/python/Cargo.toml @@ -18,6 +18,7 @@ crate-type = ["cdylib"] [dependencies] reflow_rt = { version = "0.2.0", path = "../../crates/reflow_rt", default-features = false, features = ["components-core"] } reflow_components = { version = "0.2.0", path = "../../crates/reflow_components", default-features = false, features = ["av-core"] } +reflow_tracing_protocol = { version = "0.2.1", path = "../../crates/reflow_tracing_protocol" } # abi3-py39 produces a single wheel per platform that works on any # CPython 3.9+ rather than needing a separate build per interpreter. # Forward-compatibility lets us build against newer Pythons (e.g. 3.14 diff --git a/sdk/python/src/lib.rs b/sdk/python/src/lib.rs index 37ae32e3..6921cdff 100644 --- a/sdk/python/src/lib.rs +++ b/sdk/python/src/lib.rs @@ -1554,6 +1554,51 @@ impl PyNetwork { let rx = self.inner.lock().unwrap().get_event_receiver(); PyEventStream { rx } } + + /// Subscribe to this network's live trace events. Requires tracing to be + /// enabled in the config, e.g. + /// `Network({"tracing": {"server_url": "ws://localhost:8080", "enabled": True}})`. + /// Events stream locally with no collector required. + fn traces(&self) -> PyTraceStream { + let rx = self.inner.lock().unwrap().get_trace_receiver(); + PyTraceStream { rx } + } +} + +// ─── Trace stream ────────────────────────────────────────────────────────── + +#[pyclass(module = "reflow._native", name = "TraceStream")] +pub struct PyTraceStream { + rx: flume::Receiver, +} + +#[pymethods] +impl PyTraceStream { + /// Block up to `timeout_ms` for the next trace event (as a dict). Returns + /// `None` on timeout; raises on channel close. `timeout_ms=0` blocks. + #[pyo3(signature = (timeout_ms=0))] + fn recv<'py>(&self, py: Python<'py>, timeout_ms: u32) -> PyResult>> { + let outcome = py.allow_threads(|| { + if timeout_ms == 0 { + self.rx + .recv() + .map_err(|_| flume::RecvTimeoutError::Disconnected) + } else { + self.rx + .recv_timeout(std::time::Duration::from_millis(timeout_ms as u64)) + } + }); + match outcome { + Ok(evt) => { + let v = serde_json::to_value(&evt).map_err(map_err)?; + Ok(Some(pythonize(py, &v).map_err(map_err)?)) + } + Err(flume::RecvTimeoutError::Timeout) => Ok(None), + Err(flume::RecvTimeoutError::Disconnected) => { + Err(PyRuntimeError::new_err("trace channel closed")) + } + } + } } // ─── Event stream ────────────────────────────────────────────────────────── @@ -1612,6 +1657,7 @@ fn _native(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(template_actor, m)?)?; m.add_function(wrap_pyfunction!(template_list, m)?)?; m.add_function(wrap_pyfunction!(load_pack, m)?)?; diff --git a/sdk/python/tests/test_tracing.py b/sdk/python/tests/test_tracing.py new file mode 100644 index 00000000..daffbb7b --- /dev/null +++ b/sdk/python/tests/test_tracing.py @@ -0,0 +1,83 @@ +"""First-class tracing in the Python SDK. + +Tracing is enabled via the network config; trace events are consumed locally +through ``Network.traces()`` with no collector required. Verifies correlation +(events arrive) and fidelity (data-flow snapshots carry a content checksum). +""" + +import time + +from offbit_reflow import Actor, Message, Network + + +class Doubler(Actor): + component = "doubler" + inports = ["in"] + outports = ["out"] + + def run(self, ctx): + n = ctx.inputs["in"]["data"] + ctx.emit("out", Message.integer(n * 2)) + ctx.done() + + +class Collect(Actor): + component = "collect" + inports = ["in"] + outports = [] + + def __init__(self, bucket): + super().__init__() + self.bucket = bucket + + def run(self, ctx): + self.bucket.append(ctx.inputs["in"]) + ctx.done() + + +def _event_type_names(evt): + """TraceEventType is a string for unit variants ("ActorCreated") or a + single-key dict for data-carrying variants ({"DataFlow": {...}}).""" + et = evt.get("event_type") + if isinstance(et, str): + return {et} + if isinstance(et, dict): + return set(et.keys()) + return set() + + +def test_trace_stream_yields_correlated_events_with_checksums(): + bucket = [] + # Tracing enabled. The collector URL need not be reachable: events still + # flow to the local tap. (A real deployment points server_url at a server.) + net = Network( + {"tracing": {"server_url": "ws://127.0.0.1:8080", "enabled": True}} + ) + net.register_actor("tpl_doubler", Doubler()) + net.register_actor("tpl_collect", Collect(bucket)) + net.add_node("d", "tpl_doubler") + net.add_node("c", "tpl_collect") + net.add_connection("d", "out", "c", "in") + net.add_initial("d", "in", {"type": "Integer", "data": 21}) + + traces = net.traces() # subscribe before start + net.start() + + seen_types = set() + saw_checksum = False + deadline = time.time() + 3.0 + while time.time() < deadline: + evt = traces.recv(timeout_ms=200) + if evt is None: + continue + seen_types |= _event_type_names(evt) + msg = (evt.get("data") or {}).get("message") + if msg and isinstance(msg.get("checksum"), str) and msg["checksum"].startswith("sha256:"): + saw_checksum = True + if "ActorCreated" in seen_types and saw_checksum: + break + + net.shutdown() + + assert "ActorCreated" in seen_types, f"seen event types: {seen_types}" + assert saw_checksum, "expected a data-flow snapshot carrying a sha256 checksum"