diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5c5a33c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +target +.git +.github +.idea +.vscode +.claude +tests +*.log +*.jsonl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a66ff5c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + name: Run Rust tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Cache Cargo dependencies + uses: Swatinem/rust-cache@v2 + + - name: Format code + run: cargo fmt --all + + - name: Run Clippy + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Run tests + run: cargo test --all \ No newline at end of file diff --git a/.gitignore b/.gitignore index 885c704..d0cb370 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ /target .idea -claude/ +.claude/ .vscode/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 20498a0..a9ae534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,1636 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd059f9da4f5c36b3787f65d38ccaab1cc315f07b01f89abc8359ee6a8205011" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09628afdcc538b57f3c6341e9c8e9970f18e4a481690a64974d7023bd33548b" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http", + "hyper", + "rustls", + "tokio", + "tokio-rustls", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inotify" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c033f80b2c113cdf91ab7a33faa9cbc014726dcad99880c8609af2a370edf37d" +dependencies = [ + "libc", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "kqueue" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273c0752728918e0ac4976f2b275b6fefb9ecd400585dec929419f3844cd87b5" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07293a4e297ac234359b510362495713f75ea345d5307140414f20c69ffeb087" +dependencies = [ + "bitflags 2.13.0", + "libc", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "logtap" version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "notify", + "regex", + "reqwest", + "serde", + "serde_json", + "tokio", + "toml", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "notify" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" +dependencies = [ + "bitflags 2.13.0", + "crossbeam-channel", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio 0.8.11", + "walkdir", + "windows-sys 0.48.0", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", +] + +[[package]] +name = "regex" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0e75113e14dc5acb068cd0786884f214f1312650a3d36d269f5c4f3cdee8a2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-rustls", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-rustls", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", + "winreg", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +dependencies = [ + "bytes", + "libc", + "mio 1.2.2", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c96ecdfa941c8fc4fcaed14f99ada8ebed502eef533015095a07e3301d4c3c" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index b9bba5e..3569299 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,3 +4,16 @@ version = "0.1.0" edition = "2024" [dependencies] +tokio = { version = "1", features = ["full"] } +notify = "6" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +regex = "1" +reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] } +anyhow = "1" +toml = "1.1.3+spec-1.1.0" +clap = { version = "4.6.2", features = ["derive"] } + +[profile.release] +strip = true +lto = "thin" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..9b3429d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# --- build stage --- +FROM rust:1-slim-bookworm AS builder + +WORKDIR /app + +# Cache dependency compilation separately from source changes: with only +# Cargo.toml/Cargo.lock copied, `cargo build` here only re-runs when +# dependencies actually change, not on every source edit. +COPY Cargo.toml Cargo.lock ./ +RUN mkdir src \ + && echo "fn main() {}" > src/main.rs \ + && echo "" > src/lib.rs \ + && cargo build --release \ + && rm -rf src + +COPY src ./src +# BuildKit normalizes COPY'd file timestamps, which defeats cargo's +# mtime-based rebuild detection — without this, cargo can decide nothing +# changed and silently keep linking the dummy binary from the step above. +RUN find src -name '*.rs' -exec touch {} + && cargo build --release + +# --- runtime stage --- +FROM debian:bookworm-slim + +RUN useradd --system --create-home --home-dir /app logtap +WORKDIR /app + +COPY --from=builder /app/target/release/logtap /usr/local/bin/logtap + +USER logtap + +# logtap.toml and the source log file are expected to be mounted in at +# runtime (see README) — the binary itself carries no config or state. +ENTRYPOINT ["logtap"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fa609a0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2026 logtap contributors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index f3a6924..c83d83f 100644 --- a/README.md +++ b/README.md @@ -1 +1,216 @@ -# logtap \ No newline at end of file +# logtap + +[![License: Apache 2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![CI](https://github.com/dwego/logtap/actions/workflows/ci.yml/badge.svg)](https://github.com/dwego/logtap/actions/workflows/ci.yml) + +An asynchronous, minimalist log aggregator written in Rust, built around one core idea: **real backpressure**. The pipeline should never blow up memory just because the destination got slow. + +> **Status:** early-stage / MVP. The core pipeline works end-to-end and is covered by tests, but the project is still small on purpose — see [Roadmap](#roadmap) for what's deliberately not built yet. + +## What it does + +`logtap` watches a log source (today, a file; `stdin` is planned), turns each line into a structured record, applies filter and masking rules, and delivers everything in batches to an HTTP destination — without ever letting the internal work queue grow without bound. + +Think of it as a small, readable take on tools like Fluentd or Vector: it doesn't compete on feature count, it competes on being easy to understand end to end, from the first line of code to the last. + +## Architecture + +The project is organized as a pipeline of independent stages, each running as its own async task, connected only by channels: + +``` +log file + │ + ▼ +source.rs ──String──▶ parser.rs ──LogLine──▶ filter.rs ──LogLine──▶ sink.rs +(tails the (raw text becomes (drops and (batches records + file via structured JSON) masks fields and sends them + notify/inotify) per rule) over HTTP) +``` + +Every arrow is a `tokio::sync::mpsc::channel` with a bounded capacity (`channel_capacity`, configurable). That bounded capacity is the entire design in one sentence: when a stage produces faster than the next one can consume, the channel fills up and the sending call (`send().await` / `blocking_send()`) simply waits — no unbounded queue, no runaway `Vec`, no process getting OOM-killed. + +**Why separate stages instead of one big function.** Each piece (`source`, `parser`, `filter`, `sink`) only knows about its input channel and its output channel. That means swapping the internal implementation of any stage — for example, changing how `source.rs` watches the file, or swapping the HTTP client used in `sink.rs` — doesn't require touching any other file, as long as the function signature stays the same. + +**The central type, `LogLine`.** Rather than a fixed struct with predefined fields, `LogLine` is a type alias for `serde_json::Value`. Real-world logs rarely share a single schema — every application logs different fields — and forcing a rigid struct would mean the parser silently drops or truncates whatever doesn't fit. `Value` accepts any JSON log shape without `logtap` needing to know the field set up front. + +**Filtering as configuration, not code.** What to drop and what to mask lives in `logtap.toml`, not in `if` statements scattered through the codebase. `FilterRule` entries match on a field using `Equals`, `Contains`, or `Regex`, and either `Drop` the record or `Mask` the field (replacing its value with `"***"`). This lets operators change behavior without recompiling — including masking sensitive fields such as emails or API keys — by adding a rule, rather than depending on someone remembering to write one into the source. + +## How this is meant to scale + +"Scaling" here doesn't just mean "handle more volume" — it means three distinct things, and the project is structured so each can evolve independently. + +### 1. Volume (more logs per second) + +The natural bottleneck in a staged pipeline is its slowest stage — usually the sink, since it depends on the network. The current architecture already absorbs this in two ways: + +- **Batching amortizes network cost.** Instead of one HTTP request per line, logs accumulate until `batch_size` or `flush_interval_secs` is reached, whichever comes first. The fixed cost of each request (handshake, round-trip latency) is spread across dozens or hundreds of log lines instead of paid on every single one. +- **Backpressure protects the process, not just the sink.** If inbound volume spikes (a traffic burst, an incident generating excessive logs), the bounded channels absorb the pressure without letting memory grow — the source's own read rate slows down automatically as a side effect. + +The natural next step for volume is parallelizing the sink itself: today there's a single `run_sink` task consuming from one channel. Nothing prevents running multiple sink tasks reading from the same channel (`tokio::mpsc` already supports multiple concurrent consumers), as long as the destination can handle concurrent requests. + +### 2. Surface area (more sources, more destinations, more formats) + +Today `source.rs` and `sink.rs` are single files with no trait abstraction — a deliberate choice for the MVP, to avoid paying the cost of indirection before a second real implementation exists. This is exactly where the project is expected to grow: + +``` +source/ sink/ +├── mod.rs (trait) ├── mod.rs (trait) +├── file.rs ├── http.rs +└── stdin.rs ├── stdout.rs + └── file.rs +``` + +Once there's a second source (`stdin`, alongside the file tail) or a second destination (a local audit log as backup, alongside HTTP), it's worth introducing the trait — and the refactor stays small, because the rest of the pipeline (`parser`, `filter`, the channels) doesn't change at all. The same logic applies to fanning out to multiple sinks at once — shipping the same log to HTTP and to a local audit file without duplicating filter logic. + +### 3. Reliability (the system must not silently lose data) + +This is the most important axis. Most of it is already in place: + +- **Retry with exponential backoff** in the sink before giving up on a batch. +- **A local dead-letter file** — batches that exhaust their retries get written to `logtap.failed.jsonl` instead of lost, one record per line, in the same shape `source` already produces so it can be replayed through `logtap` itself (see [Reprocessing the dead-letter file](#reprocessing-the-dead-letter-file)). +- **Size-capped, rotating dead-letter files** — `logtap.failed.jsonl` rotates into `.1`, `.2`, etc. once it crosses `dead_letter_max_bytes`, so a prolonged outage fills disk predictably instead of without limit. Discarding the oldest rotated file is real data loss, so it's always logged loudly, never silently. + +What's still missing: **log rotation detection** — when the watched file is renamed and recreated (standard `logrotate` behavior in production), `source` needs to notice and start reading the new file, instead of continuing to hold a handle to a file that no longer exists. + +### 4. Operability (visibility into the pipeline itself) + +A log shipper nobody can observe is a blind spot inside another observability system — which is a bit ironic. The natural direction here: + +- **Internal metrics**: lines read, parsed, dropped by filter, dropped by parse error, sent successfully, failed. +- **A Prometheus-format `/metrics` endpoint**, so `logtap` can be monitored by the same tooling that watches the rest of the infrastructure. +- **Degradation signals, not just outright failure** — for example, exposing when a channel is consistently near capacity, which is the earliest symptom of a downstream stage slowing down, before it turns into actual data loss. + +## What stays fixed as the project grows + +Whichever of these directions the project tackles first, two architectural decisions stay fixed, because they're the foundation everything else depends on: + +- **Communication only through bounded channels.** No unbounded queue anywhere in the pipeline — this is the guarantee that keeps the rest of the system predictable under load. +- **Stages that know nothing about each other beyond the data type they exchange.** A new stage, a new source, a new destination — all of them fit the same shape of "receive from a channel, process, send to the next one," with no cascading changes through the rest of the code. + +That discipline, more than any specific feature, is what determines whether the project can keep growing without turning into something no one can maintain. + +## Getting started + +Build the binary and point it at a config file: + +```bash +cargo build --release +./target/release/logtap --config-path logtap.toml +``` + +`--config-path` defaults to `logtap.toml` in the current directory, so with a config file already in place, `./target/release/logtap` on its own is enough. See [`logtap.toml`](logtap.toml) in this repo for a working example. + +### Exit codes + +| Code | Meaning | +|---|---| +| `0` | Clean run. (Today the pipeline runs until killed — this is reserved for a future graceful shutdown.) | +| `1` | An expected, fixable error: missing config file, invalid TOML, a malformed field. | +| `101` | An unexpected internal panic — a bug, not a config problem. | + +### Docker + +```bash +docker build -t logtap . + +docker run --rm \ + -v $(pwd)/logtap.toml:/app/logtap.toml \ + -v /path/to/your/app.log:/data/app.log \ + logtap +``` + +The image ships only the binary — `logtap.toml` and the log file it tails are expected to be mounted in at runtime. `source_path` inside `logtap.toml` should point at wherever the log file lands *inside* the container (`/data/app.log` above), not at its path on the host. + +### Configuration (`logtap.toml`) + +`Config` is deserializable from TOML via `serde`. Only `source_path` and `sink_url` are required — everything else has a default: + +```toml +source_path = "app.log" +sink_url = "http://localhost:8080/logs" +batch_size = 100 +flush_interval_secs = 5 +channel_capacity = 1000 +max_retries = 5 +retry_backoff_initial_ms = 500 +retry_backoff_max_secs = 30 +mask_common_patterns = true +dead_letter_max_bytes = 1073741824 +dead_letter_max_files = 5 + +[[filter_rules]] +field = "level" +op = "equals" +value = "debug" +action = "drop" + +[[filter_rules]] +field = "email" +op = "regex" +value = "^[^@]+@[^@]+\\.[^@]+$" +action = "mask" +``` + +| Field | Type | Default | Description | +|---|---|---|---| +| `source_path` | path | — (required) | File to tail. | +| `sink_url` | string | — (required) | HTTP endpoint the batches are `POST`ed to as a JSON array. | +| `batch_size` | integer | `50` | Max records buffered before a flush is triggered. | +| `flush_interval_secs` | integer | `5` | Max time between flushes, even if `batch_size` isn't reached. | +| `channel_capacity` | integer | `1000` | Bound applied to every internal channel — the backpressure knob. | +| `max_retries` | integer | `5` | Attempts per batch before giving up and writing it to the dead-letter file. | +| `retry_backoff_initial_ms` | integer | `500` | Backoff before the first retry; doubles on each subsequent attempt. | +| `retry_backoff_max_secs` | integer | `30` | Ceiling on the backoff, however many retries have piled up. | +| `mask_common_patterns` | boolean | `true` | Auto-mask emails, card numbers, and `sk-...`-style API keys, independent of `filter_rules`. | +| `dead_letter_max_bytes` | integer | `1073741824` (1 GiB) | Size cap on `logtap.failed.jsonl` before it's rotated into `.1`, `.2`, etc. | +| `dead_letter_max_files` | integer | `5` | How many rotated dead-letter files to keep before the oldest is discarded. | +| `filter_rules` | array | `[]` | Ordered `FilterRule` entries (`field`, `op`, `value`, `action`). | + +`op` accepts `equals`, `contains`, or `regex`. `action` accepts `drop` or `mask`. + +### Reprocessing the dead-letter file + +Batches that exhaust `max_retries` are appended to `logtap.failed.jsonl` instead of being lost — one JSON record per line, in the same shape `source` already produces, so it can be replayed through `logtap` itself rather than needing a separate tool: + +```bash +touch replay.log +./target/release/logtap --config-path replay-logtap.toml & # points source_path at replay.log +cat logtap.failed.jsonl >> replay.log +``` + +`replay-logtap.toml` is a throwaway config pointing `source_path` at the empty `replay.log` and `sink_url` at the real destination. Since `logtap` tails from the end of the file it opens, appending the dead-letter contents into `replay.log` after it's already running makes it pick them up exactly like live traffic. Keep `batch_size` small (or `flush_interval_secs` short) in that config — otherwise the replayed lines just sit buffered until a batch actually fills up or the interval ticks. Once the file stops growing and everything's been delivered, stop the process and delete `logtap.failed.jsonl` and `replay.log`. + +## Development + +```bash +cargo test --all # unit + integration tests +cargo fmt --all # formatting +cargo clippy --all-targets --all-features -- -D warnings +``` + +CI runs all three on every pull request and on pushes to `main`. + +Tests are organized by what they're proving, one file per concern under `tests/` (each file is its own binary, so this costs nothing to keep separate): + +- `filter_test.rs`, `parser_test.rs`, `source_test.rs`, `sink_test.rs`, `notify_test.rs` — one stage or behavior at a time. +- `integration_test.rs`, `cli_test.rs` — the full pipeline wired together, the second one through the actual compiled binary. +- `stress_test.rs` — load/failure-shaped: proving an outcome (nothing lost) under conditions designed to trigger backpressure, not just checking one function's return value. + +## Roadmap + +### v1 acceptance criteria + +v1 is done when all of the following hold: + +- [x] No batch is ever lost silently — every failed send is retried or persisted locally. +- [x] The process survives log rotation without manual intervention (standard rename-then-create logrotate behavior; `copytruncate` isn't handled yet). +- [x] No channel grows unbounded under load — guaranteed by construction (every channel has a fixed `channel_capacity`), and backed by [`tests/stress_test.rs`](tests/stress_test.rs), which proves the *outcome* (a burst of logs during a prolonged outage is never lost) rather than measuring memory directly. +- [ ] Every config field is validated with a clear error message at startup — never fails silently at runtime. *(deferred to v1.1 — Config::load already rejects a missing/malformed file with a clean error, just not individual bad field values like `batch_size = 0`)* +- [x] Sensitive data is masked by default, even with no user-configured rules. +- [ ] External visibility (metrics) into what the pipeline is doing, without reading logtap's own stderr output. *(deferred to v1.1 — stderr logging already covers retries, dead-letter writes, and rotation events)* +- [x] End-to-end integration test covering the full pipeline, plus unit tests per stage. +- [x] Installing and running the project requires no source reading — README and example config are enough. + +## License + +Licensed under the [Apache License, Version 2.0](LICENSE). diff --git a/logtap.toml b/logtap.toml new file mode 100644 index 0000000..b94de59 --- /dev/null +++ b/logtap.toml @@ -0,0 +1,11 @@ +# Minimal working example — see the README's Configuration section for the +# full field reference (defaults, dead-letter rotation, filter rule syntax). + +source_path = "app.log" +sink_url = "http://localhost:8080/logs" + +[[filter_rules]] +field = "level" +op = "equals" +value = "debug" +action = "drop" diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..aa3f708 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,79 @@ +use crate::filter::FilterRule; +use anyhow::Result; +use serde::Deserialize; +use std::path::PathBuf; + +#[derive(Debug, Deserialize, Clone)] +pub struct Config { + pub source_path: PathBuf, + pub sink_url: String, + + #[serde(default = "default_batch_size")] + pub batch_size: usize, + + #[serde(default = "default_flush_interval")] + pub flush_interval_secs: u64, + + #[serde(default = "default_channel_capacity")] + pub channel_capacity: usize, + + #[serde(default = "default_max_retries")] + pub max_retries: u32, + + #[serde(default = "default_retry_backoff_initial_ms")] + pub retry_backoff_initial_ms: u64, + + #[serde(default = "default_retry_backoff_max_secs")] + pub retry_backoff_max_secs: u64, + + #[serde(default)] + pub filter_rules: Vec, + + #[serde(default = "default_mask_common_patterns")] + pub mask_common_patterns: bool, + + #[serde(default = "default_dead_letter_max_bytes")] + pub dead_letter_max_bytes: usize, + + #[serde(default = "default_dead_letter_max_files")] + pub dead_letter_max_files: u64, +} + +fn default_batch_size() -> usize { + 50 +} +fn default_flush_interval() -> u64 { + 5 +} +fn default_channel_capacity() -> usize { + 1000 +} +fn default_max_retries() -> u32 { + 5 +} +fn default_retry_backoff_initial_ms() -> u64 { + 500 +} +fn default_retry_backoff_max_secs() -> u64 { + 30 +} +fn default_mask_common_patterns() -> bool { + true +} + +fn default_dead_letter_max_bytes() -> usize { + 1024 * 1024 * 1024 +} + +fn default_dead_letter_max_files() -> u64 { + 5 +} + +impl Config { + pub fn load(path: &str) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| anyhow::anyhow!("não consegui ler {path}: {e}"))?; + let cfg: Config = toml::from_str(&text)?; + Ok(cfg) + } +} diff --git a/src/filter.rs b/src/filter.rs new file mode 100644 index 0000000..d9f9e5e --- /dev/null +++ b/src/filter.rs @@ -0,0 +1,97 @@ +use crate::LogLine; +use regex::Regex; +use serde::Deserialize; +use tokio::sync::mpsc::{Receiver, Sender}; + +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "lowercase")] +pub enum RuleOp { + Equals, + Contains, + Regex, +} + +#[derive(Debug, Deserialize, Clone)] +#[serde(rename_all = "lowercase")] +pub enum RuleAction { + Drop, + Mask, +} + +#[derive(Debug, Deserialize, Clone)] +pub struct FilterRule { + pub field: String, + pub op: RuleOp, + pub value: String, + pub action: RuleAction, +} + +pub async fn run_filter( + mut rx: Receiver, + tx: Sender, + rules: Vec, + mask_common_patterns: bool, +) { + let email_re = Regex::new(r"(?i)\b([a-z0-9._%+-])[a-z0-9._%+-]*(@[a-z0-9.-]+\.[a-z]{2,})\b") + .expect("invalid regex for email"); + let card_re = Regex::new(r"\b(?:\d[ -]?){13,19}\b").expect("invalid regex for card"); + let api_key_re = Regex::new(r"\bsk-[A-Za-z0-9]{16,}\b").expect("invalid regex for API key"); + + let builtin_patterns: Vec<(&Regex, &str)> = vec![ + (&email_re, "$1***$2"), + (&card_re, "[card-masked]"), + (&api_key_re, "[api-key-masked]"), + ]; + + while let Some(mut log) = rx.recv().await { + if mask_common_patterns { + mask_builtin(&mut log, &builtin_patterns); + } + + let mut dropped = false; + + for rule in &rules { + let field_val = log.get(&rule.field).and_then(|v| v.as_str()).unwrap_or(""); + + let matched = match rule.op { + RuleOp::Equals => field_val == rule.value, + RuleOp::Contains => field_val.contains(&rule.value), + RuleOp::Regex => Regex::new(&rule.value) + .map(|re| re.is_match(field_val)) + .unwrap_or(false), + }; + + if matched { + match rule.action { + RuleAction::Drop => { + dropped = true; + break; + } + RuleAction::Mask => { + log[&rule.field] = serde_json::json!("***"); + } + } + } + } + + if !dropped && tx.send(log).await.is_err() { + break; + } + } +} + +fn mask_builtin(log: &mut LogLine, patterns: &[(&Regex, &str)]) { + if let Some(obj) = log.as_object_mut() { + for (_, v) in obj.iter_mut() { + if let Some(s) = v.as_str() { + let mut masked = s.to_string(); + for (re, replacement) in patterns { + masked = re.replace_all(&masked, *replacement).to_string(); + } + if masked != s { + *v = serde_json::Value::String(masked); + } + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..35782c0 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,40 @@ +pub mod config; +pub mod filter; +pub mod parser; +pub mod record; +pub mod sink; +pub mod source; + +pub use config::Config; +pub use record::LogLine; + +pub async fn run(cfg: Config) -> anyhow::Result<()> { + let (raw_tx, raw_rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + let (parsed_tx, parsed_rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + let (clean_tx, clean_rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + + let source_cfg = cfg.clone(); + let mask_common_patterns = source_cfg.mask_common_patterns; + let source_handle = tokio::task::spawn_blocking(move || source::run_source(source_cfg, raw_tx)); + + let parser_handle = tokio::spawn(parser::run_parser(raw_rx, parsed_tx)); + + let rules = cfg.filter_rules.clone(); + let filter_handle = tokio::spawn(filter::run_filter( + parsed_rx, + clean_tx, + rules, + mask_common_patterns, + )); + + let sink_cfg = cfg.clone(); + let sink_handle = tokio::spawn(sink::run_sink(sink_cfg, clean_rx)); + + let (src_res, _, _, _) = tokio::join!(source_handle, parser_handle, filter_handle, sink_handle); + + if let Ok(Err(e)) = src_res { + eprintln!("logtap: source encerrou com erro: {e}"); + } + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index e7a11a9..85d560b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,22 @@ -fn main() { - println!("Hello, world!"); +use clap::Parser; +use logtap::Config; + +#[derive(Parser, Debug)] +#[command(version, about = "Logtap CLI")] +struct Args { + #[arg(short, long, value_name = "logtap.toml", default_value = "logtap.toml")] + config_path: String, +} + +#[tokio::main] +async fn main() -> Result<(), String> { + let args = Args::parse(); + + let config = Config::load(&args.config_path).map_err(|e| e.to_string())?; + + println!("Loaded config: {:?}", config); + + logtap::run(config).await.map_err(|e| e.to_string())?; + + Ok(()) } diff --git a/src/parser.rs b/src/parser.rs new file mode 100644 index 0000000..18760f8 --- /dev/null +++ b/src/parser.rs @@ -0,0 +1,28 @@ +use crate::record::LogLine; +use serde_json::json; +use tokio::sync::mpsc::{Receiver, Sender}; + +pub async fn run_parser(mut rx: Receiver, tx: Sender) { + while let Some(line) = rx.recv().await { + let log_line = parse_line(&line); + if tx.send(log_line).await.is_err() { + break; + } + } +} + +pub fn parse_line(line: &str) -> LogLine { + match serde_json::from_str::(line) { + Ok(value) if value.is_object() => value, + + Ok(value) => { + eprintln!("logtap: line was valid JSON, but is not object: {value}"); + json!({ "message": line, "level": "UNKNOWN", "parse_issue": "not_an_object" }) + } + + Err(err) => { + eprintln!("logtap: falied to process as JSON: {err}"); + json!({ "message": line, "level": "UNKNOWN", "parse_issue": "invalid_json" }) + } + } +} diff --git a/src/record.rs b/src/record.rs new file mode 100644 index 0000000..f4f1a32 --- /dev/null +++ b/src/record.rs @@ -0,0 +1 @@ +pub type LogLine = serde_json::Value; diff --git a/src/sink.rs b/src/sink.rs new file mode 100644 index 0000000..8362425 --- /dev/null +++ b/src/sink.rs @@ -0,0 +1,178 @@ +use crate::config::Config; +use crate::record::LogLine; +use std::fs; +use std::fs::OpenOptions; +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; +use tokio::sync::mpsc::Receiver; +use tokio::time::{interval, sleep}; + +const DEAD_LETTER_PATH: &str = "logtap.failed.jsonl"; + +pub async fn run_sink(cfg: Config, mut rx: Receiver) { + let client = reqwest::Client::new(); + let mut batch: Vec = Vec::with_capacity(cfg.batch_size); + let mut ticker = interval(Duration::from_secs(cfg.flush_interval_secs)); + + loop { + tokio::select! { + Some(log) = rx.recv() => { + batch.push(log); + if batch.len() >= cfg.batch_size { + flush_with_retry(&client, &cfg, &mut batch).await; + } + } + _ = ticker.tick() => { + if !batch.is_empty() { + flush_with_retry(&client, &cfg, &mut batch).await; + } + } + } + } +} + +async fn flush_with_retry(client: &reqwest::Client, cfg: &Config, batch: &mut Vec) { + if batch.is_empty() { + return; + } + + let mut attempt: u32 = 0; + + loop { + match client.post(&cfg.sink_url).json(batch).send().await { + Ok(resp) if resp.status().is_success() => { + batch.clear(); + return; + } + Ok(resp) => { + eprintln!( + "logtap: attempt {}/{} failed — server responded with status {}", + attempt + 1, + cfg.max_retries, + resp.status() + ); + } + Err(err) => { + eprintln!( + "logtap: attempt {}/{} failed — error sending batch ({} items): {err}", + attempt + 1, + cfg.max_retries, + batch.len() + ); + } + } + + attempt += 1; + + if attempt >= cfg.max_retries { + eprintln!( + "logtap: abandoning batch of {} items after {} attempts — writing to {DEAD_LETTER_PATH}", + batch.len(), + cfg.max_retries + ); + + write_dead_letter(cfg, batch); + batch.clear(); + return; + } + + let backoff_ms = cfg + .retry_backoff_initial_ms + .saturating_mul(1u64 << (attempt - 1)); + let backoff = + Duration::from_millis(backoff_ms).min(Duration::from_secs(cfg.retry_backoff_max_secs)); + + eprintln!("logtap: waiting {backoff:?} before next attempt"); + sleep(backoff).await; + } +} + +// Reopens the file on every call instead of keeping a long-lived handle — +// a handle stays bound to the underlying inode even if the file is later +// renamed out from under it (e.g. for a manual replay), which would make +// new failures silently land in the renamed-away file. +fn write_dead_letter(cfg: &Config, batch: &[LogLine]) { + rotate_dead_letter_if_full(cfg); + + let mut file = match OpenOptions::new() + .create(true) + .append(true) + .open(DEAD_LETTER_PATH) + { + Ok(file) => file, + Err(err) => { + eprintln!( + "logtap: could not open {DEAD_LETTER_PATH} ({err}) — {} item(s) lost for good", + batch.len() + ); + return; + } + }; + + for log in batch { + if let Err(err) = writeln!(file, "{log}") { + eprintln!("logtap: failed writing to {DEAD_LETTER_PATH}: {err}"); + return; + } + } +} + +fn rotated_dead_letter_path(n: u64) -> PathBuf { + PathBuf::from(format!("{DEAD_LETTER_PATH}.{n}")) +} + +// Caps how big logtap.failed.jsonl is allowed to get. Same idea as +// logrotate: once the current file crosses dead_letter_max_bytes, it's +// shifted into .1, the old .1 becomes .2, and so on up to +// dead_letter_max_files — at which point the oldest file is discarded to +// make room. That eviction is real, permanent data loss, so it's always +// logged loudly rather than happening quietly. +fn rotate_dead_letter_if_full(cfg: &Config) { + let current_size = match fs::metadata(DEAD_LETTER_PATH) { + Ok(meta) => meta.len(), + Err(_) => return, // no file yet — nothing to rotate + }; + + if current_size < cfg.dead_letter_max_bytes as u64 { + return; + } + + if cfg.dead_letter_max_files == 0 { + if let Err(err) = fs::remove_file(DEAD_LETTER_PATH) { + eprintln!("logtap: failed to reset full dead-letter file: {err}"); + } else { + eprintln!( + "logtap: dead-letter file hit {current_size} bytes and dead_letter_max_files is 0 — discarding it entirely" + ); + } + return; + } + + let oldest = rotated_dead_letter_path(cfg.dead_letter_max_files); + if oldest.exists() { + eprintln!( + "logtap: dead-letter rotation limit ({} files) reached — discarding oldest file {}", + cfg.dead_letter_max_files, + oldest.display() + ); + } + + for n in (1..cfg.dead_letter_max_files).rev() { + let from = rotated_dead_letter_path(n); + if from.exists() { + let to = rotated_dead_letter_path(n + 1); + if let Err(err) = fs::rename(&from, &to) { + eprintln!( + "logtap: failed to rotate {} -> {}: {err}", + from.display(), + to.display() + ); + } + } + } + + if let Err(err) = fs::rename(DEAD_LETTER_PATH, rotated_dead_letter_path(1)) { + eprintln!("logtap: failed to rotate {DEAD_LETTER_PATH}: {err}"); + } +} diff --git a/src/source.rs b/src/source.rs new file mode 100644 index 0000000..7ab7cf3 --- /dev/null +++ b/src/source.rs @@ -0,0 +1,123 @@ +use crate::config::Config; +use anyhow::Result; +use notify::{Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use std::fs::File; +use std::io::{BufRead, BufReader, Seek, SeekFrom}; +use std::os::unix::fs::MetadataExt; +use std::path::Path; +use std::sync::mpsc as std_mpsc; +use std::time::Duration; +use tokio::sync::mpsc::Sender; + +// How often the watch loop wakes up even without a filesystem event, just to +// check whether the downstream receiver is gone, and whether the file at +// source_path has been swapped out from under us (log rotation). Without +// this, the loop can block forever on `notify_rx.recv()` once there's no +// more filesystem activity on the file it's actually still holding open. +const SHUTDOWN_POLL_INTERVAL: Duration = Duration::from_millis(200); + +/// Whether the caller should keep going or the downstream receiver is gone. +enum DrainOutcome { + Continue, + ReceiverGone, +} + +/// Reads every complete line available from `offset` onward and forwards it +/// down the channel, advancing `offset` as it goes. +fn drain_new_lines( + reader: &mut BufReader, + offset: &mut u64, + tx: &Sender, + line: &mut String, +) -> Result { + reader.seek(SeekFrom::Start(*offset))?; + + loop { + line.clear(); + let bytes_read = reader.read_line(line)?; + + if bytes_read == 0 { + return Ok(DrainOutcome::Continue); + } + + *offset += bytes_read as u64; + let trimmed = line.trim_end().to_string(); + + if tx.blocking_send(trimmed).is_err() { + return Ok(DrainOutcome::ReceiverGone); + } + } +} + +fn watch( + path: &Path, + notify_tx: std_mpsc::Sender>, +) -> Result { + let mut watcher: RecommendedWatcher = notify::recommended_watcher(move |res| { + let _ = notify_tx.send(res); + })?; + watcher.watch(path, RecursiveMode::NonRecursive)?; + Ok(watcher) +} + +pub fn run_source(cfg: Config, tx: Sender) -> Result<()> { + let file = File::open(&cfg.source_path)?; + let mut inode = file.metadata()?.ino(); + let mut reader = BufReader::new(file); + let mut offset = reader.seek(SeekFrom::End(0))?; + + let (notify_tx, notify_rx) = std_mpsc::channel::>(); + let mut _watcher = watch(&cfg.source_path, notify_tx.clone())?; + + let mut line = String::new(); + + loop { + let event = match notify_rx.recv_timeout(SHUTDOWN_POLL_INTERVAL) { + Ok(res) => res?, + Err(std_mpsc::RecvTimeoutError::Timeout) => { + if tx.is_closed() { + return Ok(()); + } + + // The path may briefly not resolve mid-rotation (rename in + // progress) — that's transient, just retry on the next tick. + let Ok(current_inode) = std::fs::metadata(&cfg.source_path).map(|m| m.ino()) else { + continue; + }; + + if current_inode == inode { + continue; + } + + // The file at source_path is no longer the one we have open + // (standard logrotate "rename + create new" behavior). Flush + // whatever was written to the old file right before the + // swap, then switch to the new one from the start. + if let DrainOutcome::ReceiverGone = + drain_new_lines(&mut reader, &mut offset, &tx, &mut line)? + { + return Ok(()); + } + + let new_file = File::open(&cfg.source_path)?; + inode = new_file.metadata()?.ino(); + reader = BufReader::new(new_file); + offset = 0; + _watcher = watch(&cfg.source_path, notify_tx.clone())?; + + continue; + } + Err(std_mpsc::RecvTimeoutError::Disconnected) => return Ok(()), + }; + + if !matches!(event.kind, EventKind::Modify(_)) { + continue; + } + + if let DrainOutcome::ReceiverGone = + drain_new_lines(&mut reader, &mut offset, &tx, &mut line)? + { + return Ok(()); + } + } +} diff --git a/tests/cli_test.rs b/tests/cli_test.rs new file mode 100644 index 0000000..cc972e3 --- /dev/null +++ b/tests/cli_test.rs @@ -0,0 +1,130 @@ +use std::fs; +use std::io::Write; +use std::path::PathBuf; +use std::process::Stdio; +use std::time::Duration; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +async fn read_http_request_body(stream: &mut TcpStream) -> String { + let mut buffer = Vec::new(); + let mut temp = [0_u8; 1024]; + + loop { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before headers were read"); + + buffer.extend_from_slice(&temp[..read]); + + if buffer.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + let headers_end = buffer + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + + let headers = String::from_utf8_lossy(&buffer[..headers_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|value| value.trim().parse::().unwrap()) + }) + .unwrap(); + + while buffer.len() < headers_end + content_length { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before body was read"); + + buffer.extend_from_slice(&temp[..read]); + } + + String::from_utf8(buffer[headers_end..headers_end + content_length].to_vec()).unwrap() +} + +// This spawns the actual compiled `logtap` binary as a subprocess — unlike +// the other integration tests, which call `logtap::run` directly, this is +// the only test that goes through `main.rs` itself (arg parsing, config +// path handling), so it's the only one that would catch the CLI wiring +// breaking even if every library-level test still passed. +#[tokio::test] +async fn cli_reads_config_file_and_delivers_a_log_line() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + + let body = read_http_request_body(&mut stream).await; + + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + body + }); + + let source_path = PathBuf::from("cli_test.log"); + fs::write(&source_path, "").unwrap(); + + let config_path = PathBuf::from("cli_test.toml"); + let config_toml = format!( + r#" +source_path = "{source}" +sink_url = "http://{address}/logs" +batch_size = 1 +flush_interval_secs = 60 +max_retries = 0 +"#, + source = source_path.display(), + ); + fs::write(&config_path, config_toml).unwrap(); + + let mut child = tokio::process::Command::new(env!("CARGO_BIN_EXE_logtap")) + .arg("--config-path") + .arg(&config_path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the logtap binary"); + + // Spawning a whole new OS process (not just a task inside the test + // binary) has more startup overhead than the other integration tests — + // give it real time to parse args, load the config, and register the + // file watcher before writing the line it needs to pick up. + tokio::time::sleep(Duration::from_secs(1)).await; + + { + let mut file = fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap(); + + writeln!(file, r#"{{"level":"info","message":"cli test worked"}}"#).unwrap(); + } + + let body = tokio::time::timeout(Duration::from_secs(3), server) + .await + .expect("timeout: the logtap binary never delivered the log line") + .unwrap(); + + let _ = child.kill().await; + let _ = child.wait().await; + + fs::remove_file(&source_path).ok(); + fs::remove_file(&config_path).ok(); + + let received: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + received, + serde_json::json!([{ "level": "info", "message": "cli test worked" }]) + ); +} diff --git a/tests/filter_test.rs b/tests/filter_test.rs new file mode 100644 index 0000000..19efc7e --- /dev/null +++ b/tests/filter_test.rs @@ -0,0 +1,182 @@ +use logtap::filter::{FilterRule, RuleAction, RuleOp, run_filter}; + +#[tokio::test] +async fn filter_drops_log_when_equals_rule_matches() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let rules = vec![FilterRule { + field: "level".to_string(), + op: RuleOp::Equals, + value: "debug".to_string(), + action: RuleAction::Drop, + }]; + + let filter = tokio::spawn(run_filter(input_rx, output_tx, rules, false)); + + input_tx + .send(serde_json::json!({ + "level": "debug", + "message": "this should be dropped" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await; + + filter.await.unwrap(); + + assert!(received.is_none()); +} + +#[tokio::test] +async fn filter_keeps_log_when_no_rule_matches() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let rules = vec![FilterRule { + field: "level".to_string(), + op: RuleOp::Equals, + value: "debug".to_string(), + action: RuleAction::Drop, + }]; + + let filter = tokio::spawn(run_filter(input_rx, output_tx, rules, false)); + + input_tx + .send(serde_json::json!({ + "level": "info", + "message": "this should pass" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await.unwrap(); + + filter.await.unwrap(); + + assert_eq!( + received, + serde_json::json!({ + "level": "info", + "message": "this should pass" + }) + ); +} + +#[tokio::test] +async fn filter_masks_field_when_contains_rule_matches() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let rules = vec![FilterRule { + field: "email".to_string(), + op: RuleOp::Contains, + value: "@".to_string(), + action: RuleAction::Mask, + }]; + + let filter = tokio::spawn(run_filter(input_rx, output_tx, rules, false)); + + input_tx + .send(serde_json::json!({ + "email": "user@example.com", + "message": "user logged in" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await.unwrap(); + + filter.await.unwrap(); + + assert_eq!(received["email"], serde_json::json!("***")); + assert_eq!(received["message"], serde_json::json!("user logged in")); +} + +#[tokio::test] +async fn filter_applies_regex_rule() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let rules = vec![FilterRule { + field: "message".to_string(), + op: RuleOp::Regex, + value: r"\d{3}-\d{2}-\d{4}".to_string(), + action: RuleAction::Mask, + }]; + + let filter = tokio::spawn(run_filter(input_rx, output_tx, rules, false)); + + input_tx + .send(serde_json::json!({ + "message": "ssn 123-45-6789" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await.unwrap(); + + filter.await.unwrap(); + + assert_eq!(received["message"], serde_json::json!("***")); +} + +#[tokio::test] +async fn filter_masks_builtin_email_pattern_when_enabled() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let filter = tokio::spawn(run_filter(input_rx, output_tx, vec![], true)); + + input_tx + .send(serde_json::json!({ + "message": "user login: user@example.com" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await.unwrap(); + + filter.await.unwrap(); + + let message = received["message"].as_str().unwrap(); + assert!(!message.contains("user@example.com")); + assert!(message.contains("***")); +} + +#[tokio::test] +async fn filter_does_not_mask_builtin_patterns_when_disabled() { + let (input_tx, input_rx) = tokio::sync::mpsc::channel(10); + let (output_tx, mut output_rx) = tokio::sync::mpsc::channel(10); + + let filter = tokio::spawn(run_filter(input_rx, output_tx, vec![], false)); + + input_tx + .send(serde_json::json!({ + "message": "user login: user@example.com" + })) + .await + .unwrap(); + + drop(input_tx); + + let received = output_rx.recv().await.unwrap(); + + filter.await.unwrap(); + + assert_eq!( + received["message"], + serde_json::json!("user login: user@example.com") + ); +} diff --git a/tests/integration_test.rs b/tests/integration_test.rs new file mode 100644 index 0000000..4031b4e --- /dev/null +++ b/tests/integration_test.rs @@ -0,0 +1,213 @@ +use std::fs; +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +use logtap::config::Config; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +async fn read_http_request_body(stream: &mut TcpStream) -> String { + let mut buffer = Vec::new(); + let mut temp = [0_u8; 1024]; + + loop { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before headers were read"); + + buffer.extend_from_slice(&temp[..read]); + + if buffer.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + let headers_end = buffer + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + + let headers = String::from_utf8_lossy(&buffer[..headers_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|value| value.trim().parse::().unwrap()) + }) + .unwrap(); + + while buffer.len() < headers_end + content_length { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before body was read"); + + buffer.extend_from_slice(&temp[..read]); + } + + String::from_utf8(buffer[headers_end..headers_end + content_length].to_vec()).unwrap() +} + +#[tokio::test] +async fn integration_source_parser_filter_sink_sends_log_to_http_server() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + + let body = read_http_request_body(&mut stream).await; + + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + body + }); + + let source_path = PathBuf::from("integration_source_parser_filter_sink.log"); + fs::write(&source_path, "").unwrap(); + + let cfg = Config { + source_path: source_path.clone(), + sink_url: format!("http://{address}/logs"), + batch_size: 1, + flush_interval_secs: 60, + channel_capacity: 10, + max_retries: 0, + retry_backoff_initial_ms: 0, + retry_backoff_max_secs: 0, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 1024 * 1024, + dead_letter_max_files: 5, + }; + + let app = tokio::spawn(async move { + logtap::run(cfg).await.unwrap(); + }); + + tokio::time::sleep(Duration::from_millis(200)).await; + + { + let mut file = fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap(); + + writeln!( + file, + r#"{{"level":"info","message":"integration test worked"}}"# + ) + .unwrap(); + } + + let body = tokio::time::timeout(Duration::from_secs(3), server) + .await + .expect("timeout: sink did not send request to test HTTP server") + .unwrap(); + + app.abort(); + + fs::remove_file(&source_path).ok(); + + let received_logs: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + received_logs, + serde_json::json!([ + { + "level": "info", + "message": "integration test worked" + } + ]) + ); +} + +#[tokio::test] +async fn integration_loads_real_config_file_and_delivers_filtered_log() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + + let body = read_http_request_body(&mut stream).await; + + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + body + }); + + let source_path = PathBuf::from("integration_config_file.log"); + fs::write(&source_path, "").unwrap(); + + let config_path = PathBuf::from("integration_config_file.toml"); + let config_toml = format!( + r#" +source_path = "{source}" +sink_url = "http://{address}/logs" +batch_size = 1 +flush_interval_secs = 60 +channel_capacity = 10 +max_retries = 0 +retry_backoff_initial_ms = 0 +retry_backoff_max_secs = 0 +mask_common_patterns = false + +[[filter_rules]] +field = "level" +op = "equals" +value = "debug" +action = "drop" +"#, + source = source_path.display(), + address = address + ); + fs::write(&config_path, config_toml).unwrap(); + + let cfg = + logtap::Config::load(config_path.to_str().unwrap()).expect("failed to load logtap.toml"); + + let app = tokio::spawn(async move { + logtap::run(cfg).await.unwrap(); + }); + + tokio::time::sleep(Duration::from_millis(200)).await; + + { + let mut file = fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap(); + + writeln!(file, r#"{{"level":"debug","message":"should be dropped"}}"#).unwrap(); + writeln!(file, r#"{{"level":"info","message":"config file worked"}}"#).unwrap(); + } + + let body = tokio::time::timeout(Duration::from_secs(3), server) + .await + .expect("timeout: sink did not send request to test HTTP server") + .unwrap(); + + app.abort(); + + fs::remove_file(&source_path).ok(); + fs::remove_file(&config_path).ok(); + + let received_logs: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + received_logs, + serde_json::json!([ + { + "level": "info", + "message": "config file worked" + } + ]) + ); +} diff --git a/tests/notify_test.rs b/tests/notify_test.rs new file mode 100644 index 0000000..0b4d66b --- /dev/null +++ b/tests/notify_test.rs @@ -0,0 +1,60 @@ +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use notify::{EventKind, RecursiveMode, Watcher}; + +#[test] +fn notify_detects_file_modification() { + let file_name = format!( + "notify_test_{}.log", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + + let path = PathBuf::from(file_name); + + fs::write(&path, "").unwrap(); + + let (tx, rx) = std::sync::mpsc::channel(); + + let mut watcher = notify::recommended_watcher(move |result| { + tx.send(result).unwrap(); + }) + .unwrap(); + + watcher.watch(&path, RecursiveMode::NonRecursive).unwrap(); + + fs::write(&path, "changed line\n").unwrap(); + + let timeout = Duration::from_secs(3); + let deadline = Instant::now() + timeout; + let mut received_kinds = Vec::new(); + let mut saw_modify = false; + + while Instant::now() < deadline { + let remaining = deadline.saturating_duration_since(Instant::now()); + + let event = rx + .recv_timeout(remaining) + .expect("timeout: notify did not detect file modification") + .expect("notify returned error"); + + received_kinds.push(event.kind); + + if matches!(event.kind, EventKind::Modify(_)) { + saw_modify = true; + break; + } + } + + fs::remove_file(&path).ok(); + + assert!( + saw_modify, + "expected Modify, but received events were: {:?}", + received_kinds + ); +} diff --git a/tests/parser_test.rs b/tests/parser_test.rs new file mode 100644 index 0000000..d6cf559 --- /dev/null +++ b/tests/parser_test.rs @@ -0,0 +1,52 @@ +use logtap::parser::parse_line; +use serde_json::Value; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_run_parse() { + let line = r#"{"level": "info", "msg": "server started"}"#; + + let result = parse_line(line); + + assert_eq!( + result.get("level"), + Some(&Value::String("info".to_string())) + ); + assert_eq!( + result.get("msg"), + Some(&Value::String("server started".to_string())) + ); + } + + #[test] + fn parseline_with_text() { + let line = "isso aqui nao e json"; + + let result = parse_line(line); + + assert_eq!( + result.get("message"), + Some(&Value::String(line.to_string())) + ); + assert_eq!( + result.get("level"), + Some(&Value::String("UNKNOWN".to_string())) + ); + assert_eq!( + result.get("parse_issue"), + Some(&Value::String("invalid_json".to_string())) + ); + } + + #[test] + fn parse_empty_json_line_returns_empty_object() { + let line = "{}"; + + let result = parse_line(line); + + assert!(result.as_object().is_some_and(|obj| obj.is_empty())); + } +} diff --git a/tests/sink_test.rs b/tests/sink_test.rs new file mode 100644 index 0000000..8a08199 --- /dev/null +++ b/tests/sink_test.rs @@ -0,0 +1,246 @@ +use std::path::PathBuf; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::Mutex; + +use logtap::config::Config; +use logtap::record::LogLine; +use logtap::sink::run_sink; + +// The dead-letter path is a fixed constant in sink.rs, not a Config field, +// so every test that exercises it shares the same file on disk. cargo test +// runs tests in the same binary concurrently by default — this lock keeps +// the dead-letter tests from stepping on each other's file. Needs to be a +// tokio Mutex (not std) since the guard is held across await points. +static DEAD_LETTER_TEST_LOCK: Mutex<()> = Mutex::const_new(()); + +async fn read_http_request_body(stream: &mut tokio::net::TcpStream) -> String { + let mut buffer = Vec::new(); + let mut temp = [0_u8; 1024]; + + loop { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before headers were read"); + + buffer.extend_from_slice(&temp[..read]); + + if buffer.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + let headers_end = buffer + .windows(4) + .position(|window| window == b"\r\n\r\n") + .unwrap() + + 4; + + let headers = String::from_utf8_lossy(&buffer[..headers_end]); + let content_length = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length:") + .map(|value| value.trim().parse::().unwrap()) + }) + .unwrap(); + + while buffer.len() < headers_end + content_length { + let read = stream.read(&mut temp).await.unwrap(); + assert!(read > 0, "connection closed before body was read"); + + buffer.extend_from_slice(&temp[..read]); + } + + String::from_utf8(buffer[headers_end..headers_end + content_length].to_vec()).unwrap() +} + +#[tokio::test] +async fn sink_posts_logs_when_batch_size_is_reached() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + + let body = read_http_request_body(&mut stream).await; + + stream + .write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 0\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + + body + }); + + let cfg = Config { + source_path: PathBuf::from("unused.log"), + sink_url: format!("http://{address}/logs"), + batch_size: 2, + flush_interval_secs: 60, + channel_capacity: 10, + max_retries: 3, + retry_backoff_initial_ms: 100, + retry_backoff_max_secs: 5, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 0, + dead_letter_max_files: 0, + }; + + let (tx, rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + + let sink = tokio::spawn(run_sink(cfg, rx)); + + tx.send(serde_json::json!({ "message": "first log" })) + .await + .unwrap(); + + tx.send(serde_json::json!({ "message": "second log" })) + .await + .unwrap(); + + let body = tokio::time::timeout(std::time::Duration::from_secs(2), server) + .await + .unwrap() + .unwrap(); + + sink.abort(); + + let received_logs: serde_json::Value = serde_json::from_str(&body).unwrap(); + + assert_eq!( + received_logs, + serde_json::json!([ + { "message": "first log" }, + { "message": "second log" } + ]) + ); +} + +#[tokio::test] +async fn sink_writes_batch_to_dead_letter_file_after_exhausting_retries() { + let _guard = DEAD_LETTER_TEST_LOCK.lock().await; + + let dead_letter_path = PathBuf::from("logtap.failed.jsonl"); + std::fs::remove_file(&dead_letter_path).ok(); + + // Bind a port, then drop the listener immediately — nothing will ever + // answer there, so every attempt fails fast with "connection refused". + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + + let cfg = Config { + source_path: PathBuf::from("unused.log"), + sink_url: format!("http://{address}/logs"), + batch_size: 1, + flush_interval_secs: 60, + channel_capacity: 10, + max_retries: 2, + retry_backoff_initial_ms: 1, + retry_backoff_max_secs: 1, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 1024 * 1024, + dead_letter_max_files: 5, + }; + + let (tx, rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + + let sink = tokio::spawn(run_sink(cfg, rx)); + + tx.send(serde_json::json!({ "message": "never delivered" })) + .await + .unwrap(); + + // 2 attempts with ~1ms backoff exhaust almost immediately; give it some slack. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + sink.abort(); + + let contents = std::fs::read_to_string(&dead_letter_path) + .expect("expected dead-letter file to have been created"); + + std::fs::remove_file(&dead_letter_path).ok(); + + let lines: Vec = contents + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + + assert_eq!( + lines, + vec![serde_json::json!({ "message": "never delivered" })] + ); +} + +#[tokio::test] +async fn sink_rotates_dead_letter_file_once_it_exceeds_max_bytes() { + let _guard = DEAD_LETTER_TEST_LOCK.lock().await; + + let current = PathBuf::from("logtap.failed.jsonl"); + let rotated = PathBuf::from("logtap.failed.jsonl.1"); + std::fs::remove_file(¤t).ok(); + std::fs::remove_file(&rotated).ok(); + + // Bind a port, then drop the listener immediately — nothing will ever + // answer there, so every attempt fails fast with "connection refused". + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + + let cfg = Config { + source_path: PathBuf::from("unused.log"), + sink_url: format!("http://{address}/logs"), + batch_size: 1, + flush_interval_secs: 60, + channel_capacity: 10, + max_retries: 1, + retry_backoff_initial_ms: 1, + retry_backoff_max_secs: 1, + filter_rules: vec![], + mask_common_patterns: false, + // Tiny on purpose: the very first failed batch already exceeds this, + // so the *second* failure is guaranteed to trigger a rotation. + dead_letter_max_bytes: 10, + dead_letter_max_files: 1, + }; + + let (tx, rx) = tokio::sync::mpsc::channel::(cfg.channel_capacity); + + let sink = tokio::spawn(run_sink(cfg, rx)); + + tx.send(serde_json::json!({ "message": "first failure" })) + .await + .unwrap(); + tx.send(serde_json::json!({ "message": "second failure" })) + .await + .unwrap(); + + // Each batch gives up after a single failed attempt (max_retries: 1), + // and the sink processes messages one at a time, so this is plenty. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + + sink.abort(); + + let current_contents = + std::fs::read_to_string(¤t).expect("expected a fresh logtap.failed.jsonl"); + let rotated_contents = + std::fs::read_to_string(&rotated).expect("expected logtap.failed.jsonl.1 from rotation"); + + std::fs::remove_file(¤t).ok(); + std::fs::remove_file(&rotated).ok(); + + let current_log: serde_json::Value = serde_json::from_str(current_contents.trim()).unwrap(); + let rotated_log: serde_json::Value = serde_json::from_str(rotated_contents.trim()).unwrap(); + + assert_eq!( + current_log, + serde_json::json!({ "message": "second failure" }) + ); + assert_eq!( + rotated_log, + serde_json::json!({ "message": "first failure" }) + ); +} diff --git a/tests/source_test.rs b/tests/source_test.rs new file mode 100644 index 0000000..db82a07 --- /dev/null +++ b/tests/source_test.rs @@ -0,0 +1,112 @@ +use std::fs; +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +use logtap::config::Config; +use logtap::source::run_source; +use tokio::sync::mpsc; +use tokio::time::timeout; + +#[tokio::test] +async fn test_run_source_emite_linhas_novas() { + let path = PathBuf::from("teste_run_source.log"); + fs::write(&path, "").unwrap(); + + let (tx, mut rx) = mpsc::channel::(100); + + let cfg = Config { + source_path: path.clone(), + sink_url: "http://localhost:8080".to_string(), + batch_size: 100, + flush_interval_secs: 5, + channel_capacity: 100, + max_retries: 3, + retry_backoff_initial_ms: 100, + retry_backoff_max_secs: 5, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 1024 * 1024, + dead_letter_max_files: 5, + }; + + let _handle = tokio::task::spawn_blocking(move || run_source(cfg, tx)); + + tokio::time::sleep(Duration::from_millis(100)).await; + + { + let mut file = fs::OpenOptions::new().append(true).open(&path).unwrap(); + writeln!(file, "linha de teste").unwrap(); + } + + let received = timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("timeout: nenhuma linha chegou em 2s") + .expect("canal fechou antes de receber a linha"); + + assert_eq!(received, "linha de teste"); + + fs::remove_file(&path).ok(); +} + +#[tokio::test] +async fn test_run_source_detects_log_rotation() { + let path = PathBuf::from("teste_run_source_rotacao.log"); + let rotated_path = PathBuf::from("teste_run_source_rotacao.log.1"); + fs::remove_file(&rotated_path).ok(); + fs::write(&path, "").unwrap(); + + let (tx, mut rx) = mpsc::channel::(100); + + let cfg = Config { + source_path: path.clone(), + sink_url: "http://localhost:8080".to_string(), + batch_size: 100, + flush_interval_secs: 5, + channel_capacity: 100, + max_retries: 3, + retry_backoff_initial_ms: 100, + retry_backoff_max_secs: 5, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 1024 * 1024, + dead_letter_max_files: 5, + }; + + let _handle = tokio::task::spawn_blocking(move || run_source(cfg, tx)); + + tokio::time::sleep(Duration::from_millis(100)).await; + + { + let mut file = fs::OpenOptions::new().append(true).open(&path).unwrap(); + writeln!(file, "antes da rotacao").unwrap(); + } + + let before = timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("timeout: linha de antes da rotação não chegou") + .expect("canal fechou antes de receber a linha"); + assert_eq!(before, "antes da rotacao"); + + // Simulate standard logrotate behavior: rename the current file away, + // then create a brand new empty file at the same path. + fs::rename(&path, &rotated_path).unwrap(); + fs::write(&path, "").unwrap(); + + // Give the 200ms rotation-detection tick room to notice the inode swap. + tokio::time::sleep(Duration::from_millis(400)).await; + + { + let mut file = fs::OpenOptions::new().append(true).open(&path).unwrap(); + writeln!(file, "depois da rotacao").unwrap(); + } + + let after = timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("timeout: linha de depois da rotação não chegou — rotação não detectada") + .expect("canal fechou antes de receber a linha"); + assert_eq!(after, "depois da rotacao"); + + fs::remove_file(&path).ok(); + fs::remove_file(&rotated_path).ok(); +} diff --git a/tests/stress_test.rs b/tests/stress_test.rs new file mode 100644 index 0000000..a5a22c1 --- /dev/null +++ b/tests/stress_test.rs @@ -0,0 +1,103 @@ +// Stress / load-shaped tests — distinct from the correctness-per-function +// tests elsewhere in this directory. These exercise the full pipeline under +// conditions meant to trigger backpressure and sustained failure, not just +// verify a single behavior in isolation. + +use std::fs; +use std::io::Write; +use std::path::PathBuf; +use std::time::Duration; + +use logtap::config::Config; +use tokio::net::TcpListener; + +const LOG_COUNT: usize = 20; + +#[tokio::test] +async fn outage_does_not_lose_logs_when_destination_is_down() { + // Bind a port, then drop the listener immediately — nothing answers + // there, so every send fails fast with "connection refused", simulating + // the destination being down for the entire test. + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + drop(listener); + + let source_path = PathBuf::from("stress_outage.log"); + let dead_letter_path = PathBuf::from("logtap.failed.jsonl"); + fs::remove_file(&dead_letter_path).ok(); + fs::write(&source_path, "").unwrap(); + + let cfg = Config { + source_path: source_path.clone(), + sink_url: format!("http://{address}/logs"), + batch_size: 1, + flush_interval_secs: 60, + // Deliberately small — with 20 logs written at once, this forces the + // channels to fill up and back the source's reads up behind them, + // rather than everything sailing through unobstructed. + channel_capacity: 5, + max_retries: 2, + retry_backoff_initial_ms: 1, + retry_backoff_max_secs: 1, + filter_rules: vec![], + mask_common_patterns: false, + dead_letter_max_bytes: 1024 * 1024, + dead_letter_max_files: 5, + }; + + let app = tokio::spawn(async move { + logtap::run(cfg).await.unwrap(); + }); + + tokio::time::sleep(Duration::from_millis(300)).await; + + // Write all 20 lines in a single burst, well beyond channel_capacity, so + // the source can't just breeze through them one at a time. + { + let mut file = fs::OpenOptions::new() + .append(true) + .open(&source_path) + .unwrap(); + + for seq in 0..LOG_COUNT { + writeln!(file, r#"{{"seq":{seq}}}"#).unwrap(); + } + } + + // Poll the dead-letter file instead of guessing a fixed sleep — every + // batch here is doomed to fail, so this is where all 20 logs should + // eventually land. + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + let mut seen = Vec::new(); + + while tokio::time::Instant::now() < deadline { + if let Ok(contents) = fs::read_to_string(&dead_letter_path) { + seen = contents + .lines() + .map(|line| serde_json::from_str::(line).unwrap()) + .collect(); + + if seen.len() >= LOG_COUNT { + break; + } + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } + + app.abort(); + fs::remove_file(&source_path).ok(); + fs::remove_file(&dead_letter_path).ok(); + + let mut seqs: Vec = seen + .iter() + .map(|v| v["seq"].as_u64().expect("dead-letter entry missing seq")) + .collect(); + seqs.sort_unstable(); + + let expected: Vec = (0..LOG_COUNT as u64).collect(); + assert_eq!( + seqs, expected, + "dead-letter file should contain exactly the {LOG_COUNT} logs written, no more, no fewer, no duplicates" + ); +}