diff --git a/.gitignore b/.gitignore index e33b326c..fc2da8df 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ .vscode .idea config.toml -node_modules \ No newline at end of file +node_modules +dedup_report.txt +crates/*/target \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 4c4f6400..9aeb8307 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -293,15 +299,14 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.3.0" +version = "1.6.1" dependencies = [ "bichon-core", + "bichon-memdb", "console", "dialoguer", "indicatif", - "itertools", - "memdb", - "mimalloc", + "itertools 0.15.0", "native_db", "native_model", "serde", @@ -310,22 +315,35 @@ dependencies = [ "tokio", ] +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "bincode", + "crc32fast", + "criterion", + "lz4_flex", + "rand 0.10.1", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tracing", + "zstd", +] + [[package]] name = "bichon-cli" -version = "1.3.0" +version = "1.6.1" dependencies = [ "base64 0.22.1", "bichon-core", "chrono", "clap", - "codepage-strings", - "compressed-rtf", "console", "dialoguer", - "hex", "indicatif", "mail-parser", - "mail-send", "memmap2", "outlook-pst", "reqwest", @@ -338,14 +356,17 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.3.0" +version = "1.6.1" dependencies = [ "async-imap", "base64 0.22.1", + "bichon-memdb", "blake3", "bytes 1.11.1", "chrono", "clap", + "codepage-strings", + "compressed-rtf", "cron", "dashmap", "deunicode", @@ -354,17 +375,19 @@ dependencies = [ "fjall", "futures", "governor", + "hex", "hickory-resolver", "html2text", - "itertools", + "itertools 0.15.0", "itoa", "lru 0.18.0", "mail-parser", "mail-send", - "memdb", + "memmap2", "murmur3", "num_cpus", "oauth2", + "outlook-pst", "poem-openapi", "quick-xml 0.40.0", "rand 0.10.1", @@ -395,17 +418,29 @@ dependencies = [ "whichlang", ] +[[package]] +name = "bichon-memdb" +version = "0.1.0" +dependencies = [ + "rand 0.9.2", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "bichon-server" -version = "1.3.0" +version = "1.6.1" dependencies = [ "bichon-core", "bichon-smtp", "chrono", "email_address", + "futures", "governor", "http", - "mimalloc", "poem", "poem-derive", "poem-openapi", @@ -422,7 +457,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.3.0" +version = "1.6.1" dependencies = [ "base64 0.22.1", "bichon-core", @@ -615,6 +650,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.60" @@ -658,9 +699,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -670,6 +711,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -870,6 +938,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -1062,7 +1163,6 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" dependencies = [ - "powerfmt", "serde_core", ] @@ -1297,9 +1397,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fjall" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62b25b4d815ae178d7d9e4aa32ee59f072efd5431c736abede1e6ee13c8c453" +checksum = "038acd422d607e0eca09e093f299f9eccf9bd097554343d93746afff81a45113" dependencies = [ "byteorder-lite", "byteview", @@ -1604,6 +1704,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -1791,9 +1902,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] name = "http" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes 1.11.1", "itoa", @@ -2130,6 +2241,24 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2139,6 +2268,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2264,15 +2402,6 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2326,9 +2455,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lsm-tree" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e447ac67ff6aef4ec07fc19e507b219336cbba90a697c0dbeb1bf51b91536b67" +checksum = "8ef86c3c797c10eefcc73407c43ae48c19d4df686131a8334b2895a513e91df4" dependencies = [ "byteorder-lite", "bytes 1.11.1", @@ -2349,9 +2478,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -2386,9 +2515,9 @@ dependencies = [ [[package]] name = "mail-parser" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2420e9ce11c2b0583ca97ddff7ab2398c8a613154e9b72e3bafdbf767f1d7" +checksum = "f2c0e7e0704500930be5b6c629f30d23fd1dde4d1800e138e04b3fa302e64d51" dependencies = [ "encoding_rs", "hashify", @@ -2456,18 +2585,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "memdb" -version = "0.1.0" -dependencies = [ - "rand 0.9.2", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.18", - "tokio", -] - [[package]] name = "memmap2" version = "0.9.10" @@ -2477,15 +2594,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.17" @@ -2719,9 +2827,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -2878,6 +2986,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl" version = "0.10.78" @@ -3110,6 +3224,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "poem" version = "3.1.12" @@ -3179,7 +3321,7 @@ dependencies = [ "email_address", "futures-util", "indexmap", - "itertools", + "itertools 0.14.0", "mime", "num-traits", "poem", @@ -3571,9 +3713,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.12.3" +version = "1.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" dependencies = [ "aho-corasick", "memchr", @@ -3594,9 +3736,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" [[package]] name = "reqwest" @@ -3752,9 +3894,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "log", @@ -4135,18 +4277,18 @@ checksum = "97c9f5dd7ec5cc6d743f33fcb96de4eb91bb1cc51c5e0ba40cb285a9012043da" [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -4298,9 +4440,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.39.2" +version = "0.39.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14311e7e9a03114cd4b65eedd54e8fed2945e17f08586ae97ef53bc0669f9581" +checksum = "2c8bd2130a9b60bee2581bf82cfe89ee836424d1f37dcfa4ce21509611684673" dependencies = [ "libc", "memchr", @@ -4340,7 +4482,7 @@ dependencies = [ "futures-channel", "futures-util", "htmlescape", - "itertools", + "itertools 0.14.0", "levenshtein_automata", "log", "lru 0.16.4", @@ -4391,7 +4533,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4443,7 +4585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ "futures-util", - "itertools", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -4543,12 +4685,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.47" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" dependencies = [ "deranged", - "itoa", "libc", "num-conv", "num_threads", @@ -4560,15 +4701,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.27" +version = "0.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" dependencies = [ "num-conv", "time-core", @@ -4594,6 +4735,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" @@ -4659,9 +4810,9 @@ dependencies = [ [[package]] name = "tokio-socks" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +checksum = "a7e2948f60dbe26b35f2c7fb74ac2854c1fddded0fe9d7548fcc674a246f7615" dependencies = [ "either", "futures-util", @@ -5059,9 +5210,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -5281,9 +5432,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] diff --git a/Cargo.toml b/Cargo.toml index 20c64da6..158e4220 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/memdb", "crates/core", + "crates/blob", "crates/server", "crates/cli", "crates/admin", @@ -12,15 +13,15 @@ members = [ resolver = "2" [workspace.package] -version = "1.3.0" +version = "1.6.1" edition = "2021" [workspace.dependencies] -chrono = "0.4.44" +chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive", "env"] } -mimalloc = "0.1.52" -memdb = { path = "crates/memdb" } -itertools = "0.14.0" +bichon-memdb = { path = "crates/memdb" } +bichon-blob = { path = "crates/blob" } +itertools = "0.15.0" ring = { version = "0.17.14", features = ["std"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" @@ -29,7 +30,7 @@ tracing = "0.1.44" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } base64 = "0.22.1" -snafu = "0.9.0" +snafu = "0.9.1" reqwest = { version = "0.12.24", default-features = false, features = [ "json", "stream", @@ -38,13 +39,13 @@ reqwest = { version = "0.12.24", default-features = false, features = [ "blocking", "socks", ] } -tokio-socks = "0.5.2" -http = "1.4.0" -regex = "1.12.3" +tokio-socks = "0.5.3" +http = "1.4.2" +regex = "1.12.4" email_address = "0.2.9" futures = "0.3.32" utf7-imap = "0.3.2" -mail-parser = { version = '0.11.3', features = ["serde"] } +mail-parser = { version = '0.11.4', features = ["serde"] } # mail-send = "0.5.2" tokio-rustls = { version = "0.26.4", default-features = false, features = [ "ring", @@ -53,12 +54,12 @@ tokio-rustls = { version = "0.26.4", default-features = false, features = [ timeago = "0.6.0" oauth2 = { version = "5.0.0", features = ["reqwest-blocking"] } url = { version = "2.5.8", features = ["serde"] } -sysinfo = "0.39.2" +sysinfo = "0.39.4" num_cpus = "1.17.0" rand = "0.10.1" encoding_rs = "0.8.35" -webpki-roots = "1.0.7" -rustls = { version = "0.23.40", default-features = false, features = ["ring"] } +webpki-roots = "1.0.8" +rustls = { version = "0.23.41", default-features = false, features = ["ring"] } rustls-pki-types = "1.14.1" tokio-io-timeout = "1.2.1" semver = "1.0.28" @@ -66,7 +67,7 @@ governor = "0.10.4" lru = "0.18.0" mime_guess = "2.0.5" hex = "0.4.3" -time = { version = "0.3.47", features = [ +time = { version = "0.3.51", features = [ "formatting", "parsing", "local-offset", @@ -85,8 +86,8 @@ mail-send = "0.6.0" rcgen = "0.14.8" rustls-pemfile = "2.2.0" blake3 = "1.8.5" -uuid = { version = "1.23.1", features = ["v4", "serde"] } -fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] } +uuid = { version = "1.23.4", features = ["v4", "serde"] } +fjall = { version = "3.1.5", features = ["lz4", "metrics", "bytes_1"] } tracing-log = "0.2.0" tokio-util = "0.7.18" indicatif = "0.18.4" diff --git a/README.md b/README.md index 7fa54d47..68ee0747 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,7 @@ Storage Layer │ └──────────────┘ └──────────────┘ └──────────────┘ ``` -- **memdb**: Key-value metadata store. Houses accounts, users, roles, OAuth2 configs, proxy settings, and system configuration. All operations wrapped in `tokio::spawn_blocking`. +- **memdb**: Key-value metadata store. Houses accounts, users, roles, OAuth2 configs, proxy settings, and system configuration. - **Tantivy**: Full-text search indices with Zstd compression support. Two separate indices: envelope (email metadata + body text) and attachment (file metadata + extracted text). Batch-committed every 1,000 documents or 60 seconds. - **Fjall**: LZ4-compressed LSM tree key-value store. Two keyspaces — `email_keyspace` and `attachments_keyspace`. Content-hash addressed (BLAKE3) with insert-time deduplication. Values larger than 1 KB stored as separate files (KV separation). @@ -485,6 +485,79 @@ Tantivy Fjall memdb - Manual sync via `POST /api/v1/accounts/:id/start-download`; cancel with `cancel-download` - Busy-check prevents overlapping manual and automatic syncs on the same account +### Content Deduplication & Attachment Storage + +``` + ┌──────────────────────────────────────────┐ + │ Raw EML bytes │ + └────────────────┬─────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────┐ + │ BLAKE3 → email_content_hash │ + └────────────────┬─────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────┐ + │ MIME parse → Message │ + └───────┬──────────────────┬──────────────┘ + │ │ + │ ┌────────────┘ + │ │ detach attachments + │ │ + ▼ ▼ + ┌─────────────────┐ ┌──────────────────────────────┐ + │ EMAIL BODY │ │ EACH ATTACHMENT │ + │ │ │ │ + │ Replace raw │ │ BLAKE3(decoded content) │ + │ attachment │ │ → attachment_content_hash │ + │ bytes with │ │ │ + │ placeholder: │ │ Store raw undecoded bytes │ + │ │ │ in Fjall attachments_ks │ + │ <> │ │ Extract text for indexing │ + │ │ │ (PDF, DOCX, etc.) │ + └───────┬─────────┘ └──────────────┬───────────────┘ + │ │ + ▼ │ + ┌──────────────────────────────┐ │ + │ Stripped EML stored in │ │ + │ Fjall email_keyspace │ │ + │ keyed by email_content_hash │ │ + │ (skip if hash exists) │ │ + └──────────────┬───────────────┘ │ + │ │ + ▼ ▼ + ┌─────────────────────────────────────────────────┐ + │ Tantivy full-text index │ + │ envelope index · attachment index │ + └─────────────────────────────────────────────────┘ + + ═══════════════════════════════════════════════════════════════ + + Dedup layers + ┌─────────────────────────────────────────────────────────────────┐ + │ Fjall (insert-time) │ + │ contains_key(hash)? → skip : store with LZ4 compression │ + │ │ + │ Tantivy (periodic, every 12 h) │ + │ Group by (account, mailbox, content_hash) │ + │ Keep latest ingest_at → soft-delete older copies │ + │ Cascade-delete orphaned attachment index entries │ + └─────────────────────────────────────────────────────────────────┘ + + Reconstruction + ┌─────────────────────────────────────────────────────────────────┐ + │ Fetch stripped EML by content_hash from Fjall │ + │ Find <> placeholders │ + │ Replace each with raw attachment blob from Fjall │ + │ Result → byte-identical original EML │ + └─────────────────────────────────────────────────────────────────┘ +``` + +Every ingested email is hashed with BLAKE3. Attachments are detached from the MIME tree, hashed independently (decoded content), and stored as raw undecoded bytes in Fjall's `attachments_keyspace`. The email body is patched with hash-based placeholders and stored in `email_keyspace`. Both keyspaces check for existing hashes before writing — identical content is never stored twice, regardless of which account or folder it arrives in. A periodic index dedup task (every 12 hours) scans Tantivy for duplicate `(account, mailbox, content_hash)` tuples, keeps the most recently ingested copy, and cascade-deletes orphaned attachment entries so UID-based incremental sync remains accurate. The original EML reconstructs byte-for-byte by swapping placeholders back with their attachment blobs. + ## Storage & Backup ### Data Directory Layout @@ -532,21 +605,21 @@ The WebUI is available in **18 languages**: Language preference and UI theme are saved to your user profile and can be changed anytime from the WebUI settings. -## Data Migration (v0.3.7 → v1.0) +## Data Migration (v0.3.7 → v1.x) -Bichon v1.0 introduced a redesigned storage architecture: +Bichon v1.x introduced a redesigned storage architecture: -| Layer | v0.3.7 (Legacy) | v1.0 | -|-------|---------------|------| -| **Index** | Tantivy (shared) | Tantivy (separate envelope + attachment indices) | -| **Raw data** | Tantivy (inline) | Fjall (LZ4-compressed key-value store) | -| **Metadata** | Tantivy (shared) | memdb (dedicated embedded DB) | +| Layer | v0.3.7 (Legacy) | v1.x | +| :--- | :--- | :--- | +| **Index** | Tantivy (shared instance, no full attachments) | Tantivy (separate envelope + attachment indices) | +| **Raw data** | Tantivy (inline, stored in another Tantivy instance) | Fjall (LZ4-compressed LSM-tree key-value store) | +| **Metadata** | Native_DB (shared, disk-based DB powered by redb) | memdb (dedicated, in-house in-memory DB) | -If you ran Bichon prior to v1.0, migrate your data: +If you ran Bichon prior to v1.x, migrate your data: ```bash ./bichon-admin -# Select "Migrate Legacy v0.3.7 Storage to v1.0" +# Select "Migrate Legacy v0.3.7 Storage to v1.x" ``` > [!NOTE] @@ -642,6 +715,31 @@ cargo test Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or join the [Discord](https://discord.gg/Bq4M2cDmF4) to discuss ideas. +#### Guidelines + +1. **AI-assisted, not AI-authored.** Use AI to help analyze, debug, or draft code when unsure — but understand and review every change yourself before submitting. Don't submit unreviewed AI-generated content. +2. **Keep PRs scoped to one issue.** Don't bundle unrelated changes (CI config, dependency bumps, fixes to other modules) into the same PR. Split them into separate PRs. +3. **Frontend/backend changes go together.** If a change affects an API, data structure, or behavior with a frontend consumer, update the frontend in the same PR (or a clearly linked companion PR). +4. **Unit tests are required.** New or fixed logic must include tests that reproduce the original issue and verify the fix. PRs without tests won't be merged. +5. **Maintain backward compatibility.** Changes to data formats, protocols, configs, or APIs must state whether they're backward compatible. If not, include a migration plan. +6. **State the blast radius.** PR descriptions must specify which modules/APIs/data are affected and any downstream impact. + +### Commit Messages + +Format: `(): ` + +- **type**: `fix`, `feat`, `refactor`, `ci`, `test`, `docs`, `chore` +- **scope**: affected module/component (e.g. `rustmailer#286`, `dedup_cache`) +- **subject**: imperative, present tense, no period + +Rules: +- One logical change per commit — don't mix a fix with CI tweaks or unrelated module changes. +- Reference the issue number when applicable (e.g. `fix(#286): ...`). +- Body explains *why*, not just *what* — include root cause and how it was verified for non-trivial fixes. +- Rebase before submitting — squash WIP/fixup commits into a clean, logical sequence. +- No vague messages like `update`, `fix bug`, `wip`. + + ## Tech Stack | Layer | Technology | @@ -656,7 +754,6 @@ Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or joi | **Frontend** | React 18, TypeScript, Vite 6, ShadCN UI, TanStack Router/Query/Table | | **Charts** | Recharts | | **i18n** | i18next (18 languages) | -| **Allocator** | mimalloc | | **Container** | Ubuntu 24.04, Docker | ## License diff --git a/config.toml b/config.toml index 2bab9bc8..e7fe0ca2 100644 --- a/config.toml +++ b/config.toml @@ -1,2 +1,2 @@ base_url = "http://localhost:15630" -api_token = "WuqNC0g8yNle7CVnxcvjUwjN" +api_token = "eErI7WN3PtKeLwWAbIfSXCP6" diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index 0d8ba94b..5184ed55 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -17,5 +17,4 @@ serde_json.workspace = true itertools.workspace = true snafu.workspace = true -memdb.workspace = true -mimalloc = "0.1.50" \ No newline at end of file +bichon-memdb.workspace = true \ No newline at end of file diff --git a/crates/admin/src/main.rs b/crates/admin/src/main.rs index 10e77520..15655a1d 100644 --- a/crates/admin/src/main.rs +++ b/crates/admin/src/main.rs @@ -18,7 +18,6 @@ use console::style; use dialoguer::{theme::ColorfulTheme, Select}; -use mimalloc::MiMalloc; use crate::{migrate::handle_migration, reset::handle_reset_password}; @@ -26,8 +25,6 @@ pub mod meta; pub mod migrate; pub mod reset; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; fn main() { run_interactive(); diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index ac9eb12c..e23edb68 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -18,9 +18,9 @@ use bichon_core::{ token::TokenType, users::{acl::AccessControl, role::RoleType}, }; +use bichon_memdb::{Durability, MemDb}; use console::style; use itertools::Itertools; -use memdb::{Durability, MemDb}; use native_db::*; use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; @@ -246,11 +246,11 @@ impl From for AccountModel { capabilities: value.capabilities, date_since: value.date_since, date_before: value.date_before, - folder_limit: value.folder_limit, download_folders: value.sync_folders, account_type: value.account_type, download_interval_min: value.sync_interval_min, download_batch_size: value.sync_batch_size, + max_email_size_bytes: None, known_folders: value.known_folders, created_at: value.created_at, updated_at: value.updated_at, @@ -262,6 +262,9 @@ impl From for AccountModel { imap_quota_bytes: None, auto_download_new_mailboxes: None, download_schedule: None, + deleting: false, + archive_rules: None, + extraction_rules: None, } } } @@ -573,6 +576,8 @@ impl From for bichon_core::users::BichonUserV2 { acl: value.acl, theme: value.theme, language: value.language, + sso_id: None, + sso_provider: None, } } } @@ -661,6 +666,7 @@ impl From for bichon_core::cache::imap::mailbox::MailBox { unseen: value.unseen, uid_next: value.uid_next, uid_validity: value.uid_validity, + highest_uid: None, } } } diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs index 4de08965..0005e5f7 100644 --- a/crates/admin/src/migrate.rs +++ b/crates/admin/src/migrate.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use bichon_core::migrate::{ count_eml_segments, do_migrate_segment, is_tantivy_index_dir, - store::{LegacyDirs, NewDirs}, + store::{LegacyDirs, NewDirs, NewIndexWriter}, }; use console::style; use dialoguer::{theme::ColorfulTheme, Confirm, Input}; @@ -326,6 +326,18 @@ pub fn handle_migration(theme: &ColorfulTheme) { .progress_chars("#>-"), ); + let mut writer = match NewIndexWriter::open(NewDirs::new( + new_index_path.clone(), + new_data_path.clone(), + )) { + Ok(w) => w, + Err(e) => { + pb.finish_with_message(format!("{}", style("Migration failed.").red())); + eprintln!("\n{} {:?}", style("✘").red().bold(), e); + return; + } + }; + let mut grand_total_migrated: usize = 0; let mut grand_total_skipped: usize = 0; @@ -337,7 +349,7 @@ pub fn handle_migration(theme: &ColorfulTheme) { match do_migrate_segment( batch_size, legacy, - NewDirs::new(new_index_path.clone(), new_data_path.clone()), + &mut writer, seg_idx, |msg| { if let Some(data) = msg.strip_prefix("TOTAL:") { @@ -407,6 +419,13 @@ pub fn handle_migration(theme: &ColorfulTheme) { pb.set_position((seg_idx + 1) as u64); } + pb.set_message(style("Finalizing indexes...").dim().to_string()); + if let Err(e) = writer.finish_writers() { + pb.finish_with_message(format!("{}", style("Migration failed.").red())); + eprintln!("\n{} {:?}", style("✘").red().bold(), e); + return; + } + pb.finish_with_message(format!( "Migration finished. Total: {}, Skipped: {}", grand_total_migrated, grand_total_skipped diff --git a/crates/blob/Cargo.lock b/crates/blob/Cargo.lock new file mode 100644 index 00000000..b31a3bdf --- /dev/null +++ b/crates/blob/Cargo.lock @@ -0,0 +1,1105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "lz4_flex", + "rand", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", + "zstd", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/blob/Cargo.toml b/crates/blob/Cargo.toml new file mode 100644 index 00000000..4a94e9b2 --- /dev/null +++ b/crates/blob/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "bichon-blob" +version = "0.1.0" +edition = "2021" +description = "Embedded KV storage engine for email" + +[dependencies] +crc32fast = "1.4" +zstd = "0.13" +lz4_flex = "0.13.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +bincode = "1" +tracing = "0.1" +thiserror = "2" + +[dev-dependencies] +tempfile = "3" +rand = "0.10.1" +criterion = { version = "0.6", features = ["html_reports"] } + +[[bench]] +name = "benchmark" +harness = false diff --git a/crates/blob/benches/benchmark.rs b/crates/blob/benches/benchmark.rs new file mode 100644 index 00000000..278531ad --- /dev/null +++ b/crates/blob/benches/benchmark.rs @@ -0,0 +1,316 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use std::time::Duration; +use tempfile::TempDir; + +use bichon_blob::{Codec, Config, Engine}; + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let mut v = Vec::with_capacity(size); + // Fill with somewhat realistic text-like data so compression works + let pattern = b"The quick brown fox jumps over the lazy dog. "; + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +pub fn bench_write_small(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024); // 1 KB + let mut counter = 0u64; + + group.bench_function("1KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_medium(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(64 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(64 * 1024); // 64 KB + let mut counter = 0u64; + + group.bench_function("64KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_large(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + let mut counter = 0u64; + + group.bench_function("1MB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_read_cache_hit(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate: 10 keys, all in same bucket → cache hit after first read + let value = make_value(4096); + for i in 0..10u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 10); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_cache_miss(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 8; // Small cache to force misses + let engine = Engine::open(dir.path(), config).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + // Write 1000 keys spread across all 16 buckets — small LRU will thrash + for i in 0..1000u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_miss", |b| { + b.iter(|| { + let key = make_key(counter % 1000); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_large_value(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + for i in 0..5u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("1MB_cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 5); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_delete(c: &mut Criterion) { + let mut group = c.benchmark_group("delete"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + group.bench_function("delete", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + let mut counter = 0u64; + + b.iter_batched( + || { + counter += 1; + let key = make_key(counter); + engine + .write("bench", key, &value, Codec::Zstd) + .unwrap(); + key + }, + |key| { + engine.delete("bench", &key).unwrap(); + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_mixed_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("mixed"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate with 500 entries + let value = make_value(8192); + for i in 0..500u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter: u64 = 500; + group.bench_function("80w_15r_5d", |b| { + b.iter(|| { + counter += 1; + let op = counter % 100; + match op { + 0..=79 => { + // 80% writes + let key = make_key(counter); + let val = make_value(4096); + engine.write("bench", key, &val, Codec::Zstd).unwrap(); + } + 80..=94 => { + // 15% reads + std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap()); + } + _ => { + // 5% deletes + if counter % 2 == 0 { + let key = make_key(counter % 500); + let _ = engine.delete("bench", &key); + } + } + } + }) + }); + group.finish(); +} + +pub fn bench_gc(c: &mut Criterion) { + let mut group = c.benchmark_group("gc"); + group.measurement_time(Duration::from_secs(30)); + group.sample_size(10); + + group.bench_function("gc_30pct_deleted", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Fill a segment with ~1000 entries, then delete 30% + let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB + let n = 1200u64; + for i in 0..n { + engine + .write("bench", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~30% + for i in (0..n).step_by(3) { + engine.delete("bench", &make_key(i)).unwrap(); + } + + b.iter(|| { + engine.gc("bench").unwrap(); + }) + }); + group.finish(); +} + +criterion_group!( + benches, + bench_write_small, + bench_write_medium, + bench_write_large, + bench_read_cache_hit, + bench_read_cache_miss, + bench_read_large_value, + bench_delete, + bench_mixed_workload, + bench_gc, +); +criterion_main!(benches); diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs new file mode 100644 index 00000000..b95794d2 --- /dev/null +++ b/crates/blob/src/account.rs @@ -0,0 +1,284 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, RwLock}; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::{Error, Result}; +use crate::file_pool::FilePool; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader, SegmentWriter}; +use crate::types::Codec; + +// ── AccountHandle ────────────────────────────────────────────────────────── + +pub struct AccountHandle { + id: String, + dir: PathBuf, + inner: RwLock, + pub(crate) write_mutex: Mutex<()>, + file_pool: FilePool, +} + +impl AccountHandle { + pub fn id(&self) -> &str { + &self.id + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + /// Open an existing account. + pub fn open(store_root: &Path, account_id: &str) -> Result> { + let dir = store_root.join("accounts").join(account_id); + if !dir.exists() { + return Err(Error::AccountNotFound(account_id.to_string())); + } + let inner = AccountInner::open(&dir)?; + Ok(Arc::new(Self { + id: account_id.to_string(), + dir, + inner: RwLock::new(inner), + write_mutex: Mutex::new(()), + file_pool: FilePool::new(8), + })) + } + + /// Create a new account. + pub fn create(store_root: &Path, account_id: &str) -> Result> { + let dir = store_root.join("accounts").join(account_id); + if dir.exists() { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + let inner = AccountInner::create(&dir, account_id)?; + Ok(Arc::new(Self { + id: account_id.to_string(), + dir, + inner: RwLock::new(inner), + write_mutex: Mutex::new(()), + file_pool: FilePool::new(8), + })) + } + + /// Lock the inner state for reading. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, AccountInner> { + self.inner.read().unwrap() + } + + /// Lock the inner state for writing. + pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> { + self.inner.write().unwrap() + } + + /// Get a cached file handle for a segment. + pub fn get_segment_file(&self, seg_id: u32, path: &Path) -> Result>> { + self.file_pool.get(seg_id, path) + } + + /// Invalidate cached file handles for a segment (after GC). + pub fn invalidate_file_cache(&self, seg_id: u32) { + self.file_pool.invalidate(seg_id); + } +} + +// ── AccountInner ─────────────────────────────────────────────────────────── + +pub struct AccountInner { + dir: PathBuf, + meta: AccountMeta, + active_writer: SegmentWriter, + readers: HashMap, +} + +impl AccountInner { + fn open(dir: &Path) -> Result { + let meta = AccountMeta::load(dir)?; + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(meta.active_segment_id)); + let active_writer = if seg_path.exists() { + SegmentWriter::open_append(seg_path, meta.active_segment_id)? + } else { + fs::create_dir_all(dir.join("segments"))?; + SegmentWriter::create(seg_path, meta.active_segment_id)? + }; + + let mut readers = HashMap::new(); + for (&seg_id, stats) in &meta.segments { + if stats.sealed { + let seg_path = dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if seg_path.exists() { + readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?); + } + } + } + + Ok(Self { + dir: dir.to_path_buf(), + meta, + active_writer, + readers, + }) + } + + fn create(dir: &Path, account_id: &str) -> Result { + fs::create_dir_all(dir.join("segments"))?; + BucketFile::ensure_dir(dir)?; + + let meta = AccountMeta::new(account_id.to_string(), 1); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let active_writer = SegmentWriter::create(seg_path, 1)?; + + meta.save(dir)?; + + Ok(Self { + dir: dir.to_path_buf(), + meta, + active_writer, + readers: HashMap::new(), + }) + } + + pub fn meta(&self) -> &AccountMeta { + &self.meta + } + + /// Mark the segment as indexed up to the given offset and persist meta. + pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> { + if let Some(stats) = self.meta.segments.get_mut(&segment_id) { + if indexed_up_to_offset > stats.indexed_up_to_offset { + stats.indexed_up_to_offset = indexed_up_to_offset; + } + } + self.meta.save(&self.dir) + } + + /// Append an entry without fsync. + pub fn append_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + if self.active_writer.is_full() { + self.seal_active()?; + } + + use crate::segment::Entry; + let entry = if flags == 1 { + Entry::tombstone(key) + } else { + Entry::new(key, data, flags, codec) + }; + + let data_size = entry.data.len() as u32; + let segment_id = self.active_writer.id(); + let offset = self.active_writer.append(&entry)?; + + let stats = self + .meta + .segments + .entry(segment_id) + .or_insert_with(|| SegmentStats::new(segment_id)); + stats.total_bytes += data_size as u64; + if flags == 1 { + stats.deleted_bytes += entry.raw_size as u64; + } + stats.recompute_ratio(); + + Ok((segment_id, offset, data_size)) + } + + /// Fsync the active segment and persist meta. + pub fn flush_active(&mut self) -> Result<()> { + self.active_writer.fsync()?; + self.meta.save(&self.dir) + } + + /// Write an entry with fsync. + pub fn write_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + let result = self.append_entry(key, data, flags, codec)?; + self.flush_active()?; + Ok(result) + } + + fn seal_active(&mut self) -> Result<()> { + let old_id = self.active_writer.id(); + let old_stats = self + .meta + .segments + .entry(old_id) + .or_insert_with(|| SegmentStats::new(old_id)); + old_stats.sealed = true; + + let seg_path = self + .dir + .join("segments") + .join(segment::segment_filename(old_id)); + self.readers + .insert(old_id, SegmentReader::open(seg_path, old_id)?); + + let new_id = old_id + 1; + self.meta.active_segment_id = new_id; + let new_path = self + .dir + .join("segments") + .join(segment::segment_filename(new_id)); + self.active_writer = SegmentWriter::create(new_path, new_id)?; + self.meta.save(&self.dir)?; + + Ok(()) + } + + /// Get the on-disk path for a segment. + pub fn segment_path(&self, segment_id: u32) -> Result { + let filename = segment::segment_filename(segment_id); + let path = self.dir.join("segments").join(&filename); + if path.exists() { + Ok(path) + } else { + Err(Error::SegmentNotFound(segment_id)) + } + } + + /// Append index record to the appropriate bucket file. + pub fn append_index(&self, record: &IndexRecord) -> Result<()> { + let bucket_id = bucket::bucket_id(&record.key); + let bf = BucketFile::open(&self.dir, bucket_id); + bf.append(record) + } + + /// Return list of sealed segment IDs. + pub fn sealed_segments(&self) -> Vec { + self.meta + .segments + .iter() + .filter(|(_, s)| s.sealed) + .map(|(id, _)| *id) + .collect() + } + + /// All segment IDs (including active). + pub fn all_segment_ids(&self) -> Vec { + let mut ids: Vec = self.meta.segments.keys().copied().collect(); + if !ids.contains(&self.meta.active_segment_id) { + ids.push(self.meta.active_segment_id); + } + ids.sort_unstable(); + ids + } +} diff --git a/crates/blob/src/bucket.rs b/crates/blob/src/bucket.rs new file mode 100644 index 00000000..208346f5 --- /dev/null +++ b/crates/blob/src/bucket.rs @@ -0,0 +1,329 @@ +use std::fs::OpenOptions; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::Result; +use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE}; + +/// On-disk format: 52 bytes per record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexRecord { + pub key: [u8; 32], + pub segment_id: u32, + pub offset: u64, + pub data_size: u32, + pub flags: u8, +} + +impl IndexRecord { + pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self { + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] { + let mut buf = [0u8; INDEX_RECORD_SIZE]; + buf[0..32].copy_from_slice(&self.key); + buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes()); + buf[36..44].copy_from_slice(&self.offset.to_le_bytes()); + buf[44..48].copy_from_slice(&self.data_size.to_le_bytes()); + buf[48] = self.flags; + // bytes 49..52 are padding (keep zero) + buf + } + + pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self { + let mut key = [0u8; 32]; + key.copy_from_slice(&buf[0..32]); + let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap()); + let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap()); + let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap()); + let flags = buf[48]; + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } +} + +/// Represents a loaded and deduplicated bucket in memory. +pub struct BucketIndex { + pub bucket_id: u16, + /// Records sorted by key, deduplicated (one record per key, latest wins). + pub records: Vec, +} + +impl BucketIndex { + /// Build from raw records: sort by key, dedup keeping the one with max offset. + pub fn from_records(mut records: Vec, bucket_id: u16) -> Self { + records.sort_by_key(|a| a.key); + // Dedup: keep last (max offset) for each key + let mut deduped = Vec::with_capacity(records.len()); + let mut i = 0; + while i < records.len() { + let mut best = i; + let mut j = i + 1; + while j < records.len() && records[j].key == records[i].key { + if records[j].offset > records[best].offset { + best = j; + } + j += 1; + } + deduped.push(records[best].clone()); + i = j; + } + Self { + bucket_id, + records: deduped, + } + } + + /// Binary search for a key. Returns the record if found. + pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> { + match self.records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => Some(&self.records[idx]), + Err(_) => None, + } + } + + /// Append a new record and maintain sorted order. + pub fn insert(&mut self, record: IndexRecord) { + match self.records.binary_search_by(|r| r.key.cmp(&record.key)) { + Ok(idx) => { + // Replace if newer (larger offset) + if record.offset > self.records[idx].offset { + self.records[idx] = record; + } + } + Err(idx) => { + self.records.insert(idx, record); + } + } + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } +} + +/// Manages a bucket index file on disk. +pub struct BucketFile { + path: PathBuf, + bucket_id: u16, +} + +impl BucketFile { + pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf { + account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id)) + } + + pub fn open(account_dir: &Path, bucket_id: u16) -> Self { + Self { + path: Self::path_for(account_dir, bucket_id), + bucket_id, + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bucket_id(&self) -> u16 { + self.bucket_id + } + + /// Ensure the buckets directory exists. + pub fn ensure_dir(account_dir: &Path) -> Result<()> { + let dir = account_dir.join("buckets"); + std::fs::create_dir_all(&dir)?; + Ok(()) + } + + /// Append a single record to the bucket file. + pub fn append(&self, record: &IndexRecord) -> Result<()> { + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + file.write_all(&record.encode())?; + Ok(()) + } + + /// Append multiple records at once. + pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> { + if records.is_empty() { + return Ok(()); + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + for r in records { + file.write_all(&r.encode())?; + } + Ok(()) + } + + /// Load all records from the bucket file. + /// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write), + /// the trailing bytes are silently ignored. + pub fn load_all(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let data = std::fs::read(&self.path)?; + let remainder = data.len() % INDEX_RECORD_SIZE; + let count = data.len() / INDEX_RECORD_SIZE; + let mut records = Vec::with_capacity(count); + for i in 0..count { + let start = i * INDEX_RECORD_SIZE; + let end = start + INDEX_RECORD_SIZE; + let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end] + .try_into() + .map_err(|_| crate::error::Error::BucketIndexCorrupt { + path: self.path.clone(), + reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE), + })?; + records.push(IndexRecord::decode(buf)); + } + if remainder > 0 { + tracing::warn!( + "Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring", + self.path, remainder, INDEX_RECORD_SIZE + ); + } + Ok(records) + } + + /// Load all records, sort, and deduplicate into a BucketIndex. + pub fn load_index(&self) -> Result { + let records = self.load_all()?; + Ok(BucketIndex::from_records(records, self.bucket_id)) + } + + /// Rewrite the bucket file with a sorted, deduplicated set of records. + /// Uses atomic temp+rename to be safe on NFS. + pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> { + let mut buf = Vec::with_capacity(records.len() * INDEX_RECORD_SIZE); + for r in records { + buf.extend_from_slice(&r.encode()); + } + crate::fs::create_atomic(&self.path, &buf) + } + + /// Delete the bucket file. + pub fn delete(&self) -> Result<()> { + if self.path.exists() { + std::fs::remove_file(&self.path)?; + } + Ok(()) + } +} + +/// Compute bucket_id from a key's first 2 bytes. +pub fn bucket_id(key: &[u8; 32]) -> u16 { + u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_index_record_encode_decode() { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&[1, 2, 3, 4]); + let rec = IndexRecord::new(key, 5, 12345, 500, 0); + let encoded = rec.encode(); + let decoded = IndexRecord::decode(&encoded); + assert_eq!(rec, decoded); + } + + #[test] + fn test_bucket_id_deterministic() { + let mut key = [0u8; 32]; + key[0] = 0x00; + key[1] = 0x0F; + assert_eq!(bucket_id(&key), 15); + key[0] = 0x00; + key[1] = 0x10; + assert_eq!(bucket_id(&key), 0); + } + + #[test] + fn test_bucket_append_and_load() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0); + + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [2u8; 32]); + } + + #[test] + fn test_bucket_index_dedup() { + let recs = vec![ + IndexRecord::new([1u8; 32], 1, 100, 50, 0), + IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins + IndexRecord::new([2u8; 32], 1, 300, 60, 0), + ]; + let idx = BucketIndex::from_records(recs, 0); + assert_eq!(idx.len(), 2); + let found = idx.find(&[1u8; 32]).unwrap(); + assert_eq!(found.segment_id, 2); + assert_eq!(found.offset, 200); + } + + #[test] + fn test_bucket_index_find_missing() { + let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)]; + let idx = BucketIndex::from_records(recs, 0); + assert!(idx.find(&[99u8; 32]).is_none()); + } + + #[test] + fn test_bucket_rewrite() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0); + let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + // Rewrite sorted + let sorted = vec![r2.clone(), r1.clone()]; + bucket.rewrite(&sorted).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [3u8; 32]); + } +} diff --git a/crates/blob/src/cache.rs b/crates/blob/src/cache.rs new file mode 100644 index 00000000..b25830dd --- /dev/null +++ b/crates/blob/src/cache.rs @@ -0,0 +1,219 @@ +use std::collections::HashMap; +use std::path::Path; +use std::sync::Mutex; + +use crate::bucket::{BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; + +type CacheKey = (String, u16); + +/// Thread-safe LRU bucket cache with single-lock interior. +/// Eliminates the TOCTOU race in the old two-Mutex design. +pub struct BucketCache { + inner: Mutex, +} + +struct CacheInner { + max_entries: usize, + entries: Vec, + index: HashMap, +} + +struct CacheEntry { + key: CacheKey, + index: BucketIndex, +} + +impl BucketCache { + pub fn new(max_entries: usize) -> Self { + Self { + inner: Mutex::new(CacheInner { + max_entries: max_entries.max(1), + entries: Vec::new(), + index: HashMap::new(), + }), + } + } + + /// Get or load a bucket index. Eliminates TOCTOU via double-checked locking. + pub fn get_or_load( + &self, + account: &str, + bucket_id: u16, + account_dir: &Path, + ) -> Result> { + let key: CacheKey = (account.to_string(), bucket_id); + + // Check cache + { + let inner = self.inner.lock().unwrap(); + if let Some(&pos) = inner.index.get(&key) { + return Ok(inner.entries[pos].index.records.clone()); + } + } + + // Load from disk + let bucket_file = BucketFile::open(account_dir, bucket_id); + let bucket_index = bucket_file.load_index()?; + let records = bucket_index.records.clone(); + + // Insert with double-check (another thread might have beaten us) + { + let mut inner = self.inner.lock().unwrap(); + if let Some(&pos) = inner.index.get(&key) { + return Ok(inner.entries[pos].index.records.clone()); + } + // Evict if full + if inner.entries.len() >= inner.max_entries { + if let Some(evicted) = inner.entries.pop() { + inner.index.remove(&evicted.key); + } + } + // Insert at front + inner.entries.insert(0, CacheEntry { + key: key.clone(), + index: bucket_index, + }); + // Rebuild index + inner.index.clear(); + for i in 0..inner.entries.len() { + let key = inner.entries[i].key.clone(); + inner.index.insert(key, i); + } + } + + Ok(records) + } + + /// Insert or update a single record in a cached bucket. + pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut inner = self.inner.lock().unwrap(); + + if let Some(&pos) = inner.index.get(&key) { + inner.entries[pos].index.insert(record); + // Move to front + let entry = inner.entries.remove(pos); + inner.entries.insert(0, entry); + // Rebuild index + inner.index.clear(); + for i in 0..inner.entries.len() { + let entry_key = inner.entries[i].key.clone(); + inner.index.insert(entry_key, i); + } + } + } + + /// Invalidate a cached bucket (after GC rewrites bucket files). + pub fn invalidate(&self, account: &str, bucket_id: u16) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut inner = self.inner.lock().unwrap(); + + if let Some(&pos) = inner.index.get(&key) { + inner.entries.remove(pos); + inner.index.clear(); + for i in 0..inner.entries.len() { + let entry_key = inner.entries[i].key.clone(); + inner.index.insert(entry_key, i); + } + } + } + + pub fn len(&self) -> usize { + self.inner.lock().unwrap().entries.len() + } + + pub fn is_empty(&self) -> bool { + self.inner.lock().unwrap().entries.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::IndexRecord; + use tempfile::TempDir; + + #[test] + fn test_cache_miss_loads_from_disk() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + } + + #[test] + fn test_cache_hit() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let _ = cache.get_or_load("test", 0, dir.path()).unwrap(); + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(cache.len(), 1); + } + + #[test] + fn test_cache_eviction() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let cache = BucketCache::new(2); + + for b in 0..4 { + let bf = BucketFile::open(dir.path(), b); + bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0)) + .unwrap(); + let _ = cache.get_or_load("test", b, dir.path()).unwrap(); + } + + assert!(cache.len() <= 2); + } + + #[test] + fn test_concurrent_get_or_load_no_deadlock() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + for i in 0..10u8 { + bf.append(&IndexRecord::new([i; 32], 1, i as u64 * 100, 50, 0)) + .unwrap(); + } + + let cache = Arc::new(BucketCache::new(10)); + let dir_path = dir.path().to_path_buf(); + + let mut handles = vec![]; + for _ in 0..4 { + let cache = cache.clone(); + let dir_path = dir_path.clone(); + handles.push(thread::spawn(move || { + for _ in 0..100 { + let records = cache + .get_or_load("test", 0, &dir_path) + .unwrap(); + assert_eq!(records.len(), 10); + } + })); + } + + for h in handles { + h.join().unwrap(); + } + } +} diff --git a/crates/blob/src/checksum.rs b/crates/blob/src/checksum.rs new file mode 100644 index 00000000..b773a2c2 --- /dev/null +++ b/crates/blob/src/checksum.rs @@ -0,0 +1,65 @@ +use crc32fast::Hasher; + +pub fn crc32(data: &[u8]) -> u32 { + let mut h = Hasher::new(); + h.update(data); + h.finalize() +} + +pub struct CrcWriter { + hasher: Hasher, +} + +impl Default for CrcWriter { + fn default() -> Self { + Self::new() + } +} + +impl CrcWriter { + pub fn new() -> Self { + Self { + hasher: Hasher::new(), + } + } + + pub fn update(&mut self, data: &[u8]) { + self.hasher.update(data); + } + + pub fn finalize(self) -> u32 { + self.hasher.finalize() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_crc32_deterministic() { + let a = crc32(b"hello"); + let b = crc32(b"hello"); + assert_eq!(a, b); + } + + #[test] + fn test_crc32_different() { + let a = crc32(b"hello"); + let b = crc32(b"world"); + assert!(a != b); + } + + #[test] + fn test_crc_writer_matches_crc32() { + let mut w = CrcWriter::new(); + w.update(b"hello"); + w.update(b" world"); + assert_eq!(w.finalize(), crc32(b"hello world")); + } + + #[test] + fn test_crc32_empty() { + assert_eq!(crc32(b""), 0); + } +} diff --git a/crates/blob/src/compress.rs b/crates/blob/src/compress.rs new file mode 100644 index 00000000..c6e0b7f0 --- /dev/null +++ b/crates/blob/src/compress.rs @@ -0,0 +1,97 @@ +use crate::types::Codec; + +pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec, Codec) { + if data.len() < threshold { + return (data.to_vec(), Codec::None); + } + let (compressed, actual_codec) = match codec { + Codec::Zstd => { + match zstd::encode_all(data, level) { + Ok(out) => (out, Codec::Zstd), + Err(e) => { + tracing::warn!("zstd compression failed, storing uncompressed: {}", e); + (data.to_vec(), Codec::None) + } + } + } + Codec::Lz4 => { + let out = lz4_flex::compress(data); + (out, Codec::Lz4) + } + Codec::None => (data.to_vec(), Codec::None), + }; + // If compression made it larger, store uncompressed + if compressed.len() >= data.len() { + (data.to_vec(), Codec::None) + } else { + (compressed, actual_codec) + } +} + +pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result> { + match codec { + Codec::None => Ok(data.to_vec()), + Codec::Zstd => { + zstd::decode_all(data) + .map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e))) + } + Codec::Lz4 => { + lz4_flex::decompress(data, raw_size) + .map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_small_data_not_compressed() { + let data = b"hi"; + let (out, codec) = compress(data, Codec::Zstd, 4096, 0); + assert_eq!(out, b"hi"); + assert_eq!(codec, Codec::None); + } + + #[test] + fn test_large_data_compressed_zstd() { + let data = vec![b'A'; 5000]; + let (out, codec) = compress(&data, Codec::Zstd, 4096, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } + + #[test] + fn test_roundtrip_zstd() { + let data = vec![b'B'; 10000]; + let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_lz4() { + let data = vec![b'C'; 10000]; + let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_none() { + let data = vec![b'D'; 100]; + let (compressed, codec) = compress(&data, Codec::None, 4096, 0); + assert_eq!(codec, Codec::None); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_threshold_zero_always_compresses() { + let data = vec![b'E'; 100]; + let (out, codec) = compress(&data, Codec::Zstd, 0, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } +} diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs new file mode 100644 index 00000000..0b71a6d1 --- /dev/null +++ b/crates/blob/src/engine.rs @@ -0,0 +1,398 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; + +use crate::account::AccountHandle; +use crate::bucket::{self, IndexRecord}; +use crate::cache::BucketCache; +use crate::compress; +use crate::error::{Error, Result}; +use crate::gc::{self, GcStats}; +use crate::meta::GlobalMeta; +use crate::segment::SegmentReader; +use crate::types::{Codec, Config, ENTRY_HEADER_SIZE}; + +pub struct Engine { + root: PathBuf, + config: Config, + cache: BucketCache, + accounts: RwLock>>, +} + +#[derive(Debug, Clone)] +pub struct AccountStats { + pub account_id: String, + pub total_keys: u64, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub segment_count: usize, +} + +impl Engine { + pub fn open(path: &Path, config: Config) -> Result { + config.validate()?; + fs::create_dir_all(path)?; + fs::create_dir_all(path.join("accounts"))?; + + let mut global = GlobalMeta::load(path)?; + global.save(path)?; + + let cache = BucketCache::new(config.lru_bucket_count); + + let accounts_dir = path.join("accounts"); + let mut accounts = HashMap::new(); + + if accounts_dir.exists() { + for entry in fs::read_dir(&accounts_dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + let account_name = entry.file_name().to_string_lossy().into_owned(); + + let _ = crate::recovery::cleanup_temp_files(&entry.path()); + + match crate::recovery::recover_account(&entry.path()) { + Ok(_meta) => { + match AccountHandle::open(path, &account_name) { + Ok(handle) => { + accounts.insert(account_name, handle); + } + Err(e) => { + tracing::warn!( + "Failed to open account {}: {}", + account_name, + e + ); + } + } + } + Err(e) => { + tracing::warn!( + "Failed to recover account {}: {}", + account_name, + e + ); + } + } + } + } + } + + global.accounts = accounts.keys().cloned().collect(); + global.save(path)?; + + Ok(Self { + root: path.to_path_buf(), + config, + cache, + accounts: RwLock::new(accounts), + }) + } + + // ── Account management ────────────────────────────────────────────── + + pub fn create_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + if accounts.contains_key(account_id) { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + let handle = AccountHandle::create(&self.root, account_id)?; + accounts.insert(account_id.to_string(), handle); + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn delete_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + let handle = accounts + .remove(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + let account_dir = handle.dir().to_path_buf(); + drop(handle); + fs::remove_dir_all(&account_dir)?; + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn list_accounts(&self) -> Vec { + let accounts = self.accounts.read().unwrap(); + accounts.keys().cloned().collect() + } + + // ── Read / Write / Delete ─────────────────────────────────────────── + + pub fn write( + &self, + account_id: &str, + key: [u8; 32], + value: &[u8], + codec: Codec, + ) -> Result<()> { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); + + let (data, actual_codec) = + compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + inner.write_entry(key, &data, 0, actual_codec)?; + + let record = IndexRecord::new(key, segment_id, offset, data_size, 0); + inner.append_index(&record)?; + + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + inner.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(&key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result>> { + let bucket_id = bucket::bucket_id(key); + + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let (record, seg_path): (IndexRecord, PathBuf) = { + let inner = handle.read(); + let records = self + .cache + .get_or_load(account_id, bucket_id, handle.dir())?; + match records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => { + let r = records[idx].clone(); + if r.is_tombstone() { + return Ok(None); + } + let seg_path = inner.segment_path(r.segment_id)?; + (r, seg_path) + } + Err(_) => return Ok(None), + } + }; + + if !seg_path.exists() { + return Err(Error::SegmentNotFound(record.segment_id)); + } + + let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?; + let file = handle.get_segment_file(record.segment_id, &seg_path)?; + let (entry, _) = reader.read_entry_at_file(record.offset, &file)?; + + let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?; + + Ok(Some(value)) + } + + pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> { + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); + + let (segment_id, offset, data_size) = + inner.write_entry(*key, &[], 1, Codec::None)?; + + let record = IndexRecord::new(*key, segment_id, offset, data_size, 1); + inner.append_index(&record)?; + + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + inner.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + // ── Batch write ───────────────────────────────────────────────────── + + pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec, Codec)]) -> Result<()> { + if entries.is_empty() { + return Ok(()); + } + + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); + + let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len()); + for (key, value, codec) in entries { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + let (data, actual_codec) = + compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + inner.append_entry(*key, &data, 0, actual_codec)?; + + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + let record = IndexRecord::new(*key, segment_id, offset, data_size, 0); + pending.push((record, entry_end)); + } + + inner.flush_active()?; + + for (record, entry_end) in &pending { + inner.append_index(record)?; + inner.mark_indexed(record.segment_id, *entry_end)?; + + let bucket_id = bucket::bucket_id(&record.key); + self.cache.update_record(account_id, bucket_id, record.clone()); + } + + Ok(()) + } + + // ── GC ────────────────────────────────────────────────────────────── + + pub fn gc(&self, account_id: &str) -> Result> { + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + // Hold write_mutex to prevent concurrent writes from racing + // with GC's bucket rebuild phase. + let _write_lock = handle.write_mutex.lock().unwrap(); + + let result = gc::gc_account(handle.dir(), self.config.gc_deleted_ratio)?; + + // Invalidate FilePool for GC'd segments (they were rewritten via rename) + if let Some(ref stats) = result { + handle.invalidate_file_cache(stats.segment_id); + } + + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(result) + } + + pub fn compact_buckets(&self, account_id: &str) -> Result<()> { + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + // Hold write_mutex — compact rewrites all bucket files. + let _write_lock = handle.write_mutex.lock().unwrap(); + + gc::compact_buckets(handle.dir())?; + + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(()) + } + + // ── Stats / Shutdown ──────────────────────────────────────────────── + + pub fn stats(&self, account_id: &str) -> Result { + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let inner = handle.read(); + let meta = inner.meta(); + let mut total_bytes = 0u64; + let mut deleted_bytes = 0u64; + + for seg in meta.segments.values() { + total_bytes += seg.total_bytes; + deleted_bytes += seg.deleted_bytes; + } + + let mut total_keys = 0u64; + for bid in 0..crate::types::BUCKET_COUNT { + if let Ok(records) = + self.cache + .get_or_load(account_id, bid, handle.dir()) + { + total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64; + } + } + + Ok(AccountStats { + account_id: account_id.to_string(), + total_keys, + total_bytes, + deleted_bytes, + segment_count: meta.segments.len(), + }) + } + + pub fn shutdown(&self) -> Result<()> { + let accounts = self.accounts.read().unwrap(); + for (_, handle) in accounts.iter() { + let mut inner = handle.write(); + inner.flush_active()?; + } + let global = GlobalMeta::load(&self.root)?; + global.save(&self.root)?; + tracing::info!("bichon-blob shut down cleanly"); + Ok(()) + } +} + +impl Drop for Engine { + fn drop(&mut self) { + if let Err(e) = self.shutdown() { + tracing::error!("bichon-blob shutdown error: {}", e); + } + } +} diff --git a/crates/blob/src/error.rs b/crates/blob/src/error.rs new file mode 100644 index 00000000..377c8402 --- /dev/null +++ b/crates/blob/src/error.rs @@ -0,0 +1,59 @@ +use std::{io, path::PathBuf}; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("CRC32 mismatch at {path}:{offset}")] + CrcMismatch { path: PathBuf, offset: u64 }, + + #[error("Corrupt entry at {path}:{offset}: {reason}")] + CorruptEntry { + path: PathBuf, + offset: u64, + reason: String, + }, + + #[error("Account not found: {0}")] + AccountNotFound(String), + + #[error("Account already exists: {0}")] + AccountAlreadyExists(String), + + #[error("Segment not found: {0}")] + SegmentNotFound(u32), + + #[error("Value too large: {size} bytes (max 100 MB)")] + ValueTooLarge { size: usize }, + + #[error("Compression error: {0}")] + Compression(String), + + #[error("Disk full: {0}")] + DiskFull(String), + + #[error("Invalid config: {0}")] + InvalidConfig(String), + + #[error("Bucket index corrupt at {path}: {reason}")] + BucketIndexCorrupt { path: PathBuf, reason: String }, + + #[error("Segment file truncated at {path}: expected {expected}, got {actual}")] + SegmentTruncated { + path: PathBuf, + expected: u64, + actual: u64, + }, + + #[error("Corrupt metadata file: {0}")] + CorruptMeta(String), + + #[error("Unsupported metadata version {version} in {path}")] + UnsupportedMetaVersion { path: PathBuf, version: u32 }, +} diff --git a/crates/blob/src/file_pool.rs b/crates/blob/src/file_pool.rs new file mode 100644 index 00000000..ec8ec5b7 --- /dev/null +++ b/crates/blob/src/file_pool.rs @@ -0,0 +1,54 @@ +use std::collections::VecDeque; +use std::fs::File; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use crate::error::Result; +use crate::fs as fs_util; + +/// Simple LRU pool of open file handles, keyed by segment_id. +/// Uses Arc> to allow safe concurrent reads from the same segment. +pub struct FilePool { + max_entries: usize, + entries: Mutex>)>>, +} + +impl FilePool { + pub fn new(max_entries: usize) -> Self { + Self { + max_entries: max_entries.max(1), + entries: Mutex::new(VecDeque::new()), + } + } + + /// Get an open File for the given segment. Reuses cached handle if available. + pub fn get(&self, seg_id: u32, path: &Path) -> Result>> { + let mut entries = self.entries.lock().unwrap(); + + // Check for existing entry + for (i, (id, _)) in entries.iter().enumerate() { + if *id == seg_id { + let (_, file) = entries.remove(i).unwrap(); + entries.push_front((seg_id, file.clone())); + return Ok(file); + } + } + + // Open new file + let file = Arc::new(Mutex::new(fs_util::open_read(path)?)); + + // Evict oldest if full + if entries.len() >= self.max_entries { + entries.pop_back(); + } + + entries.push_front((seg_id, file.clone())); + Ok(file) + } + + /// Remove a cached file handle (e.g. after GC rewrites a segment). + pub fn invalidate(&self, seg_id: u32) { + let mut entries = self.entries.lock().unwrap(); + entries.retain(|(id, _)| *id != seg_id); + } +} diff --git a/crates/blob/src/fs.rs b/crates/blob/src/fs.rs new file mode 100644 index 00000000..5e0ffc1b --- /dev/null +++ b/crates/blob/src/fs.rs @@ -0,0 +1,142 @@ +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; +use std::time::Duration; + +use crate::error::Result; + +/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.) +const MAX_RETRIES: u32 = 5; +const RETRY_DELAY: Duration = Duration::from_millis(20); + +/// Check if an I/O error is transient (retryable). +fn is_transient(err: &io::Error) -> bool { + use std::io::ErrorKind; + matches!( + err.kind(), + ErrorKind::TimedOut + | ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::UnexpectedEof + ) || err.raw_os_error() == Some(116) // ESTALE on Linux +} + +/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.). +pub fn open_read(path: &Path) -> Result { + let mut last_err = None; + for attempt in 0..MAX_RETRIES { + match File::open(path) { + Ok(f) => return Ok(f), + Err(e) if is_transient(&e) => { + last_err = Some(e); + if attempt > 0 { + std::thread::sleep(RETRY_DELAY * attempt); + } + continue; + } + Err(e) => return Err(e.into()), + } + } + Err(crate::error::Error::Io(last_err.unwrap())) +} + +/// Open an existing file for writing, with retry on transient errors. +pub fn open_write(path: &Path) -> Result { + let mut last_err = None; + for attempt in 0..MAX_RETRIES { + match OpenOptions::new().write(true).open(path) { + Ok(f) => return Ok(f), + Err(e) if is_transient(&e) => { + last_err = Some(e); + if attempt > 0 { + std::thread::sleep(RETRY_DELAY * attempt); + } + continue; + } + Err(e) => return Err(e.into()), + } + } + Err(crate::error::Error::Io(last_err.unwrap())) +} + +/// Create a new file atomically: write content to a temp file, fsync, then rename. +/// Avoids `create_new(true)` which is racy on NFS. +pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> { + let tmp = path.with_extension( + path.extension() + .map(|e| format!("{}.tmp", e.to_string_lossy())) + .unwrap_or_else(|| "tmp".to_string()), + ); + + { + let mut f = File::create(&tmp)?; + f.write_all(content)?; + f.sync_all()?; + } + + fs::rename(&tmp, path)?; + Ok(()) +} + +/// Truncate an existing file to the given size, with retry. +pub fn truncate(path: &Path, size: u64) -> Result<()> { + let f = open_write(path)?; + f.set_len(size)?; + f.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_open_read_existing() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.txt"); + std::fs::write(&path, b"hello").unwrap(); + + let mut f = open_read(&path).unwrap(); + let mut s = String::new(); + std::io::Read::read_to_string(&mut f, &mut s).unwrap(); + assert_eq!(s, "hello"); + } + + #[test] + fn test_open_read_missing() { + let dir = TempDir::new().unwrap(); + let result = open_read(&dir.path().join("nope.txt")); + assert!(result.is_err()); + } + + #[test] + fn test_create_atomic_success() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("data.bin"); + create_atomic(&path, b"hello world").unwrap(); + + let content = std::fs::read(&path).unwrap(); + assert_eq!(content, b"hello world"); + // Temp file should not exist + assert!(!dir.path().join("data.bin.tmp").exists()); + } + + #[test] + fn test_create_atomic_overwrites() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("data.bin"); + create_atomic(&path, b"first").unwrap(); + create_atomic(&path, b"second").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"second"); + } + + #[test] + fn test_truncate() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("trunc.bin"); + std::fs::write(&path, b"1234567890").unwrap(); + truncate(&path, 5).unwrap(); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 5); + } +} diff --git a/crates/blob/src/gc.rs b/crates/blob/src/gc.rs new file mode 100644 index 00000000..619d08f4 --- /dev/null +++ b/crates/blob/src/gc.rs @@ -0,0 +1,267 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; +#[cfg(test)] +use crate::meta::SegmentStats; +use crate::segment::{self, SegmentReader, SegmentWriter}; + +/// Result of a GC run. +#[derive(Debug)] +pub struct GcStats { + pub segment_id: u32, + pub bytes_before: u64, + pub bytes_after: u64, + pub entries_kept: usize, + pub entries_skipped: usize, +} + +/// Run GC on an account: pick the sealed segment with highest deleted_ratio, +/// rewrite it without deleted/overwritten entries, then rebuild all bucket files. +pub fn gc_account( + account_dir: &Path, + deleted_ratio_threshold: f64, +) -> Result> { + let meta = crate::meta::AccountMeta::load(account_dir)?; + + // Find the best candidate + let candidate = meta + .segments + .values() + .filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold) + .max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap()); + + let target = match candidate { + Some(s) => s.clone(), + None => return Ok(None), + }; + + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(target.segment_id)); + let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?; + + // Build a global view: for each key, which entry (segment_id + offset) is the latest? + let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new(); + + for &seg_id in meta.segments.keys() { + let rpath = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !rpath.exists() { + continue; + } + let r = SegmentReader::open(rpath, seg_id)?; + let _ = r.scan_entries(0, |entry, offset| { + match latest_key.get(&entry.key) { + Some((existing_seg, existing_off)) => { + if seg_id > *existing_seg + || (seg_id == *existing_seg && offset > *existing_off) + { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + None => { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + Ok(()) + })?; + } + + // Create temp segment with a unique name + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let temp_name = format!("temp_{:016x}.seg", timestamp); + let temp_path = account_dir.join("segments").join(&temp_name); + let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?; + + let mut bytes_after: u64 = 0; + let mut entries_kept: usize = 0; + let mut entries_skipped: usize = 0; + + reader.scan_entries(0, |entry, offset| { + // Skip tombstones + if entry.is_tombstone() { + entries_skipped += 1; + return Ok(()); + } + // Skip if this key has a newer entry in another segment + if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) { + if *latest_seg != target.segment_id || *latest_off != offset { + entries_skipped += 1; + return Ok(()); + } + } + // Keep this entry + writer.append(entry)?; + bytes_after += entry.data.len() as u64; + entries_kept += 1; + Ok(()) + })?; + + writer.fsync()?; + + // Atomic rename: replace old segment with new one + fs::rename(&temp_path, &seg_path)?; + + // Rebuild all bucket files + rebuild_buckets(account_dir, &meta)?; + + // Update meta + let mut meta = crate::meta::AccountMeta::load(account_dir)?; + if let Some(stats) = meta.segments.get_mut(&target.segment_id) { + stats.total_bytes = bytes_after; + stats.deleted_bytes = 0; + stats.recompute_ratio(); + } + meta.save(account_dir)?; + + Ok(Some(GcStats { + segment_id: target.segment_id, + bytes_before: target.total_bytes, + bytes_after, + entries_kept, + entries_skipped, + })) +} + +/// Rebuild all 16 bucket files from scratch by scanning all segments. +fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> { + let mut bucket_records: HashMap> = HashMap::new(); + for i in 0..crate::types::BUCKET_COUNT { + bucket_records.insert(i, Vec::new()); + } + + for &seg_id in meta.segments.keys() { + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !seg_path.exists() { + continue; + } + let reader = SegmentReader::open(seg_path, seg_id)?; + reader.scan_entries(0, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + bucket_records.entry(bid).or_default().push(rec); + Ok(()) + })?; + } + + for (bid, records) in &bucket_records { + let index = BucketIndex::from_records(records.clone(), *bid); + let bf = BucketFile::open(account_dir, *bid); + bf.rewrite(&index.records)?; + } + + Ok(()) +} + +/// Compact bucket files: load, dedup, rewrite. +pub fn compact_buckets(account_dir: &Path) -> Result<()> { + for bid in 0..crate::types::BUCKET_COUNT { + let bf = BucketFile::open(account_dir, bid); + if bf.path().exists() { + let index = bf.load_index()?; + bf.rewrite(&index.records)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::segment::Entry; + use crate::types::Codec; + use tempfile::TempDir; + + fn setup_account(dir: &Path) { + fs::create_dir_all(dir.join("segments")).unwrap(); + crate::bucket::BucketFile::ensure_dir(dir).unwrap(); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let mut writer = SegmentWriter::create(seg_path, 1).unwrap(); + + // Write 5 entries + for i in 0..5u8 { + let mut key = [0u8; 32]; + key[0] = i; + let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None); + writer.append(&entry).unwrap(); + } + + // Tombstone entry 2 + let mut key2 = [0u8; 32]; + key2[0] = 2; + let tomb = Entry::tombstone(key2); + writer.append(&tomb).unwrap(); + + writer.fsync().unwrap(); + + // Save meta + let mut meta = crate::meta::AccountMeta::new("test".into(), 2); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 6000, + deleted_bytes: 1000, + deleted_ratio: 1000.0 / 6000.0, + sealed: true, + indexed_up_to_offset: 0, + }, + ); + // Make segment 2 active so segment 1 is sealed + let seg2_path = dir + .join("segments") + .join(segment::segment_filename(2)); + SegmentWriter::create(seg2_path, 2).unwrap(); + meta.save(dir).unwrap(); + } + + #[test] + fn test_gc_removes_tombstones() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + + let result = gc_account(dir.path(), 0.01).unwrap(); + assert!(result.is_some()); + + // Verify segment 1 no longer has the tombstone'd entry + let seg_path = dir + .path() + .join("segments") + .join(segment::segment_filename(1)); + let reader = SegmentReader::open(seg_path, 1).unwrap(); + let mut count = 0; + reader.scan_entries(0, |entry, _offset| { + count += 1; + assert!(entry.key[0] != 2); + Ok(()) + }).unwrap(); + assert_eq!(count, 4); // 5 original - 1 tombstoned + } + + #[test] + fn test_compact_buckets() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + compact_buckets(dir.path()).unwrap(); + // Should not panic + } +} diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs new file mode 100644 index 00000000..78b99587 --- /dev/null +++ b/crates/blob/src/lib.rs @@ -0,0 +1,19 @@ +pub mod account; +pub mod bucket; +pub mod cache; +pub mod checksum; +pub mod compress; +pub mod engine; +pub mod error; +pub mod file_pool; +pub mod fs; +pub mod gc; +pub mod meta; +pub mod recovery; +pub mod segment; +pub mod types; + +pub use account::AccountHandle; +pub use engine::{AccountStats, Engine}; +pub use error::{Error, Result}; +pub use types::{Codec, Config}; diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs new file mode 100644 index 00000000..6238e68d --- /dev/null +++ b/crates/blob/src/meta.rs @@ -0,0 +1,284 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use crate::checksum; +use crate::error::Result; +use serde::{Deserialize, Serialize}; + +const META_VERSION: u32 = 1; + +// ── Helpers ──────────────────────────────────────────────────────────────── + +fn write_bin(path: &Path, value: &T) -> Result<()> { + let payload = bincode::serialize(value).map_err(|e| { + crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e)) + })?; + let crc = checksum::crc32(&payload); + let mut buf = Vec::with_capacity(8 + payload.len()); + buf.extend_from_slice(&crc.to_le_bytes()); + buf.extend_from_slice(&META_VERSION.to_le_bytes()); + buf.extend_from_slice(&payload); + + crate::fs::create_atomic(path, &buf)?; + Ok(()) +} + +fn read_bin Deserialize<'de>>(path: &Path) -> Result { + let data = std::fs::read(path)?; + if data.len() < 8 { + return Err(crate::error::Error::CorruptMeta(path.display().to_string())); + } + let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap()); + let version = u32::from_le_bytes(data[4..8].try_into().unwrap()); + if version != META_VERSION { + return Err(crate::error::Error::UnsupportedMetaVersion { + path: path.to_path_buf(), + version, + }); + } + let computed = checksum::crc32(&data[8..]); + if stored_crc != computed { + return Err(crate::error::Error::CorruptMeta(path.display().to_string())); + } + bincode::deserialize(&data[8..]).map_err(|e| { + crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e)) + }) +} + +// ── GlobalMeta ───────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GlobalMeta { + pub version: u32, + pub accounts: Vec, +} + +impl Default for GlobalMeta { + fn default() -> Self { + Self { + version: META_VERSION, + accounts: Vec::new(), + } + } +} + +impl GlobalMeta { + pub fn load(store_root: &Path) -> Result { + let bin_path = store_root.join("global_meta.bin"); + if bin_path.exists() { + return read_bin(&bin_path); + } + // Migration from JSON + let json_path = store_root.join("global_meta.json"); + if json_path.exists() { + let data = std::fs::read_to_string(&json_path)?; + let mut meta: Self = serde_json::from_str(&data)?; + meta.accounts.sort(); + write_bin(&bin_path, &meta)?; + let _ = std::fs::remove_file(&json_path); + return Ok(meta); + } + Ok(Self::default()) + } + + pub fn save(&self, store_root: &Path) -> Result<()> { + let path = store_root.join("global_meta.bin"); + let mut meta = self.clone(); + meta.accounts.sort(); + write_bin(&path, &meta) + } +} + +// ── SegmentStats ─────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SegmentStats { + pub segment_id: u32, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub deleted_ratio: f64, + pub sealed: bool, + /// Byte offset up to which entries have been indexed in bucket files. + /// Recovery starts scanning from here instead of 0. + pub indexed_up_to_offset: u64, +} + +impl SegmentStats { + pub fn new(segment_id: u32) -> Self { + Self { + segment_id, + total_bytes: 0, + deleted_bytes: 0, + deleted_ratio: 0.0, + sealed: false, + indexed_up_to_offset: 0, + } + } + + pub fn recompute_ratio(&mut self) { + if self.total_bytes > 0 { + self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64; + } else { + self.deleted_ratio = 0.0; + } + } +} + +// ── AccountMeta ──────────────────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccountMeta { + pub account_id: String, + pub active_segment_id: u32, + pub segments: BTreeMap, +} + +impl AccountMeta { + pub fn new(account_id: String, active_segment_id: u32) -> Self { + Self { + account_id, + active_segment_id, + segments: BTreeMap::new(), + } + } + + pub fn load(account_dir: &Path) -> Result { + let bin_path = account_dir.join("meta.bin"); + if bin_path.exists() { + return read_bin(&bin_path); + } + // Migration from JSON + let json_path = account_dir.join("meta.json"); + if json_path.exists() { + let data = std::fs::read_to_string(&json_path)?; + let meta: Self = serde_json::from_str(&data)?; + write_bin(&bin_path, &meta)?; + let _ = std::fs::remove_file(&json_path); + return Ok(meta); + } + Err(crate::error::Error::AccountNotFound( + account_dir.to_string_lossy().into(), + )) + } + + pub fn save(&self, account_dir: &Path) -> Result<()> { + write_bin(&account_dir.join("meta.bin"), self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_global_meta_bin_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = GlobalMeta::default(); + meta.accounts.push("alice".into()); + meta.save(dir.path()).unwrap(); + + let loaded = GlobalMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.accounts, vec!["alice"]); + assert!(!dir.path().join("global_meta.json").exists()); + assert!(dir.path().join("global_meta.bin").exists()); + } + + #[test] + fn test_global_meta_default_when_missing() { + let dir = TempDir::new().unwrap(); + let meta = GlobalMeta::load(dir.path()).unwrap(); + assert!(meta.accounts.is_empty()); + } + + #[test] + fn test_json_migration() { + let dir = TempDir::new().unwrap(); + // Write old JSON format + let json = r#"{"version":1,"accounts":["bob","alice"]}"#; + std::fs::write(dir.path().join("global_meta.json"), json).unwrap(); + + let meta = GlobalMeta::load(dir.path()).unwrap(); + // Should be sorted + assert_eq!(meta.accounts, vec!["alice", "bob"]); + // JSON should be removed + assert!(!dir.path().join("global_meta.json").exists()); + // BIN should exist + assert!(dir.path().join("global_meta.bin").exists()); + } + + #[test] + fn test_account_meta_bin_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = AccountMeta::new("alice".into(), 1); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 1000, + deleted_bytes: 300, + deleted_ratio: 0.3, + sealed: false, + indexed_up_to_offset: 0, + }, + ); + meta.save(dir.path()).unwrap(); + + let loaded = AccountMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.active_segment_id, 1); + assert_eq!(loaded.segments[&1].total_bytes, 1000); + } + + #[test] + fn test_corrupt_bin_detected() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap(); + let result = AccountMeta::load(dir.path()); + assert!(result.is_err()); + // 0xFFFFFFFF version triggers UnsupportedMetaVersion + assert!(matches!(result.unwrap_err(), crate::error::Error::UnsupportedMetaVersion { .. })); + } + + #[test] + fn test_crc_corruption_detected() { + let dir = TempDir::new().unwrap(); + // Write a well-formed header (version=1) but with wrong CRC bytes + let mut buf = Vec::new(); + buf.extend_from_slice(&0xDEADBEEFu32.to_le_bytes()); // wrong CRC + buf.extend_from_slice(&1u32.to_le_bytes()); // version = 1 (OK) + buf.extend_from_slice(b"some payload bytes"); // payload + std::fs::write(dir.path().join("meta.bin"), &buf).unwrap(); + let result = AccountMeta::load(dir.path()); + assert!(matches!(result.unwrap_err(), crate::error::Error::CorruptMeta(_))); + } + + #[test] + fn test_account_json_migration() { + let dir = TempDir::new().unwrap(); + // Write old JSON format for AccountMeta + let json = r#"{"account_id":"alice","active_segment_id":5,"segments":{}}"#; + std::fs::write(dir.path().join("meta.json"), json).unwrap(); + + let meta = AccountMeta::load(dir.path()).unwrap(); + assert_eq!(meta.account_id, "alice"); + assert_eq!(meta.active_segment_id, 5); + // JSON should be removed + assert!(!dir.path().join("meta.json").exists()); + // BIN should exist + assert!(dir.path().join("meta.bin").exists()); + } + + #[test] + fn test_bin_sorted_keys() { + let dir = TempDir::new().unwrap(); + let mut meta = AccountMeta::new("test".into(), 1); + meta.segments.insert(3, SegmentStats::new(3)); + meta.segments.insert(1, SegmentStats::new(1)); + meta.segments.insert(2, SegmentStats::new(2)); + meta.save(dir.path()).unwrap(); + + let loaded = AccountMeta::load(dir.path()).unwrap(); + let keys: Vec = loaded.segments.keys().copied().collect(); + assert_eq!(keys, vec![1, 2, 3]); + } +} diff --git a/crates/blob/src/recovery.rs b/crates/blob/src/recovery.rs new file mode 100644 index 00000000..b72f0932 --- /dev/null +++ b/crates/blob/src/recovery.rs @@ -0,0 +1,202 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::Result; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader}; + +/// Recover an account after a crash: scan segments, repair indices, update stats. +pub fn recover_account(account_dir: &Path) -> Result { + let meta_bin = account_dir.join("meta.bin"); + let meta_json = account_dir.join("meta.json"); + let meta_exists = meta_bin.exists() || meta_json.exists(); + let mut meta = if meta_exists { + AccountMeta::load(account_dir).unwrap_or_else(|_| { + AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + ) + }) + } else { + return Ok(AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + )); + }; + + // Discover all segment files on disk + let seg_dir = account_dir.join("segments"); + if !seg_dir.exists() { + fs::create_dir_all(&seg_dir)?; + } + + let mut disk_segments: Vec = Vec::new(); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".seg") && !name_str.contains("temp_") { + if let Some(id_str) = name_str.strip_suffix(".seg") { + if let Ok(id) = id_str.parse::() { + disk_segments.push(id); + } + } + } + } + } + disk_segments.sort_unstable(); + + if disk_segments.is_empty() { + meta.active_segment_id = 1; + } else { + let max_id = *disk_segments.last().unwrap(); + meta.active_segment_id = max_id; + } + + // Ensure buckets directory exists + let buckets_dir = account_dir.join("buckets"); + fs::create_dir_all(&buckets_dir)?; + + // For each segment, scan only the unindexed tail and update stats incrementally + for &seg_id in &disk_segments { + let seg_path = seg_dir.join(segment::segment_filename(seg_id)); + let file_size = fs::metadata(&seg_path)?.len(); + + // Preserve existing stats; start fresh if this is a newly discovered segment + let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id)); + let is_sealed = seg_id != meta.active_segment_id; + stats.sealed = is_sealed; + + // Scan start: from last indexed offset. Clamp defensively. + let scan_start = if stats.indexed_up_to_offset <= file_size { + stats.indexed_up_to_offset + } else { + 0 + }; + + // If fully indexed, skip scanning entirely + if scan_start >= file_size { + meta.segments.insert(seg_id, stats); + continue; + } + + let reader = SegmentReader::open(seg_path.clone(), seg_id)?; + let mut new_records: HashMap> = HashMap::new(); + + let truncation_point = reader.scan_entries(scan_start, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + new_records.entry(bid).or_default().push(rec); + + stats.total_bytes += entry.data.len() as u64; + if entry.is_tombstone() { + stats.deleted_bytes += entry.raw_size as u64; + } + + Ok(()) + })?; + + // Merge new records into bucket files (only the newly discovered ones) + for (bid, records) in &new_records { + let bf = BucketFile::open(account_dir, *bid); + bf.append_batch(records)?; + } + + // Truncate if tail corruption found + if truncation_point < file_size { + segment::truncate_segment(&seg_path, truncation_point)?; + } + + stats.indexed_up_to_offset = truncation_point; + stats.recompute_ratio(); + meta.segments.insert(seg_id, stats); + } + + meta.save(account_dir)?; + + Ok(meta) +} + +/// Clean up leftover temp files from interrupted GC. +pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> { + let seg_dir = account_dir.join("segments"); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with("temp_") { + let path = entry.path(); + tracing::warn!("Removing leftover temp file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + // Also cleanup temp bucket files + let buckets_dir = account_dir.join("buckets"); + if buckets_dir.exists() { + for entry in fs::read_dir(&buckets_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".tmp") { + let path = entry.path(); + tracing::warn!("Removing leftover temp bucket file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_recover_fresh_account() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(&account_dir).unwrap(); + + let meta = recover_account(&account_dir).unwrap(); + assert_eq!(meta.active_segment_id, 1); + assert!(meta.segments.is_empty()); + } + + #[test] + fn test_cleanup_temp_files() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(account_dir.join("segments")).unwrap(); + fs::create_dir_all(account_dir.join("buckets")).unwrap(); + fs::write( + account_dir.join("segments").join("temp_ABC123.seg"), + b"garbage", + ) + .unwrap(); + fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap(); + + cleanup_temp_files(&account_dir).unwrap(); + + assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists()); + } +} diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs new file mode 100644 index 00000000..06906bd6 --- /dev/null +++ b/crates/blob/src/segment.rs @@ -0,0 +1,539 @@ +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use crate::checksum; +use crate::error::{Error, Result}; +use crate::fs as fs_util; +use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE}; + +/// In-memory representation of a stored entry. +#[derive(Debug, Clone)] +pub struct Entry { + pub flags: u8, + pub codec: Codec, + pub key: [u8; 32], + pub raw_size: u32, + pub data: Vec, +} + +impl Entry { + /// Create a normal data entry. + pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self { + Self { + flags, + codec, + key, + raw_size: raw_data.len() as u32, + data: raw_data.to_vec(), + } + } + + /// Create a tombstone entry. + pub fn tombstone(key: [u8; 32]) -> Self { + Self { + flags: 1, + codec: Codec::None, + key, + raw_size: 0, + data: Vec::new(), + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + /// Total on-disk size: header + data + pub fn disk_size(&self) -> usize { + ENTRY_HEADER_SIZE + self.data.len() + } +} + +/// Write entries sequentially to a segment file. +pub struct SegmentWriter { + file: File, + path: PathBuf, + id: u32, + bytes_written: u64, +} + +impl SegmentWriter { + pub fn create(path: PathBuf, id: u32) -> Result { + // Use create+truncate instead of create_new to avoid NFS O_EXCL issues. + let file = File::create(&path)?; + Ok(Self { + file, + path, + id, + bytes_written: 0, + }) + } + + pub fn open_append(path: PathBuf, id: u32) -> Result { + let mut file = fs_util::open_write(&path)?; + file.seek(SeekFrom::End(0))?; + let bytes_written = file.stream_position()?; + Ok(Self { + file, + path, + id, + bytes_written, + }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bytes_written(&self) -> u64 { + self.bytes_written + } + + pub fn is_full(&self) -> bool { + self.bytes_written >= SEGMENT_MAX_SIZE + } + + /// Append an entry. Returns the offset where it was written. + pub fn append(&mut self, entry: &Entry) -> Result { + let offset = self.bytes_written; + self.write_entry(entry) + .map_err(|e| map_io_err(e, &self.path))?; + Ok(offset) + } + + fn write_entry(&mut self, entry: &Entry) -> Result<()> { + let data_size = entry.data.len() as u32; + + // Write magic + self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?; + + // CRC32 placeholder: write zeros, remember position + let crc_pos = self.file.stream_position()?; + self.file.write_all(&0u32.to_le_bytes())?; + + // Write flags, codec, key, raw_size, data_size + self.file.write_all(&[entry.flags])?; + self.file.write_all(&[entry.codec as u8])?; + self.file.write_all(&entry.key)?; + self.file.write_all(&entry.raw_size.to_le_bytes())?; + self.file.write_all(&data_size.to_le_bytes())?; + + // Write data + self.file.write_all(&entry.data)?; + + // Calculate CRC32 over everything after the crc32 field + let crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[entry.flags]); + hasher.update(&[entry.codec as u8]); + hasher.update(&entry.key); + hasher.update(&entry.raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&entry.data); + hasher.finalize() + }; + + // Seek back and write the real CRC32 + self.file.seek(SeekFrom::Start(crc_pos))?; + self.file.write_all(&crc.to_le_bytes())?; + + // Seek back to end + self.file.seek(SeekFrom::End(0))?; + + self.bytes_written += entry.disk_size() as u64; + Ok(()) + } + + pub fn fsync(&self) -> Result<()> { + self.file.sync_all().map_err(|e| { + if e.kind() == std::io::ErrorKind::StorageFull { + Error::DiskFull(format!("{}: {}", self.path.display(), e)) + } else { + Error::Io(e) + } + })?; + Ok(()) + } +} + +/// Read entries from a segment file. +pub struct SegmentReader { + path: PathBuf, + id: u32, +} + +impl SegmentReader { + pub fn open(path: PathBuf, id: u32) -> Result { + Ok(Self { path, id }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn file_size(&self) -> Result { + Ok(fs::metadata(&self.path)?.len()) + } + + /// Read a single entry at the given offset. Returns the entry and the offset of the next entry. + pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> { + let mut file = fs_util::open_read(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + + // Read magic + let mut magic_buf = [0u8; 4]; + file.read_exact(&mut magic_buf)?; + let magic = u32::from_le_bytes(magic_buf); + if magic != ENTRY_MAGIC { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("bad magic: 0x{:08X}", magic), + }); + } + + // Read CRC32 + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf)?; + let stored_crc = u32::from_le_bytes(crc_buf); + + // Read flags, codec + let mut flags_buf = [0u8; 1]; + file.read_exact(&mut flags_buf)?; + let flags = flags_buf[0]; + + let mut codec_buf = [0u8; 1]; + file.read_exact(&mut codec_buf)?; + let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("unknown codec: {}", codec_buf[0]), + })?; + + // Read key, raw_size, data_size + let mut key = [0u8; 32]; + file.read_exact(&mut key)?; + + let mut raw_size_buf = [0u8; 4]; + file.read_exact(&mut raw_size_buf)?; + let raw_size = u32::from_le_bytes(raw_size_buf); + + let mut data_size_buf = [0u8; 4]; + file.read_exact(&mut data_size_buf)?; + let data_size = u32::from_le_bytes(data_size_buf); + + // Read data + let mut data = vec![0u8; data_size as usize]; + file.read_exact(&mut data)?; + + // Verify CRC32 (over everything after the crc32 field) + let computed_crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[flags]); + hasher.update(&[codec as u8]); + hasher.update(&key); + hasher.update(&raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&data); + hasher.finalize() + }; + + if stored_crc != computed_crc { + return Err(Error::CrcMismatch { + path: self.path.clone(), + offset, + }); + } + + let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + + Ok(( + Entry { + flags, + codec, + key, + raw_size, + data, + }, + next_offset, + )) + } + + /// Read a single entry at the given offset using a pre-opened File (via Mutex). + /// This avoids the per-read File::open cost for hot segments. + pub fn read_entry_at_file(&self, offset: u64, file: &Mutex) -> Result<(Entry, u64)> { + + let mut file = file.lock().unwrap(); + + file.seek(SeekFrom::Start(offset))?; + + // Read magic + let mut magic_buf = [0u8; 4]; + file.read_exact(&mut magic_buf)?; + let magic = u32::from_le_bytes(magic_buf); + if magic != ENTRY_MAGIC { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("bad magic: 0x{:08X}", magic), + }); + } + + // Read CRC32 + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf)?; + let stored_crc = u32::from_le_bytes(crc_buf); + + // Read flags, codec + let mut flags_buf = [0u8; 1]; + file.read_exact(&mut flags_buf)?; + let flags = flags_buf[0]; + + let mut codec_buf = [0u8; 1]; + file.read_exact(&mut codec_buf)?; + let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("unknown codec: {}", codec_buf[0]), + })?; + + // Read key, raw_size, data_size + let mut key = [0u8; 32]; + file.read_exact(&mut key)?; + + let mut raw_size_buf = [0u8; 4]; + file.read_exact(&mut raw_size_buf)?; + let raw_size = u32::from_le_bytes(raw_size_buf); + + let mut data_size_buf = [0u8; 4]; + file.read_exact(&mut data_size_buf)?; + let data_size = u32::from_le_bytes(data_size_buf); + + // Read data + let mut data = vec![0u8; data_size as usize]; + file.read_exact(&mut data)?; + + // Verify CRC32 + let computed_crc = { + let mut hasher = crate::checksum::CrcWriter::new(); + hasher.update(&[flags]); + hasher.update(&[codec as u8]); + hasher.update(&key); + hasher.update(&raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&data); + hasher.finalize() + }; + + if stored_crc != computed_crc { + return Err(Error::CrcMismatch { + path: self.path.clone(), + offset, + }); + } + + let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + + Ok(( + Entry { + flags, + codec, + key, + raw_size, + data, + }, + next_offset, + )) + } + + /// Read data portion of an entry (for pread-style reads when you already know offset + data_size). + pub fn read_data(&self, offset: u64, data_size: u32) -> Result> { + let mut file = fs_util::open_read(&self.path)?; + // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes + let data_start = offset + ENTRY_HEADER_SIZE as u64; + file.seek(SeekFrom::Start(data_start))?; + let mut buf = vec![0u8; data_size as usize]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Read the full entry header + data for verification (used by recovery and GC). + pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result> { + let mut file = fs_util::open_read(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + let total = ENTRY_HEADER_SIZE + data_size as usize; + let mut buf = vec![0u8; total]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Iterate over all valid entries in the segment, calling f for each. + /// Stops when hitting a corrupt/incomplete entry at the tail. + pub fn scan_entries(&self, start_offset: u64, mut f: F) -> Result + where + F: FnMut(&Entry, u64) -> Result<()>, + { + let file_size = self.file_size()?; + let mut offset = start_offset; + + while offset + ENTRY_HEADER_SIZE as u64 <= file_size { + match self.read_entry_at(offset) { + Ok((entry, next)) => { + f(&entry, offset)?; + offset = next; + } + Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => { + // If near end of file (within one max entry), truncate + if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 { + // Likely a partial write at tail, stop here + break; + } else { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: "mid-file corruption detected".into(), + }); + } + } + Err(e) => return Err(e), + } + } + + Ok(offset) // return the truncation point + } +} + +/// Truncate a segment file to the given size. +pub fn truncate_segment(path: &Path, size: u64) -> Result<()> { + fs_util::truncate(path, size) +} + +/// Map an Error, converting Io(StorageFull) to DiskFull with path context. +fn map_io_err(e: Error, path: &Path) -> Error { + match e { + Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => { + Error::DiskFull(format!("{}: {}", path.display(), io)) + } + _ => e, + } +} + +/// Segment file name from id: "00000001.seg" +pub fn segment_filename(id: u32) -> String { + format!("{:08}.seg", id) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf { + dir.path().join(segment_filename(id)) + } + + #[test] + fn test_write_and_read_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xAAu8; 32]; + let data = b"hello world".to_vec(); + + let entry = Entry::new(key, &data, 0, Codec::None); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, next) = reader.read_entry_at(0).unwrap(); + + assert_eq!(read_entry.key, key); + assert_eq!(read_entry.data, data); + assert_eq!(read_entry.flags, 0); + assert_eq!(read_entry.raw_size, 11); + assert!(next > 0); + } + + #[test] + fn test_tombstone_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xBBu8; 32]; + + let entry = Entry::tombstone(key); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, _) = reader.read_entry_at(0).unwrap(); + + assert!(read_entry.is_tombstone()); + assert_eq!(read_entry.data.len(), 0); + } + + #[test] + fn test_multiple_entries() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + + let entries: Vec<_> = (0..10) + .map(|i| { + let mut key = [0u8; 32]; + key[0] = i; + Entry::new(key, &vec![i; 100], 0, Codec::None) + }) + .collect(); + + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + for e in &entries { + writer.append(e).unwrap(); + } + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let mut offset = 0u64; + for (i, expected) in entries.iter().enumerate() { + let (entry, next) = reader.read_entry_at(offset).unwrap(); + assert_eq!(entry.key[0], i as u8); + assert_eq!(entry.data, expected.data); + offset = next; + } + } + + #[test] + fn test_bad_magic_detected() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + // Write garbage + std::fs::write(&path, vec![0xFFu8; 100]).unwrap(); + + let reader = SegmentReader::open(path, 1).unwrap(); + let result = reader.read_entry_at(0); + assert!(result.is_err()); + } + + #[test] + fn test_is_full() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let writer = SegmentWriter::create(path, 1).unwrap(); + assert!(!writer.is_full()); + } +} diff --git a/crates/blob/src/types.rs b/crates/blob/src/types.rs new file mode 100644 index 00000000..33c885c9 --- /dev/null +++ b/crates/blob/src/types.rs @@ -0,0 +1,88 @@ +use serde::{Deserialize, Serialize}; + +/// Magic number for entry identification +pub const ENTRY_MAGIC: u32 = 0xB3DB_0001; + +/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) +pub const ENTRY_HEADER_SIZE: usize = 50; + +/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) +pub const INDEX_RECORD_SIZE: usize = 52; + +/// Maximum segment size (256 MB) +pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024; + +/// Number of hash buckets per account +pub const BUCKET_COUNT: u16 = 16; + +/// Maximum value size (100 MB) +pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024; + +/// Default compression threshold (4 KB) +pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096; + +/// Default LRU bucket cache size +pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256; + +/// Default GC deleted ratio threshold +pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Codec { + None = 0, + Zstd = 1, + Lz4 = 2, +} + +impl Codec { + pub fn from_u8(v: u8) -> Option { + match v { + 0 => Some(Codec::None), + 1 => Some(Codec::Zstd), + 2 => Some(Codec::Lz4), + _ => None, + } + } +} + +#[derive(Debug, Clone)] +pub struct Config { + pub compress_threshold: usize, + pub default_codec: Codec, + pub compression_level: i32, + pub lru_bucket_count: usize, + pub gc_deleted_ratio: f64, +} + +impl Default for Config { + fn default() -> Self { + Self { + compress_threshold: DEFAULT_COMPRESS_THRESHOLD, + default_codec: Codec::Zstd, + compression_level: 0, + lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT, + gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO, + } + } +} + +impl Config { + pub fn validate(&self) -> crate::error::Result<()> { + if self.lru_bucket_count == 0 { + return Err(crate::error::Error::InvalidConfig( + "lru_bucket_count must be > 0".into(), + )); + } + if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 { + return Err(crate::error::Error::InvalidConfig( + "gc_deleted_ratio must be in (0.0, 1.0)".into(), + )); + } + if self.compression_level < 0 { + return Err(crate::error::Error::InvalidConfig( + "compression_level must be >= 0".into(), + )); + } + Ok(()) + } +} diff --git a/crates/blob/tests/acid_test.rs b/crates/blob/tests/acid_test.rs new file mode 100644 index 00000000..e04dae14 --- /dev/null +++ b/crates/blob/tests/acid_test.rs @@ -0,0 +1,498 @@ +/// Crash-consistency and ACID property tests for bichon-blob. +/// +/// Since we can't kill the process mid-write in an inline test, we simulate crashes +/// by dropping the Engine without calling any cleanup (close/drop is the "crash"), +/// then re-opening and verifying recovery produced consistent state. +/// +/// For true power-loss simulation, each test writes data, drops the engine abruptly, +/// then reopens and verifies: no corruption, no lost committed data, no partial writes. + +use std::fs; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let pattern = b"The quick brown fox jumps over the lazy dog. "; + let mut v = Vec::with_capacity(size); + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +// --------------------------------------------------------------------------- +// 1. Durability: committed data survives crash +// --------------------------------------------------------------------------- + +#[test] +fn test_durability_single_write_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(42); + let value = make_value(8192); + + // Write + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // <-- Engine dropped = simulated crash + + // Recover + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_durability_many_writes_survive_crash() { + let dir = TempDir::new().unwrap(); + let n = 500; + let value = make_value(2048); + let mut keys = Vec::new(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..n { + let key = make_key(i as u64); + keys.push(key); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + } // crash + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (i, key) in keys.iter().enumerate() { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result, Some(value.clone()), "missing key at index {}", i); + } + } +} + +#[test] +fn test_durability_delete_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(99); + let value = make_value(4096); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // crash after write + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.delete("alice", &key).unwrap(); + } // crash after delete + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None, "delete should persist across crash"); + } +} + +// --------------------------------------------------------------------------- +// 2. Atomicity: no partial writes visible after crash +// --------------------------------------------------------------------------- + +#[test] +fn test_atomicity_no_partial_entries_after_crash() { + let dir = TempDir::new().unwrap(); + + // Write enough entries to fill part of a segment, then crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + let value = make_value(50_000); // big enough to notice + for i in 0..200u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + } // crash + + // Recovery should clean up any partial tail entries and all committed + // entries should be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(50_000); + for i in 0..200u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert_eq!( + result, + Some(value.clone()), + "committed key {} should be intact", + i + ); + } + } +} + +#[test] +fn test_atomicity_crash_during_segment_roll() { + let dir = TempDir::new().unwrap(); + let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + // Write enough to cross at least one segment boundary (256 MB) + for i in 0..140u64 { + engine + .write("alice", make_key(i), &big_value, Codec::None) + .unwrap(); + } + } // crash mid-way or after multiple segments + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // All committed writes (that returned Ok) must be readable + for i in 0..140u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!( + result.is_some(), + "key {} should exist after segment roll recovery", + i + ); + } + } +} + +// --------------------------------------------------------------------------- +// 3. Consistency: CRC detects corruption, no silent data loss +// --------------------------------------------------------------------------- + +#[test] +fn test_consistency_crc_detects_corruption() { + let dir = TempDir::new().unwrap(); + let key = make_key(77); + let value = make_value(8192); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Corrupt the segment file by flipping a byte + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + // Flip a byte in the data portion, not the header + let flip_pos = data.len() - 100; + data[flip_pos] ^= 0xFF; + fs::write(&seg_path, &data).unwrap(); + + // Reading should detect CRC mismatch + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key); + // Either error or None is acceptable — never silently wrong data + match result { + Err(_) => {} // CRC mismatch detected — good + Ok(None) => {} // index may point to truncated/removed data + Ok(Some(v)) => { + if v == value { + panic!("CRC corruption was NOT detected — silent data corruption!"); + } + // If value differs, index pointed elsewhere after recovery + } + } + } +} + +#[test] +fn test_consistency_corrupt_magic_truncated_on_recovery() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..10u64 { + engine + .write("alice", make_key(i), &make_value(4096), Codec::Zstd) + .unwrap(); + } + } + + // Append garbage to the segment file (simulating partial write from crash) + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + let orig_len = data.len(); + // Append garbage that doesn't start with the magic number + data.extend_from_slice(&[0xFF; 200]); + fs::write(&seg_path, &data).unwrap(); + + // Recovery should truncate the garbage + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Verify committed data is still intact + for i in 0..10u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!(result.is_some(), "committed key {} should survive tail truncation", i); + } + } + + // Verify file was actually truncated + let truncated_len = fs::metadata(&seg_path).unwrap().len(); + assert!(truncated_len <= orig_len as u64, "garbage should have been truncated"); +} + +// --------------------------------------------------------------------------- +// 4. Isolation: concurrent reader sees consistent snapshot +// --------------------------------------------------------------------------- + +#[test] +fn test_isolation_reader_sees_snapshot_not_partial_write() { + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Pre-populate a known key + let original_value = make_value(4096); + let key = make_key(100); + engine + .write("alice", key, &original_value, Codec::Zstd) + .unwrap(); + + let running = Arc::new(AtomicBool::new(true)); + let writer_done = Arc::new(AtomicBool::new(false)); + + // Spawn a writer that continuously overwrites the same key + let writer_engine = engine.clone(); + let writer_running = running.clone(); + let writer_done_flag = writer_done.clone(); + let writer_key = key; + + let writer = thread::spawn(move || { + for i in 0..1000u64 { + if !writer_running.load(Ordering::Relaxed) { + break; + } + let val = make_value(4096 + (i as usize % 100)); + writer_engine + .write("alice", writer_key, &val, Codec::Zstd) + .unwrap(); + thread::yield_now(); + } + writer_done_flag.store(true, Ordering::SeqCst); + }); + + // Concurrent reader: reads should never panic or hang + let reader_engine = engine.clone(); + let reader_running = running.clone(); + let reader = thread::spawn(move || { + let mut reads = 0; + while reads < 500 { + if !reader_running.load(Ordering::Relaxed) && reads > 0 { + break; + } + let result = reader_engine.read("alice", &key); + match result { + Ok(Some(_)) | Ok(None) => {} // OK + Err(e) => { + // Accept transient errors but report them + eprintln!("reader saw error: {:?}", e); + } + } + reads += 1; + thread::yield_now(); + } + }); + + reader.join().unwrap(); + running.store(false, Ordering::SeqCst); + writer.join().unwrap(); + + // Final read should see the last committed value (not partial) + let final_result = engine.read("alice", &key).unwrap(); + assert!(final_result.is_some(), "final read should find a value"); +} + +// --------------------------------------------------------------------------- +// 5. Crash during GC: old data intact, no corruption +// --------------------------------------------------------------------------- + +#[test] +fn test_crash_during_gc_leaves_data_intact() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let value = make_value(500_000); // 500 KB each + // Write enough entries and delete some to create GC candidate + for i in 0..500u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~40% + for i in (0..500u64).step_by(5) { + engine.delete("alice", &make_key(i)).unwrap(); + } + // Single GC run (may or may not trigger) + let _ = engine.gc("alice"); + } // crash after GC + + // All non-deleted entries must still be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(500_000); + for i in 0..500u64 { + let key = make_key(i); + let result = engine.read("alice", &key).unwrap(); + if i % 5 == 0 { + // Deleted keys + assert_eq!(result, None, "key {} should be deleted", i); + } else { + assert_eq!( + result, + Some(value.clone()), + "key {} should survive GC+crash", + i + ); + } + } + } +} + +// --------------------------------------------------------------------------- +// 6. Multiple crash-reopen cycles (torture test) +// --------------------------------------------------------------------------- + +#[test] +fn test_multiple_crash_reopen_cycles() { + use std::collections::HashSet; + + let dir = TempDir::new().unwrap(); + let value = make_value(4096); + let mut alive: HashSet = HashSet::new(); + + // Populate and crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..50u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, write more, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 100..150u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, delete some, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 0..10u64 { + engine.delete("alice", &make_key(i)).unwrap(); + alive.remove(&i); + } + } + + // Final reopen: survivors exist, deleted gone + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some(), + "key {} should exist", k); + } + for i in 0..10u64 { + assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None, + "key {} should be deleted", i); + } + } +} + +// --------------------------------------------------------------------------- +// 7. Account-level isolation +// --------------------------------------------------------------------------- + +#[test] +fn test_account_isolation_crash_one_account_does_not_affect_others() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + engine + .write("alice", make_key(1), &make_value(4096), Codec::Zstd) + .unwrap(); + engine + .write("bob", make_key(1), &make_value(8192), Codec::Zstd) + .unwrap(); + } + + // Delete alice's account dir partially to simulate corruption + // Then verify bob is intact + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Bob should be fine + let result = engine.read("bob", &make_key(1)).unwrap(); + assert!(result.is_some(), "bob should be unaffected"); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf { + let seg_dir = store_root.join("accounts").join(account).join("segments"); + for entry in fs::read_dir(&seg_dir).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".seg") && !name.contains("temp_") { + return entry.path(); + } + } + panic!("no segment found in {:?}", seg_dir); +} diff --git a/crates/blob/tests/integration_test.rs b/crates/blob/tests/integration_test.rs new file mode 100644 index 00000000..de634f17 --- /dev/null +++ b/crates/blob/tests/integration_test.rs @@ -0,0 +1,497 @@ +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +#[test] +fn test_create_and_list_accounts() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + let accounts = engine.list_accounts(); + assert!(accounts.contains(&"alice".to_string())); + assert!(accounts.contains(&"bob".to_string())); +} + +#[test] +fn test_write_and_read() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xAA; 32]; + let value = b"Hello, this is a test email!".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_read_missing_key() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xFF; 32]; + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xBB; 32]; + let value = b"Some email content".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + engine.delete("alice", &key).unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete_account() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.delete_account("alice").unwrap(); + + let accounts = engine.list_accounts(); + assert!(!accounts.contains(&"alice".to_string())); +} + +#[test] +fn test_small_value_not_compressed() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xCC; 32]; + let value = b"hi"; // Smaller than 4KB threshold + + engine + .write("alice", key, value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.to_vec())); +} + +#[test] +fn test_large_value() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xDD; 32]; + let value = vec![b'X'; 100_000]; // 100KB + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_multiple_keys() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 100; + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let value = format!("email number {}", i).into_bytes(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(format!("email number {}", i).into_bytes())); + } +} + +#[test] +fn test_gc() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + // Write many entries + let value = vec![b'Y'; 5000]; + let n = 100; + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine + .write("alice", key, &value, Codec::None) + .unwrap(); + } + + // Delete even-numbered keys + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine.delete("alice", &key).unwrap(); + } + + // Run GC + let _result = engine.gc("alice").unwrap(); + + // Verify remaining keys still readable + for i in (1..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.clone())); + } + + // Deleted keys should not exist + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); + } +} + +#[test] +fn test_reopen_persistence() { + let dir = TempDir::new().unwrap(); + let key = [0xEE; 32]; + let value = b"persistent data".to_vec(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Reopen + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_stats() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + engine + .write("alice", [1u8; 32], b"hello", Codec::None) + .unwrap(); + + let stats = engine.stats("alice").unwrap(); + assert!(stats.total_bytes > 0); +} + +#[test] +fn test_batch_write() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 50; + let entries: Vec<_> = (0..n) + .map(|i: u64| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + let value = format!("batch email {}", i).into_bytes(); + (key, value, Codec::Zstd) + }) + .collect(); + + engine.write_batch("alice", &entries).unwrap(); + + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } +} + +#[test] +fn test_batch_write_persistence() { + let dir = TempDir::new().unwrap(); + let entries: Vec<_> = (0..30u64) + .map(|i| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + (key, format!("persist {}", i).into_bytes(), Codec::Zstd) + }) + .collect(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.write_batch("alice", &entries).unwrap(); + } + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } + } +} + +#[test] +fn test_invalid_config_rejected() { + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 0; + assert!(Engine::open(dir.path(), config).is_err()); + + let mut config = Config::default(); + config.gc_deleted_ratio = 1.5; + assert!(Engine::open(dir.path(), config).is_err()); +} + +#[test] +fn test_concurrent_reads() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Write some data + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap(); + } + + // Spawn 4 threads, each reading a different subset + let mut handles = vec![]; + for t in 0..4 { + let engine = engine.clone(); + handles.push(thread::spawn(move || { + for i in (t * 12)..((t + 1) * 12) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "key {} should exist", i); + } + })); + } + for h in handles { + h.join().unwrap(); + } +} + +#[test] +fn test_concurrent_writes_different_accounts() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + + for name in &["alice", "bob", "carol"] { + engine.create_account(name).unwrap(); + } + + let mut handles = vec![]; + for (t, name) in ["alice", "bob", "carol"].iter().enumerate() { + let engine = engine.clone(); + let account_name = name.to_string(); + handles.push(thread::spawn(move || { + for i in 0..20 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes()); + let value = vec![(t * 100 + i) as u8; 512]; + engine.write(&account_name, key, &value, Codec::None).unwrap(); + } + })); + } + for h in handles { + h.join().unwrap(); + } + + // Verify all writes persisted + for (t, name) in ["alice", "bob", "carol"].iter().enumerate() { + for i in 0..20 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes()); + let read = engine.read(name, &key).unwrap(); + assert!(read.is_some(), "account {} key {} should exist", name, i); + } + } +} + +#[test] +fn test_crash_recovery() { + let dir = TempDir::new().unwrap(); + let dir_path = dir.path().to_path_buf(); + + // Phase 1: write data, then drop without shutdown (simulates crash) + { + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap(); + } + // Engine dropped here without calling shutdown() + } + + // Phase 2: reopen — recovery should run, data should be intact + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + let stats = engine.stats("alice").unwrap(); + assert!(stats.total_keys > 0, "recovery should preserve data"); + + // Verify reads work + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "key {} should survive crash recovery", i); + } +} + +#[test] +fn test_meta_bin_durability() { + // Verify meta.bin has valid CRC and can be read after a write cycle. + let dir = TempDir::new().unwrap(); + let dir_path = dir.path().to_path_buf(); + + { + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0x42u8; 32]; + engine.write("alice", key, b"durable", Codec::None).unwrap(); + } + // Engine dropped → shutdown() called → meta saved via write_bin (with fsync) + + // Verify meta.bin exists and has valid CRC + let meta_path = dir_path + .join("accounts") + .join("alice") + .join("meta.bin"); + assert!(meta_path.exists(), "meta.bin should exist after clean shutdown"); + + let data = std::fs::read(&meta_path).unwrap(); + assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)"); + + let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap()); + assert_ne!(stored_crc, 0, "stored CRC should be non-zero"); + + // Reopen and verify data is intact + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + let read = engine.read("alice", &[0x42u8; 32]).unwrap(); + assert_eq!(read, Some(b"durable".to_vec())); +} + +#[test] +fn test_gc_concurrent_with_writes() { + // GC should not lose entries that are written concurrently. + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Pre-fill: write enough to trigger eventual GC + let big_value = vec![b'X'; 8192]; + for i in 0..500u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &big_value, Codec::None).unwrap(); + } + + // Delete some to create GC candidates + for i in 0..250u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.delete("alice", &key).unwrap(); + } + + let running = Arc::new(AtomicBool::new(true)); + let engine_gc = engine.clone(); + let running_gc = running.clone(); + + // Thread 1: run GC in a loop + let gc_handle = thread::spawn(move || { + while running_gc.load(Ordering::Relaxed) { + let _ = engine_gc.gc("alice"); + thread::sleep(std::time::Duration::from_millis(10)); + } + }); + + // Thread 2: keep writing new entries + let engine_write = engine.clone(); + let running_write = running.clone(); + let write_handle = thread::spawn(move || { + let mut counter = 10000u32; + while running_write.load(Ordering::Relaxed) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&counter.to_le_bytes()); + engine_write + .write("alice", key, &vec![counter as u8; 256], Codec::None) + .unwrap(); + counter += 1; + } + counter + }); + + // Let them race for a bit + thread::sleep(std::time::Duration::from_millis(500)); + running.store(false, Ordering::Relaxed); + + gc_handle.join().unwrap(); + let final_counter = write_handle.join().unwrap(); + + // All written entries must be readable + let mut missing = 0; + for i in 0..500u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + if engine.read("alice", &key).unwrap().is_none() { + // Entries 0..250 were deleted, they should be gone + if i >= 250 { + missing += 1; + } + } + } + assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC"); + + // Entries written during the race should be readable + for i in 10000..final_counter { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "concurrently written key {} should exist after GC", i); + } +} diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 08de615c..ac7f72ea 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -16,12 +16,8 @@ reqwest.workspace = true toml = "0.9.8" memmap2 = "0.9.10" outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" } -compressed-rtf = "1.0.1" chrono.workspace = true -mail-send.workspace = true base64.workspace = true -codepage-strings = "1.0.2" -hex = "0.4.3" sysinfo.workspace = true indicatif.workspace = true serde_json.workspace = true \ No newline at end of file diff --git a/crates/cli/src/api/search.rs b/crates/cli/src/api/search.rs index 0729d59f..2a348a51 100644 --- a/crates/cli/src/api/search.rs +++ b/crates/cli/src/api/search.rs @@ -10,13 +10,17 @@ use crate::BichonCliConfig; pub async fn search_messages( client: &Client, config: &BichonCliConfig, + account_ids: Option>, page: u64, page_size: u64, ) -> Option> { let url = format!("{}/api/v1/search-messages", config.base_url); let payload = EmailSearchRequest { - filter: EmailSearchFilter::default(), + filter: EmailSearchFilter { + account_ids, + ..Default::default() + }, page, page_size, sort_by: Some(SortBy::DATE), diff --git a/crates/cli/src/export/mod.rs b/crates/cli/src/export/mod.rs index ef5fd7d0..bf632004 100644 --- a/crates/cli/src/export/mod.rs +++ b/crates/cli/src/export/mod.rs @@ -146,20 +146,23 @@ pub async fn handle_account_export( let mut total_pages; loop { - if let Some(batch) = search_messages(&client, config, current_page, page_size).await { + let account_ids = Some(std::collections::HashSet::from([account.id])); + if let Some(batch) = search_messages(&client, config, account_ids, current_page, page_size).await { total_pages = batch.total_pages.unwrap(); pb.set_message(format!("Page {}/{}", current_page, total_pages)); for envelope in batch.items { let success = - download_and_export_with_json_header(&client, config, envelope, &mut file) + download_and_export_with_json_header(&client, config, envelope.clone(), &mut file) .await; if !success { - pb.finish_with_message("Failed"); - eprintln!(" ✘ Failed to export an email. Aborting process..."); - return; + eprintln!( + " ✘ Failed to export email {}, skipping...", + envelope.id + ); + continue; } pb.inc(1); } diff --git a/crates/cli/src/mbox/mod.rs b/crates/cli/src/mbox/mod.rs index 4ed48c0f..68396ffa 100644 --- a/crates/cli/src/mbox/mod.rs +++ b/crates/cli/src/mbox/mod.rs @@ -21,7 +21,7 @@ use std::path::PathBuf; use crate::api::sender::send_batch_request; use crate::mbox::gmail::determine_folder; -use crate::mbox::reader::MboxFile; +use bichon_core::import::reader::MboxFile; use crate::BichonCliConfig; use bichon_core::base64_encode_url_safe; use bichon_core::envelope::meta::{parse_bichon_metadata, BichonMetadata}; @@ -37,7 +37,6 @@ const MAX_EMAIL_BYTES: usize = 100 * 1024 * 1024; const MAX_BUFFER_BYTES: usize = 200 * 1024 * 1024; pub mod gmail; -pub mod reader; pub async fn handle_mbox_single_file_import( config: &BichonCliConfig, diff --git a/crates/cli/src/pst/mod.rs b/crates/cli/src/pst/mod.rs index 45e31aec..9ca53b78 100644 --- a/crates/cli/src/pst/mod.rs +++ b/crates/cli/src/pst/mod.rs @@ -16,21 +16,12 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use chrono::{DateTime, TimeZone, Utc}; -use dialoguer::theme::ColorfulTheme; -use dialoguer::Input; -use mail_send::mail_builder::headers::text::Text; -use mail_send::mail_builder::MessageBuilder; -use outlook_pst::ltp::prop_context::PropertyValue; - use crate::api::sender::send_batch_request; -use crate::pst::encoding::decode_subject; use crate::BichonCliConfig; -use bichon_core::base64_encode_url_safe; -use dialoguer::Confirm; -use outlook_pst::messaging::attachment::AttachmentProperties; +use bichon_core::import::pst::build_eml_base64; +use dialoguer::theme::ColorfulTheme; +use dialoguer::{Confirm, Input}; use outlook_pst::messaging::folder::Folder; -use outlook_pst::messaging::message::{Message, MessageProperties}; use outlook_pst::ndb::node_id::NodeId; use reqwest::Client; use std::future::Future; @@ -38,28 +29,6 @@ use std::path::PathBuf; use std::pin::Pin; use std::rc::Rc; -mod encoding; - -#[derive(Debug, Default)] -pub struct EmailMetadata { - pub message_id: Option, - pub subject: Option, - pub from: Option, - pub to: Option>, - pub cc: Option>, - pub bcc: Option>, - pub html: Option, - pub text: Option, - pub in_reply_to: Option, -} - -#[derive(Debug, Default)] -pub struct EmailAttachment { - pub name: Option, - pub mime_type: Option, - pub data: Option>, -} - pub async fn handle_pst_import(config: &BichonCliConfig, account_id: u64, theme: &ColorfulTheme) { let path_str: String = Input::with_theme(theme) .with_prompt("Enter the path to your SINGLE .pst file") @@ -244,167 +213,6 @@ fn process_folder_recursively<'a>( }) } -fn build_eml_base64(message: Rc) -> Option { - let properties = message.properties(); - - let mut builder = MessageBuilder::new(); - if let Some(sub) = extract_subject(properties) { - builder = builder.subject(sub); - } - if let Some(mid) = extract_string_property(properties, 0x1035) { - builder = builder.message_id(mid); - } - if let Some(irt) = extract_string_property(properties, 0x1042) { - builder = builder.in_reply_to(irt); - } - - if let Some(refs) = extract_string_property(properties, 0x1039) { - builder = builder.header("References", Text::new(refs)); - } - - if let Some(cid_val) = properties.get(0x3013) { - if let PropertyValue::Binary(bin) = cid_val { - builder = builder.header( - "X-Bichon-Conversation-ID", - Text::new(hex::encode(bin.buffer())), - ); - } - } - - let from = extract_string_property(properties, 0x5D01) - .or_else(|| extract_string_property(properties, 0x5D02)) - .or_else(|| extract_string_property(properties, 0x0C1F)); - - if let Some(f) = from { - builder = builder.from(f); - } - - if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) { - let dt = filetime_to_datetime(filetime).timestamp(); - builder = builder.date(dt); - } - - let (to, cc, bcc) = extract_recipients_list(&message); - if !to.is_empty() { - builder = builder.to(to.iter().map(|s| s.as_str()).collect::>()); - } - if !cc.is_empty() { - builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::>()); - } - if !bcc.is_empty() { - builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::>()); - } - - if let Some(html) = extract_html(properties) { - builder = builder.html_body(html); - } - - if let Some(text) = extract_text(properties) { - builder = builder.text_body(text); - } - - if let Some(attachment_table) = message.attachment_table() { - for row in attachment_table.rows_matrix() { - let node_id = NodeId::from(u32::from(row.id())); - if let Ok(attachment) = message.clone().read_attachment(node_id, None) { - let att_props = attachment.properties(); - let name = extract_attachment_string_property(att_props, 0x3707); - let mime = extract_attachment_string_property(att_props, 0x370E) - .unwrap_or_else(|| "application/octet-stream".into()); - let cid = extract_attachment_string_property(att_props, 0x3712); - let is_inline = att_props - .get(0x3714) - .and_then(|val| { - if let PropertyValue::Integer32(f) = val { - Some(f) - } else { - None - } - }) - .map(|flag| (flag & 0x4) != 0) - .unwrap_or(false); - - if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) { - let data = bin.buffer().to_vec(); - let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string()); - - if is_inline && cid.is_some() { - let content_id = cid.unwrap(); - builder = builder.inline(mime, content_id, data); - } else { - builder = builder.attachment(mime, file_name, data); - } - } - } - } - } - - match builder.write_to_vec() { - Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)), - Err(e) => { - eprintln!("Failed to generate EML: {:?}", e); - None - } - } -} - -fn filetime_to_datetime(filetime: i64) -> DateTime { - let unix_secs = (filetime / 10_000_000) - 11_644_473_600; - let nsecs = (filetime % 10_000_000) * 100; - Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap() -} - -fn extract_recipients_list(message: &Rc) -> (Vec, Vec, Vec) { - let mut to = Vec::new(); - let mut cc = Vec::new(); - let mut bcc = Vec::new(); - - let recipient_table = message.recipient_table(); - if let Some(recipient_table) = recipient_table { - let context = recipient_table.context(); - for row in recipient_table.rows_matrix() { - if let Ok(cols) = row.columns(context) { - let mut r_type = 0; - let mut email = String::new(); - - for (col, val) in context.columns().iter().zip(cols) { - let prop_val = val - .as_ref() - .and_then(|v| recipient_table.read_column(v, col.prop_type()).ok()); - match col.prop_id() { - 0x0C15 => { - if let Some(PropertyValue::Integer32(t)) = prop_val { - r_type = t; - } - } - 0x39FE | 0x3003 => { - if let Some(s) = prop_val.and_then(|v| extract_string(&v)) { - email = s; - } - } - _ => {} - } - } - - if !email.is_empty() { - match r_type { - 1 => to.push(email), - 2 => cc.push(email), - 3 => bcc.push(email), - _ => {} - } - } - } - } - } else { - let receiver = extract_string_property(message.properties(), 0x0076); - if let Some(receiver) = receiver { - to.push(receiver); - } - } - (to, cc, bcc) -} - async fn send_to_bichon( client: &Client, config: &BichonCliConfig, @@ -414,69 +222,3 @@ async fn send_to_bichon( ) { send_batch_request(client, config, account_id, folder_path, emls).await; } - -fn extract_subject(props: &MessageProperties) -> Option { - props.get(0x0037).and_then(|val| decode_subject(val)) -} - -fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option { - properties - .get(prop_id) - .and_then(|value| extract_string(value)) -} - -fn extract_attachment_string_property( - properties: &AttachmentProperties, - prop_id: u16, -) -> Option { - properties - .get(prop_id) - .and_then(|value| extract_string(value)) -} - -fn extract_string(value: &PropertyValue) -> Option { - match value { - PropertyValue::String8(value) => Some(value.to_string()), - PropertyValue::Unicode(value) => Some(value.to_string()), - _ => None, - } -} - -fn extract_text(properties: &MessageProperties) -> Option { - properties.get(0x1000).and_then(extract_string).or_else(|| { - properties.get(0x1009).and_then(|value| match value { - PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()), - _ => None, - }) - }) -} - -fn extract_html(properties: &MessageProperties) -> Option { - properties.get(0x1013).and_then(|value| match value { - PropertyValue::Binary(value) => { - let code_page = properties - .get(0x3FDE) - .and_then(|v| { - if let PropertyValue::Integer32(cpid) = v { - Some(*cpid as u16) - } else { - None - } - }) - .unwrap_or(65001); - encoding::decode_html_body(value.buffer(), code_page) - } - PropertyValue::String8(value) => Some(value.to_string()), - PropertyValue::Unicode(value) => Some(value.to_string()), - _ => None, - }) -} - -fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option { - for &prop_id in prop_ids { - if let Some(PropertyValue::Time(value)) = properties.get(prop_id) { - return Some(*value); - } - } - None -} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 83bfc9ce..801c6301 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [ ], optional = true } chrono.workspace = true clap.workspace = true -memdb.workspace = true +bichon-memdb.workspace = true itertools.workspace = true ring.workspace = true serde.workspace = true @@ -72,3 +72,8 @@ scopeguard = "1.2.0" cron = "0.15" quick-xml = { version = "0.40.0", features = ["serialize"] } hickory-resolver = "0.26.0-alpha.1" +memmap2 = "0.9.10" +outlook-pst = { git = "https://github.com/rustmailer/outlook-pst-rs.git", branch = "main" } +compressed-rtf = "1.0.1" +codepage-strings = "1.0.2" +hex.workspace = true diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index e0250c78..cb3ea529 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -64,6 +64,216 @@ pub enum QuotaWindow { Monthly, } +/// Include/exclude filter rule. +/// +/// - `include` non-empty: only values matching these patterns pass. +/// - `exclude` non-empty: values matching these patterns are rejected. +/// - Both empty: all values pass. +/// - Both set: include checked first, then exclude. +/// +/// Extension patterns use case-insensitive exact match; all others use regex. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct FilterRule { + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub exclude: Vec, +} + +impl FilterRule { + pub fn is_empty(&self) -> bool { + self.include.is_empty() && self.exclude.is_empty() + } + + fn matches_exact(&self, value: &str) -> bool { + if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) { + return false; + } + if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) { + return false; + } + true + } + + fn matches_regex(&self, value: &str) -> bool { + if !self.include.is_empty() && !matches_any_regex(&self.include, value) { + return false; + } + if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) { + return false; + } + true + } + + fn validate_regex(&self, field: &str) -> Result<(), String> { + validate_patterns(&self.include, &format!("{field}.include"))?; + validate_patterns(&self.exclude, &format!("{field}.exclude"))?; + Ok(()) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ExtractionRules { + /// Type 0: Master switch. + #[serde(default)] + pub enabled: bool, + /// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`). + #[serde(default)] + pub extensions: FilterRule, + /// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`). + #[serde(default)] + pub folders: FilterRule, + /// Type 3: Attachment filename patterns (regex). + #[serde(default)] + pub attachment_names: FilterRule, + /// Type 4: Sender patterns (regex). + #[serde(default)] + pub senders: FilterRule, +} + +impl ExtractionRules { + /// Returns `true` if the attachment should be extracted under these rules. + pub fn should_extract( + &self, + ext: &str, + folder: Option<&str>, + attachment_name: Option<&str>, + sender: Option<&str>, + ) -> bool { + if !self.enabled { + return false; + } + if !self.extensions.matches_exact(ext) { + return false; + } + if !self.folders.is_empty() { + if let Some(folder) = folder { + if !self.folders.matches_regex(folder) { + return false; + } + } + } + if !self.attachment_names.is_empty() { + if let Some(name) = attachment_name { + if !self.attachment_names.matches_regex(name) { + return false; + } + } + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + true + } + + pub fn validate(&self) -> Result<(), String> { + self.folders.validate_regex("folders")?; + self.attachment_names.validate_regex("attachment_names")?; + self.senders.validate_regex("senders")?; + Ok(()) + } +} + +/// Archive filtering rules — skip unwanted emails before storage. +/// +/// Rule types: +/// 0 — Master switch +/// 1 — Sender filter (regex) +/// 2 — Subject filter (regex) +/// 3 — Skip emails larger than this (bytes) +/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam) +/// +/// `None` = archive everything (backward compatible). +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ArchiveRules { + /// Type 0: Master switch. `false` = archive everything. + #[serde(default)] + pub enabled: bool, + /// Type 1: Sender filter (regex, include/exclude). + #[serde(default)] + pub senders: FilterRule, + /// Type 2: Subject filter (regex, include/exclude). + #[serde(default)] + pub subjects: FilterRule, + /// Type 3: Skip emails larger than this (bytes). `None` = no size limit. + #[serde(default)] + pub skip_larger_than: Option, + /// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`). + /// When the value is `yes` or `true` (case-insensitive), the email is skipped. + /// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin), + /// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange). + #[serde(default)] + pub spam_headers: Vec, +} + +impl ArchiveRules { + /// Returns `true` if the email should be archived under these rules. + pub fn should_archive( + &self, + sender: Option<&str>, + subject: Option<&str>, + size: u32, + is_spam: bool, + ) -> bool { + if !self.enabled { + return true; + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + if !self.subjects.is_empty() { + if let Some(subject) = subject { + if !self.subjects.matches_regex(subject) { + return false; + } + } + } + if let Some(limit) = self.skip_larger_than { + if size as u64 > limit { + return false; + } + } + if !self.spam_headers.is_empty() && is_spam { + return false; + } + true + } + + /// Validate all regex patterns are well-formed. + pub fn validate(&self) -> Result<(), String> { + self.senders.validate_regex("senders")?; + self.subjects.validate_regex("subjects")?; + Ok(()) + } +} + +fn matches_any_regex(patterns: &[String], value: &str) -> bool { + patterns.iter().any(|p| { + regex::Regex::new(p) + .map(|re| re.is_match(value)) + .unwrap_or(false) + }) +} + +fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> { + for p in patterns { + regex::Regex::new(p) + .map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?; + } + Ok(()) +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct Account { @@ -80,11 +290,12 @@ pub struct Account { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, - pub folder_limit: Option, pub download_folders: Option>, pub account_type: AccountType, pub download_interval_min: Option, pub download_batch_size: Option, + #[serde(default)] + pub max_email_size_bytes: Option, pub known_folders: Option>, pub created_at: i64, pub updated_at: i64, @@ -96,6 +307,16 @@ pub struct Account { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + #[serde(default)] + pub deleting: bool, + /// Email-level filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + #[serde(default)] + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + #[serde(default)] + pub extraction_rules: Option, } impl MemDbModel for Account { @@ -125,16 +346,19 @@ impl Account { created_at: utc_now!(), updated_at: utc_now!(), use_proxy: request.use_proxy, - folder_limit: request.folder_limit, use_dangerous: request.use_dangerous, pgp_key: request.pgp_key, created_by: user_id, download_batch_size: request.download_batch_size, + max_email_size_bytes: request.max_email_size_bytes, date_before: request.date_before, auto_download_new_mailboxes: request.auto_download_new_mailboxes, imap_quota_bytes: request.imap_quota_bytes, imap_quota_window: request.imap_quota_window, download_schedule: request.download_schedule, + deleting: false, + archive_rules: request.archive_rules, + extraction_rules: request.extraction_rules, }) } @@ -222,14 +446,46 @@ impl Account { pub async fn delete(account_id: u64) -> BichonResult<()> { let account = Self::get(account_id)?; - if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { - tracing::error!( - "[CLEANUP_ACCOUNT_ERROR] Account {}: failed to cleanup resources: {:#?}", - account_id, - error - ); - return Err(error); + + // Immediately stop scheduling to prevent new downloads + if matches!(account.account_type, AccountType::IMAP) { + SYNC_TASKS.stop(account.id).await?; } + + // Mark as deleting and disabled so frontend shows status and download tasks skip it + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.deleting = true; + updated.enabled = false; + Ok(updated) + }, + )?; + + // Spawn background cleanup — heavy work (Tantivy, attachments) runs off the request path + tokio::spawn(async move { + if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { + tracing::error!( + "[CLEANUP_ACCOUNT_ERROR] Account {}: cleanup failed, reverting deleting flag: {:#?}", + account_id, + error + ); + // Revert deleting flag so the user can retry (only if account record still exists) + let _ = update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.deleting = false; + updated.enabled = true; + Ok(updated) + }, + ); + } + }); + Ok(()) } @@ -238,8 +494,8 @@ impl Account { } async fn cleanup_account_resources_sequential(account: &AccountModel) -> BichonResult<()> { + // Sync task already stopped in delete() before spawning this background task if matches!(account.account_type, AccountType::IMAP) { - SYNC_TASKS.stop(account.id).await?; DownloadState::delete(account.id)?; } OAuth2AccessToken::try_delete(account.id)?; @@ -326,6 +582,7 @@ impl Account { .map(|account: AccountModel| MinimalAccount { id: account.id, email: account.email, + name: account.account_name, }) .collect::>(); Ok(result) @@ -367,20 +624,10 @@ impl Account { } } - if let Some(folder_limit) = request.folder_limit { - new.folder_limit = Some(folder_limit); - } - if let Some(account_name) = request.account_name { new.account_name = Some(account_name); } - if let Some(clear_folder_limit) = request.clear_folder_limit { - if clear_folder_limit { - new.folder_limit = None; - } - } - if matches!(old.account_type, AccountType::IMAP) { if let Some(imap) = &request.imap { if let Some(current_imap) = &mut new.imap { @@ -407,6 +654,10 @@ impl Account { new.download_batch_size = Some(*download_batch_size); } + if let Some(max_email_size_bytes) = request.max_email_size_bytes { + new.max_email_size_bytes = Some(max_email_size_bytes); + } + if let Some(use_proxy) = request.use_proxy { new.use_proxy = Some(use_proxy); } @@ -447,7 +698,299 @@ impl Account { if request.clear_download_schedule == Some(true) { new.download_schedule = None; } + if request.extraction_rules.is_some() { + new.extraction_rules = request.extraction_rules; + } + if request.archive_rules.is_some() { + new.archive_rules = request.archive_rules; + } new.updated_at = utc_now!(); Ok(new) } } + +#[cfg(test)] +mod tests { + use super::*; + + // ── FilterRule ─────────────────────────────────────────────────── + + #[test] + fn filter_rule_include_only() { + let r = FilterRule { + include: vec![r"@ok\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("spam@bad.com")); + } + + #[test] + fn filter_rule_exclude_only() { + let r = FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("bot@spam.com")); + } + + #[test] + fn filter_rule_include_then_exclude() { + let r = FilterRule { + include: vec![r"@company\.com$".into()], + exclude: vec![r"noreply@company\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@company.com")); + assert!(!r.matches_regex("noreply@company.com")); + assert!(!r.matches_regex("spam@other.com")); + } + + #[test] + fn filter_rule_exact_match() { + let r = FilterRule { + include: vec!["pdf".into(), "docx".into()], + exclude: vec!["xlsx".into()], + ..Default::default() + }; + assert!(r.matches_exact("pdf")); + assert!(r.matches_exact("docx")); + assert!(r.matches_exact("DOCX")); // case-insensitive + assert!(!r.matches_exact("xlsx")); + assert!(!r.matches_exact("txt")); + } + + // ── ExtractionRules ───────────────────────────────────────────── + + #[test] + fn extraction_rules_master_switch() { + let rules = ExtractionRules { + enabled: false, + ..Default::default() + }; + assert!(!rules.should_extract("pdf", None, None, None)); + } + + #[test] + fn extraction_rules_extension_include() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + include: vec!["pdf".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("docx", None, None, None)); + } + + #[test] + fn extraction_rules_extension_exclude() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + exclude: vec!["xlsx".into(), "pptx".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("xlsx", None, None, None)); + } + + #[test] + fn extraction_rules_folder_regex() { + let rules = ExtractionRules { + enabled: true, + folders: FilterRule { + include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None)); + assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None)); + assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None)); + } + + #[test] + fn extraction_rules_attachment_name_regex() { + let rules = ExtractionRules { + enabled: true, + attachment_names: FilterRule { + include: vec![r"^invoice-.*\.pdf$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None)); + assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None)); + } + + #[test] + fn extraction_rules_sender_regex() { + let rules = ExtractionRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@noreply\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + // non-excluded sender passes + assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com"))); + // excluded sender blocked + assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com"))); + } + + #[test] + fn extraction_rules_empty_filters_pass_everything() { + let rules = ExtractionRules { + enabled: true, + ..Default::default() + }; + assert!(rules.should_extract( + "anything", + Some("any/folder"), + Some("any.pdf"), + Some("any@x.com") + )); + } + + // ── ArchiveRules ──────────────────────────────────────────────── + + #[test] + fn archive_rules_disabled_archives_everything() { + let rules = ArchiveRules { + enabled: false, + ..Default::default() + }; + assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false)); + } + + #[test] + fn archive_rules_sender_exclude() { + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false)); + assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false)); + } + + #[test] + fn archive_rules_subject_exclude() { + let rules = ArchiveRules { + enabled: true, + subjects: FilterRule { + exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false)); + assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false)); + assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false)); + } + + #[test] + fn archive_rules_sender_include() { + // Only archive emails from specific senders + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + include: vec![r"@partner\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false)); + assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false)); + } + + #[test] + fn archive_rules_skip_larger_than() { + let rules = ArchiveRules { + enabled: true, + skip_larger_than: Some(50_000_000), + ..Default::default() + }; + assert!(rules.should_archive(None, None, 1_000_000, false)); + assert!(!rules.should_archive(None, None, 60_000_000, false)); + } + + #[test] + fn archive_rules_skip_spam_headers() { + let rules = ArchiveRules { + enabled: true, + spam_headers: vec!["X-Spam-Flag".into()], + ..Default::default() + }; + assert!(!rules.should_archive(None, None, 100, true)); + assert!(rules.should_archive(None, None, 100, false)); + } + + // ── Validation ────────────────────────────────────────────────── + + #[test] + fn validate_extraction_rules_valid() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec![r"^INBOX/.*".into()], + ..Default::default() + }, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_extraction_rules_invalid_regex() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec!["***bad[".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } + + #[test] + fn validate_archive_rules_valid() { + let rules = ArchiveRules { + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + subjects: FilterRule { + include: vec![r"(?i)invoice".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_archive_rules_invalid_regex() { + let rules = ArchiveRules { + senders: FilterRule { + include: vec!["[unclosed".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } +} diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index 34256d1f..b2c55f71 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -19,7 +19,9 @@ use std::str::FromStr; use crate::account::entity::ImapConfig; -use crate::account::migration::{AccountModel, AccountType, QuotaWindow}; +use crate::account::migration::{ + AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow, +}; use crate::account::since::{DateSince, RelativeDate}; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -41,8 +43,6 @@ pub struct AccountCreateRequest { pub date_since: Option, pub date_before: Option, pub account_type: AccountType, - #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))] - pub folder_limit: Option, #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "10"))))] pub download_interval_min: Option, #[cfg_attr( @@ -50,6 +50,7 @@ pub struct AccountCreateRequest { oai(validator(minimum(value = "10"), maximum(value = "200"))) )] pub download_batch_size: Option, + pub max_email_size_bytes: Option, pub use_proxy: Option, pub use_dangerous: bool, pub pgp_key: Option, @@ -57,6 +58,12 @@ pub struct AccountCreateRequest { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + pub extraction_rules: Option, } impl AccountCreateRequest { @@ -107,6 +114,19 @@ impl AccountCreateRequest { } AccountType::NoSync => {} } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!( + format!("extraction_rules: {}", e), + ErrorCode::InvalidParameter + ) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(AccountModel::new(user_id, self)?) } @@ -145,12 +165,6 @@ pub struct AccountUpdateRequest { pub date_since: Option, pub date_before: Option, pub clear_date_range: Option, - /// Max emails to sync for this folder. - /// If not set, sync all emails. - /// otherwise sync up to `n` most recent emails (min 10). - #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))] - pub folder_limit: Option, - pub clear_folder_limit: Option, /// Configuration for selective folder (mailbox/label) synchronization /// /// - For IMAP/SMTP accounts: @@ -173,6 +187,7 @@ pub struct AccountUpdateRequest { oai(validator(minimum(value = "10"), maximum(value = "200"))) )] pub download_batch_size: Option, + pub max_email_size_bytes: Option, /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). /// - If `None` or not provided, the client will connect directly to the API server. /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. @@ -186,6 +201,12 @@ pub struct AccountUpdateRequest { pub auto_download_new_mailboxes: Option, pub download_schedule: Option, pub clear_download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = no change. Use `Some(ArchiveRules { .. })` to set. + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = no change. Use `Some(ExtractionRules { .. })` to set. + pub extraction_rules: Option, } impl AccountUpdateRequest { @@ -205,13 +226,6 @@ impl AccountUpdateRequest { )); } - if self.clear_folder_limit == Some(true) && self.folder_limit.is_some() { - return Err(raise_error!( - "clear_folder_limit cannot be combined with folder_limit".into(), - ErrorCode::InvalidParameter - )); - } - if self.clear_date_range == Some(true) && (self.date_since.is_some() || self.date_before.is_some()) { @@ -248,6 +262,19 @@ impl AccountUpdateRequest { validate_cron_expression(schedule)?; } } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!( + format!("extraction_rules: {}", e), + ErrorCode::InvalidParameter + ) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(()) } } @@ -274,6 +301,7 @@ fn validate_cron_expression(expr: &str) -> BichonResult<()> { pub struct MinimalAccount { pub id: u64, pub email: String, + pub name: Option, } pub fn filter_accessible_accounts<'a>( diff --git a/crates/core/src/account/since.rs b/crates/core/src/account/since.rs index 8b018d36..50809605 100644 --- a/crates/core/src/account/since.rs +++ b/crates/core/src/account/since.rs @@ -364,4 +364,43 @@ mod test { }; assert!(e.validate().is_err()); } + + // ── Sliding window tests ────────────────────────────────────── + + #[test] + fn relative_date_calculate_returns_valid_format() { + let r = RelativeDate { + unit: Unit::Years, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + // Expect format like "26-May-2025" + assert!(date_str.len() > 5); + assert!(date_str.contains('-')); + } + + #[test] + fn relative_date_one_year_ago_is_before_now() { + let r = RelativeDate { + unit: Unit::Years, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap(); + let today = chrono::Local::now().date_naive(); + assert!(parsed < today, "1 year ago ({parsed}) should be before today ({today})"); + } + + #[test] + fn relative_date_one_day_ago_is_yesterday() { + let r = RelativeDate { + unit: Unit::Days, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap(); + let today = chrono::Local::now().date_naive(); + let yesterday = today - chrono::Duration::days(1); + assert_eq!(parsed, yesterday, "1 day ago should be yesterday"); + } } diff --git a/crates/core/src/account/view.rs b/crates/core/src/account/view.rs index 6f9e34fb..6d9d9c7e 100644 --- a/crates/core/src/account/view.rs +++ b/crates/core/src/account/view.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use crate::{ account::{ entity::ImapConfig, - migration::{AccountModel, AccountType, QuotaWindow}, + migration::{AccountModel, AccountType, ArchiveRules, QuotaWindow}, since::{DateSince, RelativeDate}, }, users::UserModel, @@ -40,11 +40,11 @@ pub struct AccountResp { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, - pub folder_limit: Option, pub download_folders: Option>, pub account_type: AccountType, pub download_interval_min: Option, pub download_batch_size: Option, + pub max_email_size_bytes: Option, pub known_folders: Option>, pub created_at: i64, pub updated_at: i64, @@ -58,6 +58,8 @@ pub struct AccountResp { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + pub archive_rules: Option, + pub deleting: bool, } impl AccountResp { @@ -73,11 +75,11 @@ impl AccountResp { capabilities: account.capabilities, date_since: account.date_since, date_before: account.date_before, - folder_limit: account.folder_limit, download_folders: account.download_folders, account_type: account.account_type, download_interval_min: account.download_interval_min, download_batch_size: account.download_batch_size, + max_email_size_bytes: account.max_email_size_bytes, known_folders: account.known_folders, created_at: account.created_at, updated_at: account.updated_at, @@ -95,6 +97,8 @@ impl AccountResp { imap_quota_window: account.imap_quota_window, auto_download_new_mailboxes: account.auto_download_new_mailboxes, download_schedule: account.download_schedule, + archive_rules: account.archive_rules, + deleting: account.deleting, } } } diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs index b7f14398..e1858d0c 100644 --- a/crates/core/src/admin/meta.rs +++ b/crates/core/src/admin/meta.rs @@ -18,7 +18,7 @@ use std::path::Path; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use crate::{ database::MemDbModel, diff --git a/crates/core/src/autoconfig/client.rs b/crates/core/src/autoconfig/client.rs index dcc1d1a2..1bc5656c 100644 --- a/crates/core/src/autoconfig/client.rs +++ b/crates/core/src/autoconfig/client.rs @@ -45,6 +45,10 @@ pub struct IncomingServer { #[serde(rename = "socketType")] pub socket_type: String, pub username: String, + /// Authentication method from the XML, e.g. "OAuth2", "password-cleartext", + /// "password-encrypted", "GSSAPI", "NTLM". Absent in DNS SRV fallback. + #[serde(default)] + pub authentication: String, } #[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] @@ -137,6 +141,7 @@ async fn lookup_srv(domain: &str) -> Option { port: imap_port, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![OutgoingServer { protocol: "smtp".to_string(), @@ -153,60 +158,139 @@ async fn lookup_srv(domain: &str) -> Option { // --------------------------------------------------------------------------- /// Discover mail server configuration for a domain using the Thunderbird -/// autoconfig protocol (ISPDB) and DNS SRV fallback. +/// autoconfig protocol (ISPDB), DNS SRV, MX fallback, and finally guessing. /// /// Probe order: /// 1. `https://autoconfig.{domain}/mail/config-v1.1.xml` -/// 2. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` -/// 3. DNS SRV records (`_imaps._tcp` / `_submission._tcp`) -/// 4. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`) +/// 2. `http://autoconfig.{domain}/mail/config-v1.1.xml` +/// 3. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` +/// 4. `http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` +/// 5. DNS SRV records (`_imaps._tcp` / `_submission._tcp`) +/// 6. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`) +/// 7. MX lookup → ISPDB for MX domain +/// 8. MX lookup → ISP autoconfig for MX domain +/// 9. GuessConfig — probe common hostnames + ports pub async fn fetch(domain: &str) -> BichonResult { let client = Client::builder() .timeout(std::time::Duration::from_secs(10)) .build() .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - // 1. Try autoconfig subdomain + // ── ISP autoconfig (HTTPS, then HTTP) ────────────────────────── + if let Some(config) = + fetch_xml(&client, &format!("https://autoconfig.{domain}/mail/config-v1.1.xml")).await + { + return Ok(config); + } + if let Some(config) = + fetch_xml(&client, &format!("http://autoconfig.{domain}/mail/config-v1.1.xml")).await + { + return Ok(config); + } + + // ── Well-known path (HTTPS, then HTTP) ───────────────────────── if let Some(config) = fetch_xml( &client, - &format!("https://autoconfig.{domain}/mail/config-v1.1.xml"), + &format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), ) .await { return Ok(config); } - - // 2. Try well-known path if let Some(config) = fetch_xml( &client, - &format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), + &format!("http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), ) .await { return Ok(config); } - // 3. Try DNS SRV records + // ── DNS SRV records ──────────────────────────────────────────── if let Some(config) = lookup_srv(domain).await { return Ok(config); } - // 4. Fall back to Thunderbird central database - if let Some(config) = fetch_xml( - &client, - &format!("https://autoconfig.thunderbird.net/v1.1/{domain}"), - ) - .await + // ── Thunderbird central ISPDB ────────────────────────────────── + if let Some(config) = + fetch_xml(&client, &format!("https://autoconfig.thunderbird.net/v1.1/{domain}")).await { return Ok(config); } + // ── MX fallback ──────────────────────────────────────────────── + if let Some(config) = fetch_for_mx(&client, domain).await { + return Ok(config); + } + + // ── GuessConfig ──────────────────────────────────────────────── + if let Some(config) = crate::autoconfig::guess::guess_config(domain).await { + return Ok(config); + } + Err(raise_error!( format!("No autoconfig found for domain: {domain}"), ErrorCode::InternalError )) } +/// DNS MX lookup → retry ISPDB and ISP autoconfig for the MX domain. +/// +/// Many self-hosted domains have their MX pointed at Google, Microsoft, etc. +/// The MX domain's ISPDB entry covers the original domain. +async fn fetch_for_mx(client: &Client, domain: &str) -> Option { + let mx_domain = lookup_mx_domain(domain).await?; + if mx_domain == domain.to_ascii_lowercase() { + return None; // same domain, already tried above + } + + // Try ISPDB for the MX domain + if let Some(config) = + fetch_xml(client, &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}")).await + { + return Some(config); + } + + // Try ISP autoconfig for the MX domain (HTTPS then HTTP) + if let Some(config) = + fetch_xml(client, &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await + { + return Some(config); + } + if let Some(config) = + fetch_xml(client, &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await + { + return Some(config); + } + + None +} + +/// DNS MX lookup → extract the second-level domain of the first MX hostname. +async fn lookup_mx_domain(domain: &str) -> Option { + let resolver = TokioResolver::builder(TokioConnectionProvider::default()) + .ok()? + .build(); + let lookup = resolver.mx_lookup(domain).await.ok()?; + let record = lookup.iter().next()?; + let mx_host = record.to_string().trim_end_matches('.').to_string(); + + // Extract a reasonable base domain from the MX hostname. + // E.g., "aspmx.l.google.com" → "google.com" + // "company.mail.protection.outlook.com" → "outlook.com" + extract_base_domain(&mx_host) +} + +/// Extract the top two labels from a hostname as a rough base domain. +fn extract_base_domain(host: &str) -> Option { + let parts: Vec<&str> = host.split('.').collect(); + if parts.len() >= 2 { + Some(parts[parts.len() - 2..].join(".")) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/core/src/autoconfig/guess.rs b/crates/core/src/autoconfig/guess.rs new file mode 100644 index 00000000..9acd4cc1 --- /dev/null +++ b/crates/core/src/autoconfig/guess.rs @@ -0,0 +1,105 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::account::entity::Encryption; +use crate::autoconfig::client::{IncomingServer, MailConfig}; +use crate::imap::client::Client; +use tracing::{debug, info}; + +/// A single host:port:encryption combination to probe. +struct Guess { + hostname: String, + port: u16, + encryption: Encryption, + socket_type: &'static str, +} + +/// Generate candidates in the same order Thunderbird uses: +/// 1. imap.{domain} — most common +/// 2. mail.{domain} — fallback +/// 3. {domain} — bare domain (rare) +fn make_guesses(domain: &str) -> Vec { + let hosts = [ + format!("imap.{domain}"), + format!("mail.{domain}"), + domain.to_string(), + ]; + + let mut guesses = Vec::with_capacity(hosts.len() * 2); + for host in &hosts { + guesses.push(Guess { + hostname: host.clone(), + port: 993, + encryption: Encryption::Ssl, + socket_type: "SSL", + }); + guesses.push(Guess { + hostname: host.clone(), + port: 143, + encryption: Encryption::StartTls, + socket_type: "STARTTLS", + }); + } + guesses +} + +/// Try to open a connection, read the IMAP banner, and close. +/// Returns `true` if the server responds with an IMAP greeting. +async fn probe(hostname: &str, port: u16, encryption: &Encryption) -> bool { + match Client::connection(hostname, encryption, port, None, true).await { + Ok(_) => { + debug!("GuessConfig probe succeeded: {hostname}:{port} ({encryption:?})"); + true + } + Err(e) => { + debug!("GuessConfig probe failed for {hostname}:{port}: {e:?}"); + false + } + } +} + +/// Thunderbird-style guessing: try common hostnames and ports, probing +/// each with a real TCP connection. +/// +/// Returns the first working `MailConfig`, or `None` if nothing works. +pub async fn guess_config(domain: &str) -> Option { + let guesses = make_guesses(domain); + info!("GuessConfig: trying {} candidates for {domain}", guesses.len()); + + for g in &guesses { + if probe(&g.hostname, g.port, &g.encryption).await { + info!( + "GuessConfig: found working IMAP at {}:{} ({})", + g.hostname, g.port, g.socket_type + ); + return Some(MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: g.hostname.clone(), + port: g.port, + socket_type: g.socket_type.to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), + }], + outgoing: vec![], + }); + } + } + + None +} diff --git a/crates/core/src/autoconfig/load.rs b/crates/core/src/autoconfig/load.rs index 9cf89efc..1408681f 100644 --- a/crates/core/src/autoconfig/load.rs +++ b/crates/core/src/autoconfig/load.rs @@ -19,6 +19,7 @@ use crate::account::entity::Encryption; use crate::autoconfig::client::{self, MailConfig}; use crate::autoconfig::entity::{MailServerConfig, ServerConfig}; +use crate::autoconfig::oauth2_providers::lookup_oauth2; use crate::autoconfig::CachedMailSettings; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -54,9 +55,17 @@ pub(crate) fn mail_config_to_server_config(config: &MailConfig) -> Option field and a known + // hostname → issuer mapping determine whether the provider supports OAuth2. + let oauth2 = if imap.authentication.eq_ignore_ascii_case("OAuth2") { + lookup_oauth2(&imap.hostname) + } else { + None + }; + Some(MailServerConfig { imap: ServerConfig::new(imap.hostname.clone(), port, encryption), - oauth2: None, + oauth2, }) } diff --git a/crates/core/src/autoconfig/mod.rs b/crates/core/src/autoconfig/mod.rs index 39809747..4f8afc7c 100644 --- a/crates/core/src/autoconfig/mod.rs +++ b/crates/core/src/autoconfig/mod.rs @@ -24,7 +24,9 @@ use serde::{Deserialize, Serialize}; pub mod client; pub mod entity; +pub mod guess; pub mod load; +mod oauth2_providers; #[cfg(test)] mod tests; diff --git a/crates/core/src/autoconfig/oauth2_providers.rs b/crates/core/src/autoconfig/oauth2_providers.rs new file mode 100644 index 00000000..71b82274 --- /dev/null +++ b/crates/core/src/autoconfig/oauth2_providers.rs @@ -0,0 +1,151 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::autoconfig::entity::OAuth2Config; + +/// Per-provider OAuth2 metadata, mirroring Thunderbird's `OAuth2Providers.sys.mjs`. +/// +/// Each entry maps one or more IMAP hostname suffixes to a well-known OIDC issuer +/// and the IMAP-specific OAuth2 scopes. +struct Provider { + /// Suffixes matched case-insensitively against the end of the IMAP hostname. + host_suffixes: &'static [&'static str], + /// The OIDC issuer URL used by the provider. + issuer: &'static str, + /// OAuth2 scope(s) required for IMAP access. + scopes: &'static [&'static str], +} + +const PROVIDERS: &[Provider] = &[ + // Google + Provider { + host_suffixes: &["imap.gmail.com", ".gmail.com", ".googlemail.com"], + issuer: "https://accounts.google.com", + scopes: &["https://mail.google.com/"], + }, + // Microsoft (Outlook / Office 365 / Hotmail / Live) + Provider { + host_suffixes: &[ + "outlook.office365.com", + ".outlook.com", + ".hotmail.com", + ".live.com", + ".office365.com", + ], + issuer: "https://login.microsoftonline.com/common/v2.0", + scopes: &[ + "https://outlook.office365.com/IMAP.AccessAsUser.All", + "offline_access", + ], + }, + // Yahoo / AOL / ATT / Verizon + Provider { + host_suffixes: &[ + "imap.mail.yahoo.com", + ".yahoo.com", + ".yahoodns.net", + ".aol.com", + "imap.aol.com", + ], + issuer: "https://login.yahoo.com", + scopes: &["mail-w"], + }, + // Yandex + Provider { + host_suffixes: &["imap.yandex.ru", "imap.yandex.com", ".yandex.ru"], + issuer: "https://oauth.yandex.com", + scopes: &["imap:all"], + }, + // Mail.ru + Provider { + host_suffixes: &["imap.mail.ru", ".mail.ru", ".bk.ru", ".list.ru", ".inbox.ru"], + issuer: "https://o2.mail.ru", + scopes: &["imap"], + }, + // Fastmail + Provider { + host_suffixes: &["imap.fastmail.com", ".fastmail.com"], + issuer: "https://www.fastmail.com", + scopes: &[ + "https://www.fastmail.com/dev/imap", + "offline_access", + ], + }, + // Comcast + Provider { + host_suffixes: &["imap.comcast.net", ".comcast.net"], + issuer: "https://oauth.xfinity.com", + scopes: &["https://email.comcast.net/"], + }, +]; + +/// Try to find an OAuth2 provider that matches the given IMAP hostname. +/// +/// Matching is case-insensitive and done by suffix: a hostname "imap.gmail.com" +/// matches the suffix ".gmail.com". +pub fn lookup_oauth2(hostname: &str) -> Option { + let host = hostname.to_ascii_lowercase(); + for provider in PROVIDERS { + if provider + .host_suffixes + .iter() + .any(|suffix| host.ends_with(&suffix.to_ascii_lowercase())) + { + return Some(OAuth2Config { + issuer: provider.issuer.to_string(), + scope: provider.scopes.iter().map(|s| s.to_string()).collect(), + auth_url: String::new(), + token_url: String::new(), + }); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_known_providers() { + let cases = [ + ("imap.gmail.com", Some("https://accounts.google.com")), + ("imap.gmail.com", Some("https://accounts.google.com")), + ("outlook.office365.com", Some("https://login.microsoftonline.com/common/v2.0")), + ("imap.mail.yahoo.com", Some("https://login.yahoo.com")), + ("imap.aol.com", Some("https://login.yahoo.com")), + ("imap.yandex.ru", Some("https://oauth.yandex.com")), + ("imap.mail.ru", Some("https://o2.mail.ru")), + ("imap.fastmail.com", Some("https://www.fastmail.com")), + ("imap.comcast.net", Some("https://oauth.xfinity.com")), + ]; + for (hostname, expected_issuer) in &cases { + let result = lookup_oauth2(hostname); + assert_eq!( + result.map(|c| c.issuer), + expected_issuer.map(|s| s.to_string()), + "failed for hostname: {hostname}" + ); + } + } + + #[test] + fn test_unknown_provider() { + assert!(lookup_oauth2("mail.my-company.example").is_none()); + } +} diff --git a/crates/core/src/autoconfig/tests.rs b/crates/core/src/autoconfig/tests.rs index a7211164..b9d36708 100644 --- a/crates/core/src/autoconfig/tests.rs +++ b/crates/core/src/autoconfig/tests.rs @@ -195,6 +195,7 @@ fn make_imap_server(host: &str, port: u16, socket_type: &str) -> IncomingServer port, socket_type: socket_type.to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), } } @@ -241,6 +242,7 @@ fn convert_no_imap_only_pop3() { port: 995, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; @@ -266,6 +268,7 @@ fn convert_picks_imap_over_pop3() { port: 995, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }, make_imap_server("imap.example.com", 993, "SSL"), ], @@ -284,6 +287,7 @@ fn convert_imaps_protocol_variant() { port: 993, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; @@ -300,9 +304,65 @@ fn convert_case_insensitive_protocol() { port: 143, socket_type: "STARTTLS".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; let result = mail_config_to_server_config(&config).expect("should recognize 'IMAP'"); assert_eq!(result.imap.host, "imap.example.com"); } + +#[test] +fn convert_gmail_oauth2() { + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "imap.gmail.com".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + let oauth2 = result.oauth2.expect("Gmail should have OAuth2"); + assert_eq!(oauth2.issuer, "https://accounts.google.com"); + assert!(oauth2.scope.contains(&"https://mail.google.com/".to_string())); +} + +#[test] +fn convert_outlook_oauth2() { + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "outlook.office365.com".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + let oauth2 = result.oauth2.expect("Outlook should have OAuth2"); + assert!(oauth2.issuer.contains("microsoftonline")); +} + +#[test] +fn convert_unknown_host_no_oauth2() { + // OAuth2 auth flag on an unknown hostname → no OAuth2 returned + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "mail.random-isp.example".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + assert!(result.oauth2.is_none(), "unknown hostname → no OAuth2 mapping"); +} diff --git a/crates/core/src/cache/imap/download/download_type.rs b/crates/core/src/cache/imap/download/download_type.rs index 2c27d730..1bf691f7 100644 --- a/crates/core/src/cache/imap/download/download_type.rs +++ b/crates/core/src/cache/imap/download/download_type.rs @@ -77,10 +77,7 @@ pub async fn decide_next_download_task( } } -fn should_trigger_next_download( - last_trigger_at: i64, - sync_interval_min: i64, -) -> bool { +fn should_trigger_next_download(last_trigger_at: i64, sync_interval_min: i64) -> bool { let now = utc_now!(); now - last_trigger_at > (sync_interval_min * 60 * 1000) } @@ -107,7 +104,10 @@ fn should_trigger_scheduled(schedule_str: &str, last_trigger_at: i64) -> bool { }; let last_dt: DateTime = last_utc.with_timezone(&Local); let now = Local::now(); - schedule.after(&last_dt).next().map_or(false, |next| next <= now) + schedule + .after(&last_dt) + .next() + .map_or(false, |next| next <= now) } #[cfg(test)] @@ -153,13 +153,4 @@ mod test { let last_trigger = now.timestamp_millis() - 61 * 60 * 1000; assert!(should_trigger_scheduled("0 0 * * * *", last_trigger)); } - - #[test] - fn cron_every_hour_no_trigger_if_recent() { - // "0 0 * * * *" = every hour at minute 0, second 0 - // last_trigger was just 1 minute ago → should NOT trigger (except at :00/:01 boundary) - let now = Local::now(); - let last_trigger = now.timestamp_millis() - 60_000; - assert!(!should_trigger_scheduled("0 0 * * * *", last_trigger)); - } } diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 025a6f39..bc421b13 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -32,15 +32,18 @@ use crate::{ SEMAPHORE, }, error::{code::ErrorCode, BichonResult}, - imap::executor::ImapExecutor, + imap::executor::{ + compress_uid_list, generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, + }, store::tantivy::envelope::ENVELOPE_MANAGER, }, }; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; -pub const DEFAULT_BATCH_SIZE: u32 = 30; +const MAX_NETWORK_RETRIES: u32 = 3; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum FetchDirection { @@ -54,7 +57,7 @@ pub async fn fetch_and_save_by_date( mailbox: &MailBox, direction: FetchDirection, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { let account_id = account.id; let mut session = match ImapExecutor::create_connection(account_id).await { Ok(session) => session, @@ -108,32 +111,18 @@ pub async fn fetch_and_save_by_date( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - let folder_limit = account.folder_limit; // sort small -> bigger let mut uid_vec: Vec = uid_list.into_iter().collect(); uid_vec.sort(); - if let Some(limit) = folder_limit { - let limit = limit.max(100) as usize; - if len > limit { - uid_vec = match direction { - FetchDirection::Since => uid_vec.split_off(len - limit), - FetchDirection::Before => { - uid_vec.truncate(limit); - uid_vec - } - }; - } - } - + let max_uid = uid_vec.last().copied(); let planned = uid_vec.len() as u64; let uid_batches = generate_uid_sequence_hashset( uid_vec, account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize, - false, ); DownloadState::update_folder_progress( account_id, @@ -165,17 +154,65 @@ pub async fn fetch_and_save_by_date( break; } // Fetch metadata for the current batch of UIDs - match ImapExecutor::uid_batch_retrieve_emails( - &mut session, - account_id, - mailbox.id, - &batch.0, - token.clone(), - ) - .await - { - Ok(_) => { - current_processed += batch.1; + let mut retries = 0u32; + let batch_result = loop { + match ImapExecutor::uid_batch_retrieve_emails( + &mut session, + account_id, + mailbox.id, + &batch.0, + account.max_email_size_bytes, + token.clone(), + ) + .await + { + Ok(processed) => break Ok(processed), + Err(e) + if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => + { + retries += 1; + warn!( + account_id, + mailbox = mailbox.name, + index, + retries, + "Network error on batch, reconnecting ({}/{})", + retries, + MAX_NETWORK_RETRIES + ); + match ImapExecutor::create_connection(account_id).await { + Ok(new_session) => { + session = new_session; + if let Err(e2) = session.examine(&mailbox.encoded_name()).await + { + let err_msg = format!( + "Re-examine failed after reconnect: {:#?}", + e2 + ); + DownloadState::append_session_error( + account_id, + err_msg, + )?; + break Err(e); + } + tokio::time::sleep(Duration::from_secs( + 1 << (retries - 1), + )) + .await; + continue; + } + Err(e2) => { + error!(account_id, "Reconnection failed: {:#?}", e2); + break Err(e); + } + } + } + Err(e) => break Err(e), + } + }; + match batch_result { + Ok(processed) => { + current_processed += processed; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -212,14 +249,16 @@ pub async fn fetch_and_save_by_date( )?; } session.logout().await.ok(); - Ok(()) + Ok(max_uid) } +/// Fetches all messages from a mailbox. +/// Returns `Ok(Some(max_uid))` with the highest UID stored, or `Ok(None)` if empty. pub async fn fetch_and_save_full_mailbox( account: &AccountModel, mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { let mailbox_id = mailbox.id; let account_id = account.id; @@ -262,33 +301,17 @@ pub async fn fetch_and_save_full_mailbox( } }; - let folder_limit = account.folder_limit; - let total_to_fetch = match folder_limit { - Some(limit) if (limit as u64) < total => { - let limit64 = limit as u64; - total.min(limit64.max(100)) - } - _ => total, - }; - - let page_size = if let Some(limit) = folder_limit { - limit - .max(100) - .min(account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE)) - } else { - account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) - }; - - let total_batches = total_to_fetch.div_ceil(page_size as u64); - let desc = folder_limit.is_some(); + let page_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE); + let total_batches = total.div_ceil(page_size as u64); info!( - "Starting full mailbox download for '{}', total={}, limit={:?}, batches={}, desc={}", - mailbox.name, total, folder_limit, total_batches, desc + "Starting full mailbox download for '{}', total={}, batches={}", + mailbox.name, total, total_batches ); let mut current_processed = 0u64; let mut has_error_or_cancel = false; + let mut max_uid: Option = None; for page in 1..=total_batches { if token.is_cancelled() { @@ -300,7 +323,7 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Cancelled, None, @@ -309,25 +332,66 @@ pub async fn fetch_and_save_full_mailbox( break; } - match ImapExecutor::batch_retrieve_emails( - &mut session, - account_id, - mailbox_id, - total_to_fetch, - page as u64, - page_size as u64, - &mailbox.encoded_name(), - desc, - token.clone(), - ) - .await - { + let mut retries = 0u32; + let batch_result = loop { + match ImapExecutor::batch_retrieve_emails( + &mut session, + account_id, + mailbox_id, + total, + page as u64, + page_size as u64, + &mailbox.encoded_name(), + account.max_email_size_bytes, + token.clone(), + &mut max_uid, + ) + .await + { + Ok(count) => break Ok(count), + Err(e) + if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => + { + retries += 1; + warn!( + account_id, + mailbox = mailbox.name, + page, + retries, + "Network error on batch, reconnecting ({}/{})", + retries, + MAX_NETWORK_RETRIES + ); + match ImapExecutor::create_connection(account_id).await { + Ok(new_session) => { + session = new_session; + if let Err(e2) = session.examine(&mailbox.encoded_name()).await { + let err_msg = format!( + "Re-examine failed after reconnect: {:#?}", + e2 + ); + DownloadState::append_session_error(account_id, err_msg)?; + break Err(e); + } + tokio::time::sleep(Duration::from_secs(1 << (retries - 1))).await; + continue; + } + Err(e2) => { + error!(account_id, "Reconnection failed: {:#?}", e2); + break Err(e); + } + } + } + Err(e) => break Err(e), + } + }; + match batch_result { Ok(count) => { current_processed += count as u64; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Downloading, None, @@ -339,7 +403,7 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Failed, Some(err_msg), @@ -354,71 +418,292 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Success, None, )?; } session.logout().await.ok(); - Ok(()) + Ok(max_uid) } -pub fn generate_uid_sequence_hashset( - unique_nums: Vec, - chunk_size: usize, - desc: bool, -) -> Vec<(String, u64)> { - assert!(!unique_nums.is_empty()); - let mut nums = unique_nums; - if desc { - nums.reverse(); - } +/// Generates a synthetic UIDVALIDITY for IMAP servers that don't provide it. +/// Uses a stable hash of the mailbox name to ensure consistent IDs across sessions. +fn generate_synthetic_uidvalidity(mailbox_name: &str) -> u32 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; - let mut result = Vec::new(); + let mut hasher = DefaultHasher::new(); + mailbox_name.hash(&mut hasher); + (hasher.finish() as u32).wrapping_add(1) // Avoid 0, which might be reserved +} - for chunk in nums.chunks(chunk_size) { - let size = chunk.len() as u64; - let compressed = compress_uid_list(chunk.to_vec()); - result.push((compressed, size)); +/// Retry fetching UIDVALIDITY via STATUS command when the initial listing returned None. +/// A None UIDVALIDITY while other STATUS fields (MESSAGES, UNSEEN, UIDNEXT) are present +/// can be caused by transient network issues corrupting just the UIDVALIDITY portion of +/// the response. Retrying avoids unnecessarily triggering a full reconcile. +async fn fetch_uid_validity_with_retry( + account_id: u64, + mailbox_name: &str, + max_retries: u32, +) -> BichonResult> { + let mailbox_name = mailbox_name.to_string(); + fetch_uid_validity_with_retry_inner(max_retries, move || { + let account_id = account_id; + let mailbox_name = mailbox_name.clone(); + async move { + let mut session = ImapExecutor::create_connection(account_id).await?; + let result = session + .status(&mailbox_name, "(UIDVALIDITY)") + .await + .map(|r| r.uid_validity) + .map_err(|e| { + let msg = format!("STATUS failed during UIDVALIDITY retry: {:#?}", e); + raise_error!(msg, ErrorCode::InternalError) + }); + session.logout().await.ok(); + result + } + }) + .await +} + +/// Generic retry loop: calls `fetch_fn` up to `max_retries` times with +/// backoff (500ms × attempt). Returns the first `Some(uid)`, or `Ok(None)` +/// if all attempts return `None` or error. +async fn fetch_uid_validity_with_retry_inner( + max_retries: u32, + mut fetch_fn: F, +) -> BichonResult> +where + F: FnMut() -> Fut, + Fut: std::future::Future>>, +{ + for attempt in 0..max_retries { + if attempt > 0 { + tokio::time::sleep(std::time::Duration::from_millis(500 * attempt as u64)).await; + } + + match fetch_fn().await { + Ok(Some(uid)) => return Ok(Some(uid)), + Ok(None) => { + warn!( + attempt = attempt + 1, + max_retries, + "STATUS returned no UIDVALIDITY" + ); + } + Err(e) => { + warn!( + attempt = attempt + 1, + max_retries, + "UIDVALIDITY fetch attempt failed: {:#?}", e + ); + } + } } - result + Ok(None) } -pub fn compress_uid_list(nums: Vec) -> String { - if nums.is_empty() { - return String::new(); +/// Handle uid_validity change without deleting local data. +/// Compares remote Message-IDs with local Tantivy index, downloads only +/// truly missing emails. DedupCache catches any remaining duplicates. +async fn reconcile_uid_validity_change( + account: &AccountModel, + local_mailbox: &MailBox, + remote_mailbox: &MailBox, + token: CancellationToken, +) -> BichonResult> { + let account_id = account.id; + + // Phase 1: connect + examine + let mut session = ImapExecutor::create_connection(account_id).await?; + session + .examine(&remote_mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // Phase 2: collect remote UIDs, respecting date constraints + let remote_uid_list: Vec = if let Some(date_since) = &account.date_since { + let date = date_since.since_date()?; + let results = session + .uid_search(&format!("SINCE {date}")) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + } else if let Some(date_before) = &account.date_before { + let date = date_before.calculate_date()?; + let results = session + .uid_search(&format!("BEFORE {date}")) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + } else { + let results = session + .uid_search("ALL") + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + }; + + if remote_uid_list.is_empty() { + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + Some("UIDVALIDITY changed but remote mailbox is empty.".into()), + )?; + session.logout().await.ok(); + return Ok(None); } - let mut sorted_nums = nums; - sorted_nums.sort(); + let max_uid = remote_uid_list.last().copied(); + + // Phase 3: fetch remote Message-IDs (headers only, no bodies) + let uid_set = compress_uid_list(remote_uid_list.clone()); + let remote_msg_ids = + ImapExecutor::fetch_uid_metadata(&mut session, &uid_set, token.clone()).await?; + session.logout().await.ok(); - let mut result = Vec::new(); - let mut current_range_start = sorted_nums[0]; - let mut current_range_end = sorted_nums[0]; + // Phase 4: query local Message-IDs from Tantivy. + // For a large mailbox this can allocate 50-100 MB of HashSet. + let local_msg_ids = + ENVELOPE_MANAGER.get_message_ids_for_mailbox(account_id, local_mailbox.id)?; - for &n in sorted_nums.iter().skip(1) { - if n == current_range_end + 1 { - current_range_end = n; - } else { - if current_range_start == current_range_end { - result.push(current_range_start.to_string()); - } else { - result.push(format!("{}:{}", current_range_start, current_range_end)); + // Phase 5: compute missing UIDs + let mut missing_uids: Vec = Vec::new(); + for uid in &remote_uid_list { + if token.is_cancelled() { + return Err(raise_error!("Cancelled".into(), ErrorCode::InternalError)); + } + match remote_msg_ids.get(uid) { + Some(Some(msg_id)) if local_msg_ids.contains(msg_id) => { + // already have this email locally } - current_range_start = n; - current_range_end = n; + _ => missing_uids.push(*uid), } } - if current_range_start == current_range_end { - result.push(current_range_start.to_string()); + // Phase 6: download missing + if missing_uids.is_empty() { + info!( + account_id, + mailbox = remote_mailbox.name, + "UIDVALIDITY changed but all {} emails already exist locally", + remote_uid_list.len() + ); + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + None, + )?; } else { - result.push(format!("{}:{}", current_range_start, current_range_end)); + let planned = missing_uids.len() as u64; + info!( + account_id, + mailbox = remote_mailbox.name, + total = remote_uid_list.len(), + missing = planned, + "UIDVALIDITY changed, downloading missing emails" + ); + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + 0, + FolderStatus::Downloading, + Some("UIDVALIDITY changed, downloading missing emails...".into()), + )?; + + let mut session2 = ImapExecutor::create_connection(account_id).await?; + session2 + .examine(&remote_mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let batch_size = account + .download_batch_size + .unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let batches = generate_uid_sequence_hashset(missing_uids, batch_size); + + let mut downloaded = 0u64; + for (index, batch) in batches.into_iter().enumerate() { + if token.is_cancelled() { + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Cancelled, + None, + )?; + session2.logout().await.ok(); + return Err(raise_error!("Cancelled".into(), ErrorCode::InternalError)); + } + + match ImapExecutor::uid_batch_retrieve_emails( + &mut session2, + account_id, + remote_mailbox.id, + &batch.0, + account.max_email_size_bytes, + token.clone(), + ) + .await + { + Ok(processed) => { + downloaded += processed; + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Downloading, + None, + )?; + } + Err(e) => { + let err_msg = format!("Batch {} failed: {:#?}", index, e); + DownloadState::append_session_error(account_id, err_msg.clone())?; + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Failed, + Some(err_msg), + )?; + session2.logout().await.ok(); + return Err(e); + } + } + } + + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Success, + None, + )?; + session2.logout().await.ok(); } - result.join(",") + Ok(max_uid) } pub async fn reconcile_mailboxes( @@ -448,82 +733,83 @@ pub async fn reconcile_mailboxes( break; } - if local_mailbox.uid_validity != remote_mailbox.uid_validity { - if remote_mailbox.uid_validity.is_none() { - let err_msg = format!( - "Mailbox '{}' logic error: Server did not provide UIDVALIDITY.", - local_mailbox.name - ); - - warn!("Account {}: {}", account_id, err_msg); - - DownloadState::update_folder_progress( - account_id, - remote_mailbox.name.clone(), - 0, - 0, - FolderStatus::Failed, - Some(err_msg.clone()), - )?; - DownloadState::append_session_error(account_id, err_msg)?; - continue; + // Handle missing UIDVALIDITY from non-compliant IMAP servers + // (e.g., Tencent Enterprise Mail, etc.) + let remote_uid_validity = match remote_mailbox.uid_validity { + Some(uid) => uid, + None => { + if local_mailbox.uid_validity.is_some() { + // We had a real UIDVALIDITY before; a None now is likely + // transient (network jitter). Retry before falling back. + warn!( + "Account {}: Mailbox '{}' - STATUS returned no UIDVALIDITY, retrying to rule out network jitter...", + account_id, remote_mailbox.name + ); + match fetch_uid_validity_with_retry( + account_id, + &remote_mailbox.encoded_name(), + 3, + ) + .await? + { + Some(uid) => { + info!( + "Account {}: Mailbox '{}' - UIDVALIDITY recovered after retry: {}", + account_id, remote_mailbox.name, uid + ); + uid + } + None => { + // All retries exhausted; keep the local value. + // Safer than synthetic because we know the server had one. + let fallback = local_mailbox.uid_validity.unwrap(); + warn!( + "Account {}: Mailbox '{}' - all retries failed, keeping local UIDVALIDITY {}", + account_id, remote_mailbox.name, fallback + ); + fallback + } + } + } else { + // First sync and server genuinely doesn't provide UIDVALIDITY + let synthetic_uid = generate_synthetic_uidvalidity(&remote_mailbox.name); + warn!( + "Account {}: Mailbox '{}' - Server did not provide UIDVALIDITY. \ + Using synthetic UIDVALIDITY {} based on mailbox name. \ + This mailbox will be synced but may require periodic rebuilds if the server's mailbox structure changes.", + account_id, remote_mailbox.name, synthetic_uid + ); + synthetic_uid + } } + }; + + let new_highest_uid = if local_mailbox.uid_validity != Some(remote_uid_validity) { info!( "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \ - The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.", - account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_mailbox.uid_validity + Comparing by Message-ID to find missing emails.", + account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_uid_validity ); - DownloadState::update_folder_progress( - account_id, - local_mailbox.name.clone(), - remote_mailbox.exists as u64, - 0, - FolderStatus::Downloading, - Some("UID validity changed, rebuilding...".into()), - )?; - - match &account.date_since { - Some(date_since) => { - rebuild_mailbox_cache_by_date( - account, - local_mailbox.id, - &date_since.since_date()?, - remote_mailbox, - FetchDirection::Since, - token.clone(), - ) - .await?; - } - None => match &account.date_before { - Some(r) => { - rebuild_mailbox_cache_by_date( - account, - local_mailbox.id, - &r.calculate_date()?, - remote_mailbox, - FetchDirection::Before, - token.clone(), - ) - .await?; - } - None => { - rebuild_mailbox_cache( - account, - local_mailbox, - remote_mailbox, - token.clone(), - ) - .await? - } - }, - } + reconcile_uid_validity_change( + account, + local_mailbox, + remote_mailbox, + token.clone(), + ) + .await? } else { perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone()) - .await?; - } + .await? + }; - mailboxes_to_update.push(remote_mailbox.clone()); + let mut updated = remote_mailbox.clone(); + updated.highest_uid = new_highest_uid; + // Update uid_validity with the resolved value (either from server or synthetic) + if updated.uid_validity.is_none() { + updated.uid_validity = Some(remote_uid_validity); + } + mailboxes_to_update.push(updated); } //The metadata of this mailbox must only be updated after a successful synchronization; //otherwise, it may cause synchronization errors and result in missing emails in the local sync results. @@ -598,7 +884,11 @@ pub async fn reconcile_mailboxes( }; match result { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -622,64 +912,750 @@ pub async fn reconcile_mailboxes( } //only check new emails and sync +/// Incrementally syncs a mailbox. +/// Returns the new highest UID after sync, or `None` if nothing changed. async fn perform_incremental_sync( account: &AccountModel, local_mailbox: &MailBox, remote_mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { if remote_mailbox.exists > 0 { - let local_max_uid = ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?; - tracing::info!( - "[account {}][mailbox {}] perform_incremental_sync: local_max_uid={:?}, remote.exists={}", - account.id, - local_mailbox.name, - local_max_uid, - remote_mailbox.exists - ); - match local_max_uid { - Some(max_uid) => { - let mut session = ImapExecutor::create_connection(account.id).await?; - let before_date = account - .date_before - .as_ref() - .map(|r| r.calculate_date()) - .transpose()?; - - ImapExecutor::fetch_new_mail( - &mut session, - account, - local_mailbox, - max_uid + 1, - before_date.as_deref(), - token, - ) - .await?; - session.logout().await.ok(); + // Use stored highest_uid if available; otherwise fall back to Tantivy + // query once (backward compatibility with pre-existing databases). + let start_uid = match local_mailbox.highest_uid { + Some(uid) => { + tracing::info!( + "[account {}][mailbox {}] perform_incremental_sync: stored highest_uid={}, remote.exists={}", + account.id, + local_mailbox.name, + uid, + remote_mailbox.exists + ); + uid as u64 + 1 } None => { - info!( - "No maximum UID found in index for mailbox, assuming local cache is missing." + let local_max_uid = + ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?; + tracing::info!( + "[account {}][mailbox {}] perform_incremental_sync: highest_uid unset, Tantivy max_uid={:?}, remote.exists={}", + account.id, + local_mailbox.name, + local_max_uid, + remote_mailbox.exists ); - - match &account.date_since { - Some(date_since) => { - fetch_and_save_by_date( - account, - date_since.since_date()?.as_str(), - remote_mailbox, - FetchDirection::Since, - token, - ) - .await?; - } + match local_max_uid { + Some(uid) => uid + 1, None => { - fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; + info!( + "No maximum UID found in index for mailbox, assuming local storage is missing." + ); + + let result = match &account.date_since { + Some(date_since) => { + fetch_and_save_by_date( + account, + date_since.since_date()?.as_str(), + remote_mailbox, + FetchDirection::Since, + token, + ) + .await? + } + None => match &account.date_before { + Some(r) => { + fetch_and_save_by_date( + account, + &r.calculate_date()?, + remote_mailbox, + FetchDirection::Before, + token, + ) + .await? + } + None => { + fetch_and_save_full_mailbox( + account, remote_mailbox, token, + ) + .await? + } + }, + }; + return Ok(result); } } } + }; + + let mut session = ImapExecutor::create_connection(account.id).await?; + let before_date = account + .date_before + .as_ref() + .map(|r| r.calculate_date()) + .transpose()?; + + let new_max_uid = ImapExecutor::fetch_new_mail( + &mut session, + account, + local_mailbox, + start_uid, + remote_mailbox + .uid_next + .and_then(|uid_next| uid_next.checked_sub(1)), + before_date.as_deref(), + token, + ) + .await?; + session.logout().await.ok(); + + // Keep existing highest_uid if no new mail was fetched. + Ok(new_max_uid.or(local_mailbox.highest_uid)) + } else { + Ok(local_mailbox.highest_uid) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::imap::session::SessionStream; + use std::sync::Arc; + use tokio_io_timeout::TimeoutStream; + + // ============================================================ + // Pure unit tests (no network) + // ============================================================ + + #[test] + fn test_generate_synthetic_uidvalidity_deterministic() { + let a = generate_synthetic_uidvalidity("INBOX"); + let b = generate_synthetic_uidvalidity("INBOX"); + assert_eq!(a, b, "same mailbox name must produce same uid_validity"); + } + + #[test] + fn test_generate_synthetic_uidvalidity_different_mailboxes() { + let inbox = generate_synthetic_uidvalidity("INBOX"); + let sent = generate_synthetic_uidvalidity("Sent"); + assert_ne!(inbox, sent, "different mailboxes should have different uid_validity"); + } + + #[test] + fn test_generate_synthetic_uidvalidity_non_zero() { + let uid = generate_synthetic_uidvalidity("INBOX"); + assert_ne!(uid, 0, "UIDVALIDITY should not be 0 (reserved)"); + } + + // ============================================================ + // Integration tests (real IMAP server required) + // ============================================================ + // Fill in your IMAP server details below to run these tests. + // cargo test -p bichon-core -- --ignored + + const TEST_IMAP_HOST: &str = "imap.zoho.com"; + const TEST_IMAP_PORT: u16 = 993; + const TEST_IMAP_USERNAME: &str = ""; + const TEST_IMAP_PASSWORD: &str = ""; + const TEST_MAILBOX: &str = "INBOX"; + + /// Build a direct IMAP session bypassing the database-dependent + /// ImapConnectionManager, for local testing with real credentials. + async fn direct_connect( + host: &str, + port: u16, + username: &str, + password: &str, + ) -> Result>, String> { + use rustls::ClientConfig; + use rustls_pki_types::ServerName; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + + // Ensure a rustls crypto provider is installed (ring). + // May already be installed by production code; ignore duplicate. + rustls::crypto::CryptoProvider::install_default( + rustls::crypto::ring::default_provider(), + ) + .ok(); + + let tcp = TcpStream::connect((host, port)) + .await + .map_err(|e| format!("TCP connect error: {e}"))?; + + // Wrap in Pin>> to satisfy SessionStream, + // matching the production path in establish_tcp_connection_with_timeout. + let timeout_stream = TimeoutStream::new(tcp); + let pinned = Box::pin(timeout_stream); + + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|e| format!("Invalid hostname: {e}"))?; + + let config = ClientConfig::builder() + .with_root_certificates(rustls::RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.into(), + }) + .with_no_client_auth(); + + let connector = TlsConnector::from(Arc::new(config)); + let tls_stream = connector + .connect(server_name, pinned) + .await + .map_err(|e| format!("TLS error: {e}"))?; + + let client = async_imap::Client::new(Box::new(tls_stream) as Box); + let session = client + .login(username, password) + .await + .map_err(|(e, _)| format!("Login error: {e}"))?; + + Ok(session) + } + + /// Test STATUS UIDVALIDITY directly — verifies your server returns + /// a valid UIDVALIDITY for the given mailbox. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_status_uid_validity_present() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(UIDVALIDITY)") + .await + .expect("STATUS command should succeed"); + + session.logout().await.ok(); + + match status.uid_validity { + Some(uid) => { + println!("[OK] Server returned UIDVALIDITY: {uid}"); + assert_ne!(uid, 0); + } + None => { + println!("[INFO] Server returned no UIDVALIDITY in STATUS response"); + println!(" This server may need the synthetic fallback or the retry logic."); + } } } - Ok(()) + /// Test STATUS with full attributes (MESSAGES UNSEEN UIDNEXT UIDVALIDITY), + /// mimicking the actual listing flow in mailbox::list::fetch_remote_with_progress. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_status_full_attributes() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(MESSAGES UNSEEN UIDNEXT UIDVALIDITY)") + .await + .expect("STATUS with full attributes should succeed"); + + session.logout().await.ok(); + + println!("MESSAGES: {:?}", status.exists); + println!("UNSEEN: {:?}", status.unseen); + println!("UIDNEXT: {:?}", status.uid_next); + println!("UIDVALIDITY: {:?}", status.uid_validity); + } + + /// Simulate the retry flow: call STATUS multiple times and verify + /// UIDVALIDITY is consistently returned (or consistently absent). + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_uid_validity_consistency_over_multiple_status_calls() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let results: Vec> = Vec::with_capacity(5); + let results = std::cell::RefCell::new(results); + + for i in 0..5 { + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(UIDVALIDITY)") + .await + .expect("STATUS should succeed"); + + session.logout().await.ok(); + + println!( + "Call {}: UIDVALIDITY = {:?}", + i + 1, + status.uid_validity + ); + results.borrow_mut().push(status.uid_validity); + } + + let results = results.into_inner(); + let first = results[0]; + let all_same = results.iter().all(|r| *r == first); + if all_same { + println!("[OK] All 5 STATUS calls returned consistent UIDVALIDITY: {first:?}"); + } else { + println!("[WARN] Inconsistent UIDVALIDITY across calls: {results:?}"); + println!(" Network jitter or server-side changes detected."); + } + } + + /// Test `fetch_uid_metadata` against a real IMAP server. + /// Fetches a few UIDs from the configured mailbox and verifies + /// that every returned UID has a valid, non-empty Message-ID. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_fetch_uid_metadata_real_server() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + // Examine to enter the mailbox + session + .examine(TEST_MAILBOX) + .await + .expect("EXAMINE should succeed"); + + // Find actual UIDs via SEARCH (don't assume contiguous 1..N) + let all_uids: Vec = { + let mut v: Vec = session + .uid_search("ALL") + .await + .expect("UID SEARCH ALL should succeed") + .into_iter() + .collect(); + v.sort(); + v + }; + + if all_uids.is_empty() { + eprintln!("SKIP: mailbox is empty, nothing to fetch"); + session.logout().await.ok(); + return; + } + + // Take up to 5 UIDs for a quick test + let sample: Vec = all_uids.into_iter().take(5).collect(); + let uid_set = compress_uid_list(sample.clone()); + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + &uid_set, + CancellationToken::new(), + ) + .await + .expect("fetch_uid_metadata should succeed"); + + session.logout().await.ok(); + + println!( + "[OK] Fetched {} UIDs from mailbox '{}':", + result.len(), + TEST_MAILBOX + ); + for uid in sample.iter() { + let mid = result + .get(uid) + .map(|o| o.as_deref().unwrap_or("")) + .unwrap_or(""); + println!(" UID {uid}: {mid}"); + } + + // Every UID we requested must be present with a valid Message-ID + for uid in &sample { + let msg_id = result + .get(uid) + .unwrap_or_else(|| panic!("UID {uid} missing from result")); + let mid = msg_id + .as_deref() + .unwrap_or_else(|| panic!("UID {uid} returned None Message-ID")); + assert!(!mid.is_empty(), "UID {uid} returned empty Message-ID"); + assert!( + mid.contains('@'), + "UID {uid}: Message-ID '{mid}' does not look like a valid Message-ID" + ); + } + } + + // ============================================================ + // Unit tests for fetch_uid_validity_with_retry_inner + // (pure logic, no network needed) + // ============================================================ + + /// Mock helper: returns the given results in sequence, then always None. + fn mock_results( + results: Vec>>, + ) -> impl FnMut() -> std::future::Ready>> { + let mut iter = results.into_iter(); + move || std::future::ready(iter.next().unwrap_or(Ok(None))) + } + + #[tokio::test] + async fn test_retry_first_attempt_succeeds() { + let result = fetch_uid_validity_with_retry_inner(3, mock_results(vec![Ok(Some(42))])) + .await; + assert_eq!(result.unwrap(), Some(42)); + } + + #[tokio::test] + async fn test_retry_succeeds_after_two_nones() { + // First two attempts return None, third returns Some + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![Ok(None), Ok(None), Ok(Some(99))]), + ) + .await; + assert_eq!(result.unwrap(), Some(99)); + } + + #[tokio::test] + async fn test_retry_succeeds_after_error_then_none() { + // Error, then None, then success + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![ + Err(raise_error!("boom".into(), ErrorCode::NetworkError)), + Ok(None), + Ok(Some(7)), + ]), + ) + .await; + assert_eq!(result.unwrap(), Some(7)); + } + + #[tokio::test] + async fn test_retry_returns_none_after_all_retries_exhausted() { + // All attempts return None + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![Ok(None), Ok(None), Ok(None)]), + ) + .await; + assert_eq!(result.unwrap(), None); + } + + #[tokio::test] + async fn test_retry_returns_none_when_all_attempts_error() { + // All attempts return Err — still returns Ok(None), not propagating the error + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![ + Err(raise_error!("e1".into(), ErrorCode::NetworkError)), + Err(raise_error!("e2".into(), ErrorCode::NetworkError)), + Err(raise_error!("e3".into(), ErrorCode::NetworkError)), + ]), + ) + .await; + assert!(result.is_ok()); + assert_eq!(result.unwrap(), None); + } + + #[tokio::test] + async fn test_retry_respects_max_retries() { + // max_retries=5, success on 5th attempt + let result = fetch_uid_validity_with_retry_inner( + 5, + mock_results(vec![ + Ok(None), + Ok(None), + Ok(None), + Ok(None), + Ok(Some(5)), + ]), + ) + .await; + assert_eq!(result.unwrap(), Some(5)); + } + + #[tokio::test] + async fn test_retry_stops_at_max_retries() { + // max_retries=2, third value is Some but should never be reached + let result = fetch_uid_validity_with_retry_inner( + 2, + mock_results(vec![Ok(None), Ok(None), Ok(Some(42))]), + ) + .await; + assert_eq!(result.unwrap(), None, "should stop after max_retries (2)"); + } + + #[tokio::test] + async fn test_retry_max_retries_zero() { + // max_retries=0 means no attempts at all + let result = fetch_uid_validity_with_retry_inner( + 0, + mock_results(vec![Ok(Some(42))]), + ) + .await; + assert_eq!(result.unwrap(), None); + } + + // ============================================================ + // Mock IMAP server integration tests + // ============================================================ + + use crate::imap::mock_server::{ + examine_response, uid_fetch_metadata_response, uid_fetch_rfc822_response, + minimal_eml, MockImapServer, MockImapServerHandle, + }; + + /// Build an `async_imap::Session` connected to the mock server, + /// authenticated and with the given mailbox examined. + async fn mock_session( + handle: &MockImapServerHandle, + ) -> async_imap::Session> { + let tcp = tokio::net::TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let timeout_stream = TimeoutStream::new(tcp); + let pinned: std::pin::Pin>> = + Box::pin(timeout_stream); + let stream: Box = Box::new(pinned); + let mut client = async_imap::Client::new(stream); + + // Read greeting + client.read_response().await.unwrap(); + + // Login + let mut session = client.login("user", "pass").await.map_err(|(e, _)| { + panic!("Login failed: {e:?}") + }).unwrap(); + + // Examine + session.examine("INBOX").await.unwrap(); + + session + } + + #[tokio::test] + async fn fetch_uid_metadata_with_mock_server() { + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 3, 42, 4)) + .respond("UID FETCH", uid_fetch_metadata_response(&[ + (1, ""), + (2, ""), + (3, ""), + ])) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:3", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!( + result.get(&1).unwrap().as_deref(), + Some("msg-a@test.com") + ); + assert_eq!( + result.get(&2).unwrap().as_deref(), + Some("msg-b@test.com") + ); + assert_eq!( + result.get(&3).unwrap().as_deref(), + Some("msg-c@test.com") + ); + + session.logout().await.ok(); + } + + /// Verify basic UID FETCH (FLAGS only, no body) works with the mock server. + #[tokio::test] + async fn fetch_uid_flags_with_mock_server() { + // Response with just UID and FLAGS — no body literal parsing needed. + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 2, 42, 3)) + .respond( + "UID FETCH", + b"* 1 FETCH (UID 1 FLAGS (\\Seen))\r\n\ +* 2 FETCH (UID 2 FLAGS (\\Flagged))\r\n\ +{TAG} OK FETCH completed\r\n" + .to_vec(), + ) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let uids: Vec = { + let mut stream = session + .uid_fetch("1:2", "(UID FLAGS)") + .await + .unwrap(); + + use futures::TryStreamExt; + let mut uids = Vec::new(); + while let Some(fetch) = stream.try_next().await.unwrap() { + uids.push(fetch.uid.unwrap_or(0)); + } + uids + }; + + assert_eq!(uids, vec![1, 2]); + + session.logout().await.ok(); + } + + /// Verify UID FETCH with BODY[] (full RFC822) works with the mock server. + #[tokio::test] + async fn fetch_uid_rfc822_with_mock_server() { + let eml = minimal_eml("Test Subject", "test@example.com"); + let eml_len = eml.len(); + + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 1, 42, 2)) + .respond("UID FETCH", uid_fetch_rfc822_response(1, &eml)) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let bodies: Vec<(u32, Vec)> = { + let mut stream = session + .uid_fetch("1:1", "(UID BODY[])") + .await + .unwrap(); + + use futures::TryStreamExt; + let mut bodies = Vec::new(); + while let Some(fetch) = stream.try_next().await.unwrap() { + let uid = fetch.uid.unwrap_or(0); + let body = fetch.body().map(|b| b.to_vec()).unwrap_or_default(); + bodies.push((uid, body)); + } + bodies + }; + + assert_eq!(bodies.len(), 1); + assert_eq!(bodies[0].0, 1); + assert_eq!(bodies[0].1.len(), eml_len); + assert!(String::from_utf8_lossy(&bodies[0].1).contains("Test Subject")); + + session.logout().await.ok(); + } + + #[tokio::test] + async fn fetch_uid_metadata_empty_mailbox() { + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 0, 42, 1)) + .respond("UID FETCH", b"{TAG} OK FETCH completed\r\n".to_vec()) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:*", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert!( + result.is_empty(), + "empty mailbox should return empty map" + ); + + session.logout().await.ok(); + } + + #[tokio::test] + async fn fetch_uid_metadata_missing_message_id() { + // One entry has a Message-ID, the other has no header at all. + let header_with_msgid = + "From: sender@example.com\r\n\ + Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ + Message-ID: \r\n\r\n"; + let header_without_msgid = "\r\n"; + let len1 = header_with_msgid.len(); + let len2 = header_without_msgid.len(); + + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 2, 42, 3)) + .respond( + "UID FETCH", + format!( + "* 1 FETCH (UID 1 BODY[HEADER] {{{len1}}}\r\n\ +{header_with_msgid})\r\n\ +* 2 FETCH (UID 2 BODY[HEADER] {{{len2}}}\r\n\ +{header_without_msgid})\r\n\ +{{TAG}} OK FETCH completed\r\n" + ) + .into_bytes(), + ) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:2", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!( + result.get(&1).unwrap().as_deref(), + Some("ok@test.com") + ); + // UID 2 has no Message-ID header → None + assert_eq!(result.get(&2).unwrap().as_deref(), None); + + session.logout().await.ok(); + } } diff --git a/crates/core/src/cache/imap/download/rebuild.rs b/crates/core/src/cache/imap/download/rebuild.rs index 85c175d3..96d31d9e 100644 --- a/crates/core/src/cache/imap/download/rebuild.rs +++ b/crates/core/src/cache/imap/download/rebuild.rs @@ -89,7 +89,11 @@ pub async fn rebuild_cache( }; match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -169,7 +173,11 @@ pub async fn rebuild_cache_by_date( match fetch_and_save_by_date(&account, date.as_str(), &mailbox, direction, token.clone()) .await { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -196,7 +204,7 @@ pub async fn rebuild_mailbox_cache( local_mailbox: &MailBox, remote_mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { ENVELOPE_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox.id]) .await?; @@ -217,11 +225,11 @@ pub async fn rebuild_mailbox_cache( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; - Ok(()) + let result = fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; + Ok(result) } pub async fn rebuild_mailbox_cache_by_date( @@ -231,7 +239,7 @@ pub async fn rebuild_mailbox_cache_by_date( remote: &MailBox, direction: FetchDirection, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { ENVELOPE_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox_id]) .await?; @@ -252,9 +260,9 @@ pub async fn rebuild_mailbox_cache_by_date( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - fetch_and_save_by_date(account, date, remote, direction, token).await?; - Ok(()) + let result = fetch_and_save_by_date(account, date, remote, direction, token).await?; + Ok(result) } diff --git a/crates/core/src/cache/imap/mailbox.rs b/crates/core/src/cache/imap/mailbox.rs index b85e7121..db11e4e2 100644 --- a/crates/core/src/cache/imap/mailbox.rs +++ b/crates/core/src/cache/imap/mailbox.rs @@ -56,6 +56,10 @@ pub struct MailBox { /// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions. /// If `None`, the IMAP server has not provided this information. pub uid_validity: Option, + /// The highest UID that has been successfully downloaded and stored locally. + /// Used for incremental sync: next fetch starts from `highest_uid + 1`. + /// If `None`, a fallback query against the Tantivy index will be performed once. + pub highest_uid: Option, } impl MemDbModel for MailBox { diff --git a/crates/core/src/cache/imap/task.rs b/crates/core/src/cache/imap/task.rs index 0bac419c..66659ef1 100644 --- a/crates/core/src/cache/imap/task.rs +++ b/crates/core/src/cache/imap/task.rs @@ -113,6 +113,9 @@ impl AccountDownTask { let account = AccountModel::get(account_id).ok(); match account { Some(account) => { + if account.deleting { + return Ok(()); + } if !account.enabled { let last = LAST_WARN_TIME.load(Ordering::Relaxed); let now = utc_now!(); @@ -246,6 +249,10 @@ impl AccountDownTask { } }; + if account.deleting { + return; + } + if let Err(e) = process_imap_download(&account, token_clone, TriggerType::Manual).await { error!("Manual download failed for {}: {:?}", account_id, e); diff --git a/crates/core/src/database/manager.rs b/crates/core/src/database/manager.rs index afd0b98f..6e6f258f 100644 --- a/crates/core/src/database/manager.rs +++ b/crates/core/src/database/manager.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::settings::dir::DATA_DIR_MANAGER; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use std::sync::LazyLock; use std::time::Duration; diff --git a/crates/core/src/database/mod.rs b/crates/core/src/database/mod.rs index c7a6cc82..778fca1f 100644 --- a/crates/core/src/database/mod.rs +++ b/crates/core/src/database/mod.rs @@ -20,7 +20,7 @@ use crate::common::paginated::Paginated; use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::raise_error; -use memdb::{MemDb, Transaction}; +use bichon_memdb::{MemDb, Transaction}; use serde::de::DeserializeOwned; use serde::Serialize; diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 0839bf73..88195a72 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use crate::account::migration::AccountModel; use crate::cache::imap::mailbox::MailBox; use crate::common::AddrVec; use crate::envelope::meta::parse_bichon_metadata; @@ -26,6 +27,7 @@ use crate::imap::executor::ImapExecutor; use crate::message::content::AttachmentInfo; use crate::store::blob::{DetachedEmail, BLOB_MANAGER}; use crate::store::tantivy::attachment::ATTACHMENT_MANAGER; +use crate::store::tantivy::dedup_cache::DEDUP_CACHE; use crate::store::tantivy::envelope::ENVELOPE_MANAGER; use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments}; use crate::utils::html::extract_text; @@ -50,9 +52,17 @@ pub async fn extract_envelope_and_store_it( .map(|d| d.timestamp_millis()) .unwrap_or(0); let uid = fetch.uid.unwrap_or(0); - let body = fetch - .body() - .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?; + let body = match fetch.body() { + Some(b) => b, + None => { + tracing::warn!( + account_id, + uid = fetch.uid, + "FETCH response has no body, skipping message" + ); + return Ok(()); + } + }; let size = fetch.size.unwrap_or(body.len() as u32); extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id).await } @@ -91,6 +101,11 @@ async fn extract_envelope_core( ) -> BichonResult<()> { //The content hash of the original raw EML let email_content_hash = compute_content_hash(body); + if DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) { + tracing::debug!("Duplicate email detected"); + //println!("Duplicate email detected"); + return Ok(()); + } let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| { raise_error!( "Email header parse result is not available".into(), @@ -98,6 +113,34 @@ async fn extract_envelope_core( ) })?; + if let Ok(account) = AccountModel::get(account_id) { + if let Some(ref rules) = account.archive_rules { + let sender = message.from().and_then(|addr| { + AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address) + }); + let subject = message.subject().map(|s| s.to_string()); + + let is_spam = !rules.spam_headers.is_empty() + && rules.spam_headers.iter().any(|h| { + message + .header_raw(h.clone()) + .map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true")) + .unwrap_or(false) + }); + + if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) { + tracing::debug!( + account_id, + uid, + sender = sender.as_deref().unwrap_or("?"), + subject = subject.as_deref().unwrap_or("?"), + "Email filtered out by archive rules" + ); + return Ok(()); + } + } + } + let preview_limit = 100; let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) { text @@ -159,7 +202,7 @@ async fn extract_envelope_core( .and_then(|add| add.address) .unwrap_or_else(|| "unknown".to_string()); let attachment_count = message.attachment_count(); - let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await; + let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await; let envelope_id = Uuid::new_v4().to_string(); let now = utc_now!(); @@ -252,7 +295,8 @@ async fn extract_envelope_core( tags: (!final_tags.is_empty()).then_some(final_tags), account_email: None, mailbox_name: None, - content_hash: email_content_hash, + content_hash: email_content_hash.clone(), + account_name: None, }; // 'attachments' contains both regular and inline attachments let ea = EnvelopeWithAttachments { @@ -269,6 +313,7 @@ async fn extract_envelope_core( &ea.envelope.content_hash, ); ENVELOPE_MANAGER.queue(doc).await; + DEDUP_CACHE.insert(account_id, mailbox_id, &email_content_hash); for doc in attachment_docs { ATTACHMENT_MANAGER.queue(doc).await; } @@ -346,6 +391,7 @@ pub fn extract_envelope_from_nested_message( regular_attachment_count: Default::default(), tags: Default::default(), account_email: Default::default(), + account_name: Default::default(), mailbox_name: Default::default(), content_hash: Default::default(), }; @@ -384,7 +430,27 @@ pub async fn detach_and_store_attachments( original_body: &[u8], message: &Message<'_>, eml_content_hash: &str, + account_id: u64, + mailbox_id: u64, ) -> Vec { + let rules = if account_id > 0 { + AccountModel::get(account_id) + .ok() + .and_then(|a| a.extraction_rules) + } else { + None + }; + + let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) { + Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name), + _ => None, + }; + + let sender = message + .from() + .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) + .and_then(|add| add.address); + let mut stripped_eml = original_body.to_vec(); let mut attachment_infos = Vec::new(); // Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity @@ -412,23 +478,36 @@ pub async fn detach_and_store_attachments( let mut text_candidates: Vec = Vec::new(); for (raw_start, raw_end, att) in ranges { - // Step 2: Extract raw bytes and store them as standalone documents - let raw_bytes = &original_body[raw_start..raw_end]; - //This is the content hash of the decoded attachment, not the undecoded one. + // mail-parser may report attachment offsets past the body end for + // malformed messages; clamp the range to avoid a slice panic. + let body_len = original_body.len(); + let raw_start = raw_start.min(body_len); + let raw_end = raw_end.min(body_len); + let range_valid = raw_start < raw_end; + + // content hash is computed from the decoded attachment contents, + // which is always available regardless of raw offset validity. let content_hash = compute_content_hash(att.contents()); - //"The actual content stored in the blob is the raw undecoded data, to avoid the reconstructed EML differing from the original due to decoding and re-encoding. - attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes)));// + if range_valid { + let raw_bytes = &original_body[raw_start..raw_end]; + // The actual content stored in the blob is the raw undecoded data. + attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes))); - // Step 3: Replace raw attachment content with a hash-based placeholder - let placeholder = format!("<>", &content_hash); - let p_bytes = placeholder.as_bytes(); - stripped_eml.splice(raw_start..raw_end, p_bytes.iter().cloned()); + // Replace raw attachment content with a hash-based placeholder + let placeholder = format!("<>", &content_hash); + stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + } else { + // Invalid range: store a zero-length blob so the consistency + // check passes; reattachment will log a warning for the missing + // blob data but won't panic. + attachments.push((content_hash.clone(), Bytes::new())); + } let inline = att .content_disposition() .map(|d| d.is_inline()) - .unwrap_or(false); + .unwrap_or_else(|| att.content_id().is_some()); let file_type = att .content_type() .map(|ct| { @@ -440,19 +519,30 @@ pub async fn detach_and_store_attachments( }) .unwrap_or_else(|| "application/octet-stream".to_string()); let has_cid = att.content_id().is_some(); - let ext = att - .attachment_name() + let att_name = att.attachment_name().map(|n| n.to_string()); + let ext = att_name + .as_deref() .and_then(|n| { - std::path::Path::new(&n) + std::path::Path::new(n) .extension() .and_then(|e| e.to_str()) .map(|s| s.to_ascii_lowercase()) }) .unwrap_or_default(); + let should_extract = rules.as_ref().map_or(true, |r| { + r.should_extract( + &ext, + mailbox_name.as_deref(), + att_name.as_deref(), + sender.as_deref(), + ) + }); + if !inline || !has_cid { let decoded_len = att.contents().len(); - if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES + if should_extract + && decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES && crate::ext::text_extractor::should_try_extract(&file_type, &ext) { text_candidates.push(TextCandidate { @@ -703,7 +793,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult { ErrorCode::InternalError ) })?; - detach_and_store_attachments(&raw_body, &message, &fetched_hash).await; + detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await; Ok(Bytes::from(raw_body)) } @@ -755,4 +845,58 @@ mod test { } } } + + /// Verifies that [`super::detach_and_store_attachments`] does not panic + /// when mail-parser reports attachment offsets past the raw body length. + /// + /// Regression test for: "range end index X out of range for slice of + /// length Y" panic caused by a malformed email whose attachment + /// `raw_end_offset` exceeded the actual body size. + #[tokio::test] + async fn detach_attachments_bounds_check() { + let raw = concat!( + "From: sender@example.com\r\n", + "To: recipient@example.com\r\n", + "Subject: Test\r\n", + "MIME-Version: 1.0\r\n", + "Content-Type: multipart/mixed; boundary=\"bnd\"\r\n", + "\r\n", + "--bnd\r\n", + "Content-Type: text/plain\r\n", + "\r\n", + "Hello\r\n", + "--bnd\r\n", + "Content-Type: application/octet-stream\r\n", + "Content-Disposition: attachment; filename=\"test.bin\"\r\n", + "\r\n", + "AAAAABBBBBCCCCCDDDDDEEEEEAAAAABBBBBCCCCCDDDDDEEEEE\r\n", + "--bnd--\r\n", + ) + .as_bytes() + .to_vec(); + + let message = mail_parser::MessageParser::new() + .parse(&raw) + .expect("parse valid MIME message"); + assert_eq!(message.attachment_count(), 1); + + // Truncate the raw body so the attachment's raw_end_offset lies + // past the body end — exactly the scenario reported by users. + let truncated = &raw[..raw.len() - 20]; + assert!(truncated.len() < raw.len()); + + // Must not panic. + let infos = super::detach_and_store_attachments( + truncated, + &message, + "test_content_hash", + 0, + 0, + ) + .await; + + // The attachment count must still match so the consistency check + // in reattach_eml_content doesn't fail later. + assert_eq!(infos.len(), 1); + } } diff --git a/crates/core/src/error/mod.rs b/crates/core/src/error/mod.rs index edf360cf..27fad8ea 100644 --- a/crates/core/src/error/mod.rs +++ b/crates/core/src/error/mod.rs @@ -16,4 +16,12 @@ pub enum BichonError { }, } +impl BichonError { + pub fn code(&self) -> ErrorCode { + match self { + BichonError::Generic { code, .. } => *code, + } + } +} + pub type BichonResult = std::result::Result; diff --git a/crates/core/src/ext/text_extractor.rs b/crates/core/src/ext/text_extractor.rs index baf1243e..3a3a4af1 100644 --- a/crates/core/src/ext/text_extractor.rs +++ b/crates/core/src/ext/text_extractor.rs @@ -63,8 +63,18 @@ pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024; pub fn should_try_extract(content_type: &str, ext: &str) -> bool { matches!( ext, - "pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" - | "txt" | "rtf" | "odt" | "ods" | "odp" + "pdf" + | "doc" + | "docx" + | "xls" + | "xlsx" + | "ppt" + | "pptx" + | "txt" + | "rtf" + | "odt" + | "ods" + | "odp" ) || content_type.starts_with("text/") } diff --git a/crates/core/src/imap/client.rs b/crates/core/src/imap/client.rs index 1fd70b04..85cbceea 100644 --- a/crates/core/src/imap/client.rs +++ b/crates/core/src/imap/client.rs @@ -34,6 +34,25 @@ use std::ops::DerefMut; use tokio::io::BufWriter; use tracing::debug; +/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors. +/// `UnexpectedEof` is treated as a network error because many servers skip +/// the TLS `close_notify` alert, causing rustls to emit this error when the +/// TCP connection is dropped normally. +fn classify_io_error(e: &std::io::Error) -> ErrorCode { + use std::io::ErrorKind; + matches!( + e.kind(), + ErrorKind::BrokenPipe + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::TimedOut + | ErrorKind::UnexpectedEof + | ErrorKind::NotConnected + ) + .then_some(ErrorCode::NetworkError) + .unwrap_or(ErrorCode::ImapCommandFailed) +} + #[derive(Debug)] pub(crate) struct Client { inner: ImapClient>, @@ -141,7 +160,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(), @@ -171,7 +190,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "failed to read greeting".into(), @@ -202,7 +221,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(), diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 422dfedc..fa74cacd 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -17,8 +17,7 @@ // along with this program. If not, see . use crate::account::migration::AccountModel; -use crate::account::state::{DownloadState, FolderStatus}; -use crate::cache::imap::download::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE}; +use crate::account::state::{DownloadState, DownloadStatus, FolderStatus}; use crate::cache::imap::mailbox::MailBox; use crate::envelope::extractor::extract_envelope_and_store_it; use crate::error::code::ErrorCode; @@ -28,11 +27,29 @@ use crate::{error::BichonResult, imap::manager::ImapConnectionManager}; use async_imap::types::Name; use async_imap::Session; use futures::TryStreamExt; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tokio_util::sync::CancellationToken; use tracing::info; const BODY_FETCH_COMMAND: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[])"; +const SIZE_ONLY_FETCH: &str = "(UID RFC822.SIZE)"; + +fn classify_imap_error(e: &async_imap::error::Error) -> ErrorCode { + match e { + async_imap::error::Error::Io(io) => matches!( + io.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + ) + .then_some(ErrorCode::NetworkError) + .unwrap_or(ErrorCode::ImapCommandFailed), + async_imap::error::Error::ConnectionLost => ErrorCode::NetworkError, + _ => ErrorCode::ImapCommandFailed, + } +} pub struct ImapExecutor; @@ -43,11 +60,11 @@ impl ImapExecutor { let list = session .list(Some(""), Some("*")) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let result = list .try_collect::>() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; Ok(result) } @@ -59,11 +76,11 @@ impl ImapExecutor { session .examine(mailbox_name) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let result = session .uid_search(query) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; Ok(result) } @@ -77,132 +94,337 @@ impl ImapExecutor { session .append(mailbox_name, flags, internaldate, content) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)) + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e))) } + /// Fetches new mail for a mailbox. + /// + /// When `before` is `Some(date)`, a two-step approach is used: + /// `UID SEARCH` to find matching UIDs (standard IMAP), then batch `UID FETCH` + /// for the specific UIDs. When `before` is `None`, bounded ranges are batched + /// when the server supplied `UIDNEXT`; otherwise a direct ranged + /// `UID FETCH {start}:*` is issued and results are streamed. + /// + /// Returns `Ok(Some(max_uid))` with the highest UID fetched, or `Ok(None)` + /// if no new mail was found. pub async fn fetch_new_mail( session: &mut Session>, account: &AccountModel, mailbox: &MailBox, start_uid: u64, + end_uid: Option, before: Option<&str>, token: CancellationToken, - ) -> BichonResult<()> { + ) -> BichonResult> { assert!(start_uid > 0, "start_uid must be greater than 0"); - let query = match before { - Some(date) => format!("UID {start_uid}:* BEFORE {date}"), - None => format!("UID {start_uid}:*"), - }; + session + .examine(&mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; - let uid_list = match Self::uid_search(session, &mailbox.encoded_name(), &query).await { - Ok(uid_list) => uid_list, - Err(e) => { - let err_msg = format!("UID search failed in [{}]: {:#?}", mailbox.name, e); - DownloadState::update_folder_progress( - account.id, - mailbox.name.clone(), - 0, - 0, - FolderStatus::Failed, - Some(err_msg.clone()), - )?; - DownloadState::append_session_error(account.id, err_msg)?; - return Err(e); + match before { + Some(date) => { + Self::fetch_new_mail_with_before(session, account, mailbox, start_uid, date, token) + .await } - }; + None => { + Self::fetch_new_mail_range(session, account, mailbox, start_uid, end_uid, token) + .await + } + } + } - let len = uid_list.len(); - if len == 0 { - let msg = match before { - Some(date) => format!("No emails found before {}.", date), - None => "No new emails found.".into(), - }; + /// Two-step approach for date-filtered incremental fetch: UID SEARCH first, + /// then batch UID FETCH for matching UIDs. Uses standard IMAP syntax that + /// works across all compliant servers. + async fn fetch_new_mail_with_before( + session: &mut Session>, + account: &AccountModel, + mailbox: &MailBox, + start_uid: u64, + date: &str, + token: CancellationToken, + ) -> BichonResult> { + let query = format!("UID {start_uid}:* BEFORE {date}"); + info!( + "[account {}][mailbox {}] fetch_new_mail: UID SEARCH {}", + account.id, mailbox.name, query + ); + let results = session.uid_search(&query).await.map_err(|e| { + let err_msg = format!("UID SEARCH failed in [{}]: {:#?}", mailbox.name, e); + let _ = DownloadState::append_session_error(account.id, err_msg); + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) + })?; + + if results.is_empty() { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), 0, 0, FolderStatus::Success, - Some(msg), + Some("No new emails found.".into()), )?; - return Ok(()); + return Ok(None); } - info!( - "[account {}][mailbox {}] {} envelopes need to be fetched (start_uid={})", - account.id, mailbox.name, len, start_uid - ); - tracing::debug!( - "[account {}][mailbox {}] fetch_new_mail UID range: {}..{} ({} uids)", - account.id, - mailbox.name, - start_uid, - uid_list.iter().max().unwrap_or(&0), - len - ); - let mut uid_vec: Vec = uid_list.into_iter().collect(); + let mut uid_vec: Vec = results.into_iter().collect(); uid_vec.sort(); - let uid_batches = generate_uid_sequence_hashset( - uid_vec, - account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize, - false, - ); - let mut current_processed = 0u64; - let mut has_error_or_cancel = false; - for (index, batch) in uid_batches.into_iter().enumerate() { + let max_uid = uid_vec.last().copied(); + let planned = uid_vec.len() as u64; + let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let uid_batches = generate_uid_sequence_hashset(uid_vec, batch_size); + + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + 0, + FolderStatus::Pending, + None, + )?; + + let mut count = 0u64; + for batch in uid_batches { if token.is_cancelled() { - break; + DownloadState::update_session_status( + account.id, + DownloadStatus::Cancelled, + Some("User stopped or system shutdown".to_string()), + )?; + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + count, + FolderStatus::Cancelled, + None, + )?; + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); } - match Self::uid_batch_retrieve_emails( + let processed = Self::uid_batch_retrieve_emails( session, account.id, mailbox.id, &batch.0, + account.max_email_size_bytes, token.clone(), ) - .await - { - Ok(_) => { - current_processed += batch.1; - DownloadState::update_folder_progress( + .await?; + count += processed; + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + count, + FolderStatus::Downloading, + None, + )?; + } + + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + count, + count, + FolderStatus::Success, + None, + )?; + + Ok(max_uid) + } + + /// Ranged UID FETCH without date filtering. If `end_uid` is known, split the + /// range into bounded batches so large backlogs cannot occupy one long-lived + /// IMAP stream. If `end_uid` is unavailable, preserve the legacy unbounded + /// streaming behavior. + async fn fetch_new_mail_range( + session: &mut Session>, + account: &AccountModel, + mailbox: &MailBox, + start_uid: u64, + end_uid: Option, + token: CancellationToken, + ) -> BichonResult> { + if let Some(end_uid) = end_uid { + if start_uid > end_uid as u64 { + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + Some("No new emails found.".into()), + )?; + return Ok(None); + } + + let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let uid_batches = generate_uid_range_batches(start_uid, end_uid, batch_size); + let planned = uid_batches.iter().map(|(_, count, _)| *count).sum(); + + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + 0, + FolderStatus::Pending, + None, + )?; + + let mut count = 0u64; + let mut max_uid: Option = None; + for (uid_range, _, batch_end) in uid_batches { + if token.is_cancelled() { + tracing::info!("Account {}: fetch_new_mail batch interrupted.", account.id); + DownloadState::update_session_status( account.id, - mailbox.name.clone(), - len as u64, - current_processed, - FolderStatus::Downloading, - None, + DownloadStatus::Cancelled, + Some("User stopped or system shutdown".to_string()), )?; - } - Err(e) => { - let err_msg = format!("Batch {} failed: {:#?}", index, e); - DownloadState::append_session_error(account.id, err_msg.clone())?; DownloadState::update_folder_progress( account.id, mailbox.name.clone(), - len as u64, - current_processed, - FolderStatus::Failed, - Some(err_msg), + planned, + count, + FolderStatus::Cancelled, + None, )?; - has_error_or_cancel = true; - break; + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); + } + + info!( + "[account {}][mailbox {}] fetch_new_mail: batched UID FETCH {}", + account.id, mailbox.name, uid_range + ); + let processed = Self::uid_batch_retrieve_emails( + session, + account.id, + mailbox.id, + &uid_range, + account.max_email_size_bytes, + token.clone(), + ) + .await?; + count += processed; + if processed > 0 { + max_uid = Some(batch_end); } + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + count, + FolderStatus::Downloading, + None, + )?; } - } - if !has_error_or_cancel { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), - len as u64, - current_processed, + count, + count, FolderStatus::Success, None, )?; + + return Ok(max_uid); + } + + let uid_range = format!("{start_uid}:*"); + info!( + "[account {}][mailbox {}] fetch_new_mail: direct UID FETCH {}", + account.id, mailbox.name, uid_range + ); + + let mut stream = session + .uid_fetch(&uid_range, BODY_FETCH_COMMAND) + .await + .map_err(|e| { + let err_msg = format!("UID FETCH failed in [{}]: {:#?}", mailbox.name, e); + let _ = DownloadState::append_session_error(account.id, err_msg); + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) + })?; + + let mut count = 0u64; + let mut skipped = 0u64; + let mut max_uid: Option = None; + let size_limit = account + .max_email_size_bytes + .unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + while let Some(fetch) = stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { + if token.is_cancelled() { + tracing::info!("Account {}: fetch_new_mail stream interrupted.", account.id); + DownloadState::update_session_status( + account.id, + DownloadStatus::Cancelled, + Some("User stopped or system shutdown".to_string()), + )?; + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); + } + + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size > 0 && msg_size > size_limit { + tracing::warn!( + account_id = account.id, + mailbox_id = mailbox.id, + uid = fetch.uid, + size = msg_size, + limit = size_limit, + "Skipping oversized email (streaming mode)" + ); + skipped += 1; + continue; + } + + if let Some(uid) = fetch.uid { + max_uid = Some(max_uid.unwrap_or(0).max(uid)); + } + extract_envelope_and_store_it(fetch, account.id, mailbox.id).await?; + count += 1; + } + + let total = count + skipped; + if total == 0 { + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + Some("No new emails found.".into()), + )?; + } else { + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + total, + count, + FolderStatus::Success, + if skipped > 0 { + Some(format!("{skipped} email(s) skipped due to size limit")) + } else { + None + }, + )?; } - Ok(()) + Ok(max_uid) } pub async fn batch_retrieve_emails( @@ -213,48 +435,75 @@ impl ImapExecutor { page: u64, page_size: u64, encoded_mailbox_name: &str, - desc: bool, + max_email_size_bytes: Option, token: CancellationToken, + max_uid: &mut Option, ) -> BichonResult { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let (start, end) = if desc { - // Fetch messages starting from the newest (descending order) - let end = total.saturating_sub((page - 1) * page_size); - if end == 0 { - return Ok(0); - } - // Calculate start as end - page_size + 1 to avoid off-by-one errors - let start = end.saturating_sub(page_size - 1).max(1); - (start, end) - } else { - // Fetch messages starting from the oldest (ascending order) - let start = (page - 1) * page_size + 1; - if start > total { - return Ok(0); - } - // Calculate end, capped by the total number of messages - let end = (start + page_size - 1).min(total); - (start, end) - }; + // Fetch messages starting from the oldest (ascending order). + let start = (page - 1) * page_size + 1; + if start > total { + return Ok(0); + } + let end = (start + page_size - 1).min(total); let sequence_set = format!("{}:{}", start, end); info!( - "Fetching mailbox '{}' messages: sequence {} (page {}, page_size {}, desc={})", - encoded_mailbox_name, sequence_set, page, page_size, desc + "Fetching mailbox '{}' messages: sequence {} (page {}, page_size {})", + encoded_mailbox_name, sequence_set, page, page_size ); - let mut stream = session - .fetch(sequence_set.as_str(), BODY_FETCH_COMMAND) + let limit = max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + + // PASS 1: fetch only SIZE to identify oversized messages + let acceptable_uids = { + let mut size_stream = session + .fetch(sequence_set.as_str(), SIZE_ONLY_FETCH) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; + + let mut uids: Vec = Vec::new(); + while let Some(fetch) = size_stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { + let uid = fetch.uid.unwrap_or(0); + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size == 0 || msg_size <= limit { + uids.push(uid); + } else { + tracing::warn!( + account_id, + mailbox_id, + uid, + size = msg_size, + limit, + "Skipping oversized email" + ); + } + } + uids + }; + + if acceptable_uids.is_empty() { + return Ok(0); + } + + // PASS 2: fetch bodies only for acceptable UIDs + let filtered = compress_uid_list(acceptable_uids); + let mut body_stream = session + .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut count = 0; - while let Some(fetch) = stream + while let Some(fetch) = body_stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? { if token.is_cancelled() { tracing::info!("Account {}: UID fetch stream interrupted.", account_id); @@ -263,6 +512,9 @@ impl ImapExecutor { ErrorCode::InternalError )); } + if let Some(uid) = fetch.uid { + *max_uid = Some((*max_uid).unwrap_or(0).max(uid)); + } extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?; count += 1; } @@ -274,16 +526,58 @@ impl ImapExecutor { account_id: u64, mailbox_id: u64, uid_set: &str, + max_email_size_bytes: Option, token: CancellationToken, - ) -> BichonResult<()> { - let mut stream = session - .uid_fetch(uid_set, BODY_FETCH_COMMAND) + ) -> BichonResult { + let limit = max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + + // PASS 1: fetch only SIZE to identify oversized messages + let acceptable_uids = { + let mut size_stream = session + .uid_fetch(uid_set, SIZE_ONLY_FETCH) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; + + let mut uids: Vec = Vec::new(); + while let Some(fetch) = size_stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { + let uid = fetch.uid.unwrap_or(0); + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size == 0 || msg_size <= limit { + uids.push(uid); + } else { + tracing::warn!( + account_id, + mailbox_id, + uid, + size = msg_size, + limit, + "Skipping oversized email" + ); + } + } + uids + }; + + if acceptable_uids.is_empty() { + return Ok(0); + } + + // PASS 2: fetch bodies only for acceptable UIDs + let filtered = compress_uid_list(acceptable_uids); + let mut body_stream = session + .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; - while let Some(fetch) = stream + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; + + let mut count = 0u64; + while let Some(fetch) = body_stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? { if token.is_cancelled() { tracing::info!("Account {}: UID fetch stream interrupted.", account_id); @@ -293,8 +587,9 @@ impl ImapExecutor { )); } extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?; + count += 1; } - Ok(()) + Ok(count) } /// Fetches the raw RFC822 body of a single message by UID. @@ -310,17 +605,17 @@ impl ImapExecutor { session .examine(encoded_mailbox_name) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut stream = session .uid_fetch(uid.to_string(), BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let fetch = stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? .ok_or_else(|| { raise_error!( format!("UID {uid} not found on IMAP server"), @@ -342,7 +637,7 @@ impl ImapExecutor { // while stream // .try_next() // .await - // .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + // .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? // .is_some() // {} @@ -354,4 +649,302 @@ impl ImapExecutor { ) -> BichonResult>> { ImapConnectionManager::build(account_id).await } + + /// Fetch UID → Message-ID mapping without downloading bodies. + /// `uid_set` is an IMAP sequence-set string (e.g. "1:100" or "1,3,5"). + pub async fn fetch_uid_metadata( + session: &mut Session>, + uid_set: &str, + token: CancellationToken, + ) -> BichonResult>> { + let mut stream = session + .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER])") + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; + + let mut result = HashMap::new(); + while let Some(fetch) = stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { + if token.is_cancelled() { + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); + } + let uid = fetch.uid.unwrap_or(0); + let msg_id = fetch.header().and_then(parse_message_id_header); + result.insert(uid, msg_id); + } + Ok(result) + } +} + +pub const DEFAULT_BATCH_SIZE: u32 = 30; +pub const DEFAULT_MAX_EMAIL_SIZE: u64 = 100 * 1024 * 1024; + +/// Compresses a sorted list of UIDs into an IMAP sequence-set string. +/// Consecutive UIDs become ranges (e.g. `1:5`), non-consecutive are +/// comma-separated (e.g. `1:5,10,12:15`). +pub fn compress_uid_list(nums: Vec) -> String { + if nums.is_empty() { + return String::new(); + } + + let mut sorted_nums = nums; + sorted_nums.sort(); + + let mut result = Vec::new(); + let mut current_range_start = sorted_nums[0]; + let mut current_range_end = sorted_nums[0]; + + for &n in sorted_nums.iter().skip(1) { + if n == current_range_end + 1 { + current_range_end = n; + } else { + if current_range_start == current_range_end { + result.push(current_range_start.to_string()); + } else { + result.push(format!("{}:{}", current_range_start, current_range_end)); + } + current_range_start = n; + current_range_end = n; + } + } + + if current_range_start == current_range_end { + result.push(current_range_start.to_string()); + } else { + result.push(format!("{}:{}", current_range_start, current_range_end)); + } + + result.join(",") +} + +/// Splits a sorted list of unique UIDs into compressed sequence-set batches. +/// Returns `Vec<(sequence_set_string, batch_count)>`. +pub fn generate_uid_sequence_hashset( + unique_nums: Vec, + chunk_size: usize, +) -> Vec<(String, u64)> { + assert!(!unique_nums.is_empty()); + + let mut result = Vec::new(); + let nums = unique_nums; + + for chunk in nums.chunks(chunk_size) { + let size = chunk.len() as u64; + let compressed = compress_uid_list(chunk.to_vec()); + result.push((compressed, size)); + } + + result +} + +/// Splits a bounded inclusive UID range into IMAP sequence-set batches. +/// Returns `Vec<(sequence_set_string, batch_count, batch_end_uid)>`. +fn generate_uid_range_batches( + start_uid: u64, + end_uid: u32, + chunk_size: usize, +) -> Vec<(String, u64, u32)> { + assert!(chunk_size > 0); + + if start_uid == 0 || start_uid > end_uid as u64 || start_uid > u32::MAX as u64 { + return Vec::new(); + } + + let mut result = Vec::new(); + let mut start = start_uid as u32; + + while start <= end_uid { + let end = start + .saturating_add(chunk_size as u32) + .saturating_sub(1) + .min(end_uid); + let count = end as u64 - start as u64 + 1; + let sequence_set = if start == end { + start.to_string() + } else { + format!("{start}:{end}") + }; + result.push((sequence_set, count, end)); + + if end == u32::MAX { + break; + } + start = end + 1; + } + + result +} + +fn parse_message_id_header(header_bytes: &[u8]) -> Option { + let header = std::str::from_utf8(header_bytes).ok()?; + for line in header.lines() { + if let Some(value) = line + .strip_prefix("Message-ID:") + .or_else(|| line.strip_prefix("Message-Id:")) + .or_else(|| line.strip_prefix("Message-id:")) + { + // mail_parser strips angle brackets, so we must do the same + // to ensure comparisons against the Tantivy index match. + let trimmed = value.trim(); + let stripped = trimmed.strip_prefix('<').unwrap_or(trimmed); + let stripped = stripped.strip_suffix('>').unwrap_or(stripped); + if !stripped.is_empty() { + return Some(stripped.to_string()); + } + } + } + None +} + +#[cfg(test)] +mod test { + use super::*; + + // ── compress_uid_list ────────────────────────────────────────── + + #[test] + fn compress_empty() { + assert_eq!(compress_uid_list(vec![]), ""); + } + + #[test] + fn compress_single_uid() { + assert_eq!(compress_uid_list(vec![42]), "42"); + } + + #[test] + fn compress_consecutive_range() { + assert_eq!(compress_uid_list(vec![1, 2, 3, 4, 5]), "1:5"); + } + + #[test] + fn compress_mixed_ranges() { + assert_eq!( + compress_uid_list(vec![1, 2, 3, 5, 7, 8, 9, 10]), + "1:3,5,7:10" + ); + } + + #[test] + fn compress_gap_at_boundary() { + assert_eq!(compress_uid_list(vec![1, 2, 4, 5]), "1:2,4:5"); + } + + // ── generate_uid_sequence_hashset ────────────────────────────── + + #[test] + fn batch_single_chunk() { + let batches = generate_uid_sequence_hashset(vec![1, 2, 3], 10); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].0, "1:3"); + assert_eq!(batches[0].1, 3); + } + + #[test] + fn batch_multiple_chunks() { + let batches = generate_uid_sequence_hashset(vec![1, 2, 3, 4, 5], 2); + assert_eq!(batches.len(), 3); + assert_eq!(batches[0].0, "1:2"); + assert_eq!(batches[0].1, 2); + assert_eq!(batches[1].0, "3:4"); + assert_eq!(batches[1].1, 2); + assert_eq!(batches[2].0, "5"); + assert_eq!(batches[2].1, 1); + } + + #[test] + fn uid_range_batches_do_not_use_unbounded_fetch() { + let batches = generate_uid_range_batches(195_225, 195_426, 100); + + assert_eq!( + batches, + vec![ + ("195225:195324".to_string(), 100, 195_324), + ("195325:195424".to_string(), 100, 195_424), + ("195425:195426".to_string(), 2, 195_426), + ] + ); + assert!(batches + .iter() + .all(|(sequence_set, _, _)| !sequence_set.contains('*'))); + } + + // ── parse_message_id_header ───────────────────────────────────── + + #[test] + fn parse_standard_message_id() { + let header = b"Message-ID: \r\n"; + assert_eq!( + parse_message_id_header(header), + Some("abc123@example.com".into()) + ); + } + + #[test] + fn parse_message_id_lowercase() { + let header = b"Message-Id: \r\n"; + assert_eq!(parse_message_id_header(header), Some("foo@bar.com".into())); + } + + #[test] + fn parse_message_id_extra_whitespace() { + let header = b"Message-ID: \r\n"; + assert_eq!( + parse_message_id_header(header), + Some("spaces@test.com".into()) + ); + } + + #[test] + fn parse_empty_message_id_returns_none() { + let header = b"Message-ID: <>\r\n"; + assert_eq!(parse_message_id_header(header), None); + } + + #[test] + fn parse_missing_header_returns_none() { + let header = b"X-Custom: something\r\n"; + assert_eq!(parse_message_id_header(header), None); + } + + #[test] + fn parse_empty_body_returns_none() { + assert_eq!(parse_message_id_header(b""), None); + } + + #[test] + fn parse_message_id_in_full_header() { + // The Message-ID line is in the middle, not at the start. + let header = b"From: sender@example.com\r\n\ +Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ +Subject: test\r\n\ +Message-ID: \r\n\ +To: recipient@example.com\r\n\r\n"; + assert_eq!( + parse_message_id_header(header), + Some("mid@example.com".into()) + ); + } + + #[test] + fn parse_message_id_only_in_full_header() { + // Only a few headers, Message-ID is among them. + let header = b"From: a@b.com\r\nMessage-ID: \r\n\r\n"; + assert_eq!(parse_message_id_header(header), Some("x@y.com".into())); + } + + #[test] + fn parse_message_id_no_brackets_still_works() { + let header = b"Message-ID: plain@example.com\r\n"; + assert_eq!( + parse_message_id_header(header), + Some("plain@example.com".into()) + ); + } } diff --git a/crates/core/src/imap/manager.rs b/crates/core/src/imap/manager.rs index d32f5176..e048edd0 100644 --- a/crates/core/src/imap/manager.rs +++ b/crates/core/src/imap/manager.rs @@ -95,16 +95,40 @@ impl ImapConnectionManager { pub async fn build(account_id: u64) -> BichonResult>> { let account = AccountModel::get(account_id)?; - let client = match Self::create_client(&account).await { - Ok(client) => client, - Err(error) => { - error!( - "Failed to create IMAP {}'s client: {:#?}", - &account.email, error - ); - return Err(error); + let account_email = account.email.clone(); + + let mut client = None; + for attempt in 0..3u32 { + match Self::create_client(&account).await { + Ok(c) => { + client = Some(c); + break; + } + Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => { + warn!( + "IMAP connection attempt {}/3 to {} failed (network error), retrying...", + attempt + 1, + account_email + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + Err(error) => { + error!( + "Failed to create IMAP {}'s client: {:#?}", + account_email, error + ); + return Err(error); + } } - }; + } + + let client = client.ok_or_else(|| { + raise_error!( + format!("Failed to create IMAP {}'s client after 3 attempts", account_email), + ErrorCode::NetworkError + ) + })?; let mut session = match Self::authenticate(client, &account).await { Ok(session) => session, diff --git a/crates/core/src/imap/mock_server.rs b/crates/core/src/imap/mock_server.rs new file mode 100644 index 00000000..c37c3455 --- /dev/null +++ b/crates/core/src/imap/mock_server.rs @@ -0,0 +1,538 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! A minimal scriptable IMAP server for integration testing. +//! +//! Each instance listens on a random localhost port and responds to a +//! pre-configured script of (expected_command, response) pairs. Commands +//! are matched by substring — the first matching pattern wins. +//! +//! # Example +//! ```ignore +//! let server = MockImapServer::new() +//! .greeting("* OK ready\r\n") +//! .respond("LOGIN", "A0 OK logged in\r\n") +//! .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n") +//! .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n") +//! .respond("LOGOUT", "* BYE\r\nA0 OK\r\n") +//! .start() +//! .await; +//! +//! let (host, port) = server.addr(); +//! // connect to host:port with Encryption::None +//! ``` + +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpListener, TcpStream}; + +type Response = Vec; + +pub struct MockImapServer { + greeting: Vec, + script: Vec<(String, Response)>, +} + +impl MockImapServer { + pub fn new() -> Self { + Self { + greeting: b"* OK Mock IMAP server ready\r\n".to_vec(), + script: Vec::new(), + } + } + + /// Set the greeting banner sent immediately after connection. + pub fn greeting(mut self, banner: impl Into>) -> Self { + self.greeting = banner.into(); + self + } + + /// Add a script step: when a client command *contains* `pattern` (case-insensitive), + /// respond with `response`. Steps are checked in insertion order. + pub fn respond(mut self, pattern: impl Into, response: impl Into>) -> Self { + self.script.push((pattern.into(), response.into())); + self + } + + /// Start the server on a random port. Returns a handle whose `addr()` gives + /// the `(host, port)` to connect to. + pub async fn start(self) -> MockImapServerHandle { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + + let server = Arc::new(self); + + tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let srv = server.clone(); + tokio::spawn(async move { + srv.handle_connection(stream).await; + }); + } + Err(_) => break, + } + } + }); + + MockImapServerHandle { addr } + } + + async fn handle_connection(&self, mut stream: TcpStream) { + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Send greeting + if writer.write_all(&self.greeting).await.is_err() { + return; + } + + let mut line = String::new(); + loop { + line.clear(); + match reader.read_line(&mut line).await { + Ok(0) => break, // EOF + Ok(_) => {} + Err(_) => break, + } + + let tag = extract_tag(&line).unwrap_or("A0"); + let matched = self.find_match(&line); + if let Some(response) = matched { + let substituted = substitute_tag(response, tag); + if writer.write_all(&substituted).await.is_err() { + break; + } + } else { + // Default: send tagged OK for commands we don't handle + let fallback = format!("{tag} OK done\r\n"); + if writer.write_all(fallback.as_bytes()).await.is_err() { + break; + } + } + } + } + + fn find_match(&self, line: &str) -> Option<&[u8]> { + let line_lower = line.to_lowercase(); + for (pattern, response) in &self.script { + if line_lower.contains(&pattern.to_lowercase()) { + return Some(response); + } + } + None + } +} + +impl Default for MockImapServer { + fn default() -> Self { + Self::new() + } +} + +/// Handle to a running mock IMAP server. The server stops when this handle +/// is dropped. +pub struct MockImapServerHandle { + addr: SocketAddr, +} + +impl MockImapServerHandle { + pub fn host(&self) -> String { + self.addr.ip().to_string() + } + + pub fn port(&self) -> u16 { + self.addr.port() + } +} + +fn extract_tag(line: &str) -> Option<&str> { + line.split_whitespace().next() +} + +/// Replace `{TAG}` placeholders in `response` with `tag`. +fn substitute_tag(response: &[u8], tag: &str) -> Vec { + let placeholder = b"{TAG}"; + if response.is_empty() || !contains_slice(response, placeholder) { + return response.to_vec(); + } + let tag_bytes = tag.as_bytes(); + let mut result = Vec::with_capacity(response.len()); + let mut pos = 0; + while let Some(idx) = find_slice(&response[pos..], placeholder) { + result.extend_from_slice(&response[pos..pos + idx]); + result.extend_from_slice(tag_bytes); + pos += idx + placeholder.len(); + } + result.extend_from_slice(&response[pos..]); + result +} + +fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +fn find_slice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|w| w == needle) +} + +// ============================================================ +// Pre-built response helpers +// ============================================================ + +/// Build a tagged OK response. +pub fn ok(tag: impl AsRef, msg: impl AsRef) -> Vec { + format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes() +} + +/// Build a STATUS response line. +pub fn status_response( + mailbox: &str, + messages: u32, + unseen: u32, + uid_next: u32, + uid_validity: Option, +) -> Vec { + let uv = uid_validity + .map(|v| format!(" UIDVALIDITY {v}")) + .unwrap_or_default(); + let text = format!( + "* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n" + ); + // Clients expect a tagged response after the untagged STATUS line. + // We produce a generic OK that works for any tag. + let mut out = text.into_bytes(); + out.extend_from_slice(b"{TAG} OK STATUS completed\r\n"); + out +} + +/// Build an EXAMINE response with mailbox data. +pub fn examine_response( + _mailbox: &str, + exists: u32, + uid_validity: u32, + uid_next: u32, +) -> Vec { + format!( + "* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\ + * OK [PERMANENTFLAGS ()]\r\n\ + * {exists} EXISTS\r\n\ + * 0 RECENT\r\n\ + * OK [UIDVALIDITY {uid_validity}]\r\n\ + * OK [UIDNEXT {uid_next}]\r\n\ + * OK [HIGHESTMODSEQ 1]\r\n\ + {{TAG}} OK [READ-ONLY] EXAMINE completed\r\n" + ) + .into_bytes() +} + +/// Build a UID SEARCH response for the given UID list. +pub fn uid_search_response(uids: &[u32]) -> Vec { + let uid_str = uids + .iter() + .map(|u| u.to_string()) + .collect::>() + .join(" "); + format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes() +} + +/// Build a UID FETCH response returning full headers (for BODY[HEADER]). +/// Each entry: (uid, message_id) +pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec { + let mut out = Vec::new(); + for (uid, msg_id) in entries { + // Build a minimal header that contains the Message-ID line. + let header_data = format!( + "From: sender@example.com\r\n\ +To: recipient@example.com\r\n\ +Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ +Subject: test\r\n\ +Message-ID: {msg_id}\r\n\r\n" + ); + let header_len = header_data.len(); + let line = format!( + "* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\ +{header_data}\ +)\r\n", + ); + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(b"{TAG} OK FETCH completed\r\n"); + out +} + +/// Build a UID FETCH RFC822 response with a full email body. +pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec { + let header = format!( + "* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n", + len = eml.len() + ); + let mut out = header.into_bytes(); + out.extend_from_slice(eml); + out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n"); + out +} + +/// A minimal RFC822 email fixture for testing. +pub fn minimal_eml(subject: &str, message_id: &str) -> Vec { + format!( + "From: sender@example.com\r\n\ + To: recipient@example.com\r\n\ + Subject: {subject}\r\n\ + Message-ID: <{message_id}>\r\n\ + Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ + MIME-Version: 1.0\r\n\ + Content-Type: text/plain; charset=utf-8\r\n\ + \r\n\ + This is a test email: {subject}.\r\n" + ) + .into_bytes() +} + +// ============================================================ +// Self-tests for the mock server itself +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect_and_read_greeting(host: &str, port: u16) -> String { + let mut stream = TcpStream::connect((host, port)).await.unwrap(); + let (reader, _writer) = stream.split(); + let mut reader = BufReader::new(reader); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + line + } + + async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String { + let mut stream = TcpStream::connect((host, port)).await.unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + + // Send command + writer.write_all(cmd.as_bytes()).await.unwrap(); + writer.write_all(b"\r\n").await.unwrap(); + + // Read response (may be multi-line; read until tagged response) + let mut out = String::new(); + loop { + line.clear(); + reader.read_line(&mut line).await.unwrap(); + out.push_str(&line); + if line.starts_with("A0") || line.starts_with("A1") { + break; + } + } + out + } + + #[tokio::test] + async fn test_mock_greeting() { + let handle = MockImapServer::new().start().await; + let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await; + assert!(greeting.starts_with("* OK")); + } + + #[tokio::test] + async fn test_mock_scripted_response() { + let handle = MockImapServer::new() + .respond( + "LOGIN", + "A0 OK LOGIN completed\r\n", + ) + .start() + .await; + + let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await; + assert!(resp.contains("LOGIN completed")); + } + + #[tokio::test] + async fn test_mock_fallback_on_unmatched() { + let handle = MockImapServer::new().start().await; + + // Send a command that has no scripted response + let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await; + assert!(resp.contains("OK done"), "unmatched command should get fallback OK"); + } + + #[tokio::test] + async fn test_status_response_helper() { + let resp = status_response("INBOX", 10, 2, 11, Some(42)); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("MESSAGES 10")); + assert!(text.contains("UNSEEN 2")); + assert!(text.contains("UIDNEXT 11")); + assert!(text.contains("UIDVALIDITY 42")); + } + + #[tokio::test] + async fn test_status_response_without_uidvalidity() { + let resp = status_response("INBOX", 10, 2, 11, None); + let text = String::from_utf8(resp).unwrap(); + assert!(!text.contains("UIDVALIDITY")); + assert!(text.contains("MESSAGES 10")); + } + + #[tokio::test] + async fn test_examine_response() { + let resp = examine_response("INBOX", 10, 42, 11); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("UIDVALIDITY 42")); + assert!(text.contains("10 EXISTS")); + } + + #[tokio::test] + async fn test_uid_search_response() { + let resp = uid_search_response(&[1, 3, 5]); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("SEARCH 1 3 5")); + } + + #[tokio::test] + async fn test_uid_fetch_metadata_response() { + let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("Message-ID: msg-a@x.com")); + assert!(text.contains("Message-ID: msg-b@x.com")); + } + + #[tokio::test] + async fn test_multiple_commands_in_sequence() { + let handle = MockImapServer::new() + .respond("LOGIN", "A0 OK LOGIN\r\n") + .respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99))) + .respond("LOGOUT", "* BYE\r\nA0 OK\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // LOGIN + writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + assert!(buf.contains("LOGIN")); + + // STATUS + writer + .write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n") + .await + .unwrap(); + buf.clear(); + // Read multi-line STATUS response (untagged line + tagged OK) + loop { + reader.read_line(&mut buf).await.unwrap(); + if buf.contains("UIDVALIDITY 99") { + // Consume the tagged OK line that follows + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + break; + } + } + + // LOGOUT + writer.write_all(b"A0 LOGOUT\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + assert!(buf.contains("BYE")); + } + + #[tokio::test] + async fn test_tag_substitution_in_response() { + // Use {TAG} placeholder in the response and verify it gets the + // client's actual tag ("A5") substituted in. + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // Send LOGIN with non-standard tag + writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + + assert!( + buf.contains("A5 OK LOGIN succeeded"), + "expected 'A5 OK LOGIN succeeded', got '{buf}'" + ); + } + + #[tokio::test] + async fn test_tag_substitution_multiple_placeholders() { + let handle = MockImapServer::new() + .respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // Send with tag "B99" + writer.write_all(b"B99 NOOP\r\n").await.unwrap(); + + // Read all lines + let mut all = String::new(); + loop { + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + all.push_str(&buf); + if buf.starts_with("B99") { + break; + } + } + + assert!(all.contains("* 0 RECENT\r\n")); + assert!(all.contains("B99 OK NOOP done\r\n")); + } +} diff --git a/crates/core/src/imap/mod.rs b/crates/core/src/imap/mod.rs index 479c9548..0866f1a1 100644 --- a/crates/core/src/imap/mod.rs +++ b/crates/core/src/imap/mod.rs @@ -26,3 +26,5 @@ pub mod session; pub mod stats; #[cfg(test)] mod tests; +#[cfg(test)] +pub mod mock_server; diff --git a/crates/core/src/import/history.rs b/crates/core/src/import/history.rs new file mode 100644 index 00000000..a17d0b11 --- /dev/null +++ b/crates/core/src/import/history.rs @@ -0,0 +1,127 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +use crate::database::MemDbModel; +use crate::import::{ImportProgress, ImportStatus}; +use serde::{Deserialize, Serialize}; + +/// Maximum number of import history entries to keep per user. +pub const MAX_HISTORY_PER_USER: usize = 5; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ImportHistory { + /// Composite key: "{user_id}:{import_id}" + pub id: String, + pub user_id: u64, + pub import_id: String, + pub account_id: u64, + pub folder: String, + pub format: String, + pub status: String, + pub total: usize, + pub success: usize, + pub duplicates: usize, + pub failed: usize, + pub failed_details: Vec, + /// Unix timestamp in milliseconds. + pub created_at: i64, +} + +impl MemDbModel for ImportHistory { + fn collection() -> &'static str { + "import_history" + } + fn key(&self) -> String { + self.id.clone() + } +} + +impl ImportHistory { + pub fn from_progress( + user_id: u64, + import_id: &str, + account_id: u64, + folder: &str, + progress: &ImportProgress, + ) -> Self { + Self { + id: format!("{}:{}", user_id, import_id), + user_id, + import_id: import_id.to_string(), + account_id, + folder: folder.to_string(), + format: progress.format.clone(), + status: match progress.status { + ImportStatus::Pending => "pending", + ImportStatus::Processing => "processing", + ImportStatus::Completed => "completed", + ImportStatus::Failed => "failed", + } + .to_string(), + total: progress.total, + success: progress.success, + duplicates: progress.duplicates, + failed: progress.failed, + failed_details: progress.failed_details.clone(), + created_at: crate::utc_now!(), + } + } +} + +/// Prune old entries for a user so only the latest `MAX_HISTORY_PER_USER` remain. +pub fn prune_user_history(user_id: u64) -> crate::error::BichonResult<()> { + use crate::database::manager::DB_MANAGER; + use crate::database::batch_delete_impl; + use crate::raise_error; + use crate::error::code::ErrorCode; + let db = DB_MANAGER.db(); + let coll = db.collection(ImportHistory::collection()); + let prefix = format!("{}:", user_id); + + let mut entries: Vec = coll + .scan_prefix(&prefix) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + if entries.len() <= MAX_HISTORY_PER_USER { + return Ok(()); + } + + // Sort by created_at descending (newest first), keep the first N + entries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + let to_delete: Vec = entries + .iter() + .skip(MAX_HISTORY_PER_USER) + .map(|e| e.id.clone()) + .collect(); + + if !to_delete.is_empty() { + batch_delete_impl::(db, to_delete)?; + } + + Ok(()) +} + +/// Save an import history record and prune old entries for the user. +pub fn save_import_history( + user_id: u64, + account_id: u64, + folder: &str, + progress: &ImportProgress, +) { + use crate::database::manager::DB_MANAGER; + use crate::database::upsert_impl; + + let entry = ImportHistory::from_progress(user_id, &progress.import_id, account_id, folder, progress); + let db = DB_MANAGER.db(); + + if let Err(e) = upsert_impl::(db, entry) { + tracing::error!("Failed to save import history: {:?}", e); + return; + } + + if let Err(e) = prune_user_history(user_id) { + tracing::warn!("Failed to prune import history: {:?}", e); + } +} diff --git a/crates/core/src/import/mod.rs b/crates/core/src/import/mod.rs index 8cd533ea..bd8db2ae 100644 --- a/crates/core/src/import/mod.rs +++ b/crates/core/src/import/mod.rs @@ -18,7 +18,16 @@ //use poem_openapi::Object; +pub mod history; +pub mod reader; +pub mod pst; +pub use history::ImportHistory; use serde::{Deserialize, Serialize}; +use std::{ + collections::HashMap, + path::Path, + sync::RwLock, +}; use crate::{ base64_decode_url_safe, @@ -27,15 +36,20 @@ use crate::{ cache::imap::mailbox::{Attribute, AttributeEnum, MailBox}, envelope::extractor::extract_envelope_from_eml, error::{BichonResult, code::ErrorCode}, + settings::dir::DATA_DIR_MANAGER, utils::create_hash, }, raise_error, }; -/// Skip individual emails larger than this after decoding (100 MB). +/// Maximum byte size of an individual email message after splitting (100 MB). const MAX_SINGLE_EML_BYTES: usize = 100 * 1024 * 1024; -#[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] +/// Max file size accepted via the web upload endpoint. +pub const MAX_WEB_EML_BYTES: usize = 100 * 1024 * 1024; // 100 MB +pub const MAX_WEB_MBOX_BYTES: usize = 1024 * 1024 * 1024; // 1 GB + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct BatchEmlRequest { pub account_id: u64, @@ -46,24 +60,26 @@ pub struct BatchEmlRequest { #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] -pub struct FailedEmlDetail { - /// The 0-based index of the failed EML in the request list +pub struct FailedItemDetail { + /// The index (0-based) of the failed item. pub index: usize, - /// The error message that caused the import to fail + /// The error message that caused the import to fail. pub error_message: String, } #[derive(Debug, Clone, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct BatchEmlResult { - /// Total number of emails processed + /// Total number of emails processed. pub total: usize, - /// Number of emails successfully imported + /// Number of emails successfully imported. pub success: usize, - /// Number of emails failed to import + /// Number of duplicate emails skipped (content hash already existed). + pub duplicates: usize, + /// Number of emails failed to import. pub failed: usize, - /// A list of details for failed imports - pub failed_details: Vec, + /// A list of details for failed imports. + pub failed_details: Vec, } pub struct ImportEmls; @@ -105,6 +121,7 @@ impl ImportEmls { unseen: None, uid_next: None, uid_validity: None, + highest_uid: None, }; let mailbox_id = mailbox.id; // Upsert the mailbox, creating it if it doesn't exist @@ -115,7 +132,7 @@ impl ImportEmls { let account_id = account.id; let mut success_count = 0; - let mut failed_details: Vec = Vec::new(); // Store failure details + let mut failed_details: Vec = Vec::new(); // Store failure details let total = request.emls.len(); let mut index: usize = 0; @@ -126,7 +143,7 @@ impl ImportEmls { let error_msg = format!("Failed to decode base64 EML at index {}: {:?}", index, e); tracing::error!("{}", error_msg); - failed_details.push(FailedEmlDetail { + failed_details.push(FailedItemDetail { index, error_message: error_msg, }); @@ -143,7 +160,7 @@ impl ImportEmls { index, size_mb, ); tracing::warn!("{}", error_msg); - failed_details.push(FailedEmlDetail { + failed_details.push(FailedItemDetail { index, error_message: error_msg, }); @@ -161,7 +178,7 @@ impl ImportEmls { index, e ); tracing::error!("{}", error_msg); - failed_details.push(FailedEmlDetail { + failed_details.push(FailedItemDetail { index, error_message: error_msg, }); @@ -177,8 +194,650 @@ impl ImportEmls { Ok(BatchEmlResult { total, success: success_count, + duplicates: 0, failed: failed_count, failed_details, // Return the list of failure details }) } } + +// ── File upload import ────────────────────────────────────────────── + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Enum))] +pub enum ImportStatus { + Pending, + Processing, + Completed, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ImportProgress { + pub import_id: String, + pub status: ImportStatus, + pub format: String, + pub total: usize, + pub success: usize, + pub duplicates: usize, + pub failed: usize, + pub failed_details: Vec, +} + +static PROGRESS_STORE: std::sync::LazyLock>> = + std::sync::LazyLock::new(|| RwLock::new(HashMap::new())); + +pub fn get_import_progress(import_id: &str) -> Option { + PROGRESS_STORE.read().ok()?.get(import_id).cloned() +} + +pub fn update_progress(import_id: &str, progress: ImportProgress) { + if let Ok(mut store) = PROGRESS_STORE.write() { + store.insert(import_id.to_string(), progress); + } +} + +/// Check free disk space (in bytes) on the temp directory's filesystem. +pub fn check_temp_disk_space() -> BichonResult { + use sysinfo::Disks; + let disks = Disks::new_with_refreshed_list(); + let temp_path = &DATA_DIR_MANAGER.temp_dir; + // Use the canonical path so we can match mount points + let canonical = std::fs::canonicalize(temp_path).unwrap_or_else(|_| temp_path.clone()); + for disk in disks.list() { + if canonical.starts_with(disk.mount_point()) { + return Ok(disk.available_space()); + } + } + // Fallback: if we can't find the mount point, report plenty of space + Ok(u64::MAX) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum FileFormat { + Eml, + Mbox, + Pst, +} + +pub fn detect_format(bytes: &[u8], file_name: &str) -> Option { + // PST files start with OLE2 compound document magic bytes + if bytes.len() >= 8 && &bytes[..8] == b"\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1" { + return Some(FileFormat::Pst); + } + + // MBOX files start with "From " (note the trailing space after From) + if bytes.starts_with(b"From ") { + // Double-check: look for a valid date after the first "From " line + // MBOX format: "From sender@host DayOfWeek Mon DD HH:MM:SS YYYY" + if let Some(first_newline) = bytes.iter().position(|&b| b == b'\n') { + let from_line = std::str::from_utf8(&bytes[..first_newline]).unwrap_or(""); + let parts: Vec<&str> = from_line.split_whitespace().collect(); + if parts.len() >= 7 { + return Some(FileFormat::Mbox); + } + } + } + // EML: starts with a header line or "Return-Path:", "Received:", "From:", "Date:", etc. + // Or check extension + if bytes.starts_with(b"Return-Path:") + || bytes.starts_with(b"Received:") + || bytes.starts_with(b"Date:") + || bytes.starts_with(b"From:") + || bytes.starts_with(b"Subject:") + || bytes.starts_with(b"To:") + || bytes.starts_with(b"Message-ID:") + { + return Some(FileFormat::Eml); + } + // Fallback: check file extension + let lower = file_name.to_lowercase(); + if lower.ends_with(".eml") { + Some(FileFormat::Eml) + } else if lower.ends_with(".mbox") { + Some(FileFormat::Mbox) + } else if lower.ends_with(".pst") { + Some(FileFormat::Pst) + } else { + None + } +} + +/// Check whether `bytes` looks like a text file by inspecting the first chunk. +/// Returns `true` if it passes, `false` if it appears to be binary (video, executable, etc.). +/// +/// Email files (EML/MBOX) are text-based with printable ASCII, whitespace, and +/// optional UTF-8. Binary files like video contain null bytes and high ratios of +/// non-printable control characters. +pub fn detect_text_file(bytes: &[u8]) -> bool { + let check_len = bytes.len().min(8192); + if check_len == 0 { + return false; + } + let sample = &bytes[..check_len]; + + // Null bytes are a strong binary indicator + if sample.contains(&0x00) { + return false; + } + + let mut printable = 0usize; + let mut total = 0usize; + + let mut i = 0; + while i < sample.len() { + total += 1; + let b = sample[i]; + + if b.is_ascii_graphic() || b.is_ascii_whitespace() { + // Printable ASCII + whitespace (space, tab, CR, LF) + printable += 1; + } else if b == 0x1b { + // ESC — common in terminal sequences, rare in email + // Count as printable to avoid false positives + printable += 1; + } else if b >= 0x80 { + // UTF-8 continuation or multi-byte lead byte — allow. + // Check that we have a valid UTF-8 sequence ahead. + let seq_len = match b { + b if b & 0xE0 == 0xC0 => 2, + b if b & 0xF0 == 0xE0 => 3, + b if b & 0xF8 == 0xF0 => 4, + _ => 0, + }; + if seq_len > 0 && i + seq_len <= sample.len() { + let valid = std::str::from_utf8(&sample[i..i + seq_len]).is_ok(); + if valid { + printable += 1; + i += 1; // lead byte counted, continuations counted in loop + } + // if invalid, don't count as printable + } + // standalone continuation byte — not printable + } + // Other control characters (0x01-0x1F except whitespace/Esc) are not counted as printable + + i += 1; + } + + // Require at least 90% printable characters + printable as f64 / total as f64 >= 0.90 +} + +/// Validate that the target account exists, is enabled, and is NoSync type. +fn validate_import_account(account_id: u64) -> BichonResult { + let account = AccountModel::check_account_exists(account_id)?; + if !account.enabled { + return Err(raise_error!( + "The account is disabled.".into(), + ErrorCode::InvalidParameter + )); + } + if !matches!(account.account_type, AccountType::NoSync) { + return Err(raise_error!( + "Import is only allowed for NoSync accounts. IMAP accounts sync from the server.".into(), + ErrorCode::InvalidParameter + )); + } + Ok(account) +} + +/// Resolve or create a mailbox/folder for the given account. +pub(super) fn resolve_mailbox(account: &AccountModel, folder: &str) -> BichonResult { + match account.account_type { + AccountType::IMAP => { + // Shouldn't reach here (validated above), but handle gracefully + let all_mailboxes = MailBox::list_all(account.id)?; + let mailbox = all_mailboxes.into_iter().find(|m| m.name == folder); + match mailbox { + Some(m) => Ok(m.id), + None => Err(raise_error!( + format!("Mail folder '{}' not found.", folder).into(), + ErrorCode::ResourceNotFound + )), + } + } + AccountType::NoSync => { + let mailbox = MailBox { + id: create_hash(account.id, folder), + account_id: account.id, + name: folder.to_string(), + delimiter: Some("/".to_string()), + attributes: vec![Attribute { + attr: AttributeEnum::Extension, + extension: Some("CreatedByBichon".into()), + }], + exists: 0, + unseen: None, + uid_next: None, + uid_validity: None, + highest_uid: None, + }; + let mailbox_id = mailbox.id; + MailBox::batch_upsert(&[mailbox])?; + Ok(mailbox_id) + } + } +} + +/// Resolve or create a mailbox for a given account_id and folder name. +/// Used by PST import to create per-folder mailboxes. +pub fn resolve_mailbox_by_account_id(account_id: u64, folder: &str) -> BichonResult { + let account = AccountModel::check_account_exists(account_id)?; + resolve_mailbox(&account, folder) +} + +/// Process an uploaded file (EML or MBOX) and import into the given account/folder. +/// This runs synchronously and should be spawned on a background thread. +/// +/// For MBOX files, the file is memory-mapped via `memmap2` and messages are yielded +/// one at a time — the full file is never loaded into RAM. Individual messages +/// exceeding `MAX_SINGLE_EML_BYTES` (100 MB) are skipped. +pub fn process_uploaded_file( + import_id: &str, + file_path: &Path, + file_name: &str, + account_id: u64, + folder: &str, + user_id: u64, +) { + let account = match validate_import_account(account_id) { + Ok(a) => a, + Err(e) => { + let progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Failed, + format: "unknown".to_string(), + total: 0, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![FailedItemDetail { + index: 0, + error_message: format!("Account validation failed: {:?}", e), + }], + }; + update_progress(import_id, progress.clone()); + history::save_import_history(user_id, account_id, folder, &progress); + return; + } + }; + + let mailbox_id = match resolve_mailbox(&account, folder) { + Ok(id) => id, + Err(e) => { + let progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Failed, + format: "unknown".to_string(), + total: 0, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![FailedItemDetail { + index: 0, + error_message: format!("Mailbox resolution failed: {:?}", e), + }], + }; + update_progress(import_id, progress.clone()); + history::save_import_history(user_id, account_id, folder, &progress); + return; + } + }; + + // Read a small prefix for format detection + let format = match detect_format_from_file(file_path, file_name) { + Ok(f) => f, + Err(e) => { + let progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Failed, + format: "unknown".to_string(), + total: 0, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![FailedItemDetail { + index: 0, + error_message: format!("{:?}", e), + }], + }; + update_progress(import_id, progress.clone()); + history::save_import_history(user_id, account_id, folder, &progress); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + match format { + FileFormat::Eml => process_eml_file(import_id, file_path, account_id, mailbox_id, user_id, folder), + FileFormat::Mbox => process_mbox_file(import_id, file_path, account_id, mailbox_id, user_id, folder), + FileFormat::Pst => process_pst_upload(import_id, file_path, account_id, mailbox_id, user_id, folder), + } +} + +/// Detect format from a file by reading only the first few KB. +fn detect_format_from_file(file_path: &Path, file_name: &str) -> BichonResult { + use std::io::Read; + let mut file = std::fs::File::open(file_path).map_err(|e| { + raise_error!(format!("Failed to open file: {}", e), ErrorCode::InternalError) + })?; + let mut buf = vec![0u8; 8192]; + let n = file.read(&mut buf).unwrap_or(0); + buf.truncate(n); + + detect_format(&buf, file_name).ok_or_else(|| { + raise_error!( + "Unknown file format. Supported: .eml, .mbox, .pst".into(), + ErrorCode::InvalidParameter + ) + }) +} + +/// Process a single EML file. The file is at most `MAX_WEB_EML_BYTES` (100 MB), +/// so reading it entirely is safe. +fn process_eml_file( + import_id: &str, + file_path: &Path, + account_id: u64, + mailbox_id: u64, + user_id: u64, + folder: &str, +) { + let file_bytes = match std::fs::read(file_path) { + Ok(b) => b, + Err(e) => { + fail_progress(import_id, "eml", &format!("Failed to read file: {}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + let total = 1; + update_progress(import_id, ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Processing, + format: "eml".to_string(), + total, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![], + }); + + let (success_count, failed_details) = process_single_eml(&file_bytes, 0, account_id, mailbox_id); + + // Clean up + let _ = std::fs::remove_file(file_path); + + let final_progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Completed, + format: "eml".to_string(), + total, + success: success_count, + duplicates: 0, + failed: failed_details.len(), + failed_details, + }; + history::save_import_history(user_id, account_id, folder, &final_progress); + update_progress(import_id, final_progress); +} + +/// Process an MBOX file using memory-mapped I/O. Messages are yielded one at a +/// time by `MboxReader` — the full file is never loaded into RAM. +fn process_mbox_file( + import_id: &str, + file_path: &Path, + account_id: u64, + mailbox_id: u64, + user_id: u64, + folder: &str, +) { + let mbox = match reader::MboxFile::from_file(file_path) { + Ok(m) => m, + Err(e) => { + fail_progress(import_id, "mbox", &format!("Failed to open MBOX file: {}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + // First pass: count total messages (MboxReader is lazy, so this is O(n) but cheap) + let total = mbox.iter().count(); + + update_progress(import_id, ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Processing, + format: "mbox".to_string(), + total, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![], + }); + + let mut success_count = 0usize; + let mut failed_details: Vec = Vec::new(); + + for (index, entry) in mbox.iter().enumerate() { + let eml_bytes = entry.data; + + if eml_bytes.len() > MAX_SINGLE_EML_BYTES { + let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0; + failed_details.push(FailedItemDetail { + index, + error_message: format!( + "Email at index {} is {:.1} MB (limit {} MB). Skipping.", + index, + size_mb, + MAX_SINGLE_EML_BYTES / 1024 / 1024 + ), + }); + continue; + } + + match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) { + Ok(_) => { + success_count += 1; + } + Err(e) => { + failed_details.push(FailedItemDetail { + index, + error_message: format!("{:?}", e), + }); + } + }; + + // Update progress every 100 items + if index % 100 == 0 || index == total - 1 { + update_progress(import_id, ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Processing, + format: "mbox".to_string(), + total, + success: success_count, + duplicates: 0, + failed: failed_details.len(), + failed_details: failed_details.clone(), + }); + } + } + + // Clean up temp file (drop the mmap first — MboxFile owns it) + drop(mbox); + let _ = std::fs::remove_file(file_path); + + let final_progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Completed, + format: "mbox".to_string(), + total, + success: success_count, + duplicates: 0, + failed: failed_details.len(), + failed_details, + }; + history::save_import_history(user_id, account_id, folder, &final_progress); + update_progress(import_id, final_progress); +} + +/// Process a single EML byte slice and return (success_count, failed_details). +fn process_single_eml( + eml_bytes: &[u8], + index: usize, + account_id: u64, + mailbox_id: u64, +) -> (usize, Vec) { + if eml_bytes.len() > MAX_SINGLE_EML_BYTES { + let size_mb = eml_bytes.len() as f64 / 1024.0 / 1024.0; + return (0, vec![FailedItemDetail { + index, + error_message: format!( + "Email is {:.1} MB (limit {} MB). Skipping.", + size_mb, + MAX_SINGLE_EML_BYTES / 1024 / 1024 + ), + }]); + } + + match futures::executor::block_on(extract_envelope_from_eml(eml_bytes, account_id, mailbox_id)) { + Ok(_) => (1, vec![]), + Err(e) => (0, vec![FailedItemDetail { + index, + error_message: format!("{:?}", e), + }]), + } +} + +/// Process a PST file uploaded via the web UI. +/// Two-pass approach: count messages first, then process with periodic progress updates. +fn process_pst_upload( + import_id: &str, + file_path: &Path, + account_id: u64, + _mailbox_id: u64, // ignored; PST creates its own mailboxes per folder + user_id: u64, + folder: &str, +) { + // Pass 1: count total messages + let total = match pst::count_pst_messages(file_path) { + Ok(n) => n, + Err(e) => { + fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + update_progress(import_id, ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Processing, + format: "pst".to_string(), + total, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![], + }); + + // Pass 2: process messages with progress updates + let mut success_count: usize = 0; + let mut failed_details: Vec = Vec::new(); + let mut index: usize = 0; + + let pst_store = match outlook_pst::open_store(file_path) { + Ok(s) => s, + Err(e) => { + fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + let ipm_sub_tree = match pst_store.properties().ipm_sub_tree_entry_id() { + Ok(id) => id, + Err(e) => { + fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + let ipm_subtree_folder = match pst_store.open_folder(&ipm_sub_tree) { + Ok(f) => f, + Err(e) => { + fail_progress(import_id, "pst", &format!("{:?}", e), user_id, account_id, folder); + let _ = std::fs::remove_file(file_path); + return; + } + }; + + // Progress callback: update progress every 50 messages + let import_id = import_id.to_string(); + let format_str = "pst".to_string(); + pst::process_folder_with_progress( + &ipm_subtree_folder, + "", // parent_path starts empty + account_id, + total, // pass pre-counted total for accurate progress + &mut success_count, + &mut failed_details, + &mut index, + &|processed, actual_failed| { + update_progress(&import_id, ImportProgress { + import_id: import_id.clone(), + status: ImportStatus::Processing, + format: format_str.clone(), + total, + success: processed - actual_failed, + duplicates: 0, + failed: actual_failed, + failed_details: vec![], + }); + }, + ); + + // Clean up temp file + let _ = std::fs::remove_file(file_path); + + let final_progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Completed, + format: "pst".to_string(), + total, + success: success_count, + duplicates: 0, + failed: failed_details.len(), + failed_details, + }; + history::save_import_history(user_id, account_id, folder, &final_progress); + update_progress(&import_id, final_progress); +} + +/// Record a fatal failure and save history. +fn fail_progress( + import_id: &str, + format: &str, + message: &str, + user_id: u64, + account_id: u64, + folder: &str, +) { + let progress = ImportProgress { + import_id: import_id.to_string(), + status: ImportStatus::Failed, + format: format.to_string(), + total: 0, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![FailedItemDetail { + index: 0, + error_message: message.to_string(), + }], + }; + update_progress(import_id, progress.clone()); + history::save_import_history(user_id, account_id, folder, &progress); +} diff --git a/crates/cli/src/pst/encoding/mod.rs b/crates/core/src/import/pst/encoding/mod.rs similarity index 97% rename from crates/cli/src/pst/encoding/mod.rs rename to crates/core/src/import/pst/encoding/mod.rs index 1e0830fe..af61640b 100644 --- a/crates/cli/src/pst/encoding/mod.rs +++ b/crates/core/src/import/pst/encoding/mod.rs @@ -16,8 +16,6 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - -use compressed_rtf::*; use outlook_pst::ltp::prop_context::PropertyValue; pub fn decode_subject(value: &PropertyValue) -> Option { @@ -60,5 +58,5 @@ pub fn decode_html_body(buffer: &[u8], code_page: u16) -> Option { } pub fn decode_rtf_compressed(buffer: &[u8]) -> Option { - decompress_rtf(buffer).ok() + compressed_rtf::decompress_rtf(buffer).ok() } diff --git a/crates/core/src/import/pst/mod.rs b/crates/core/src/import/pst/mod.rs new file mode 100644 index 00000000..2fe7992e --- /dev/null +++ b/crates/core/src/import/pst/mod.rs @@ -0,0 +1,486 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::base64_encode_url_safe; +use crate::envelope::extractor::extract_envelope_from_eml; +use chrono::{DateTime, TimeZone, Utc}; +use mail_send::mail_builder::headers::text::Text; +use mail_send::mail_builder::MessageBuilder; +use outlook_pst::ltp::prop_context::PropertyValue; +use outlook_pst::messaging::attachment::AttachmentProperties; +use outlook_pst::messaging::folder::Folder; +use outlook_pst::messaging::message::{Message, MessageProperties}; +use outlook_pst::ndb::node_id::NodeId; +use std::rc::Rc; + +mod encoding; + +/// Convert a PST Message into a base64-encoded EML string. +pub fn build_eml_base64(message: Rc) -> Option { + let properties = message.properties(); + + let mut builder = MessageBuilder::new(); + if let Some(sub) = extract_subject(properties) { + builder = builder.subject(sub); + } + if let Some(mid) = extract_string_property(properties, 0x1035) { + builder = builder.message_id(mid); + } + if let Some(irt) = extract_string_property(properties, 0x1042) { + builder = builder.in_reply_to(irt); + } + + if let Some(refs) = extract_string_property(properties, 0x1039) { + builder = builder.header("References", Text::new(refs)); + } + + if let Some(cid_val) = properties.get(0x3013) { + if let PropertyValue::Binary(bin) = cid_val { + builder = builder.header( + "X-Bichon-Conversation-ID", + Text::new(hex::encode(bin.buffer())), + ); + } + } + + let from = extract_string_property(properties, 0x5D01) + .or_else(|| extract_string_property(properties, 0x5D02)) + .or_else(|| extract_string_property(properties, 0x0C1F)); + + if let Some(f) = from { + builder = builder.from(f); + } + + if let Some(filetime) = extract_i64_property(properties, &[0x0039, 0x0E06]) { + let dt = filetime_to_datetime(filetime).timestamp(); + builder = builder.date(dt); + } + + let (to, cc, bcc) = extract_recipients_list(&message); + if !to.is_empty() { + builder = builder.to(to.iter().map(|s| s.as_str()).collect::>()); + } + if !cc.is_empty() { + builder = builder.cc(cc.iter().map(|s| s.as_str()).collect::>()); + } + if !bcc.is_empty() { + builder = builder.bcc(bcc.iter().map(|s| s.as_str()).collect::>()); + } + + if let Some(html) = extract_html(properties) { + builder = builder.html_body(html); + } + + if let Some(text) = extract_text(properties) { + builder = builder.text_body(text); + } + + if let Some(attachment_table) = message.attachment_table() { + for row in attachment_table.rows_matrix() { + let node_id = NodeId::from(u32::from(row.id())); + if let Ok(attachment) = message.clone().read_attachment(node_id, None) { + let att_props = attachment.properties(); + let name = extract_attachment_string_property(att_props, 0x3707); + let mime = extract_attachment_string_property(att_props, 0x370E) + .unwrap_or_else(|| "application/octet-stream".into()); + let cid = extract_attachment_string_property(att_props, 0x3712); + let is_inline = att_props + .get(0x3714) + .and_then(|val| { + if let PropertyValue::Integer32(f) = val { + Some(f) + } else { + None + } + }) + .map(|flag| (flag & 0x4) != 0) + .unwrap_or(false); + + if let Some(PropertyValue::Binary(bin)) = att_props.get(0x3701) { + let data = bin.buffer().to_vec(); + let file_name = name.unwrap_or_else(|| "unnamed_attachment".to_string()); + + if is_inline && cid.is_some() { + let content_id = cid.unwrap(); + builder = builder.inline(mime, content_id, data); + } else { + builder = builder.attachment(mime, file_name, data); + } + } + } + } + } + + match builder.write_to_vec() { + Ok(eml_vec) => Some(base64_encode_url_safe!(eml_vec)), + Err(e) => { + tracing::error!("Failed to generate EML from PST message: {:?}", e); + None + } + } +} + +fn filetime_to_datetime(filetime: i64) -> DateTime { + let unix_secs = (filetime / 10_000_000) - 11_644_473_600; + let nsecs = (filetime % 10_000_000) * 100; + Utc.timestamp_opt(unix_secs, nsecs as u32).unwrap() +} + +fn extract_recipients_list(message: &Rc) -> (Vec, Vec, Vec) { + let mut to = Vec::new(); + let mut cc = Vec::new(); + let mut bcc = Vec::new(); + + let recipient_table = message.recipient_table(); + if let Some(recipient_table) = recipient_table { + let context = recipient_table.context(); + for row in recipient_table.rows_matrix() { + if let Ok(cols) = row.columns(context) { + let mut r_type = 0; + let mut email = String::new(); + + for (col, val) in context.columns().iter().zip(cols) { + let prop_val = val + .as_ref() + .and_then(|v| recipient_table.read_column(v, col.prop_type()).ok()); + match col.prop_id() { + 0x0C15 => { + if let Some(PropertyValue::Integer32(t)) = prop_val { + r_type = t; + } + } + 0x39FE | 0x3003 => { + if let Some(s) = prop_val.and_then(|v| extract_string(&v)) { + email = s; + } + } + _ => {} + } + } + + if !email.is_empty() { + match r_type { + 1 => to.push(email), + 2 => cc.push(email), + 3 => bcc.push(email), + _ => {} + } + } + } + } + } else { + let receiver = extract_string_property(message.properties(), 0x0076); + if let Some(receiver) = receiver { + to.push(receiver); + } + } + (to, cc, bcc) +} + +fn extract_subject(props: &MessageProperties) -> Option { + props.get(0x0037).and_then(|val| encoding::decode_subject(val)) +} + +fn extract_string_property(properties: &MessageProperties, prop_id: u16) -> Option { + properties + .get(prop_id) + .and_then(|value| extract_string(value)) +} + +fn extract_attachment_string_property( + properties: &AttachmentProperties, + prop_id: u16, +) -> Option { + properties + .get(prop_id) + .and_then(|value| extract_string(value)) +} + +fn extract_string(value: &PropertyValue) -> Option { + match value { + PropertyValue::String8(value) => Some(value.to_string()), + PropertyValue::Unicode(value) => Some(value.to_string()), + _ => None, + } +} + +fn extract_text(properties: &MessageProperties) -> Option { + properties.get(0x1000).and_then(extract_string).or_else(|| { + properties.get(0x1009).and_then(|value| match value { + PropertyValue::Binary(value) => encoding::decode_rtf_compressed(value.buffer()), + _ => None, + }) + }) +} + +fn extract_html(properties: &MessageProperties) -> Option { + properties.get(0x1013).and_then(|value| match value { + PropertyValue::Binary(value) => { + let code_page = properties + .get(0x3FDE) + .and_then(|v| { + if let PropertyValue::Integer32(cpid) = v { + Some(*cpid as u16) + } else { + None + } + }) + .unwrap_or(65001); + encoding::decode_html_body(value.buffer(), code_page) + } + PropertyValue::String8(value) => Some(value.to_string()), + PropertyValue::Unicode(value) => Some(value.to_string()), + _ => None, + }) +} + +fn extract_i64_property(properties: &MessageProperties, prop_ids: &[u16]) -> Option { + for &prop_id in prop_ids { + if let Some(PropertyValue::Time(value)) = properties.get(prop_id) { + return Some(*value); + } + } + None +} + +/// Open a PST file and count total messages across all folders. +/// Called from the web upload flow to get the total before processing. +pub fn count_pst_messages(pst_path: &std::path::Path) -> crate::error::BichonResult { + let pst_store = outlook_pst::open_store(pst_path).map_err(|e| { + crate::raise_error!( + format!("Failed to open PST file: {:?}", e), + crate::error::code::ErrorCode::InvalidParameter + ) + })?; + + let ipm_sub_tree = pst_store.properties().ipm_sub_tree_entry_id().map_err(|e| { + crate::raise_error!( + format!("Could not find IPM_SUBTREE in PST: {:?}", e), + crate::error::code::ErrorCode::InvalidParameter + ) + })?; + + let ipm_subtree_folder = pst_store.open_folder(&ipm_sub_tree).map_err(|e| { + crate::raise_error!( + format!("Failed to open root mailbox folder: {:?}", e), + crate::error::code::ErrorCode::InvalidParameter + ) + })?; + + Ok(count_folder_messages(&ipm_subtree_folder)) +} + +fn count_folder_messages(folder: &Rc) -> usize { + let mut count = 0usize; + + if let Some(contents_table) = folder.contents_table() { + for row in contents_table.rows_matrix() { + let store = folder.store().clone(); + let entry_id = match store + .properties() + .make_entry_id(NodeId::from(u32::from(row.id()))) + { + Ok(id) => id, + Err(_) => continue, + }; + + if store.open_message(&entry_id, None).is_ok() { + count += 1; + } + } + } + + if let Some(hierarchy_table) = folder.hierarchy_table() { + for row in hierarchy_table.rows_matrix() { + let node = NodeId::from(u32::from(row.id())); + if let Ok(entry_id) = folder.store().properties().make_entry_id(node) { + if let Ok(sub_folder) = folder.store().open_folder(&entry_id) { + count += count_folder_messages(&sub_folder); + } + } + } + } + + count +} + +/// Walk all folders and process messages, calling the progress callback +/// every 50 messages. Used by the web upload flow. +pub fn process_folder_with_progress( + folder: &Rc, + parent_path: &str, + account_id: u64, + total: usize, + success_count: &mut usize, + failed_details: &mut Vec, + index: &mut usize, + progress_cb: &F, +) where + F: Fn(usize, usize), // (processed, failed) +{ + process_folder_with_progress_inner( + folder, + parent_path, + account_id, + total, + success_count, + failed_details, + index, + progress_cb, + ); +} + +fn process_folder_with_progress_inner( + folder: &Rc, + parent_path: &str, + account_id: u64, + total: usize, + success_count: &mut usize, + failed_details: &mut Vec, + index: &mut usize, + progress_cb: &F, +) where + F: Fn(usize, usize), +{ + let folder_name = folder + .properties() + .display_name() + .unwrap_or_else(|_| "Unknown".to_string()); + + let mail_folder = if parent_path.is_empty() { + folder_name + } else { + format!("{}/{}", parent_path, folder_name) + }; + + tracing::debug!("Processing PST folder: {}", mail_folder); + + let mailbox_id = match super::resolve_mailbox_by_account_id(account_id, &mail_folder) { + Ok(id) => id, + Err(e) => { + tracing::error!("Failed to resolve mailbox '{}': {:?}", mail_folder, e); + // Still recurse into sub-folders even if this folder's mailbox creation fails + if let Some(hierarchy_table) = folder.hierarchy_table() { + for row in hierarchy_table.rows_matrix() { + let node = NodeId::from(u32::from(row.id())); + if let Ok(entry_id) = folder.store().properties().make_entry_id(node) { + if let Ok(sub_folder) = folder.store().open_folder(&entry_id) { + process_folder_with_progress_inner( + &sub_folder, + &mail_folder, + account_id, + total, + success_count, + failed_details, + index, + progress_cb, + ); + } + } + } + } + return; + } + }; + + let mut batch_size = 0usize; + + if let Some(contents_table) = folder.contents_table() { + for row in contents_table.rows_matrix() { + let store = folder.store().clone(); + + let entry_id = match store + .properties() + .make_entry_id(NodeId::from(u32::from(row.id()))) + { + Ok(id) => id, + Err(e) => { + tracing::warn!("Skip PST row {}: {:?}", row.unique(), e); + continue; + } + }; + + match store.open_message(&entry_id, None) { + Ok(message) => match build_eml_base64(message) { + Some(base64_eml) => { + let decoded = match crate::base64_decode_url_safe!(base64_eml.as_bytes()) { + Ok(bytes) => bytes, + Err(e) => { + failed_details.push(super::FailedItemDetail { + index: *index, + error_message: format!( + "Failed to decode base64 EML at index {}: {:?}", + *index, e + ), + }); + *index += 1; + batch_size += 1; + continue; + } + }; + + match futures::executor::block_on( + extract_envelope_from_eml(&decoded, account_id, mailbox_id) + ) { + Ok(_) => { + *success_count += 1; + } + Err(e) => { + failed_details.push(super::FailedItemDetail { + index: *index, + error_message: format!("{:?}", e), + }); + } + }; + *index += 1; + batch_size += 1; + } + None => {} + }, + Err(e) => { + tracing::warn!("Open PST message error: {:?}", e); + } + } + + // Report progress every 50 messages + if batch_size % 50 == 0 { + progress_cb(*success_count + failed_details.len(), failed_details.len()); + } + } + } + + if let Some(hierarchy_table) = folder.hierarchy_table() { + for row in hierarchy_table.rows_matrix() { + let node = NodeId::from(u32::from(row.id())); + if let Ok(entry_id) = folder.store().properties().make_entry_id(node) { + if let Ok(sub_folder) = folder.store().open_folder(&entry_id) { + process_folder_with_progress_inner( + &sub_folder, + &mail_folder, + account_id, + total, + success_count, + failed_details, + index, + progress_cb, + ); + } + } + } + } +} diff --git a/crates/cli/src/mbox/reader.rs b/crates/core/src/import/reader.rs similarity index 88% rename from crates/cli/src/mbox/reader.rs rename to crates/core/src/import/reader.rs index b5e2730f..b2b6aa35 100644 --- a/crates/cli/src/mbox/reader.rs +++ b/crates/core/src/import/reader.rs @@ -21,6 +21,8 @@ use std::fs; use std::io; use std::path::Path; +/// Memory-mapped MBOX file. Messages are yielded one at a time without +/// loading the entire file into RAM. pub struct MboxFile { map: Mmap, } @@ -127,10 +129,6 @@ impl<'a> Iterator for MboxReader<'a> { #[cfg(test)] mod tests { - use mail_parser::MessageParser; - - use crate::mbox::gmail::determine_folder; - use super::*; fn collect_entries(data: &[u8]) -> Vec<&[u8]> { @@ -144,6 +142,7 @@ mod tests { let e = collect_entries(data); assert_eq!(e, vec![b"mail1\n", b"mail2\n"]); } + #[test] fn no_trailing_newline() { let data = b"From a\nmail1"; @@ -204,22 +203,4 @@ mod tests { let e = collect_entries(&data); assert_eq!(e.len(), 1000); } - - #[test] - fn test11() { - let mbox = MboxFile::from_file(Path::new("e:\\test.mbox")).unwrap(); - - for e in mbox.iter() { - let body = e.data; - - let message = MessageParser::new().parse(body).unwrap(); - let labels = message.header("X-Gmail-Labels").unwrap().as_text().unwrap(); - //println!("offset={} X-Gmail-Labels={:?}", e.offset, labels); - println!( - "X-Gmail-Labels={:?}, determine_folder={}", - labels, - determine_folder(labels) - ) - } - } } diff --git a/crates/core/src/mailbox/list.rs b/crates/core/src/mailbox/list.rs index 667e2201..5fb1068b 100644 --- a/crates/core/src/mailbox/list.rs +++ b/crates/core/src/mailbox/list.rs @@ -157,8 +157,13 @@ async fn fetch_remote_with_progress(account_id: u64) -> BichonResult, /// Whether the extracted text came from OCR. + #[serde(default)] pub extracted_is_ocr: bool, } @@ -205,7 +206,9 @@ pub fn retrieve_email_content( content_type.c_subtype.as_deref().unwrap_or("") ); - let inline = disposition.map(|d| d.is_inline()).unwrap_or(false); + let inline = disposition + .map(|d| d.is_inline()) + .unwrap_or_else(|| attachment.content_id().is_some()); if inline { if let Some(html1) = html.as_deref() { @@ -301,7 +304,9 @@ pub fn retrieve_nested_eml_content( for attachment in nested_message.attachments() { let cid = attachment.content_id(); let disposition = attachment.content_disposition(); - let is_inline = disposition.map(|d| d.is_inline()).unwrap_or(false); + let is_inline = disposition + .map(|d| d.is_inline()) + .unwrap_or_else(|| cid.is_some()); if has_html && is_inline && cid.is_some() { let content_id = cid.unwrap(); @@ -361,3 +366,102 @@ pub fn retrieve_nested_eml_content( has_remote_content, }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Simulates JSON written by a version before `extracted_text`, `extracted_page_count`, + /// and `extracted_is_ocr` were added to [`AttachmentInfo`]. Deserialization must + /// succeed and fill the missing fields with their defaults. + #[test] + fn attachment_info_backward_compat_no_extracted_fields() { + let old_json = r#"[ + { + "file_type": "application/pdf", + "inline": false, + "filename": "report.pdf", + "size": 12345, + "content_id": null, + "content_hash": "abc123", + "is_message": false + }, + { + "file_type": "image/png", + "inline": true, + "filename": "logo.png", + "size": 6789, + "content_id": "cid:logo@example.com", + "content_hash": "def456", + "is_message": false + } + ]"#; + + let attachments: Vec = + serde_json::from_str(old_json).expect("should deserialize legacy JSON"); + + assert_eq!(attachments.len(), 2); + + // First attachment (regular file) + assert_eq!(attachments[0].file_type, "application/pdf"); + assert!(!attachments[0].inline); + assert_eq!(attachments[0].filename.as_deref(), Some("report.pdf")); + assert_eq!(attachments[0].size, 12345); + assert_eq!(attachments[0].content_id, None); + assert_eq!(attachments[0].content_hash, "abc123"); + assert!(!attachments[0].is_message); + // Fields added after the legacy format — must default correctly + assert_eq!(attachments[0].extracted_text, None); + assert_eq!(attachments[0].extracted_page_count, None); + assert!(!attachments[0].extracted_is_ocr); + + // Second attachment (inline image with content-id) + assert_eq!(attachments[1].file_type, "image/png"); + assert!(attachments[1].inline); + assert_eq!(attachments[1].filename.as_deref(), Some("logo.png")); + assert_eq!(attachments[1].size, 6789); + assert_eq!(attachments[1].content_id.as_deref(), Some("cid:logo@example.com")); + assert_eq!(attachments[1].content_hash, "def456"); + assert!(!attachments[1].is_message); + assert_eq!(attachments[1].extracted_text, None); + assert_eq!(attachments[1].extracted_page_count, None); + assert!(!attachments[1].extracted_is_ocr); + } + + /// Current struct must round-trip through serde_json without data loss. + #[test] + fn attachment_info_round_trip() { + let attachments = vec![ + AttachmentInfo { + file_type: "text/html".into(), + inline: false, + filename: Some("page.html".into()), + size: 42, + content_id: None, + content_hash: "hash1".into(), + is_message: true, + extracted_text: Some("hello world".into()), + extracted_page_count: Some(1), + extracted_is_ocr: false, + }, + AttachmentInfo { + file_type: "application/zip".into(), + inline: false, + filename: Some("archive.zip".into()), + size: 99999, + content_id: None, + content_hash: "hash2".into(), + is_message: false, + extracted_text: None, + extracted_page_count: None, + extracted_is_ocr: true, + }, + ]; + + let json = serde_json::to_string(&attachments).expect("serialize"); + let round_tripped: Vec = + serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(attachments, round_tripped); + } +} diff --git a/crates/core/src/message/search.rs b/crates/core/src/message/search.rs index 910fc158..f7cdbcdb 100644 --- a/crates/core/src/message/search.rs +++ b/crates/core/src/message/search.rs @@ -43,6 +43,10 @@ pub struct EmailSearchFilter { pub to: Option, pub cc: Option, pub bcc: Option, + /// Matches if the address appears in `to`, `cc`, or `bcc` (OR semantics). + pub any_recipient: Option, + /// Matches if the address appears in `from`, `to`, `cc`, or `bcc` (OR semantics). + pub any_participant: Option, pub since: Option, pub before: Option, /// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp. diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs index 02f15aa4..05308ef6 100644 --- a/crates/core/src/migrate/mod.rs +++ b/crates/core/src/migrate/mod.rs @@ -4,7 +4,7 @@ use crate::{ error::{code::ErrorCode, BichonResult}, migrate::{ legacy::schema::SchemaTools, - store::{LegacyDirs, NewDirs, NewIndexWriter}, + store::{LegacyDirs, NewIndexWriter}, }, raise_error, settings::cli::SETTINGS, @@ -121,7 +121,7 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result { pub fn do_migrate_segment( batch_size: u32, legacy: LegacyDirs, - new_dirs: NewDirs, + writer: &mut NewIndexWriter, segment_index: usize, mut on_progress: F, ) -> BichonResult<()> @@ -226,8 +226,6 @@ where drop(envelope_index); // ── Phase 2: process EML docs, streaming one at a time ───────────── - let mut writer = NewIndexWriter::open(new_dirs)?; - let mut total_migrated = 0usize; let mut total_skipped = 0usize; @@ -308,7 +306,6 @@ where chunk_start = chunk_end; } - writer.finish_writers()?; on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped)); Ok(()) } diff --git a/crates/core/src/migrate/store.rs b/crates/core/src/migrate/store.rs index c453e9bf..8dc203f9 100644 --- a/crates/core/src/migrate/store.rs +++ b/crates/core/src/migrate/store.rs @@ -86,13 +86,22 @@ pub fn detach_attachments_standalone( for (raw_start, raw_end, att) in ranges { let content_hash = compute_content_hash(att.contents()); - blobs.push(( - content_hash.clone(), - Bytes::copy_from_slice(&original_body[raw_start..raw_end]), - )); + let body_len = original_body.len(); + let raw_start = raw_start.min(body_len); + let raw_end = raw_end.min(body_len); + let range_valid = raw_start < raw_end; + + if range_valid { + blobs.push(( + content_hash.clone(), + Bytes::copy_from_slice(&original_body[raw_start..raw_end]), + )); + } - let placeholder = format!("<>", &content_hash); - stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + if range_valid { + let placeholder = format!("<>", &content_hash); + stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + } infos.push(AttachmentInfo { filename: att.attachment_name().map(|n| n.to_string()), @@ -100,7 +109,7 @@ pub fn detach_attachments_standalone( inline: att .content_disposition() .map(|d| d.is_inline()) - .unwrap_or(false), + .unwrap_or_else(|| att.content_id().is_some()), file_type: att .content_type() .map(|ct| { @@ -395,6 +404,7 @@ impl NewIndexWriter { regular_attachment_count: attachment_docs.len(), tags: None, account_email: None, + account_name: None, mailbox_name: None, content_hash: email_content_hash, }; diff --git a/crates/core/src/oauth2/flow.rs b/crates/core/src/oauth2/flow.rs index 72f74a2e..fa5fa807 100644 --- a/crates/core/src/oauth2/flow.rs +++ b/crates/core/src/oauth2/flow.rs @@ -20,6 +20,7 @@ use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::oauth2::{entity::OAuth2, pending::OAuth2PendingEntity, token::OAuth2AccessToken}; use crate::settings::proxy::Proxy; +use crate::utils::net::parse_proxy_url; use crate::{decrypt, encrypt, raise_error}; use oauth2::{ basic::BasicClient, AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, @@ -265,14 +266,12 @@ impl OAuth2Flow { fn build_http_client(use_proxy: Option) -> BichonResult { if let Some(proxy_id) = use_proxy { let proxy = Proxy::get(proxy_id)?; + let proxy_url = parse_proxy_url(&proxy.url)?.standard_url(); return oauth2::reqwest::ClientBuilder::new() .redirect(oauth2::reqwest::redirect::Policy::none()) - .proxy(reqwest::Proxy::all(&proxy.url).map_err(|e| { + .proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| { raise_error!( - format!( - "Failed to configure SOCKS5 proxy ({}): {:#?}. Please check", - &proxy.url, e - ), + "Failed to configure proxy. Please check the proxy configuration.".into(), ErrorCode::InternalError ) })?) diff --git a/crates/core/src/settings/cli.rs b/crates/core/src/settings/cli.rs index 3c1f0917..1e6c0e73 100644 --- a/crates/core/src/settings/cli.rs +++ b/crates/core/src/settings/cli.rs @@ -308,6 +308,56 @@ pub struct Settings { help = "Enable SMTP authentication requirement" )] pub bichon_smtp_auth_required: bool, + + /// Enable OIDC-based Single Sign-On (Pro/Enterprise feature). + #[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")] + pub bichon_oidc_enabled: bool, + + /// OIDC issuer URL (e.g. https://keycloak.example.com/realms/myorg). + #[clap(long, env, help = "OpenID Connect issuer URL")] + pub bichon_oidc_issuer_url: Option, + + /// OIDC client ID registered with the IdP. + #[clap(long, env, help = "OpenID Connect client ID")] + pub bichon_oidc_client_id: Option, + + /// OIDC client secret registered with the IdP. + #[clap(long, env, help = "OpenID Connect client secret")] + pub bichon_oidc_client_secret: Option, + + /// OIDC redirect URI (must match what's registered with the IdP). + #[clap(long, env, help = "OpenID Connect redirect URI")] + pub bichon_oidc_redirect_uri: Option, + + /// Maximum HTTP request body size in MB for file uploads (default: 1100 MB). + /// Requests exceeding this limit are rejected at the framework level before + /// the application reads the body, preventing memory exhaustion attacks. + #[clap( + long, + default_value = "1100", + env, + help = "Maximum HTTP request body size in MB for file uploads" + )] + pub bichon_upload_body_limit_mb: u64, + + /// Maximum per-file size in MB for MBOX uploads via the web UI (default: 1024 MB = 1 GB). + /// Individual EML files are always capped at 100 MB regardless of this setting. + #[clap( + long, + default_value = "1024", + env, + help = "Maximum per-file size in MB for MBOX uploads via the web UI" + )] + pub bichon_web_mbox_upload_limit_mb: u64, + + /// Maximum per-file size in MB for PST uploads via the web UI (default: 2048 MB = 2 GB). + #[clap( + long, + default_value = "2048", + env, + help = "Maximum per-file size in MB for PST uploads via the web UI" + )] + pub bichon_web_pst_upload_limit_mb: u64, } impl Settings { @@ -317,9 +367,8 @@ impl Settings { // rejects it, fall back to parsing with only the binary name so that // the settings come entirely from environment variables. let args: Vec = std::env::args().collect(); - let s = Self::try_parse_from(&args).unwrap_or_else(|_| { - Self::parse_from(std::iter::once(args[0].clone())) - }); + let s = Self::try_parse_from(&args) + .unwrap_or_else(|_| Self::parse_from(std::iter::once(args[0].clone()))); if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() { panic!( "One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set" diff --git a/crates/core/src/settings/mod.rs b/crates/core/src/settings/mod.rs index 827e8368..6e843e49 100644 --- a/crates/core/src/settings/mod.rs +++ b/crates/core/src/settings/mod.rs @@ -60,6 +60,17 @@ pub struct SystemConfigurations { pub bichon_smtp_auth_required: bool, pub bichon_smtp_tls_key_path: Option, pub bichon_smtp_tls_cert_path: Option, + + pub bichon_oidc_enabled: bool, + pub bichon_oidc_issuer_url: Option, + pub bichon_oidc_client_id: Option, + pub bichon_oidc_redirect_uri: Option, + + pub bichon_upload_body_limit_mb: u64, + + pub bichon_web_mbox_upload_limit_mb: u64, + + pub bichon_web_pst_upload_limit_mb: u64, } impl From<&Settings> for SystemConfigurations { @@ -94,6 +105,13 @@ impl From<&Settings> for SystemConfigurations { bichon_smtp_auth_required: s.bichon_smtp_auth_required, bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(), bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(), + bichon_oidc_enabled: s.bichon_oidc_enabled, + bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(), + bichon_oidc_client_id: s.bichon_oidc_client_id.clone(), + bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(), + bichon_upload_body_limit_mb: s.bichon_upload_body_limit_mb, + bichon_web_mbox_upload_limit_mb: s.bichon_web_mbox_upload_limit_mb, + bichon_web_pst_upload_limit_mb: s.bichon_web_pst_upload_limit_mb, } } } diff --git a/crates/core/src/settings/proxy.rs b/crates/core/src/settings/proxy.rs index 23592401..ddfbc038 100644 --- a/crates/core/src/settings/proxy.rs +++ b/crates/core/src/settings/proxy.rs @@ -18,6 +18,7 @@ //use poem_openapi::Object; use serde::{Deserialize, Serialize}; +use std::{error::Error, time::Duration}; use crate::{ database::{ @@ -26,9 +27,30 @@ use crate::{ }, error::{code::ErrorCode, BichonResult}, id, raise_error, utc_now, - utils::net::parse_proxy_addr, + utils::net::parse_proxy_url, }; +const PROXY_TEST_TIMEOUT: Duration = Duration::from_secs(8); +const GEO_PROVIDERS: &[GeoProvider] = &[ + GeoProvider { + name: "ip-api.com", + url: "http://ip-api.com/json/?fields=status,message,query,country,countryCode,regionName,city,isp,timezone,lat,lon", + }, + GeoProvider { + name: "ipwho.is", + url: "https://ipwho.is/", + }, + GeoProvider { + name: "ipapi.co", + url: "https://ipapi.co/json/", + }, +]; + +struct GeoProvider { + name: &'static str, + url: &'static str, +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct Proxy { @@ -45,6 +67,16 @@ pub struct Proxy { pub updated_at: i64, } +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ProxyTestResult { + pub ip: Option, + pub country: Option, + pub region: Option, + pub city: Option, + pub isp: Option, +} + impl MemDbModel for Proxy { fn collection() -> &'static str { "proxies" @@ -85,9 +117,10 @@ impl Proxy { pub fn update(id: u64, url: String) -> BichonResult<()> { update_impl(DB_MANAGER.db(), &id.to_string(), move |current: Proxy| { - let mut updated = current.clone(); + let mut updated = current; updated.url = url; updated.updated_at = utc_now!(); + updated.validate()?; Ok(updated) })?; Ok(()) @@ -98,11 +131,185 @@ impl Proxy { insert_impl(DB_MANAGER.db(), self.to_owned()) } - /// Validate that the URL is a valid SOCKS5 proxy URL. + /// Validate that the URL is a valid proxy URL. pub fn validate(&self) -> BichonResult<()> { - parse_proxy_addr(&self.url)?; + parse_proxy_url(&self.url)?; Ok(()) } + + pub async fn test_connectivity(&self) -> BichonResult { + test_proxy_url(&self.url).await + } + + pub async fn test(id: u64) -> BichonResult { + let proxy = Self::get(id)?; + proxy.test_connectivity().await + } +} + +async fn test_proxy_url(url: &str) -> BichonResult { + let proxy_url = parse_proxy_url(url)?.standard_url(); + let client = reqwest::Client::builder() + .timeout(PROXY_TEST_TIMEOUT) + .proxy(reqwest::Proxy::all(&proxy_url).map_err(|_| { + raise_error!( + "Failed to configure proxy. Please check the proxy configuration.".into(), + ErrorCode::InvalidParameter + ) + })?) + .build() + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut last_error = None; + for provider in GEO_PROVIDERS { + match test_geo_provider(&client, provider).await { + Ok(result) => return Ok(result), + Err(err) => last_error = Some(err.to_string()), + } + } + + Err(raise_error!( + format!( + "Proxy check failed with all geo providers: {}", + last_error.unwrap_or_else(|| "unknown error".into()) + ), + ErrorCode::NetworkError + )) +} + +async fn test_geo_provider( + client: &reqwest::Client, + provider: &GeoProvider, +) -> BichonResult { + let value = client + .get(provider.url) + .send() + .await + .map_err(|e| { + raise_error!( + proxy_request_error_message( + &format!("Proxy check request failed via {}", provider.name), + &e + ), + ErrorCode::NetworkError + ) + })? + .error_for_status() + .map_err(|e| { + raise_error!( + proxy_request_error_message( + &format!("Proxy check request failed via {}", provider.name), + &e + ), + ErrorCode::NetworkError + ) + })? + .json::() + .await + .map_err(|e| { + raise_error!( + proxy_request_error_message( + &format!("Failed to read proxy check response from {}", provider.name), + &e + ), + ErrorCode::NetworkError + ) + })?; + + proxy_test_result_from_value(provider.name, &value) +} + +fn proxy_test_result_from_value( + provider: &str, + value: &serde_json::Value, +) -> BichonResult { + if provider == "ip-api.com" && value["status"].as_str() == Some("fail") { + return Err(raise_error!( + format!( + "ip-api.com proxy check failed: {}", + value["message"].as_str().unwrap_or("unknown error") + ), + ErrorCode::NetworkError + )); + } + if provider == "ipwho.is" && value["success"].as_bool() == Some(false) { + return Err(raise_error!( + format!( + "ipwho.is proxy check failed: {}", + value["message"].as_str().unwrap_or("unknown error") + ), + ErrorCode::NetworkError + )); + } + if provider == "ipapi.co" && value["error"].as_bool() == Some(true) { + return Err(raise_error!( + format!( + "ipapi.co proxy check failed: {}", + value["reason"].as_str().unwrap_or("unknown error") + ), + ErrorCode::NetworkError + )); + } + + let ip_key = if provider == "ip-api.com" { + "query" + } else { + "ip" + }; + let ip = value[ip_key].as_str().ok_or_else(|| { + raise_error!( + format!("{provider} did not return an IP address"), + ErrorCode::NetworkError + ) + })?; + let connection = &value["connection"]; + + Ok(ProxyTestResult { + ip: Some(ip.to_string()), + country: value[if provider == "ipapi.co" { + "country_name" + } else { + "country" + }] + .as_str() + .map(str::to_string), + region: value[if provider == "ip-api.com" { + "regionName" + } else { + "region" + }] + .as_str() + .map(str::to_string), + city: value["city"].as_str().map(str::to_string), + isp: if provider == "ipapi.co" { + value["org"].as_str().map(str::to_string) + } else if provider == "ip-api.com" { + value["isp"].as_str().map(str::to_string) + } else { + connection["isp"].as_str().map(str::to_string) + }, + }) +} + +fn proxy_request_error_message(context: &str, err: &reqwest::Error) -> String { + let kind = if err.is_timeout() { + "timed out" + } else if err.is_connect() { + "could not connect through the proxy" + } else if err.is_status() { + "received an error response" + } else { + "request failed" + }; + let mut message = format!("{context}: {kind}: {err}"); + let mut source = err.source(); + + while let Some(err) = source { + message.push_str(&format!(": {err}")); + source = err.source(); + } + + message } #[cfg(test)] @@ -111,11 +318,54 @@ mod tests { #[test] fn test_valid_proxy_urls() { - let urls = vec!["socks5://127.0.0.1:1080", "http://127.0.0.1:8080"]; + let urls = vec![ + "socks5://127.0.0.1:1080", + "http://127.0.0.1:8080", + "socks5://proxy.example.com:1080", + "socks5://user:pass@proxy.example.com:1080", + "http://user:pass@proxy.example.com:8080", + "socks5://[::1]:1080", + "socks5://user:pass@[::1]:1080", + // Non-standard format: host:port:user:pass + "socks5://server.nodeprovider.com:8080:username123:passwordhere", + "http://server.nodeprovider.com:8080:username123:passwordhere", + ]; for url in urls { let proxy = Proxy::new(url.to_string()); assert!(proxy.validate().is_ok(), "URL should be valid: {}", url); } } + + #[test] + fn test_invalid_proxy_urls() { + for url in ["socks5://user@proxy.example.com:1080", "socks5://::1:1080"] { + let proxy = Proxy::new(url.to_string()); + assert!(proxy.validate().is_err(), "URL should be invalid: {}", url); + } + } + + #[test] + fn test_ipv6_proxy_urls_render_with_brackets() { + let addr = parse_proxy_url("socks5://[::1]:1080").unwrap(); + assert_eq!(addr.standard_url(), "socks5://[::1]:1080"); + + let addr = parse_proxy_url("socks5://user:pass@[::1]:1080").unwrap(); + assert_eq!(addr.standard_url(), "socks5://user:pass@[::1]:1080"); + } + + #[test] + fn proxy_test_result_rejects_empty_provider_response() { + let result = proxy_test_result_from_value("ipwho.is", &serde_json::json!({})); + assert!(result.is_err()); + } + + #[test] + fn proxy_test_result_rejects_provider_error_response() { + let result = proxy_test_result_from_value( + "ipwho.is", + &serde_json::json!({ "success": false, "message": "reserved range" }), + ); + assert!(result.is_err()); + } } diff --git a/crates/core/src/store/envelope.rs b/crates/core/src/store/envelope.rs index 6767f29f..ca39460e 100644 --- a/crates/core/src/store/envelope.rs +++ b/crates/core/src/store/envelope.rs @@ -27,6 +27,7 @@ pub struct Envelope { pub message_id: String, pub account_id: u64, pub account_email: Option, + pub account_name: Option, pub mailbox_id: u64, pub mailbox_name: Option, pub uid: u32, diff --git a/crates/core/src/store/tantivy/dedup_cache.rs b/crates/core/src/store/tantivy/dedup_cache.rs new file mode 100644 index 00000000..836a7ec3 --- /dev/null +++ b/crates/core/src/store/tantivy/dedup_cache.rs @@ -0,0 +1,641 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{LazyLock, Mutex}; + +use crate::store::tantivy::envelope::ENVELOPE_MANAGER; +use crate::store::tantivy::fields::{F_ACCOUNT_ID, F_CONTENT_HASH, F_INGEST_AT, F_MAILBOX_ID}; +use crate::utc_now; + +/// Max entries before evicting the oldest. +/// At ~152 bytes/entry, 300_000 ≈ 45 MB, within the 50 MB budget. +const MAX_ENTRIES: usize = 300_000; + +/// Fraction of entries to keep when evicting (newest 3/4). +const KEEP_FRACTION_NUM: usize = 3; +const KEEP_FRACTION_DEN: usize = 4; + +/// Populate only loads entries ingested within this window. +const POPULATE_WINDOW_MS: i64 = 7 * 24 * 60 * 60 * 1000; // 7 days + +pub static DEDUP_CACHE: LazyLock = LazyLock::new(DedupCache::new); + +pub struct DedupCache { + entries: Mutex>, + max_entries: usize, + populated: AtomicBool, +} + +impl DedupCache { + fn new() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries: MAX_ENTRIES, + populated: AtomicBool::new(false), + } + } + + #[cfg(test)] + fn new_for_test() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries: MAX_ENTRIES, + populated: AtomicBool::new(true), + } + } + + #[cfg(test)] + fn new_for_test_small(max_entries: usize) -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries, + populated: AtomicBool::new(true), + } + } + + /// Returns true if this `(account_id, mailbox_id, content_hash)` triple + /// has already been seen. + /// + /// On the very first call the cache is populated from the Tantivy index + /// FAST columns (only entries ingested within [`POPULATE_WINDOW_MS`]). + /// If that scan fails the cache starts empty and still operates correctly + /// for newly-arriving emails. + pub fn contains(&self, account_id: u64, mailbox_id: u64, hash: &str) -> bool { + self.ensure_populated(); + + let entries = self.entries.lock().unwrap(); + entries.contains_key(&(account_id, mailbox_id, hash.to_string())) + } + + /// Insert a triple into the cache after it has been queued for indexing. + /// + /// Each entry is stamped with the current time. When the cache exceeds + /// [`MAX_ENTRIES`], the oldest entries are evicted, keeping the newest + /// `MAX_ENTRIES * 3/4`. + pub fn insert(&self, account_id: u64, mailbox_id: u64, hash: &str) { + let mut entries = self.entries.lock().unwrap(); + let now = utc_now!(); + entries.insert((account_id, mailbox_id, hash.to_string()), now); + + if entries.len() > self.max_entries { + let keep = self.max_entries * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN; + let mut vec: Vec<_> = entries.drain().collect(); + // Sort descending by timestamp (newest first) + vec.sort_by(|a, b| b.1.cmp(&a.1)); + for (k, v) in vec.into_iter().take(keep) { + entries.insert(k, v); + } + tracing::warn!( + "DedupCache evicted oldest entries, kept {}/{}", + entries.len(), + keep + ); + } + } + + // ── private ────────────────────────────────────────────────────────────── + + fn ensure_populated(&self) { + if self.populated.load(Ordering::Acquire) { + return; + } + self.do_populate(); + } + + fn do_populate(&self) { + if self + .populated + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + let reader = match ENVELOPE_MANAGER.create_reader() { + Ok(r) => r, + Err(e) => { + tracing::warn!("DedupCache: failed to create reader for populate: {e}"); + return; + } + }; + + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = self.entries.lock().unwrap(); + + for segment_reader in searcher.segment_readers() { + let account_col = match segment_reader.fast_fields().u64(F_ACCOUNT_ID) { + Ok(c) => c, + Err(_) => continue, + }; + let mailbox_col = match segment_reader.fast_fields().u64(F_MAILBOX_ID) { + Ok(c) => c, + Err(_) => continue, + }; + let hash_col = match segment_reader.fast_fields().str(F_CONTENT_HASH) { + Ok(Some(c)) => c, + _ => continue, + }; + let ingest_col = match segment_reader.fast_fields().i64(F_INGEST_AT) { + Ok(c) => c, + Err(_) => continue, + }; + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); + let mut hash_buf = String::new(); + if hash_col.ord_to_str(hash_ord, &mut hash_buf).is_err() { + continue; + } + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + + tracing::info!( + "DedupCache populated with {} entries from index (cutoff {}d ago)", + entries.len(), + POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000), + ); + } + + /// Remove all entries for a specific account. + pub fn remove_by_account(&self, account_id: u64) { + let mut entries = self.entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(aid, _, _), _| *aid != account_id); + let removed = before - entries.len(); + if removed > 0 { + tracing::info!( + "DedupCache: removed {} entries for account {}", + removed, + account_id + ); + } + } + + /// Remove all entries for a specific mailbox (across all accounts). + pub fn remove_by_mailbox(&self, mailbox_id: u64) { + let mut entries = self.entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(_, mid, _), _| *mid != mailbox_id); + let removed = before - entries.len(); + if removed > 0 { + tracing::info!( + "DedupCache: removed {} entries for mailbox {}", + removed, + mailbox_id + ); + } + } + + /// Remove a specific triple (most precise removal). + pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) { + let mut entries = self.entries.lock().unwrap(); + if entries + .remove(&(account_id, mailbox_id, hash.to_string())) + .is_some() + { + tracing::debug!( + "DedupCache: removed specific entry ({}, {}, {})", + account_id, + mailbox_id, + hash + ); + } + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::tantivy::fields::EmailFields; + use crate::store::tantivy::schema::SchemaTools; + use crate::store::tantivy::tokenizers::EuroTokenizer; + use std::fs; + use tantivy::{Index, TantivyDocument}; + + fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir() + .join("bichon-dedup-cache-test") + .join(name) + .join(uuid::Uuid::new_v4().to_string()); + fs::create_dir_all(&dir).unwrap(); + dir + } + + // ── basic contains / insert ───────────────────────────────────────────── + + #[test] + fn contains_after_insert() { + let cache = DedupCache::new_for_test(); + assert!(!cache.contains(1, 10, "hash-aaa")); + cache.insert(1, 10, "hash-aaa"); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + #[test] + fn different_hash_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(1, 10, "hash-bbb")); + } + + #[test] + fn different_account_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(2, 10, "hash-aaa")); + } + + #[test] + fn different_mailbox_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(1, 20, "hash-aaa")); + } + + #[test] + fn cross_account_allowed() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + cache.insert(2, 10, "hash-aaa"); + assert!(cache.contains(2, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + #[test] + fn cross_mailbox_allowed() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + cache.insert(1, 20, "hash-aaa"); + assert!(cache.contains(1, 20, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + // ── time-based eviction ───────────────────────────────────────────────── + + #[test] + fn eviction_keeps_newest() { + let cap = 100; + let cache = DedupCache::new_for_test_small(cap); + + // Fill to exact capacity. Entry hash-0 is oldest. + for i in 0..cap { + cache.insert(1, 1, &format!("hash-{}", i)); + std::thread::sleep(std::time::Duration::from_micros(100)); + } + assert!(cache.contains(1, 1, "hash-0")); + assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1))); + + // One more triggers eviction + cache.insert(1, 1, "hash-overflow"); + + // Newest survives, oldest evicted + assert!(cache.contains(1, 1, "hash-overflow")); + assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1))); + assert!(!cache.contains(1, 1, "hash-0")); + + let keep = cap * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN; + assert!(cache.entries.lock().unwrap().len() <= keep); + } + + // ── memory bound ──────────────────────────────────────────────────────── + + #[test] + fn memory_bound_within_budget() { + let cache = DedupCache::new_for_test(); + + for i in 0..MAX_ENTRIES { + cache.insert(1, 1, &format!("{:064x}", i)); + } + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), MAX_ENTRIES); + + let capacity = entries.capacity(); + // HashMap with (u64,u64,String) key + i64 value ≈ 112 + map overhead + let approx_bytes = capacity * (104 + 8 + 8); + let approx_mb = approx_bytes as f64 / (1024.0 * 1024.0); + println!( + "DedupCache: {} entries, {} buckets, ~{:.1} MB", + MAX_ENTRIES, capacity, approx_mb + ); + assert!( + approx_mb < 55.0, + "memory estimate {:.1} MB exceeds 55 MB buffer", + approx_mb + ); + } + + // ── populate guard ────────────────────────────────────────────────────── + + #[test] + fn populate_cas_is_idempotent() { + let cache = DedupCache::new_for_test(); + assert!(cache.populated.load(Ordering::Acquire)); + cache.ensure_populated(); + assert!(cache.populated.load(Ordering::Acquire)); + cache.do_populate(); + } + + // ── populate from test index ──────────────────────────────────────────── + + fn build_test_index() -> (Index, &'static EmailFields) { + let dir = temp_dir("populate"); + let schema = SchemaTools::email_schema(); + let fields = SchemaTools::email_fields(); + let index = Index::create_in_dir(&dir, schema).unwrap(); + index.tokenizers().register("euro", EuroTokenizer::new()); + (index, fields) + } + + fn add_email_doc( + fields: &EmailFields, + writer: &mut tantivy::IndexWriter, + account: u64, + mailbox: u64, + hash: &str, + ingest_at: i64, + ) { + let mut doc = TantivyDocument::new(); + doc.add_u64(fields.f_account_id, account); + doc.add_u64(fields.f_mailbox_id, mailbox); + doc.add_text(fields.f_content_hash, hash); + doc.add_i64(fields.f_ingest_at, ingest_at); + doc.add_text(fields.f_id, &uuid::Uuid::new_v4().to_string()); + doc.add_text(fields.f_subject, "test"); + doc.add_text(fields.f_body, "test body"); + doc.add_u64(fields.f_uid, 1); + doc.add_i64(fields.f_date, 1); + doc.add_i64(fields.f_internal_date, 1); + doc.add_u64(fields.f_size, 100); + writer.add_document(doc).unwrap(); + } + + #[test] + fn populate_reads_all_docs_in_window() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 2, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 1, 20, "hash-recent", recent); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert_eq!(cache.entries.lock().unwrap().len(), 3); + assert!(cache.contains(1, 10, "hash-recent")); + assert!(cache.contains(2, 10, "hash-recent")); + assert!(cache.contains(1, 20, "hash-recent")); + } + + #[test] + fn populate_skips_old_entries() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + let old = recent - POPULATE_WINDOW_MS - 60_000; // 1 minute past the window + add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 1, 10, "hash-old", old); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert!(cache.contains(1, 10, "hash-recent")); + assert!(!cache.contains(1, 10, "hash-old")); + assert_eq!(cache.entries.lock().unwrap().len(), 1); + } + + #[test] + fn populate_skips_deleted_docs() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + add_email_doc(&fields, &mut writer, 1, 10, "hash-keep", recent); + add_email_doc(&fields, &mut writer, 1, 10, "hash-delete", recent); + writer.commit().unwrap(); + + let term = tantivy::Term::from_field_text(fields.f_content_hash, "hash-delete"); + writer.delete_term(term); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert!(cache.contains(1, 10, "hash-keep")); + assert!(!cache.contains(1, 10, "hash-delete")); + } + + // ── removal methods ───────────────────────────────────────────────────── + + #[test] + fn remove_by_account_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-a1"); + cache.insert(1, 20, "hash-a2"); + cache.insert(2, 10, "hash-b1"); + cache.insert(2, 30, "hash-b2"); + cache.insert(1, 10, "hash-a3"); + + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + cache.remove_by_account(1); + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), 2); + assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string()))); + assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string()))); + assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string()))); + + assert!(entries.contains_key(&(2, 10, "hash-b1".to_string()))); + assert!(entries.contains_key(&(2, 30, "hash-b2".to_string()))); + } + + #[test] + fn remove_by_mailbox_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-1"); + cache.insert(1, 20, "hash-2"); + cache.insert(2, 10, "hash-3"); + cache.insert(3, 20, "hash-4"); + cache.insert(1, 10, "hash-5"); + + cache.remove_by_mailbox(10); + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), 2); + + assert!(entries.contains_key(&(1, 20, "hash-2".to_string()))); + assert!(entries.contains_key(&(3, 20, "hash-4".to_string()))); + + assert!(!entries.contains_key(&(1, 10, "hash-1".to_string()))); + assert!(!entries.contains_key(&(2, 10, "hash-3".to_string()))); + } + + #[test] + fn remove_specific_triple_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-aaa"); + cache.insert(1, 10, "hash-bbb"); + cache.insert(2, 20, "hash-aaa"); + + assert!(cache.contains(1, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-bbb")); + assert!(cache.contains(2, 20, "hash-aaa")); + + cache.remove(1, 10, "hash-aaa"); + + assert!(!cache.contains(1, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-bbb")); + assert!(cache.contains(2, 20, "hash-aaa")); + } +} diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index d5af7d2a..b47d8f6a 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -40,6 +40,7 @@ use crate::{ envelope::Envelope, tantivy::{ attachment::ATTACHMENT_MANAGER, + dedup_cache::DEDUP_CACHE, fatal_commit, fields::{ F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE, @@ -51,9 +52,11 @@ use crate::{ }, }, utc_now, + utils::html::extract_text, }; use chrono::Utc; +use mail_parser::MessageParser; use serde_json::json; use tantivy::{ aggregation::{ @@ -275,6 +278,80 @@ impl IndexManager { Box::new(boolean_query) } + /// Return all Message-IDs stored in Tantivy for a given mailbox. + /// Prefer `mailbox_contains_message_id` for existence checks on large + /// mailboxes — this method loads everything into a HashSet. + pub fn get_message_ids_for_mailbox( + &self, + account_id: u64, + mailbox_id: u64, + ) -> BichonResult> { + let query = self.mailbox_query(account_id, mailbox_id); + let fields = SchemaTools::email_fields(); + let searcher = self.create_searcher()?; + + let docs = searcher + .search(&query, &DocSetCollector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut result = HashSet::new(); + for doc_address in docs { + let doc = searcher + .doc::(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if let Some(v) = doc.get_first(fields.f_message_id) { + if let Some(s) = v.as_str() { + if !s.is_empty() { + result.insert(s.to_string()); + } + } + } + } + Ok(result) + } + + /// Check whether a specific Message-ID exists in a mailbox. + /// Uses a TermQuery — O(1) per call, no allocation proportional to + /// mailbox size. Suitable for large mailboxes where + /// `get_message_ids_for_mailbox` would allocate too much memory. + pub fn mailbox_contains_message_id( + &self, + account_id: u64, + mailbox_id: u64, + message_id: &str, + ) -> BichonResult { + let fields = SchemaTools::email_fields(); + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_account_id, account_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_mailbox_id, mailbox_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(fields.f_message_id, message_id), + IndexRecordOption::Basic, + )), + ), + ]); + let searcher = self.create_searcher()?; + let count = searcher + .search(&query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count > 0) + } + fn envelope_query(&self, account_id: u64, eid: &str) -> Box { let account_id_query = TermQuery::new( Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id), @@ -384,6 +461,34 @@ impl IndexManager { } } + // any_recipient: OR across to, cc, bcc + if let Some(ref v) = filter.any_recipient { + let mut recipient_queries: Vec<(Occur, Box)> = Vec::new(); + for field in [f.f_to_text, f.f_cc_text, f.f_bcc_text] { + let query_parser = QueryParser::for_index(&self.index, vec![field]); + if let Ok(q) = query_parser.parse_query(v) { + recipient_queries.push((Occur::Should, q)); + } + } + if !recipient_queries.is_empty() { + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(recipient_queries)))); + } + } + + // any_participant: OR across from, to, cc, bcc + if let Some(ref v) = filter.any_participant { + let mut participant_queries: Vec<(Occur, Box)> = Vec::new(); + for field in [f.f_from_text, f.f_to_text, f.f_cc_text, f.f_bcc_text] { + let query_parser = QueryParser::for_index(&self.index, vec![field]); + if let Ok(q) = query_parser.parse_query(v) { + participant_queries.push((Occur::Should, q)); + } + } + if !participant_queries.is_empty() { + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(participant_queries)))); + } + } + if let Some(has) = filter.has_attachment { let lower: Bound; let upper: Bound; @@ -790,6 +895,8 @@ impl IndexManager { attachments_content_hashes, )?; } + + DEDUP_CACHE.remove_by_account(account_id); Ok(()) } @@ -813,8 +920,8 @@ impl IndexManager { } let mut queries: Vec> = Vec::with_capacity(mailbox_ids.len()); - for mailbox_id in mailbox_ids { - queries.push(self.mailbox_query(account_id, mailbox_id)); + for mailbox_id in &mailbox_ids { + queries.push(self.mailbox_query(account_id, *mailbox_id)); } let mut writer = self.index_writer.lock().await; for query in queries { @@ -833,6 +940,11 @@ impl IndexManager { attachments_content_hashes, )?; } + + for mailbox_id in mailbox_ids { + DEDUP_CACHE.remove_by_mailbox(mailbox_id); + } + Ok(()) } @@ -840,6 +952,21 @@ impl IndexManager { &self, query: Box, ) -> BichonResult<(HashSet, HashSet)> { + let (eml_with_mailbox, attachments_content_hashes) = + self.collect_content_hashes_with_mailbox(query)?; + + let eml_content_hashes = eml_with_mailbox + .into_iter() + .map(|(hash, _mailbox_id)| hash) + .collect(); + + Ok((eml_content_hashes, attachments_content_hashes)) + } + + fn collect_content_hashes_with_mailbox( + &self, + query: Box, + ) -> BichonResult<(HashSet<(String, u64)>, HashSet)> { let mut eml_content_hashes = HashSet::new(); let mut attachments_content_hashes = HashSet::new(); @@ -855,10 +982,14 @@ impl IndexManager { .doc::(doc_address) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mailbox_id = doc.get_first(fields.f_mailbox_id).and_then(|v| v.as_u64()); + // Extract content_hash if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) { - if let Some(str) = content_hash_value.as_str() { - eml_content_hashes.insert(str.to_string()); + if let (Some(hash_str), Some(mailbox_id)) = + (content_hash_value.as_str(), mailbox_id) + { + eml_content_hashes.insert((hash_str.to_string(), mailbox_id)); } } @@ -931,7 +1062,7 @@ impl IndexManager { return Ok(()); } - let mut eml_content_hashes: HashSet = HashSet::new(); + let mut eml_content_hash_triples: HashSet<(u64, u64, String)> = HashSet::new(); let mut attachments_content_hashes: HashSet = HashSet::new(); for (account_id, envelope_ids) in &deletes { @@ -942,8 +1073,14 @@ impl IndexManager { for eid in unique_ids { let query = self.envelope_query(*account_id, eid); - let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?; - eml_content_hashes.extend(eml_hashes); + let (eml_hashes_with_mailbox, attachment_hashes) = + self.collect_content_hashes_with_mailbox(query)?; + + eml_content_hash_triples.extend( + eml_hashes_with_mailbox + .into_iter() + .map(|(hash, mailbox_id)| (*account_id, mailbox_id, hash)), + ); attachments_content_hashes.extend(attachment_hashes); } } @@ -966,7 +1103,12 @@ impl IndexManager { .commit() .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { + if !eml_content_hash_triples.is_empty() || !attachments_content_hashes.is_empty() { + let eml_content_hashes: HashSet = eml_content_hash_triples + .iter() + .map(|(_, _, hash)| hash.clone()) + .collect(); + self.cleanup_unused_content( &mut writer, eml_content_hashes, @@ -974,6 +1116,10 @@ impl IndexManager { )?; } + for (aid, mid, hash) in eml_content_hash_triples { + DEDUP_CACHE.remove(aid, mid, &hash); + } + Ok(()) } @@ -1073,8 +1219,9 @@ impl IndexManager { let searcher = self.create_searcher()?; let mut writer = self.index_writer.lock().await; - let f_tags = SchemaTools::email_fields().f_tags; - let f_id = SchemaTools::email_fields().f_id; + let f = SchemaTools::email_fields(); + let f_tags = f.f_tags; + let f_id = f.f_id; let deduplicated_updates: HashMap> = request .updates .into_iter() @@ -1119,13 +1266,113 @@ impl IndexManager { let mut new_doc = TantivyDocument::new(); + // Copy stored fields, excluding f_tags (handled separately). for (field, value) in old_doc.field_values() { if field != f_tags { new_doc.add_field_value(field, value); } } - for tag in current_tags { - new_doc.add_facet(f_tags, &tag); + + // Reconstruct non-stored text-search fields from their + // stored counterparts. f_from_text / f_to_text / f_cc_text / + // f_bcc_text carry the same content as f_from / f_to / f_cc / f_bcc. + for val in old_doc.get_all(f.f_from) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_from_text, s); + } + } + for val in old_doc.get_all(f.f_to) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_to_text, s); + } + } + for val in old_doc.get_all(f.f_cc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_cc_text, s); + } + } + for val in old_doc.get_all(f.f_bcc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_bcc_text, s); + } + } + + // Reconstruct attachment-name fields from the stored + // f_attachments JSON blob. + if let Some(attrs_val) = old_doc.get_first(f.f_attachments) { + if let Some(json_str) = attrs_val.as_str() { + if let Ok(parsed) = serde_json::from_str::(json_str) + { + if let Some(arr) = parsed.as_array() { + for att in arr { + let is_inline = att + .get("inline") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let has_cid = att + .get("content_id") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if is_inline && has_cid { + continue; + } + if let Some(filename) = att + .get("filename") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { + new_doc.add_text(f.f_attachment_name_text, filename); + new_doc.add_text(f.f_attachment_name_exact, filename); + } + } + } + } + } + } + + // Reconstruct body text from the original EML stored in the + // blob store, referenced by f_content_hash. + if let Some(hash_val) = old_doc.get_first(f.f_content_hash) { + if let Some(content_hash) = hash_val.as_str() { + match BLOB_MANAGER.get_email(content_hash) { + Ok(Some(eml_bytes)) => { + if let Some(message) = MessageParser::new().parse(&eml_bytes) { + let text = message + .body_text(0) + .map(|cow| cow.into_owned()) + .or_else(|| { + message + .body_html(0) + .map(|cow| extract_text(cow.into_owned())) + }) + .unwrap_or_default(); + let body_text = + text.split_whitespace().collect::>().join(" "); + if !body_text.is_empty() { + new_doc.add_text(f.f_body, &body_text); + } + } + } + Ok(None) => { + tracing::warn!( + content_hash, + "EML not found in blob store during tag update" + ); + } + Err(e) => { + tracing::warn!( + content_hash, + error = %e, + "Failed to fetch EML during tag update" + ); + } + } + } + } + + for tag in ¤t_tags { + new_doc.add_facet(f_tags, tag); } let delete_term = Term::from_field_text(f_id, eid); @@ -1568,3 +1815,718 @@ impl IndexManager { Ok(stats) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::tantivy::tokenizers::EuroTokenizer; + use serde_json::json; + use tantivy::{ + collector::Count, + query::{QueryParser, TermQuery}, + schema::IndexRecordOption, + Index, Term, + }; + + /// Build a complete test document with known values across all + /// stored and non-stored fields so we can verify reconstruction. + fn build_test_doc() -> TantivyDocument { + let f = SchemaTools::email_fields(); + let mut doc = TantivyDocument::new(); + + doc.add_text(f.f_id, "test-eid-001"); + doc.add_text(f.f_message_id, ""); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_u64(f.f_uid, 100); + doc.add_text(f.f_subject, "Test Subject Line"); + doc.add_text(f.f_body, "the quick brown fox jumps over the lazy dog"); + doc.add_text(f.f_preview, "the quick brown fox..."); + doc.add_text(f.f_content_hash, "test-content-hash-001"); + // f_from / f_from_text carry the same data + doc.add_text(f.f_from, "alice@example.com"); + doc.add_text(f.f_from_text, "alice@example.com"); + doc.add_text(f.f_to, "bob@example.com"); + doc.add_text(f.f_to_text, "bob@example.com"); + doc.add_text(f.f_cc, "carol@example.com"); + doc.add_text(f.f_cc_text, "carol@example.com"); + doc.add_text(f.f_bcc, "dave@example.com"); + doc.add_text(f.f_bcc_text, "dave@example.com"); + doc.add_i64(f.f_date, 1_700_000_000_000); + doc.add_i64(f.f_internal_date, 1_700_000_000_000); + doc.add_i64(f.f_ingest_at, 1_700_000_000_000); + doc.add_u64(f.f_size, 999); + doc.add_text(f.f_thread_id, "thread-xyz"); + + // Attachment metadata (stored as JSON). + let atts = json!([{ + "filename": "invoice.pdf", + "file_type": "application/pdf", + "inline": false, + "size": 5000, + "content_id": null, + "content_hash": "att-hash-pdf", + "is_message": false + }]); + doc.add_text(f.f_attachments, atts.to_string()); + doc.add_text(f.f_attachment_name_text, "invoice.pdf"); + doc.add_text(f.f_attachment_name_exact, "invoice.pdf"); + doc.add_text(f.f_attachment_ext, "pdf"); + doc.add_text(f.f_attachment_category, "document"); + doc.add_text(f.f_attachment_content_type, "application/pdf"); + doc.add_text(f.f_attachment_content_hash, "att-hash-pdf"); + doc.add_u64(f.f_attachment_count, 1); + doc.add_u64(f.f_regular_attachment_count, 1); + doc.add_u64(f.f_shard_id, 0); + + // Initial tags. + doc.add_facet(f.f_tags, "/inbox"); + doc.add_facet(f.f_tags, "/unread"); + + doc + } + + /// Reconstruct a new tantivy document from `old_doc`, preserving all + /// fields (including non-stored ones) and replacing tags with + /// `new_tags`. Body text is reconstructed from the supplied `eml_cache` + /// (a stand-in for the blob store) rather than from + /// `old_doc.field_values()` because `f_body` is not STORED. + fn reconstruct_for_test( + old_doc: &TantivyDocument, + new_tags: &HashSet, + eml_cache: &HashMap>, + ) -> TantivyDocument { + let f = SchemaTools::email_fields(); + let mut new_doc = TantivyDocument::new(); + + // ── stored fields (except f_tags) ────────────────────────── + for (field, value) in old_doc.field_values() { + if field != f.f_tags { + new_doc.add_field_value(field, value); + } + } + + // ── non-stored text-search fields ────────────────────────── + for val in old_doc.get_all(f.f_from) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_from_text, s); + } + } + for val in old_doc.get_all(f.f_to) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_to_text, s); + } + } + for val in old_doc.get_all(f.f_cc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_cc_text, s); + } + } + for val in old_doc.get_all(f.f_bcc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_bcc_text, s); + } + } + + // ── attachment-name fields ───────────────────────────────── + if let Some(attrs_val) = old_doc.get_first(f.f_attachments) { + if let Some(json_str) = attrs_val.as_str() { + if let Ok(parsed) = serde_json::from_str::(json_str) { + if let Some(arr) = parsed.as_array() { + for att in arr { + let is_inline = + att.get("inline").and_then(|v| v.as_bool()).unwrap_or(false); + let has_cid = att + .get("content_id") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if is_inline && has_cid { + continue; + } + if let Some(filename) = att + .get("filename") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { + new_doc.add_text(f.f_attachment_name_text, filename); + new_doc.add_text(f.f_attachment_name_exact, filename); + } + } + } + } + } + } + + // ── body text (from eml cache – stands in for BLOB_MANAGER) ── + if let Some(hash_val) = old_doc.get_first(f.f_content_hash) { + if let Some(content_hash) = hash_val.as_str() { + if let Some(eml_bytes) = eml_cache.get(content_hash) { + if let Some(message) = MessageParser::new().parse(eml_bytes) { + let text = message + .body_text(0) + .map(|cow| cow.into_owned()) + .or_else(|| { + message + .body_html(0) + .map(|cow| extract_text(cow.into_owned())) + }) + .unwrap_or_default(); + let body_text = text.split_whitespace().collect::>().join(" "); + if !body_text.is_empty() { + new_doc.add_text(f.f_body, &body_text); + } + } + } + } + } + + // ── updated tags ─────────────────────────────────────────── + for tag in new_tags { + new_doc.add_facet(f.f_tags, tag); + } + + new_doc + } + + #[test] + fn update_tags_preserves_non_stored_fields() { + let f = SchemaTools::email_fields(); + + // ---- setup: in-memory index + document -------------------- + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + let doc = build_test_doc(); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } // drop writer so the next one can acquire the lock + + // ---- build a minimal EML so body reconstruction works ------ + let eml = b"From: alice@example.com\r\n\ + To: bob@example.com\r\n\ + Subject: Test\r\n\ + Date: Thu, 01 Jan 2023 00:00:00 +0000\r\n\ + Message-ID: \r\n\ + \r\n\ + the quick brown fox jumps over the lazy dog\r\n"; + let mut eml_cache = HashMap::new(); + eml_cache.insert("test-content-hash-001".to_string(), eml.to_vec()); + + // ---- read old doc, reconstruct, delete + add -------------- + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + let query = TermQuery::new( + Term::from_field_text(f.f_id, "test-eid-001"), + IndexRecordOption::Basic, + ); + let hits = searcher + .search(&query, &TopDocs::with_limit(1).order_by_score()) + .unwrap(); + assert_eq!(hits.len(), 1); + + let old_doc: TantivyDocument = searcher.doc(hits[0].1).unwrap(); + + let mut new_tags = HashSet::new(); + new_tags.insert("/important".to_string()); + new_tags.insert("/inbox".to_string()); + + let new_doc = reconstruct_for_test(&old_doc, &new_tags, &eml_cache); + + let mut writer2 = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer2"); + writer2.delete_term(Term::from_field_text(f.f_id, "test-eid-001")); + writer2.add_document(new_doc).unwrap(); + writer2.commit().unwrap(); + + // ---- verify: search for non-stored fields still works ----- + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // Body text (tokenized via "euro") + let body_parser = QueryParser::for_index(&index, vec![f.f_body]); + let body_hits = searcher + .search(&body_parser.parse_query("quick brown fox").unwrap(), &Count) + .unwrap(); + assert_eq!(body_hits, 1, "body text should survive tag update"); + + // from_text (tokenized via "euro") + let from_parser = QueryParser::for_index(&index, vec![f.f_from_text]); + let from_hits = searcher + .search( + &from_parser.parse_query("alice@example.com").unwrap(), + &Count, + ) + .unwrap(); + assert_eq!(from_hits, 1, "from_text should survive tag update"); + + // to_text + let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]); + let to_hits = searcher + .search(&to_parser.parse_query("bob@example.com").unwrap(), &Count) + .unwrap(); + assert_eq!(to_hits, 1, "to_text should survive tag update"); + + // attachment_name_exact (STRING — not tokenized) + let att_hits = searcher + .search( + &TermQuery::new( + Term::from_field_text(f.f_attachment_name_exact, "invoice.pdf"), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!( + att_hits, 1, + "attachment_name_exact should survive tag update" + ); + + // Updated tags + let tags_hits = searcher + .search( + &TermQuery::new( + Term::from_facet(f.f_tags, &Facet::from_text("/important").unwrap()), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!(tags_hits, 1, "new tag /important should be present"); + + // Old tag /unread should be gone since we overwrote with new_tags + let old_tag_hits = searcher + .search( + &TermQuery::new( + Term::from_facet(f.f_tags, &Facet::from_text("/unread").unwrap()), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!(old_tag_hits, 0, "old tag /unread should have been removed"); + } + + #[test] + fn body_reconstruction_from_eml_cache() { + // Verify the EML → body_text extraction used inside + // reconstruct_for_test (and therefore update_envelope_tags). + let eml = b"From: x@y\r\n\ + Subject: testing\r\n\ + Date: Thu, 01 Jan 2023 00:00:00 +0000\r\n\ + \r\n\ + hello world from the test suite\r\n"; + + let mut cache = HashMap::new(); + cache.insert("hash-abc".to_string(), eml.to_vec()); + + let f = SchemaTools::email_fields(); + let mut old = TantivyDocument::new(); + old.add_text(f.f_content_hash, "hash-abc"); + + let reconstructed = reconstruct_for_test(&old, &HashSet::new(), &cache); + + // The body should have been extracted from the EML and added + // back to the document. Search for it. + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + writer.add_document(reconstructed).unwrap(); + writer.commit().unwrap(); + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + let parser = QueryParser::for_index(&index, vec![f.f_body]); + let hits = searcher + .search(&parser.parse_query("hello world").unwrap(), &Count) + .unwrap(); + assert_eq!(hits, 1, "body text should be reconstructed from EML"); + } + + #[test] + fn body_reconstruction_missing_eml_is_graceful() { + // When the EML is not in the cache (simulating a blob-store + // miss), the document should still be produced without body. + let f = SchemaTools::email_fields(); + let mut old = TantivyDocument::new(); + old.add_text(f.f_content_hash, "nonexistent-hash"); + + let cache = HashMap::new(); // empty + let reconstructed = reconstruct_for_test(&old, &HashSet::new(), &cache); + + // The document exists but has no body field. + let body_vals: Vec<_> = reconstructed.get_all(f.f_body).collect(); + assert!( + body_vals.is_empty(), + "body should be absent when EML is missing" + ); + } + + // ── get_message_ids_for_mailbox ───────────────────────────────── + + #[test] + fn get_message_ids_returns_stored_ids() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + // Insert two docs for mailbox 10, one for mailbox 20 + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc1 = TantivyDocument::new(); + doc1.add_u64(f.f_account_id, 1); + doc1.add_u64(f.f_mailbox_id, 10); + doc1.add_text(f.f_message_id, ""); + doc1.add_text(f.f_id, "id-a"); + doc1.add_u64(f.f_uid, 1); + doc1.add_text(f.f_content_hash, "hash-a"); + writer.add_document(doc1).unwrap(); + + let mut doc2 = TantivyDocument::new(); + doc2.add_u64(f.f_account_id, 1); + doc2.add_u64(f.f_mailbox_id, 10); + doc2.add_text(f.f_message_id, ""); + doc2.add_text(f.f_id, "id-b"); + doc2.add_u64(f.f_uid, 2); + doc2.add_text(f.f_content_hash, "hash-b"); + writer.add_document(doc2).unwrap(); + + let mut doc3 = TantivyDocument::new(); + doc3.add_u64(f.f_account_id, 1); + doc3.add_u64(f.f_mailbox_id, 20); + doc3.add_text(f.f_message_id, ""); + doc3.add_text(f.f_id, "id-c"); + doc3.add_u64(f.f_uid, 3); + doc3.add_text(f.f_content_hash, "hash-c"); + writer.add_document(doc3).unwrap(); + + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // We can't easily call ENVELOPE_MANAGER.get_message_ids_for_mailbox + // because it reads from ENVELOPE_MANAGER's own index, not our in-memory one. + // Instead, test the query pattern directly. + let query: Box = { + let account_query = TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + ); + let mailbox_query = TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + ); + Box::new(BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(mailbox_query)), + ])) + }; + + let docs = searcher + .search(&query, &DocSetCollector) + .unwrap(); + + let mut ids: Vec = Vec::new(); + for addr in docs { + let doc: TantivyDocument = searcher.doc(addr).unwrap(); + if let Some(v) = doc.get_first(f.f_message_id) { + if let Some(s) = v.as_str() { + ids.push(s.to_string()); + } + } + } + ids.sort(); + + assert_eq!(ids, vec!["", ""]); + } + + #[test] + fn get_message_ids_empty_mailbox_returns_empty() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + // Doc for a different mailbox + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 99); + doc.add_text(f.f_message_id, ""); + doc.add_text(f.f_id, "id-other"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-other"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query: Box = { + let account_query = TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + ); + let mailbox_query = TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + ); + Box::new(BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(mailbox_query)), + ])) + }; + + let docs = searcher.search(&query, &DocSetCollector).unwrap(); + assert!(docs.is_empty()); + } + + // ── mailbox_contains_message_id ─────────────────────────────── + + #[test] + fn mailbox_contains_message_id_finds_existing() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_text(f.f_message_id, "abc@example.com"); + doc.add_text(f.f_id, "id-1"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + // We test the query pattern directly (can't call ENVELOPE_MANAGER + // which uses a different index). + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "abc@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + + let count = searcher.search(&query, &Count).unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn mailbox_contains_message_id_returns_zero_for_missing() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_text(f.f_message_id, "existing@example.com"); + doc.add_text(f.f_id, "id-1"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "nonexistent@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + + let count = searcher.search(&query, &Count).unwrap(); + assert_eq!(count, 0); + } + + #[test] + fn mailbox_contains_message_id_respects_mailbox_boundary() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + // Same Message-ID in mailbox 10 + let mut doc1 = TantivyDocument::new(); + doc1.add_u64(f.f_account_id, 1); + doc1.add_u64(f.f_mailbox_id, 10); + doc1.add_text(f.f_message_id, "shared@example.com"); + doc1.add_text(f.f_id, "id-1"); + doc1.add_u64(f.f_uid, 1); + doc1.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc1).unwrap(); + + // Same Message-ID in mailbox 20 (different mailbox) + let mut doc2 = TantivyDocument::new(); + doc2.add_u64(f.f_account_id, 1); + doc2.add_u64(f.f_mailbox_id, 20); + doc2.add_text(f.f_message_id, "shared@example.com"); + doc2.add_text(f.f_id, "id-2"); + doc2.add_u64(f.f_uid, 2); + doc2.add_text(f.f_content_hash, "hash-2"); + writer.add_document(doc2).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // Query mailbox 10: should find 1 + let q10 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q10, &Count).unwrap(), 1); + + // Query mailbox 20: should find 1 + let q20 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 20), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q20, &Count).unwrap(), 1); + + // Query mailbox 99 (no docs): should find 0 + let q99 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 99), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q99, &Count).unwrap(), 0); + } +} diff --git a/crates/core/src/store/tantivy/mod.rs b/crates/core/src/store/tantivy/mod.rs index 0e1460fe..ce957191 100644 --- a/crates/core/src/store/tantivy/mod.rs +++ b/crates/core/src/store/tantivy/mod.rs @@ -25,6 +25,7 @@ use crate::{ pub mod attachment; pub mod dedup; +pub mod dedup_cache; pub mod envelope; pub mod fields; pub mod filter; diff --git a/crates/core/src/store/tantivy/model.rs b/crates/core/src/store/tantivy/model.rs index f05fffa6..61364c32 100644 --- a/crates/core/src/store/tantivy/model.rs +++ b/crates/core/src/store/tantivy/model.rs @@ -155,7 +155,8 @@ impl EnvelopeWithAttachments { id: extract_string_field(doc, fields.f_id, F_ID)?, message_id: extract_string_field(doc, fields.f_message_id, F_MESSAGE_ID)?, account_id, - account_email: Some(account.email), + account_email: Some(account.email), //https://github.com/rustmailer/bichon/issues/306 + account_name: account.account_name, mailbox_id, mailbox_name: Some(mailbox.name), uid: extract_u64_field(doc, fields.f_uid, F_UID)? as u32, diff --git a/crates/core/src/token/mod.rs b/crates/core/src/token/mod.rs index ba6dd70b..578cdca5 100644 --- a/crates/core/src/token/mod.rs +++ b/crates/core/src/token/mod.rs @@ -156,10 +156,16 @@ impl AccessTokenModel { })?; if matches!(token_model.token_type, TokenType::WebUI) { - let life = utc_now!() - token_model.created_at; - let max_life = SETTINGS.bichon_webui_token_expiration_hours * 60 * 60 * 1000; + // Use last_access_at if set, otherwise fall back to created_at + let last_active = if token_model.last_access_at > 0 { + token_model.last_access_at + } else { + token_model.created_at + }; + let idle = utc_now!() - last_active; + let max_life = SETTINGS.bichon_webui_token_expiration_hours as i64 * 60 * 60 * 1000; - if life > (max_life as i64) { + if idle > max_life { return Err(raise_error!( "Permission denied: the WebUI token has expired.".into(), ErrorCode::PermissionDenied @@ -176,13 +182,15 @@ impl AccessTokenModel { )); } } - update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| { - let mut updated = current.clone(); - updated.last_access_at = utc_now!(); - Ok(updated) - })?; } + // Update last_access_at on every successful use for both token types + update_impl(DB_MANAGER.db(), &token_str, |current: AccessTokenModel| { + let mut updated = current.clone(); + updated.last_access_at = utc_now!(); + Ok(updated) + })?; + let user = UserModel::find(token_model.user_id) ? .ok_or_else(|| raise_error!("The user associated with this access token does not exist or may have been deleted.".into(), ErrorCode::ResourceNotFound))?; diff --git a/crates/core/src/users/mod.rs b/crates/core/src/users/mod.rs index 9e6d9948..84b74d7a 100644 --- a/crates/core/src/users/mod.rs +++ b/crates/core/src/users/mod.rs @@ -87,6 +87,11 @@ pub struct BichonUserV2 { pub theme: Option, pub language: Option, + + /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim). + pub sso_id: Option, + /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`. + pub sso_provider: Option, } impl MemDbModel for BichonUserV2 { @@ -192,6 +197,8 @@ impl BichonUserV2 { global_permissions, theme: self.theme, language: self.language, + sso_id: self.sso_id, + sso_provider: self.sso_provider, } } @@ -226,6 +233,8 @@ impl BichonUserV2 { acl: None, theme: None, language: None, + sso_id: None, + sso_provider: None, }; // 3. Generate and insert an initial access token for the first-time setup @@ -382,6 +391,8 @@ impl BichonUserV2 { account_access_map: request.account_access_map, theme: request.theme, language: request.language, + sso_id: None, + sso_provider: None, }; let user_clone = user.clone(); diff --git a/crates/core/src/users/view.rs b/crates/core/src/users/view.rs index 28396353..4ef909d9 100644 --- a/crates/core/src/users/view.rs +++ b/crates/core/src/users/view.rs @@ -52,4 +52,9 @@ pub struct UserView { pub acl: Option, pub theme: Option, pub language: Option, + + /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim). + pub sso_id: Option, + /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`. + pub sso_provider: Option, } diff --git a/crates/core/src/utils/mod.rs b/crates/core/src/utils/mod.rs index 35ef0a67..49c463aa 100644 --- a/crates/core/src/utils/mod.rs +++ b/crates/core/src/utils/mod.rs @@ -138,7 +138,7 @@ macro_rules! generate_token { }}; } -pub(crate) fn generate_token_impl(bit_strength: usize) -> String { +pub fn generate_token_impl(bit_strength: usize) -> String { let byte_length = (bit_strength + 23) / 24 * 3; let random_bytes: Vec = (0..byte_length).map(|_| rand::random::()).collect(); let mut encoded = general_purpose::URL_SAFE.encode(&random_bytes); diff --git a/crates/core/src/utils/net.rs b/crates/core/src/utils/net.rs index 7249d519..caa449a3 100644 --- a/crates/core/src/utils/net.rs +++ b/crates/core/src/utils/net.rs @@ -21,9 +21,11 @@ use crate::raise_error; use crate::settings::proxy::Proxy; use crate::utils::tls::establish_tls_stream; use crate::{error::BichonResult, imap::session::SessionStream}; +use base64::{engine::general_purpose, Engine as _}; use std::net::SocketAddr; use std::pin::Pin; use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::time::timeout; use tokio_io_timeout::TimeoutStream; @@ -32,6 +34,54 @@ use tracing::error; pub(crate) const TIMEOUT: Duration = Duration::from_secs(30); +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum ProxyScheme { + Socks5, + Http, +} + +impl ProxyScheme { + fn as_str(self) -> &'static str { + match self { + Self::Socks5 => "socks5", + Self::Http => "http", + } + } +} + +/// Parsed proxy address components. +#[derive(Debug, Clone)] +pub struct ProxyAddr { + pub scheme: ProxyScheme, + pub host: String, + pub port: u16, + pub username: Option, + pub password: Option, +} + +impl ProxyAddr { + pub fn standard_url(&self) -> String { + let host = if self.host.contains(':') { + format!("[{}]", self.host) + } else { + self.host.clone() + }; + + if let (Some(user), Some(pass)) = (&self.username, &self.password) { + format!( + "{}://{}:{}@{}:{}", + self.scheme.as_str(), + user, + pass, + host, + self.port + ) + } else { + format!("{}://{}:{}", self.scheme.as_str(), host, self.port) + } + } +} + pub(crate) async fn establish_tcp_connection_with_timeout( address: SocketAddr, use_proxy: Option, @@ -66,74 +116,247 @@ pub async fn establish_tls_connection( Ok(tls_stream) } -pub fn parse_proxy_addr(input: &str) -> BichonResult { - // Normalize and check protocol prefix +/// Parse a proxy URL into its components. +/// +/// Supports two formats: +/// - **Standard**: `[scheme://][user:pass@]host:port` +/// - **Non-standard** (some proxy providers): `[scheme://]host:port:username:password` +/// +/// The distinguishing feature is the `@` sign in the standard format. +pub fn parse_proxy_url(input: &str) -> BichonResult { + // Normalize and strip scheme prefix let (scheme, stripped) = if let Some(rest) = input .strip_prefix("socks5://") .or_else(|| input.strip_prefix("SOCKS5://")) .or_else(|| input.strip_prefix("Socks5://")) { - ("socks5", rest) + (ProxyScheme::Socks5, rest) } else if let Some(rest) = input .strip_prefix("http://") .or_else(|| input.strip_prefix("HTTP://")) .or_else(|| input.strip_prefix("Http://")) { - ("http", rest) + (ProxyScheme::Http, rest) } else { return Err(raise_error!( - format!( - "Invalid proxy URL: must start with 'http://' or 'socks5://', got '{}'", - input - ), + "Invalid proxy URL: must start with 'http://' or 'socks5://'".into(), ErrorCode::InvalidParameter )); }; - // Parse the remaining address - let addr = stripped.parse::().map_err(|e| { + if stripped.is_empty() { + return Err(raise_error!( + "Proxy URL has empty address after scheme.".into(), + ErrorCode::InvalidParameter + )); + } + + // Check for standard format: user:pass@host:port + if let Some(at_pos) = stripped.rfind('@') { + let userinfo = &stripped[..at_pos]; + let hostport = &stripped[at_pos + 1..]; + + let (username, password) = split_userinfo(userinfo)?; + let (host, port) = split_hostport(hostport)?; + + return Ok(ProxyAddr { + scheme, + host, + port, + username, + password, + }); + } + + // No '@' — check for non-standard format: host:port:user:pass + if stripped.starts_with('[') { + let (host, port) = split_hostport(stripped)?; + return Ok(ProxyAddr { + scheme, + host, + port, + username: None, + password: None, + }); + } + + let mut parts = stripped.split(':'); + match ( + parts.next(), + parts.next(), + parts.next(), + parts.next(), + parts.next(), + ) { + (Some(_), Some(_), None, None, None) => { + // host:port, no auth + let (host, port) = split_hostport(stripped)?; + Ok(ProxyAddr { + scheme, + host, + port, + username: None, + password: None, + }) + } + (Some(host), Some(port), Some(username), Some(password), None) => { + // Non-standard: host:port:username:password + let port = port.parse::().map_err(|_| { + raise_error!( + format!("Invalid port '{}' in proxy URL.", port), + ErrorCode::InvalidParameter + ) + })?; + + if host.is_empty() { + return Err(raise_error!( + "Empty hostname in proxy URL.".into(), + ErrorCode::InvalidParameter + )); + } + if host.contains(':') || host.contains('[') || host.contains(']') { + return Err(raise_error!( + "IPv6 proxy hosts are not supported.".into(), + ErrorCode::InvalidParameter + )); + } + if username.is_empty() { + return Err(raise_error!( + "Empty username in proxy URL.".into(), + ErrorCode::InvalidParameter + )); + } + if password.is_empty() { + return Err(raise_error!( + "Empty password in proxy URL.".into(), + ErrorCode::InvalidParameter + )); + } + + Ok(ProxyAddr { + scheme, + host: host.to_string(), + port, + username: Some(username.to_string()), + password: Some(password.to_string()), + }) + } + _ => Err(raise_error!( + "Invalid proxy URL format. Expected '[scheme://][user:pass@]host:port' or 'scheme://host:port:user:pass'.".into(), + ErrorCode::InvalidParameter + )), + } +} + +/// Split "user:pass" into (Some(user), Some(pass)). +fn split_userinfo(userinfo: &str) -> BichonResult<(Option, Option)> { + if userinfo.is_empty() { + return Ok((None, None)); + } + if let Some(colon_pos) = userinfo.find(':') { + let user = &userinfo[..colon_pos]; + let pass = &userinfo[colon_pos + 1..]; + if user.is_empty() { + return Err(raise_error!( + "Empty username in proxy URL credentials.".into(), + ErrorCode::InvalidParameter + )); + } + if pass.is_empty() { + return Err(raise_error!( + "Empty password in proxy URL credentials.".into(), + ErrorCode::InvalidParameter + )); + } + Ok((Some(user.to_string()), Some(pass.to_string()))) + } else { + Err(raise_error!( + "Password cannot be empty when username is provided.".into(), + ErrorCode::InvalidParameter + )) + } +} + +/// Split "host:port" into (host, port). Bracketed IPv6 is accepted. +fn split_hostport(hostport: &str) -> BichonResult<(String, u16)> { + if hostport.is_empty() { + return Err(raise_error!( + "Empty host:port in proxy URL.".into(), + ErrorCode::InvalidParameter + )); + } + + if let Some(rest) = hostport.strip_prefix('[') { + let Some(close_bracket) = rest.find(']') else { + return Err(raise_error!( + format!("Invalid IPv6 address in proxy URL: '{}'.", hostport), + ErrorCode::InvalidParameter + )); + }; + let host = &rest[..close_bracket]; + let port_text = rest[close_bracket + 1..].strip_prefix(':').ok_or_else(|| { + raise_error!( + format!( + "Missing port after IPv6 address in proxy URL: '{}'.", + hostport + ), + ErrorCode::InvalidParameter + ) + })?; + let port = port_text.parse::().map_err(|_| { + raise_error!( + format!("Invalid port in proxy URL: '{}'.", hostport), + ErrorCode::InvalidParameter + ) + })?; + + return Ok((host.to_string(), port)); + } + + // hostname:port or ip:port — split from right + let last_colon = hostport.rfind(':').ok_or_else(|| { raise_error!( - format!( - "Failed to parse {} proxy address '{}': {}", - scheme, stripped, e - ), + format!("Missing port in proxy URL: '{}'.", hostport), + ErrorCode::InvalidParameter + ) + })?; + let host = hostport[..last_colon].to_string(); + let port = hostport[last_colon + 1..].parse::().map_err(|_| { + raise_error!( + format!("Invalid port in proxy URL: '{}'.", hostport), ErrorCode::InvalidParameter ) })?; - Ok(addr) + if host.is_empty() { + return Err(raise_error!( + "Empty hostname in proxy URL.".into(), + ErrorCode::InvalidParameter + )); + } + if host.contains(':') || host.contains('[') || host.contains(']') { + return Err(raise_error!( + "IPv6 proxy hosts are not supported.".into(), + ErrorCode::InvalidParameter + )); + } + + Ok((host, port)) } -/// Try to connect via SOCKS5 proxy or TCP with timeout +/// Try to connect via SOCKS5 proxy or TCP with timeout. async fn connect_with_optional_proxy( use_proxy: Option, address: SocketAddr, ) -> BichonResult { - // Try if proxy is enabled if let Some(proxy_id) = use_proxy { let proxy = Proxy::get(proxy_id)?; - let proxy = parse_proxy_addr(&proxy.url)?; - return timeout(TIMEOUT, Socks5Stream::connect(proxy, address)) - .await - .map_err(|_| { - error!( - "SOCKS5 proxy connection to {} via {} timed out after {}s", - address, - proxy, - TIMEOUT.as_secs() - ); - raise_error!( - format!( - "SOCKS5 proxy connection to {} via {} timed out after {}s", - address, - proxy, - TIMEOUT.as_secs() - ), - ErrorCode::ConnectionTimeout - ) - })? - .map(|s| s.into_inner()) - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)); + let addr = parse_proxy_url(&proxy.url)?; + return if addr.scheme == ProxyScheme::Http { + connect_via_http_proxy(&addr, address).await + } else { + connect_via_socks5_proxy(&addr, address).await + }; } // Fallback to direct TCP connection timeout(TIMEOUT, TcpStream::connect(address)) @@ -155,3 +378,135 @@ async fn connect_with_optional_proxy( })? .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)) } + +async fn connect_via_socks5_proxy( + addr: &ProxyAddr, + address: SocketAddr, +) -> BichonResult { + let proxy_addr = (addr.host.as_str(), addr.port); + let result = if let (Some(user), Some(pass)) = (&addr.username, &addr.password) { + timeout( + TIMEOUT, + Socks5Stream::connect_with_password(proxy_addr, address, user.as_str(), pass.as_str()), + ) + .await + } else { + timeout(TIMEOUT, Socks5Stream::connect(proxy_addr, address)).await + }; + + result + .map_err(|_| { + error!( + "SOCKS5 proxy connection to {} via {}:{} timed out after {}s", + address, + addr.host, + addr.port, + TIMEOUT.as_secs() + ); + raise_error!( + format!( + "SOCKS5 proxy connection to {} via {}:{} timed out after {}s", + address, + addr.host, + addr.port, + TIMEOUT.as_secs() + ), + ErrorCode::ConnectionTimeout + ) + })? + .map(|s| s.into_inner()) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError)) +} + +async fn connect_via_http_proxy(addr: &ProxyAddr, address: SocketAddr) -> BichonResult { + let mut stream = timeout(TIMEOUT, TcpStream::connect((addr.host.as_str(), addr.port))) + .await + .map_err(|_| { + raise_error!( + format!( + "HTTP proxy connection to {}:{} timed out after {}s", + addr.host, + addr.port, + TIMEOUT.as_secs() + ), + ErrorCode::ConnectionTimeout + ) + })? + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?; + + let mut request = format!( + "CONNECT {address} HTTP/1.1\r\nHost: {address}\r\nProxy-Connection: keep-alive\r\n" + ); + if let (Some(user), Some(pass)) = (&addr.username, &addr.password) { + let auth = general_purpose::STANDARD.encode(format!("{user}:{pass}")); + request.push_str(&format!("Proxy-Authorization: Basic {auth}\r\n")); + } + request.push_str("\r\n"); + + timeout(TIMEOUT, stream.write_all(request.as_bytes())) + .await + .map_err(|_| { + raise_error!( + format!( + "HTTP proxy CONNECT to {} via {}:{} timed out after {}s", + address, + addr.host, + addr.port, + TIMEOUT.as_secs() + ), + ErrorCode::ConnectionTimeout + ) + })? + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::NetworkError))?; + + let mut response = Vec::new(); + timeout(TIMEOUT, async { + let mut byte = [0u8; 1]; + while !response.ends_with(b"\r\n\r\n") { + stream.read_exact(&mut byte).await?; + response.push(byte[0]); + if response.len() > 8192 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "HTTP proxy CONNECT response headers are too large", + )); + } + } + Ok::<(), std::io::Error>(()) + }) + .await + .map_err(|_| { + raise_error!( + format!( + "HTTP proxy CONNECT response from {}:{} timed out after {}s", + addr.host, + addr.port, + TIMEOUT.as_secs() + ), + ErrorCode::ConnectionTimeout + ) + })? + .map_err(|e| { + if e.kind() == std::io::ErrorKind::InvalidData { + raise_error!(e.to_string(), ErrorCode::NetworkError) + } else { + raise_error!(format!("{:#?}", e), ErrorCode::NetworkError) + } + })?; + + let response = String::from_utf8_lossy(&response); + if response.starts_with("HTTP/1.1 200") || response.starts_with("HTTP/1.0 200") { + Ok(stream) + } else { + Err(raise_error!( + format!( + "HTTP proxy CONNECT to {} via {}:{} failed: {}", + address, + addr.host, + addr.port, + response.lines().next().unwrap_or("invalid response") + ), + ErrorCode::NetworkError + )) + } +} diff --git a/crates/memdb/Cargo.toml b/crates/memdb/Cargo.toml index c0700e20..b3c6876f 100644 --- a/crates/memdb/Cargo.toml +++ b/crates/memdb/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "memdb" +name = "bichon-memdb" version = "0.1.0" edition = "2021" diff --git a/crates/memdb/tests/integration.rs b/crates/memdb/tests/integration.rs index ef0d5868..3058795f 100644 --- a/crates/memdb/tests/integration.rs +++ b/crates/memdb/tests/integration.rs @@ -1,4 +1,4 @@ -use memdb::{DbError, MemDb, Page}; +use bichon_memdb::{DbError, MemDb, Page}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; @@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() { // Verify WAL only contains seq=3. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 2).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].seq, 3); @@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() { // Verify WAL seq is strictly monotonic. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 50); let mut last = 0u64; for e in &entries { diff --git a/crates/memdb/tests/stress.rs b/crates/memdb/tests/stress.rs index 31702c8e..803b1a82 100644 --- a/crates/memdb/tests/stress.rs +++ b/crates/memdb/tests/stress.rs @@ -1,4 +1,4 @@ -use memdb::{Durability, MemDb, Page}; +use bichon_memdb::{Durability, MemDb, Page}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() { } let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), n); let mut last = 0u64; for e in &entries { @@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() { // Verify strict seq ordering in WAL under concurrent load. let wal_path = db_path.join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), total as usize); let mut last = 0u64; for e in &entries { @@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() { // The entire transaction should be a single WAL entry. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].ops.len(), n as usize); diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 0e814f61..4d854e3d 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -31,9 +31,9 @@ timeago.workspace = true chrono.workspace = true tracing.workspace = true tokio.workspace = true +futures.workspace = true http.workspace = true urlencoding.workspace = true -mimalloc.workspace = true [dev-dependencies] poem = { version = "3.1.12", features = ["test"] } diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index e3802de5..3cfd511d 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -16,12 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use bichon_core::error::BichonResult; -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; #[tokio::main] async fn main() -> BichonResult<()> { diff --git a/crates/server/src/rest/api/import.rs b/crates/server/src/rest/api/import.rs index 4be7e3b9..5ab53581 100644 --- a/crates/server/src/rest/api/import.rs +++ b/crates/server/src/rest/api/import.rs @@ -16,14 +16,33 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::time::{SystemTime, UNIX_EPOCH}; + use crate::common::auth::WrappedContext; use crate::rest::api::ApiTags; use crate::rest::ApiResult; -use bichon_core::import::BatchEmlResult; -use bichon_core::import::{BatchEmlRequest, ImportEmls}; +use bichon_core::account::migration::AccountModel; +use bichon_core::database::manager::DB_MANAGER; +use bichon_core::database::MemDbModel; +use bichon_core::import::{ + check_temp_disk_space, get_import_progress, process_uploaded_file, update_progress, + BatchEmlRequest, BatchEmlResult, ImportEmls, ImportHistory, ImportProgress, ImportStatus, + MAX_WEB_EML_BYTES, +}; +use bichon_core::import::history::{save_import_history, MAX_HISTORY_PER_USER}; +use bichon_core::raise_error; +use bichon_core::error::code::ErrorCode; +use bichon_core::settings::cli::SETTINGS; +use bichon_core::settings::dir::DATA_DIR_MANAGER; use bichon_core::users::permissions::Permission; -use poem_openapi::payload::Json; +use bichon_core::import::detect_text_file; +use bichon_core::import::FileFormat; +use futures::StreamExt; +use poem::Body; +use poem_openapi::param::{Path, Query}; +use poem_openapi::payload::{Json, Binary}; use poem_openapi::OpenApi; +use tokio::io::AsyncWriteExt; pub struct ImportApi; @@ -44,7 +63,362 @@ impl ImportApi { payload: Json, context: WrappedContext, ) -> ApiResult> { - context.require_permission(Some(payload.0.account_id), Permission::DATA_IMPORT_BATCH)?; - Ok(Json(ImportEmls::do_import(payload.0).await?)) + let account_id = payload.0.account_id; + let folder = payload.0.mail_folder.clone(); + context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?; + let result = ImportEmls::do_import(payload.0).await?; + + // Save import history + let progress = ImportProgress { + import_id: format!( + "batch_{:x}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ), + status: if result.failed == 0 { + ImportStatus::Completed + } else if result.success == 0 { + ImportStatus::Failed + } else { + ImportStatus::Completed + }, + format: "eml".to_string(), + total: result.total, + success: result.success, + duplicates: result.duplicates, + failed: result.failed, + failed_details: result.failed_details.clone(), + }; + save_import_history(context.user.id, account_id, &folder, &progress); + + Ok(Json(result)) } + + /// Upload an EML or MBOX file for import into a NoSync account. + /// + /// The file is sent as the raw request body. Both `account_id` and `mail_folder` + /// must be provided as query parameters, along with the original `file_name` for + /// extension validation. + /// + /// Returns an `import_id` to poll for progress via `/import-progress/:import_id`. + #[oai(path = "/upload-import", method = "post", operation_id = "upload_import")] + async fn upload_import( + &self, + /// Target account ID (must be NoSync type). + account_id: Query, + /// Target mail folder name. + mail_folder: Query, + /// Original file name, used for extension validation (e.g. "export.eml"). + file_name: Query, + /// The raw file bytes (.eml or .mbox). + data: Binary, + context: WrappedContext, + ) -> ApiResult> { + let account_id = account_id.0; + context.require_permission(Some(account_id), Permission::DATA_IMPORT_BATCH)?; + + // Basic account validation (fails fast) + AccountModel::check_account_exists(account_id)?; + + let folder = mail_folder.0.trim().to_string(); + if folder.is_empty() { + return Err(raise_error!( + "mail_folder is required.".into(), + ErrorCode::InvalidParameter + ))?; + } + + let file_name = file_name.0.trim().to_string(); + if file_name.is_empty() { + return Err(raise_error!( + "file_name is required.".into(), + ErrorCode::InvalidParameter + ))?; + } + + // Validate file extension + let ext_lower = std::path::Path::new(&file_name) + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()) + .unwrap_or_default(); + let is_mbox_ext = ext_lower == "mbox"; + let is_eml_ext = ext_lower == "eml"; + let is_pst_ext = ext_lower == "pst"; + if !is_mbox_ext && !is_eml_ext && !is_pst_ext { + return Err(raise_error!( + format!( + "Unsupported file type '.{}'. Only .eml, .mbox and .pst files are allowed.", + ext_lower + ), + ErrorCode::InvalidParameter + ))?; + } + + // Check disk space (fail fast before streaming) + let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024; + let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024; + let min_required = if is_mbox_ext { + max_mbox + } else if is_pst_ext { + max_pst + } else { + MAX_WEB_EML_BYTES + }; + let free = check_temp_disk_space()?; + if free < min_required as u64 * 2 { + let free_gb = free as f64 / 1024.0 / 1024.0 / 1024.0; + let need_gb = (min_required as f64 * 2.0) / 1024.0 / 1024.0 / 1024.0; + return Err(raise_error!( + format!( + "Insufficient disk space. Free: {:.1} GB. Need at least {:.1} GB.", + free_gb, need_gb + ), + ErrorCode::InvalidParameter + ))?; + } + + // Stream body to temp file, enforcing size limits and validating content + let import_id = format!( + "imp_{:x}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let temp_path = DATA_DIR_MANAGER.temp_dir.join(format!("import_{}.tmp", import_id)); + + let (format_detected, file_len) = stream_body_to_temp( + data.0, + &temp_path, + is_mbox_ext, + is_pst_ext, + ).await?; + + let format = format_detected.unwrap_or_else(|| { + if is_mbox_ext { + FileFormat::Mbox + } else if is_pst_ext { + FileFormat::Pst + } else { + FileFormat::Eml + } + }); + + let format_str = match format { + FileFormat::Mbox => "mbox".to_string(), + FileFormat::Eml => "eml".to_string(), + FileFormat::Pst => "pst".to_string(), + }; + + let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024; + let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024; + let max_size = match format { + FileFormat::Mbox => max_mbox, + FileFormat::Pst => max_pst, + FileFormat::Eml => MAX_WEB_EML_BYTES, + }; + if file_len > max_size { + let _ = std::fs::remove_file(&temp_path); + let max_mb = max_size as f64 / 1024.0 / 1024.0; + let actual_mb = file_len as f64 / 1024.0 / 1024.0; + return Err(raise_error!( + format!( + "File too large ({:.1} MB). Maximum for {} is {:.0} MB. Use the CLI for larger files.", + actual_mb, format_str.to_uppercase(), max_mb + ), + ErrorCode::InvalidParameter + ))?; + } + + // Record initial progress + let initial = ImportProgress { + import_id: import_id.clone(), + status: ImportStatus::Pending, + format: format_str.clone(), + total: 0, + success: 0, + duplicates: 0, + failed: 0, + failed_details: vec![], + }; + + // Store initial progress so polling can find it immediately + update_progress(&import_id, initial.clone()); + + // Spawn background processing + let id = import_id.clone(); + let folder_clone = folder.clone(); + let user_id = context.user.id; + tokio::task::spawn_blocking(move || { + process_uploaded_file(&id, &temp_path, &file_name, account_id, &folder_clone, user_id); + }); + + Ok(Json(initial)) + } + + /// Poll import progress by import ID. + #[oai( + path = "/import-progress/:import_id", + method = "get", + operation_id = "get_import_progress" + )] + async fn get_import_progress( + &self, + import_id: Path, + context: WrappedContext, + ) -> ApiResult> { + let _ = context; // progress queries don't need per-account auth + match get_import_progress(&import_id.0) { + Some(progress) => Ok(Json(progress)), + None => Err(raise_error!( + format!("Import {} not found.", import_id.0), + ErrorCode::ResourceNotFound + ))?, + } + } + + /// Check available disk space on the server's temp directory. + #[oai( + path = "/check-disk-space", + method = "get", + operation_id = "check_disk_space" + )] + async fn check_disk_space(&self, _context: WrappedContext) -> ApiResult> { + let free = check_temp_disk_space()?; + Ok(Json(free)) + } + + /// List import history for the current user (latest first, up to 5 entries). + #[oai( + path = "/import-history", + method = "get", + operation_id = "list_import_history" + )] + async fn list_import_history( + &self, + context: WrappedContext, + ) -> ApiResult>> { + let prefix = format!("{}:", context.user.id); + let coll = DB_MANAGER.db().collection(ImportHistory::collection()); + let mut entries: Vec = coll + .scan_prefix(&prefix) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + // Sort by created_at descending (newest first), keep at most N per user + entries.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + entries.truncate(MAX_HISTORY_PER_USER); + Ok(Json(entries)) + } +} + +/// Stream a poem `Body` to a temp file while enforcing size limits and +/// validating that the content looks like a text-based email file. +/// +/// PST files are binary (OLE2) — text detection is skipped for them. +/// +/// Returns the detected format (if any) and the total bytes written. +async fn stream_body_to_temp( + body: Body, + temp_path: &std::path::Path, + is_mbox_ext: bool, + is_pst_ext: bool, +) -> ApiResult<(Option, usize)> { + let max_mbox = SETTINGS.bichon_web_mbox_upload_limit_mb as usize * 1024 * 1024; + let max_pst = SETTINGS.bichon_web_pst_upload_limit_mb as usize * 1024 * 1024; + let max_stream = if is_mbox_ext { + max_mbox + } else if is_pst_ext { + max_pst + } else { + MAX_WEB_EML_BYTES + }; + + let mut file = tokio::fs::File::create(temp_path).await.map_err(|e| { + raise_error!( + format!("Failed to create temp file: {}", e), + ErrorCode::InternalError + ) + })?; + + let mut body_stream = body.into_bytes_stream(); + let mut total: usize = 0; + let mut first_chunk: Vec = Vec::new(); + let mut format_detected: Option = None; + let mut text_checked = false; + + while let Some(chunk_result) = body_stream.next().await { + let chunk = chunk_result.map_err(|e| { + raise_error!( + format!("Failed to read request body: {}", e), + ErrorCode::InternalError + ) + })?; + + total += chunk.len(); + + // Enforce size limit during streaming + if total > max_stream { + // Clean up partial temp file + drop(file); + let _ = tokio::fs::remove_file(temp_path).await; + let max_mb = max_stream as f64 / 1024.0 / 1024.0; + return Err(raise_error!( + format!( + "Upload exceeds maximum size of {:.0} MB. Use the CLI for larger files.", + max_mb + ), + ErrorCode::InvalidParameter + ))?; + } + + // Accumulate first ~8 KB for format & text detection + if first_chunk.len() < 8192 { + let remaining = 8192 - first_chunk.len(); + first_chunk.extend_from_slice(&chunk[..chunk.len().min(remaining)]); + } + + // Once we have enough data, validate format and text + if first_chunk.len() >= 512 && !text_checked { + text_checked = true; + format_detected = bichon_core::import::detect_format(&first_chunk, "upload"); + + // If extension is .eml but content looks like MBOX (or vice versa), that's OK. + // PST files are binary — skip text detection. + if !is_pst_ext && !detect_text_file(&first_chunk) { + drop(file); + let _ = tokio::fs::remove_file(temp_path).await; + return Err(raise_error!( + "The uploaded file appears to be binary (not a valid email file). Only .eml, .mbox and .pst files are accepted.".into(), + ErrorCode::InvalidParameter + ))?; + } + } + + file.write_all(&chunk).await.map_err(|e| { + raise_error!( + format!("Failed to write temp file: {}", e), + ErrorCode::InternalError + ) + })?; + } + + file.flush().await.map_err(|e| { + raise_error!( + format!("Failed to flush temp file: {}", e), + ErrorCode::InternalError + ) + })?; + + // If file is empty, reject + if total == 0 { + let _ = tokio::fs::remove_file(temp_path).await; + return Err(raise_error!( + "Empty file is not allowed.".into(), + ErrorCode::InvalidParameter + ))?; + } + + Ok((format_detected, total)) } diff --git a/crates/server/src/rest/api/message.rs b/crates/server/src/rest/api/message.rs index e4f44777..b717d997 100644 --- a/crates/server/src/rest/api/message.rs +++ b/crates/server/src/rest/api/message.rs @@ -280,6 +280,33 @@ impl MessageApi { Ok(attachment) } + /// Returns raw attachment content for in-browser preview with + /// `Content-Disposition: inline` and the correct MIME type. + #[oai( + path = "/preview-attachment/:account_id/:envelope_id", + method = "get", + operation_id = "preview_attachment" + )] + async fn preview_attachment( + &self, + /// The ID of the account. + account_id: Path, + /// The ID of the message containing the attachment. + envelope_id: Path, + /// The content_hash of the attachment to preview. + content_hash: Query, + context: WrappedContext, + ) -> ApiResult> { + let account_id = account_id.0; + let envelope_id = envelope_id.0.trim().to_string(); + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; + let content_hash = content_hash.0.trim(); + let reader = retrieve_attachment_content(account_id, envelope_id, content_hash)?; + let body = Body::from_async_read(reader); + Ok(Attachment::new(body).attachment_type(AttachmentType::Inline)) + } + /// Downloads an attachment from within a nested email (EML file). #[oai( path = "/download-nested-attachment/:account_id/:envelope_id", diff --git a/crates/server/src/rest/api/system.rs b/crates/server/src/rest/api/system.rs index 9a27bd9d..3205c7ae 100644 --- a/crates/server/src/rest/api/system.rs +++ b/crates/server/src/rest/api/system.rs @@ -23,7 +23,7 @@ use bichon_core::dashboard::DashboardStats; use bichon_core::error::code::ErrorCode; use bichon_core::raise_error; use bichon_core::settings::cli::SETTINGS; -use bichon_core::settings::proxy::Proxy; +use bichon_core::settings::proxy::{Proxy, ProxyTestResult}; use bichon_core::settings::SystemConfigurations; use bichon_core::users::permissions::Permission; use bichon_core::version::{fetch_notifications, Notifications}; @@ -70,7 +70,7 @@ impl SystemApi { Ok(Json(stats)) } - /// Get the full list of SOCKS5 proxy configurations. + /// Get the full list of proxy configurations. #[oai(method = "get", path = "/list-proxy", operation_id = "list_proxy")] async fn list_proxy(&self, _context: WrappedContext) -> ApiResult>> { //The proxy list is visible to all users. @@ -103,6 +103,17 @@ impl SystemApi { Ok(Json(Proxy::get(id.0)?)) } + /// Test whether a proxy can reach a geo lookup service. Requires root permission. + #[oai(path = "/proxy/:id/test", method = "post", operation_id = "test_proxy")] + async fn test_proxy( + &self, + id: Path, + context: WrappedContext, + ) -> ApiResult> { + context.require_permission(None, Permission::ROOT)?; + Ok(Json(Proxy::test(id.0).await?)) + } + /// Create a new proxy configuration. Requires root permission. #[oai(path = "/proxy", method = "post", operation_id = "create_proxy")] async fn create_proxy(&self, url: PlainText, context: WrappedContext) -> ApiResult<()> { @@ -122,6 +133,7 @@ impl SystemApi { context.require_permission(None, Permission::ROOT)?; Ok(Proxy::update(id.0, url.0)?) } + /// Get system configurations. /// /// Returns a read-only snapshot of the server configuration diff --git a/crates/smtp/src/server.rs b/crates/smtp/src/server.rs index a953eb2f..1f8d12ba 100644 --- a/crates/smtp/src/server.rs +++ b/crates/smtp/src/server.rs @@ -21,6 +21,7 @@ use std::net::SocketAddr; use std::time::Duration; use base64::{prelude::BASE64_STANDARD, Engine as _}; +use bichon_core::account::migration::AccountType; use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum}; use bichon_core::common::signal::SIGNAL_MANAGER; use bichon_core::envelope::extractor::extract_envelope_from_smtp; @@ -429,8 +430,20 @@ where } if is_allowed { - session.rcpt_to.push(account); - stream.write_all(b"250 OK\r\n").await?; + if !matches!(account.account_type, AccountType::NoSync) { + tracing::warn!( + "SMTP: Rejected journaling attempt to IMAP account <{}>", + addr + ); + let err = format!( + "550 5.7.1 <{}>: Not a Bichon local account, journaling is not supported\r\n", + account.email + ); + stream.write_all(err.as_bytes()).await?; + } else { + session.rcpt_to.push(account); + stream.write_all(b"250 OK\r\n").await?; + } } } Ok(None) => { @@ -615,25 +628,40 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> { return Ok(()); } }; - let mailbox = MailBox { - id: create_hash(rcpt.id, "INBOX"), - account_id: rcpt.id, - name: "INBOX".into(), - delimiter: Some("/".to_string()), - attributes: vec![Attribute { - attr: AttributeEnum::Extension, - extension: Some("CreatedByBichon".into()), - }], - exists: 0, - unseen: None, - uid_next: None, - uid_validity: None, - }; - let mailbox_id = mailbox.id; + let mailbox_id = create_hash(rcpt.id, "INBOX"); + + // The INBOX row is owned by the IMAP sync, which maintains `uid_validity`, + // `highest_uid` and `uid_next` on it. `batch_upsert` replaces the *whole* + // row, so blindly upserting here (with those fields = None) clobbers the + // IMAP-maintained state back to None. The next reconcile then sees + // `uid_validity` change from Some -> None, treats the mailbox as invalid, + // and wipes + rebuilds it — silently losing the local copy of a large + // mailbox when that rebuild is interrupted (see #297). + // + // We only need the row to *exist* so the journaled envelope can attach to + // it, so create it only when it is missing and otherwise leave the + // IMAP-owned row untouched. + if MailBox::find_mailbox(rcpt.id, mailbox_id)?.is_none() { + let mailbox = MailBox { + id: mailbox_id, + account_id: rcpt.id, + name: "INBOX".into(), + delimiter: Some("/".to_string()), + attributes: vec![Attribute { + attr: AttributeEnum::Extension, + extension: Some("CreatedByBichon".into()), + }], + exists: 0, + unseen: None, + uid_next: None, + uid_validity: None, + highest_uid: None, + }; - if let Err(e) = MailBox::batch_upsert(&[mailbox]) { - tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e); - return Err(e.into()); + if let Err(e) = MailBox::batch_upsert(&[mailbox]) { + tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e); + return Err(e.into()); + } } extract_envelope_from_smtp(data, rcpt.id, mailbox_id) diff --git a/funding.json b/funding.json new file mode 100644 index 00000000..7622d1cd --- /dev/null +++ b/funding.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://fundingjson.org/schema/v1.1.0.json", + "version": "v1.0.0", + "entity": { + "type": "individual", + "role": "maintainer", + "name": "rustmailer", + "email": "rustmailer.git@gmail.com", + "phone": "", + "description": "I'm an indie developer and the sole maintainer of Bichon, a lightweight open-source email archiver built in Rust. I believe in privacy, data ownership, and the right to self-host your own digital life.", + "webpageUrl": { + "url": "https://github.com/rustmailer" + } + }, + "projects": [ + { + "guid": "bichon", + "name": "Bichon", + "description": "Bichon is a lightweight, high-performance, self-hosted email archiver built in Rust. It synchronizes emails from IMAP servers, indexes them for full-text search, and provides a clean WebUI and REST API for access.\n\nBichon requires no external database and runs as a single binary — making it easy to deploy and maintain. It supports multiple accounts, OAuth2, SOCKS5 proxy, scheduled sync, bulk import (EML/MBOX/PST), and multi-user RBAC.\n\nAs the sole maintainer, I develop and support Bichon in my personal time. With 1.8k GitHub stars and 327k+ Docker pulls, the project has grown well beyond a personal tool and is actively used by individuals and teams worldwide — including a real-world deployment archiving 1.15 million emails across 28 accounts (800 GB original data, compressed to 421 GB on disk).", + "webpageUrl": { + "url": "https://github.com/rustmailer/bichon" + }, + "repositoryUrl": { + "url": "https://github.com/rustmailer/bichon" + }, + "licenses": ["spdx:AGPL-3.0"], + "tags": ["email", "rust", "self-hosted", "archiver", "imap", "full-text-search", "privacy", "webui"] + } + ], + "funding": { + "channels": [ + { + "guid": "buymeacoffee", + "type": "payment-provider", + "address": "https://buymeacoffee.com/rustmailer", + "description": "Support via Buy Me a Coffee." + }, + { + "guid": "bank", + "type": "bank", + "address": "", + "description": "Direct bank transfer also accepted. Please email rustmailer.git@gmail.com for details." + } + ], + "plans": [ + { + "guid": "maintainer-time", + "status": "active", + "name": "Maintainer Time", + "description": "Cover the cost of dedicated development and maintenance time for Bichon — including bug fixes, feature development, security updates, issue triage, and community support.", + "amount": 10000, + "currency": "USD", + "frequency": "yearly", + "channels": ["bank"] + } + ] + } +} diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index e7bb249b..f3e4eb7a 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -23,6 +23,7 @@ import { PaginatedResponse } from ".."; export interface MinimalAccount { id: number; email: string; + name?: string; } export const minimal_account_list = async () => { @@ -117,6 +118,20 @@ interface DateSelection { export type QuotaWindow = 'hourly' | 'daily' | 'weekly' | 'monthly' + +export interface FilterRule { + include: string[]; + exclude: string[]; +} + +export interface ArchiveRules { + enabled: boolean; + senders: FilterRule; + subjects: FilterRule; + skip_larger_than?: number; + spam_headers: string[]; +} + export interface AccountModel { id: number; account_type: AccountType; @@ -128,10 +143,10 @@ export interface AccountModel { capabilities?: string[]; date_since?: DateSelection; date_before?: RelativeDate; - folder_limit?: number, download_folders: string[]; download_interval_min?: number; download_batch_size?: number; + max_email_size_bytes?: number; created_by: number; created_user_name: string; created_user_email: string; @@ -144,6 +159,8 @@ export interface AccountModel { imap_quota_bytes?: number; auto_download_new_mailboxes?: boolean; download_schedule?: string; + archive_rules?: ArchiveRules; + deleting?: boolean; } export const download_state = async (account_id: number) => { diff --git a/web/src/api/import/api.ts b/web/src/api/import/api.ts new file mode 100644 index 00000000..48e5f113 --- /dev/null +++ b/web/src/api/import/api.ts @@ -0,0 +1,82 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project + +import axiosInstance from '@/api/axiosInstance'; +import { list_accounts } from '@/api/account/api'; +import type { AccountModel } from '@/api/account/api'; + +export interface ImportProgress { + import_id: string; + status: 'Pending' | 'Processing' | 'Completed' | 'Failed'; + format: string; + total: number; + success: number; + duplicates: number; + failed: number; + failed_details: { index: number; error_message: string }[]; +} + +export const upload_import = async ( + accountId: number, + mailFolder: string, + fileName: string, + file: File, + onProgress?: (pct: number) => void +): Promise => { + const response = await axiosInstance.post( + `api/v1/upload-import`, + file, + { + params: { account_id: accountId, mail_folder: mailFolder, file_name: fileName }, + headers: { 'Content-Type': 'application/octet-stream' }, + onUploadProgress: (e) => { + if (e.total && onProgress) onProgress(Math.round((e.loaded / e.total) * 100)); + }, + } + ); + return response.data; +}; + +export const get_import_progress = async (importId: string): Promise => { + const response = await axiosInstance.get( + `api/v1/import-progress/${importId}` + ); + return response.data; +}; + +export const check_disk_space = async (): Promise => { + const response = await axiosInstance.get('api/v1/check-disk-space'); + return response.data; +}; + +export const get_nosync_accounts = async (): Promise => { + const data = await list_accounts(); + return (data.items || []).filter( + (a) => a.account_type === 'NoSync' && a.enabled + ); +}; + +// ── Import history ──────────────────────────────────────────────── + +export interface ImportHistory { + id: string; + user_id: number; + import_id: string; + account_id: number; + folder: string; + format: string; + status: 'pending' | 'processing' | 'completed' | 'failed'; + total: number; + success: number; + duplicates: number; + failed: number; + failed_details: { index: number; error_message: string }[]; + created_at: number; +} + +export const list_import_history = async (): Promise => { + const response = await axiosInstance.get('api/v1/import-history'); + return response.data; +}; diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 9d06c44f..3484257c 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -32,6 +32,7 @@ export interface EmailEnvelope { account_id: number; mailbox_id: number; account_email: string; + account_name?: string; mailbox_name: string; uid: number; subject: string; diff --git a/web/src/api/mailbox/envelope/api.ts b/web/src/api/mailbox/envelope/api.ts index a5a3dabe..02cee154 100644 --- a/web/src/api/mailbox/envelope/api.ts +++ b/web/src/api/mailbox/envelope/api.ts @@ -41,6 +41,18 @@ export const download_attachment = async (accountId: number, id: string, content saveAs(blob, fileName); }; +/** Fetch raw attachment content for in-browser preview (Content-Disposition: inline). */ +export const preview_attachment = async (accountId: number, id: string, content_hash: string) => { + const response = await axiosInstance.get( + `api/v1/preview-attachment/${accountId}/${id}`, + { + params: { content_hash }, + responseType: 'blob', + } + ); + return response.data as Blob; +}; + export const download_nested_attachment = async (accountId: number, id: string, content_hash: string, nested_content_hash: string, fileName: string) => { const response = await axiosInstance.get(`api/v1/download-nested-attachment/${accountId}/${id}?content_hash=${content_hash}&nested_content_hash=${nested_content_hash}`, { responseType: 'blob' }); const blob = new Blob([response.data]); diff --git a/web/src/api/system/api.ts b/web/src/api/system/api.ts index 21645dc2..e2da91b5 100644 --- a/web/src/api/system/api.ts +++ b/web/src/api/system/api.ts @@ -105,6 +105,14 @@ export interface Proxy { updated_at: number; } +export interface ProxyTestResult { + ip?: string | null; + country?: string | null; + region?: string | null; + city?: string | null; + isp?: string | null; +} + export type ServerConfigurations = { bichon_log_level: string bichon_http_port: number @@ -138,6 +146,10 @@ export type ServerConfigurations = { bichon_smtp_auth_required: boolean bichon_smtp_tls_key_path?: string | null bichon_smtp_tls_cert_path?: string | null + + bichon_upload_body_limit_mb: number + bichon_web_mbox_upload_limit_mb: number + bichon_web_pst_upload_limit_mb: number } export const get_dashboard_stats = async () => { @@ -164,6 +176,11 @@ export const update_proxy = async (id: number, url: string) => { return response.data; }; +export const test_proxy = async (id: number) => { + const response = await axiosInstance.post(`api/v1/proxy/${id}/test`); + return response.data; +}; + export const add_proxy = async (url: string) => { const response = await axiosInstance.post(`api/v1/proxy`, url, { headers: { @@ -177,4 +194,4 @@ export const add_proxy = async (url: string) => { export const get_system_configurations = async () => { const response = await axiosInstance.get(`api/v1/system-configurations`); return response.data; -}; \ No newline at end of file +}; diff --git a/web/src/components/layout/data/sidebar-data.ts b/web/src/components/layout/data/sidebar-data.ts index 2012f482..ff9bcb42 100644 --- a/web/src/components/layout/data/sidebar-data.ts +++ b/web/src/components/layout/data/sidebar-data.ts @@ -22,7 +22,7 @@ import { IconLayoutDashboard, IconSettings } from '@tabler/icons-react' -import { IdCard, Inbox, Paperclip, Search, Users2 } from 'lucide-react' +import { IdCard, Inbox, Paperclip, Search, Upload, Users2 } from 'lucide-react' import { type SidebarData } from '../types' import { useTranslation } from 'react-i18next' import { useCurrentUser } from '@/hooks/use-current-user' @@ -57,6 +57,12 @@ export function useSidebarData(): SidebarData { url: '/search', icon: Search, }, + { + title: t('import.title', 'Import'), + url: '/import', + icon: Upload, + visible: require_any_permission(['data:import:batch']), + }, { title: t('navigation.attachment'), url: '/attachment', diff --git a/web/src/components/layout/github.tsx b/web/src/components/layout/github.tsx index b308493b..cec10d18 100644 --- a/web/src/components/layout/github.tsx +++ b/web/src/components/layout/github.tsx @@ -28,18 +28,54 @@ interface GithubLinkButtonProps { title?: string; } +const CACHE_KEY = "github_stars_cache"; +const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours + +interface StarsCache { + stars: number; + fetchedAt: number; +} + +function getCachedStars(repo: string): number | null { + try { + const raw = localStorage.getItem(`${CACHE_KEY}_${repo}`); + if (!raw) return null; + const cache: StarsCache = JSON.parse(raw); + if (Date.now() - cache.fetchedAt > CACHE_TTL) return null; + return cache.stars; + } catch { + return null; + } +} + +function setCachedStars(repo: string, stars: number) { + try { + localStorage.setItem( + `${CACHE_KEY}_${repo}`, + JSON.stringify({ stars, fetchedAt: Date.now() }) + ); + } catch { } +} + export const GithubLinkButton: React.FC = ({ href = "https://github.com/rustmailer/bichon", repo = "rustmailer/bichon", size = 18, title = "View on GitHub", }) => { - const [stars, setStars] = useState(null); + const [stars, setStars] = useState(() => getCachedStars(repo)); useEffect(() => { + if (stars !== null) return; // already have cached value, skip fetch fetch(`https://api.github.com/repos/${repo}`) .then(res => res.json()) - .then(data => setStars(data.stargazers_count)) + .then(data => { + const count = data.stargazers_count; + if (typeof count === "number") { + setStars(count); + setCachedStars(repo, count); + } + }) .catch(() => { }); }, [repo]); diff --git a/web/src/components/ui/breadcrumb.tsx b/web/src/components/ui/breadcrumb.tsx new file mode 100644 index 00000000..6f31b635 --- /dev/null +++ b/web/src/components/ui/breadcrumb.tsx @@ -0,0 +1,67 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { ChevronRight, Home } from "lucide-react"; +import { Link } from "@tanstack/react-router"; +import { cn } from "@/lib/utils"; + +export interface BreadcrumbItem { + label: string; + to?: string; +} + +interface BreadcrumbProps { + items: BreadcrumbItem[]; + className?: string; +} + +export function Breadcrumb({ items, className }: BreadcrumbProps) { + return ( + + ); +} diff --git a/web/src/components/ui/dialog.tsx b/web/src/components/ui/dialog.tsx index 1cacb705..faad0a98 100644 --- a/web/src/components/ui/dialog.tsx +++ b/web/src/components/ui/dialog.tsx @@ -30,8 +30,11 @@ DialogOverlay.displayName = DialogPrimitive.Overlay.displayName const DialogContent = React.forwardRef< React.ElementRef, - React.ComponentPropsWithoutRef ->(({ className, children, ...props }, ref) => { + React.ComponentPropsWithoutRef & { + hideClose?: boolean; + hideFullscreen?: boolean; + } +>(({ className, children, hideClose, hideFullscreen, ...props }, ref) => { const [isFullscreen, setIsFullscreen] = React.useState(false); return @@ -45,17 +48,23 @@ const DialogContent = React.forwardRef< {...props} > {children} -
- {isFullscreen ? ( - setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' /> - ) : ( - setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' /> - )} - - - Close - -
+ {(!hideClose || !hideFullscreen) && ( +
+ {!hideFullscreen && ( + isFullscreen ? ( + setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' /> + ) : ( + setIsFullscreen(!isFullscreen)} className='rounded-sm opacity-70 ring-offset-background transition-opacity hover:opacity-100 focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2 disabled:pointer-events-none data-[state=open]:bg-accent data-[state=open]:text-muted-foreground h-4 w-4' /> + ) + )} + {!hideClose && ( + + + Close + + )} +
+ )} diff --git a/web/src/features/accounts/account-new.tsx b/web/src/features/accounts/account-new.tsx new file mode 100644 index 00000000..53503298 --- /dev/null +++ b/web/src/features/accounts/account-new.tsx @@ -0,0 +1,260 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useCallback, useState } from "react"; +import { useForm, FormProvider } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useTranslation } from "react-i18next"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { useNavigate, Link } from "@tanstack/react-router"; +import { ArrowLeft, Loader2 } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Form } from "@/components/ui/form"; +import { useToast } from "@/hooks/use-toast"; +import { ToastAction } from "@/components/ui/toast"; +import { Breadcrumb } from "@/components/ui/breadcrumb"; +import { FixedHeader } from "@/components/layout/fixed-header"; +import { Main } from "@/components/layout/main"; +import { TabGeneral } from "./components/tab-general"; +import { TabServer } from "./components/tab-server"; +import { TabDownload } from "./components/tab-download"; +import { TabFilters } from "./components/tab-filters"; +import { create_account, autoconfig } from "@/api/account/api"; +import { getAccountSchema, type AccountFormValues } from "./components/schema"; +import type { AxiosError } from "axios"; + +const defaultValues: AccountFormValues = { + login_name: undefined, + account_name: undefined, + email: '', + imap: { + host: "", + port: 993, + encryption: 'Ssl', + auth: { auth_type: 'Password', password: undefined }, + use_proxy: undefined, + }, + enabled: true, + use_dangerous: false, + date_since: undefined, + date_before: undefined, + download_interval_min: 60, + download_batch_size: 30, + max_email_size_bytes: 100 * 1024 * 1024, + auto_download_new_mailboxes: true, + download_schedule: undefined, + archive_rules: undefined, +}; + +function SectionHeader({ title, description }: { title: string; description?: string }) { + return ( +
+

{title}

+ {description &&

{description}

} +
+ ); +} + +export function AccountNewPage() { + const { t } = useTranslation(); + const navigate = useNavigate(); + const { toast } = useToast(); + const queryClient = useQueryClient(); + const [autoConfigLoading, setAutoConfigLoading] = useState(false); + + const accountSchema = getAccountSchema(false, t); + const form = useForm({ + mode: "onChange", + defaultValues, + resolver: zodResolver(accountSchema), + }); + + const createMutation = useMutation({ + mutationFn: create_account, + onSuccess: () => { + toast({ + title: t('accounts.accountCreated'), + description: t('accounts.accountCreatedDesc'), + action: {t('common.close')}, + }); + queryClient.invalidateQueries({ queryKey: ['account-list'] }); + navigate({ to: '/accounts' }); + }, + onError: (error: AxiosError) => { + const errorMessage = + (error.response?.data as { message?: string })?.message || + error.message || + t('accounts.creationFailed'); + toast({ + variant: "destructive", + title: t('accounts.accountCreationFailed'), + description: errorMessage as string, + action: {t('common.tryAgain')}, + }); + }, + }); + + const onSubmit = useCallback( + (data: AccountFormValues) => { + createMutation.mutate({ + email: data.email, + account_name: data.account_name, + login_name: data.login_name, + imap: { + ...data.imap, + auth: { + ...data.imap.auth, + password: data.imap.auth.auth_type === 'OAuth2' ? undefined : data.imap.auth.password, + }, + }, + enabled: data.enabled, + use_dangerous: data.use_dangerous, + date_since: data.date_since, + date_before: data.date_before, + download_interval_min: data.download_interval_min, + download_batch_size: data.download_batch_size, + max_email_size_bytes: data.max_email_size_bytes, + auto_download_new_mailboxes: data.auto_download_new_mailboxes, + download_schedule: data.download_schedule || null, + account_type: "IMAP", + archive_rules: data.archive_rules || null, + }); + }, + [createMutation] + ); + + const handleAutoConfig = async () => { + const email = form.getValues('email'); + if (!email) return; + const imap = form.getValues('imap'); + if (imap.host.trim() !== "" && imap.port > 0) return; + + setAutoConfigLoading(true); + try { + const result = await autoconfig(email); + if (result) { + form.setValue('imap.host', result.imap.host); + form.setValue('imap.port', result.imap.port); + form.setValue('imap.encryption', result.imap.encryption); + if (result.oauth2) form.setValue('imap.auth.auth_type', 'OAuth2'); + } + } catch (error) { + console.error('Auto-configuration failed:', error); + } + setAutoConfigLoading(false); + }; + + return ( + <> + +
+
+
+ + + {t('accounts.settings.backToAccounts')} + + +
+ +
+
+

{t('accounts.addAccount')}

+

{t('accounts.addNewEmailAccountHere')}

+
+ + +
+ +
+ + +
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+ + +
+ +
+ +
+ + +
+ +
+ + +
+
+ + +
+
+
+ + ); +} diff --git a/web/src/features/accounts/account-settings-page.tsx b/web/src/features/accounts/account-settings-page.tsx new file mode 100644 index 00000000..8857fbe5 --- /dev/null +++ b/web/src/features/accounts/account-settings-page.tsx @@ -0,0 +1,279 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useCallback, useEffect } from "react"; +import { useForm, FormProvider } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useTranslation } from "react-i18next"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Link } from "@tanstack/react-router"; +import { ArrowLeft } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { Form } from "@/components/ui/form"; +import { useToast } from "@/hooks/use-toast"; +import { ToastAction } from "@/components/ui/toast"; +import { Breadcrumb } from "@/components/ui/breadcrumb"; +import { FixedHeader } from "@/components/layout/fixed-header"; +import { Main } from "@/components/layout/main"; +import { TabGeneral } from "./components/tab-general"; +import { TabServer } from "./components/tab-server"; +import { TabDownload } from "./components/tab-download"; +import { TabFilters } from "./components/tab-filters"; +import { update_account, list_accounts, type AccountModel } from "@/api/account/api"; +import { getAccountSchema, type AccountFormValues } from "./components/schema"; +import type { AxiosError } from "axios"; +import { useQuery } from "@tanstack/react-query"; + +const emptyImap = { + host: "", + port: 0, + encryption: "None" as const, + auth: { auth_type: "Password" as const, password: undefined }, + use_proxy: undefined, +}; + +function mapAccountToFormValues(account: AccountModel): AccountFormValues { + const imap = { ...(account.imap ?? emptyImap) }; + imap.auth = { ...imap.auth, password: undefined }; + if ((imap as any).use_proxy === null) { + (imap as any).use_proxy = undefined; + } + + return { + account_name: account.account_name ?? undefined, + login_name: account.login_name ?? undefined, + email: account.email, + imap, + enabled: account.enabled, + use_dangerous: account.use_dangerous, + date_since: account.date_since ?? undefined, + date_before: account.date_before ?? undefined, + download_interval_min: account.download_interval_min ?? 60, + download_batch_size: account.download_batch_size ?? 30, + max_email_size_bytes: account.max_email_size_bytes ?? 100 * 1024 * 1024, + auto_download_new_mailboxes: account.auto_download_new_mailboxes ?? true, + download_schedule: account.download_schedule ?? undefined, + archive_rules: account.archive_rules ?? undefined, + }; +} + +function SectionHeader({ title, description }: { title: string; description?: string }) { + return ( +
+

{title}

+ {description &&

{description}

} +
+ ); +} + +interface AccountSettingsPageProps { + accountId: number; +} + +export function AccountSettingsPage({ accountId }: AccountSettingsPageProps) { + const { t } = useTranslation(); + //const navigate = useNavigate(); + const { toast } = useToast(); + const queryClient = useQueryClient(); + + const { data: accountList } = useQuery({ + queryKey: ['account-list'], + queryFn: list_accounts, + }); + + const account = accountList?.items?.find((a) => a.id === accountId); + + const accountSchema = getAccountSchema(true, t); + const form = useForm({ + mode: "onChange", + defaultValues: account ? mapAccountToFormValues(account) : undefined, + resolver: zodResolver(accountSchema), + }); + + useEffect(() => { + if (account) { + form.reset(mapAccountToFormValues(account)); + } + }, [account?.id]); + + const updateMutation = useMutation({ + mutationFn: (data: Record) => update_account(accountId, data), + onSuccess: () => { + toast({ + title: t('accounts.settings.saved'), + description: t('accounts.settings.savedDesc'), + action: {t('common.close')}, + }); + queryClient.invalidateQueries({ queryKey: ['account-list'] }); + }, + onError: (error: AxiosError) => { + const errorMessage = + (error.response?.data as { message?: string })?.message || + error.message || + t('accounts.updateFailed'); + toast({ + variant: "destructive", + title: t('accounts.accountUpdateFailed'), + description: errorMessage as string, + action: {t('common.tryAgain')}, + }); + }, + }); + + const onSubmit = useCallback( + (data: AccountFormValues) => { + const payload: Record = { + email: data.email, + account_name: data.account_name, + login_name: data.login_name, + imap: { + ...data.imap, + auth: { + ...data.imap.auth, + password: data.imap.auth.auth_type === 'OAuth2' + ? undefined + : (data.imap.auth.password ? data.imap.auth.password : undefined), + }, + }, + enabled: data.enabled, + use_dangerous: data.use_dangerous, + date_since: data.date_since, + date_before: data.date_before, + download_interval_min: data.download_interval_min, + download_batch_size: data.download_batch_size, + max_email_size_bytes: data.max_email_size_bytes, + auto_download_new_mailboxes: data.auto_download_new_mailboxes, + download_schedule: data.download_schedule || null, + archive_rules: data.archive_rules || null, + }; + + if (!data.date_since && !data.date_before) { + payload.clear_date_range = true; + } + if (!data.download_schedule && account?.download_schedule) { + payload.clear_download_schedule = true; + } + + updateMutation.mutate(payload); + }, + [updateMutation, account] + ); + + if (!account) { + return ( + <> + +
+
+ {t('accounts.settings.loading')} +
+
+ + ); + } + + return ( + <> + +
+
+
+ + + {t('accounts.settings.backToAccounts')} + + +
+ +
+
+

{account.email}

+

{t('accounts.updateTheEmailAccountHere')}

+
+ + +
+ +
+ + +
+ +
+ +
+ + +
+ +
+ +
+ + +
+ +
+ +
+ + +
+ +
+ + +
+
+ + +
+
+
+ + ); +} diff --git a/web/src/features/accounts/components/account-detail.tsx b/web/src/features/accounts/components/account-detail.tsx index 918ab111..302ee9f7 100644 --- a/web/src/features/accounts/components/account-detail.tsx +++ b/web/src/features/accounts/components/account-detail.tsx @@ -97,6 +97,10 @@ export function AccountDetailDrawer({ open, onOpenChange, currentRow }: Props) { {t('accounts.downloadBatchSize')}: {currentRow.download_batch_size} +
+ {t('accounts.maxEmailSizeBytes')}: + {currentRow.max_email_size_bytes ? `${(currentRow.max_email_size_bytes / 1024 / 1024).toFixed(0)} MB` : t('accounts.maxEmailSizeBytesUnlimited')} +
{t('accounts.capabilities')}: diff --git a/web/src/features/accounts/components/action-dialog.tsx b/web/src/features/accounts/components/action-dialog.tsx deleted file mode 100644 index ac7ef2f0..00000000 --- a/web/src/features/accounts/components/action-dialog.tsx +++ /dev/null @@ -1,365 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -import { zodResolver } from '@hookform/resolvers/zod'; -import * as React from 'react'; -import { useForm } from 'react-hook-form'; -import { Button } from '@/components/ui/button'; -import { Form } from '@/components/ui/form'; -import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from '@/components/ui/dialog'; -import { ScrollArea } from '@/components/ui/scroll-area'; -import { useToast } from '@/hooks/use-toast'; -import Step1 from './step1'; -import Step2 from './step2'; -import Step3 from './step3'; -import Step4 from './step4'; -import { create_account, autoconfig, update_account, AccountModel, ImapConfig } from '@/api/account/api'; -import { useMutation, useQueryClient } from '@tanstack/react-query'; -import { ToastAction } from '@/components/ui/toast'; -import { AxiosError } from 'axios'; -import { useTranslation } from 'react-i18next'; -import { cn } from "@/lib/utils"; -import { getAccountSchema, type AccountFormValues } from './schema'; - -export type Account = AccountFormValues; - -type Step = { - id: `step-${number}`; - name: string; - fields: (keyof Account)[]; -}; - -export type Steps = [...Step[]]; - -const getSteps = (t: (key: string) => string): Steps => [ - { id: "step-1", name: t('accounts.steps.emailAddress'), fields: ["email", "account_name"] }, - { id: "step-2", name: t('accounts.steps.imap'), fields: ["imap", "use_dangerous", "login_name"] }, - { id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "download_interval_min", "download_batch_size", "auto_download_new_mailboxes", "download_schedule"] }, - { id: "step-4", name: t('accounts.steps.summary'), fields: [] }, -]; - -const LAST_STEP = 4; - -interface Props { - currentRow?: AccountModel; - open: boolean; - onOpenChange: (open: boolean) => void; -} - -const defaultValues: Account = { - login_name: undefined, - account_name: undefined, - email: '', - imap: { - host: "", - port: 993, - encryption: 'Ssl', - auth: { - auth_type: 'Password', - password: undefined, - }, - use_proxy: undefined - }, - enabled: true, - use_dangerous: false, - date_since: undefined, - date_before: undefined, - download_interval_min: 60, - download_batch_size: 30, - auto_download_new_mailboxes: true, - download_schedule: undefined, -}; - -const emptyImap: ImapConfig = { - host: "", - port: 0, - encryption: "None", - auth: { auth_type: "Password", password: undefined }, - use_proxy: undefined, -}; - -const mapCurrentRowToFormValues = (currentRow: AccountModel): Account => { - const imap = { ...(currentRow.imap ?? emptyImap) }; - imap.auth = { ...imap.auth, password: undefined }; - if (imap.use_proxy === null) { - imap.use_proxy = undefined; - } - - return { - account_name: currentRow.account_name ?? undefined, - login_name: currentRow.login_name ?? undefined, - email: currentRow.email, - imap, - enabled: currentRow.enabled, - use_dangerous: currentRow.use_dangerous, - date_since: currentRow.date_since ?? undefined, - date_before: currentRow.date_before ?? undefined, - download_interval_min: currentRow.download_interval_min ?? 60, - download_batch_size: currentRow.download_batch_size ?? 30, - auto_download_new_mailboxes: currentRow.auto_download_new_mailboxes ?? true, - download_schedule: currentRow.download_schedule ?? undefined, - }; -}; - -export function AccountActionDialog({ currentRow, open, onOpenChange }: Props) { - const { t } = useTranslation(); - const steps = getSteps(t); - const isEdit = !!currentRow; - const [currentStep, setCurrentStep] = React.useState(1); - const { toast } = useToast(); - const [autoConfigLoading, setAutoConfigLoading] = React.useState(false); - - const accountSchema = getAccountSchema(isEdit, t); - const form = useForm({ - mode: "onChange", - defaultValues: isEdit ? mapCurrentRowToFormValues(currentRow) : defaultValues, - resolver: zodResolver(accountSchema), - }); - - const queryClient = useQueryClient(); - - const createMutation = useMutation({ - mutationFn: create_account, - onSuccess: handleSuccess, - onError: handleError, - }); - - const updateMutation = useMutation({ - mutationFn: (data: Record) => update_account(currentRow?.id!, data), - onSuccess: handleSuccess, - onError: handleError, - }); - - function handleSuccess() { - toast({ - title: isEdit ? t('accounts.accountUpdated') : t('accounts.accountCreated'), - description: isEdit ? t('accounts.accountUpdatedDesc') : t('accounts.accountCreatedDesc'), - action: {t('common.close')}, - }); - - queryClient.invalidateQueries({ queryKey: ['account-list'] }); - form.reset(); - onOpenChange(false); - } - - function handleError(error: AxiosError) { - const errorMessage = - (error.response?.data as { message?: string })?.message || - error.message || - (isEdit ? t('accounts.updateFailed') : t('accounts.creationFailed')); - - toast({ - variant: "destructive", - title: isEdit ? t('accounts.accountUpdateFailed') : t('accounts.accountCreationFailed'), - description: errorMessage as string, - action: {t('common.tryAgain')}, - }); - console.error(error); - } - - const onSubmit = React.useCallback( - (data: Account) => { - const commonData = { - email: data.email, - account_name: data.account_name, - login_name: data.login_name, - imap: { - ...data.imap, - auth: { - ...data.imap.auth, - password: data.imap.auth.auth_type === 'OAuth2' - ? undefined - : (isEdit && !data.imap.auth.password ? undefined : data.imap.auth.password), - }, - }, - enabled: data.enabled, - use_dangerous: data.use_dangerous, - date_since: data.date_since, - date_before: data.date_before, - download_interval_min: data.download_interval_min, - download_batch_size: data.download_batch_size, - auto_download_new_mailboxes: data.auto_download_new_mailboxes, - download_schedule: data.download_schedule || null, - }; - if (isEdit) { - const isAllMode = !data.date_since && !data.date_before; - const clear_download_schedule = !data.download_schedule && currentRow?.download_schedule; - updateMutation.mutate({ - ...commonData, - ...(isAllMode ? { clear_date_range: true } : {}), - ...(clear_download_schedule ? { clear_download_schedule: true } : {}) - }); - } else { - createMutation.mutate({ ...commonData, account_type: "IMAP" }); - } - }, - [isEdit, updateMutation, createMutation] - ); - - const handleNav = async (index: number) => { - let isValid = true; - let failedStep = currentStep; - for (let i = currentStep - 1; i < index - 1 && isValid; i++) { - isValid = await form.trigger(steps[i].fields); - if (!isValid) failedStep = i; - } - if (isValid) setCurrentStep(index); - else setCurrentStep(failedStep); - }; - - async function handleContinue() { - const isValid = await form.trigger(steps[currentStep - 1].fields); - if (!isValid) return; - - if (currentStep === 1) { - let allValues = form.getValues(); - if (allValues.imap.host.trim() !== "" && allValues.imap.port > 0) { - handleNav(currentStep + 1); - return; - } - setAutoConfigLoading(true); - const email = form.getValues('email'); - form.setValue('login_name', email); - try { - const result = await autoconfig(email); - if (result) { - form.setValue('imap.host', result.imap.host); - form.setValue('imap.port', result.imap.port); - form.setValue('imap.encryption', result.imap.encryption); - if (result.oauth2) form.setValue('imap.auth.auth_type', 'OAuth2'); - } - } catch (error) { - console.error('Auto-configuration failed:', error); - } - setAutoConfigLoading(false); - handleNav(currentStep + 1); - } else { - handleNav(currentStep + 1); - } - } - - return ( - { - if (!state) { - form.reset(); - setCurrentStep(1); - } - onOpenChange(state); - }} - > - -
- - {isEdit ? t('accounts.updateAccount') : t('accounts.addAccount')} - - {isEdit ? t('accounts.updateAccount') : t('accounts.addAccount')} - {t('accounts.clickSaveWhenDone')} - - -
- -
-
- {steps.map((step, index) => ( -
- - {step.name} -
- ))} -
- -
- {steps.map((step, index) => ( -
- -
- {t('accounts.step', { index: index + 1 })} - - {step.name} - -
-
- ))} -
- -
- -
-
- - {currentStep === 1 && } - {currentStep === 2 && } - {currentStep === 3 && } - {currentStep === 4 && } - - -
- -
-
- - - {currentStep > 1 && ( - - )} - {currentStep < LAST_STEP && ( - - )} - {currentStep === LAST_STEP && ( - - )} - - - - ); -} diff --git a/web/src/features/accounts/components/add-account-dialog.tsx b/web/src/features/accounts/components/add-account-dialog.tsx deleted file mode 100644 index a26ccb0b..00000000 --- a/web/src/features/accounts/components/add-account-dialog.tsx +++ /dev/null @@ -1,159 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . -// - -import * as React from 'react' -import { Mail, Database } from 'lucide-react' -import { useTranslation } from 'react-i18next' - -import { cn } from '@/lib/utils' -import { Button } from '@/components/ui/button' -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from '@/components/ui/dialog' -import { Label } from '@/components/ui/label' -import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group' -import { useAccountContext } from '../context' - -export type AddAccountType = 'IMAP' | 'NoSync' - -interface Props { - open: boolean; - onOpenChange: (open: boolean) => void; -} - -export function AddAccountDialog({ - open, - onOpenChange -}: Props) { - const { t } = useTranslation() - const { setOpen } = useAccountContext() - const [value, setValue] = React.useState('IMAP') - - function handleContinue() { - if (value === 'IMAP') { - setOpen('add-imap') - } else { - setOpen('add-nosync') - } - } - - return ( - - - - - {t('accounts.add')} - - - - {t('accounts.selectAccountType')} - - - - setValue(v as AddAccountType)} - className="space-y-4 py-2" - > - - - - - - - - - - - - - ) -} \ No newline at end of file diff --git a/web/src/features/accounts/components/data-table-row-actions.tsx b/web/src/features/accounts/components/data-table-row-actions.tsx index 51878829..039097e2 100644 --- a/web/src/features/accounts/components/data-table-row-actions.tsx +++ b/web/src/features/accounts/components/data-table-row-actions.tsx @@ -30,11 +30,12 @@ import { DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' import { useAccountContext } from '../context' -import { Mailbox, MessageSquareMore } from 'lucide-react' +import { Mailbox, MessageSquareMore, Settings } from 'lucide-react' import { useTranslation } from 'react-i18next' import { useCurrentUser } from '@/hooks/use-current-user' import { AccountModel, cancel_account_download, start_account_download } from '@/api/account/api' import { toast } from '@/hooks/use-toast' +import { useNavigate } from '@tanstack/react-router' interface DataTableRowActionsProps { row: Row @@ -43,6 +44,7 @@ interface DataTableRowActionsProps { export function DataTableRowActions({ row }: DataTableRowActionsProps) { const { t } = useTranslation() const { setOpen, setCurrentRow } = useAccountContext() + const navigate = useNavigate() const account_type = row.original.account_type; const { require_any_permission } = useCurrentUser() @@ -50,13 +52,16 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) { const hasPermission = require_any_permission(['system:root', 'account:manage'], row.original.id); const hasReadPermission = require_any_permission(['system:root', 'account:read_details'], row.original.id); + const isDeleting = row.original.deleting === true; const canShowAnyAction = + !isDeleting && ( (hasPermission) || (account_type === 'IMAP' && hasPermission) || - (account_type === 'IMAP' && hasReadPermission); + (account_type === 'IMAP' && hasReadPermission) + ); - const showDownload = account_type === 'IMAP' && hasPermission; + const showDownload = !isDeleting && account_type === 'IMAP' && hasPermission; const handleStartDownload = async () => { try { @@ -100,11 +105,10 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) { {hasPermission && { - setCurrentRow(row.original) if (account_type === "IMAP") { - setOpen("edit-imap"); - } - if (account_type === "NoSync") { + navigate({ to: '/accounts/$id/settings', params: { id: String(row.original.id) } }); + } else { + setCurrentRow(row.original) setOpen("edit-nosync"); } }} @@ -114,6 +118,16 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) { } + {account_type === "IMAP" && hasPermission && { + navigate({ to: '/accounts/$id/settings', params: { id: String(row.original.id) } }); + }} + > + {t('accounts.settings.settings')} + + + + } {account_type === "IMAP" && hasPermission && { setCurrentRow(row.original) diff --git a/web/src/features/accounts/components/delete-dialog.tsx b/web/src/features/accounts/components/delete-dialog.tsx index 41e63ddd..77a08804 100644 --- a/web/src/features/accounts/components/delete-dialog.tsx +++ b/web/src/features/accounts/components/delete-dialog.tsx @@ -43,8 +43,8 @@ export function AccountDeleteDialog({ open, onOpenChange, currentRow }: Props) { const queryClient = useQueryClient(); function handleSuccess() { toast({ - title: t('dialogs.accountDeleted'), - description: t('dialogs.accountDeletedDesc'), + title: t('dialogs.accountDeletionStarted'), + description: t('dialogs.accountDeletionStartedDesc'), action: {t('common.close')}, }); diff --git a/web/src/features/accounts/components/enable-action.tsx b/web/src/features/accounts/components/enable-action.tsx index e26aac2e..887dfbf2 100644 --- a/web/src/features/accounts/components/enable-action.tsx +++ b/web/src/features/accounts/components/enable-action.tsx @@ -77,7 +77,7 @@ export function EnableAction({ row }: DataTableRowActionsProps) { setOpen(true)} - disabled={!hasPermission || updateMutation.isPending} + disabled={!hasPermission || updateMutation.isPending || row.original.deleting} /> . + +import { Input } from "@/components/ui/input"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Button } from "@/components/ui/button"; +import { X } from "lucide-react"; +import { useTranslation } from "react-i18next"; +import type { MatchType, PatternEntry } from "@/lib/pattern-utils"; +import { isValidRegex } from "@/lib/pattern-utils"; + +interface PatternInputProps { + entry: PatternEntry; + onChange: (id: string, entry: Partial) => void; + onRemove: (id: string) => void; +} + +export function PatternInput({ entry, onChange, onRemove }: PatternInputProps) { + const { t } = useTranslation(); + + const matchTypeLabels: Record = { + contains: t("accounts.filters.matchType.contains"), + starts_with: t("accounts.filters.matchType.startsWith"), + ends_with: t("accounts.filters.matchType.endsWith"), + is_exactly: t("accounts.filters.matchType.isExactly"), + regex: t("accounts.filters.matchType.regex"), + }; + + const regexValid = entry.matchType === 'regex' ? isValidRegex(entry.value) : true; + const showRegexHint = entry.matchType === 'regex' && entry.value && !regexValid; + + return ( +
+ +
+ onChange(entry.id, { value: e.target.value })} + /> + {showRegexHint && ( + + Invalid regex + + )} +
+ +
+ ); +} diff --git a/web/src/features/accounts/components/running-state-action.tsx b/web/src/features/accounts/components/running-state-action.tsx index 2556582d..5816aef2 100644 --- a/web/src/features/accounts/components/running-state-action.tsx +++ b/web/src/features/accounts/components/running-state-action.tsx @@ -35,6 +35,9 @@ export function RunningStateCellAction({ row }: Props) { const { setOpen, setCurrentRow } = useAccountContext() const { require_any_permission } = useCurrentUser() + if (row.original.deleting) { + return Deleting... + } let account_type = row.original.account_type; if (account_type === "NoSync") { return n/a diff --git a/web/src/features/accounts/components/schema.ts b/web/src/features/accounts/components/schema.ts index 50d6cdfd..35b504f2 100644 --- a/web/src/features/accounts/components/schema.ts +++ b/web/src/features/accounts/components/schema.ts @@ -67,6 +67,19 @@ const dateSelectionSchema = (t: (key: string) => string) => }) .optional() +const filterRuleSchema = z.object({ + include: z.array(z.string()), + exclude: z.array(z.string()), +}) + +const archiveRulesSchema = z.object({ + enabled: z.boolean(), + senders: filterRuleSchema, + subjects: filterRuleSchema, + skip_larger_than: z.number().int().positive().optional(), + spam_headers: z.array(z.string()), +}) + export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) => z.object({ account_name: z.string().optional(), @@ -100,6 +113,13 @@ export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) => .max(200, { message: t('validation.singleRequestBatchSizeTooLarge'), }), + max_email_size_bytes: z + .number({ + invalid_type_error: t('validation.maxEmailSizeMustBeNumber'), + }) + .int() + .min(1 * 1024 * 1024, { message: t('validation.maxEmailSizeTooSmall') }) + .max(100 * 1024 * 1024, { message: t('validation.maxEmailSizeTooLarge') }), auto_download_new_mailboxes: z.boolean(), download_schedule: z .string() @@ -113,6 +133,7 @@ export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) => }, { message: t('validation.invalidCronExpression') } ), + archive_rules: archiveRulesSchema.optional(), }) export type AccountFormValues = z.infer< diff --git a/web/src/features/accounts/components/step1.tsx b/web/src/features/accounts/components/step1.tsx deleted file mode 100644 index 949e3f0a..00000000 --- a/web/src/features/accounts/components/step1.tsx +++ /dev/null @@ -1,88 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -import { useFormContext } from "react-hook-form"; -import { - FormField, - FormItem, - FormLabel, - FormMessage, - FormControl, - FormDescription, -} from "@/components/ui/form"; - -import { Input } from "@/components/ui/input"; -import { Account } from "./action-dialog"; -import { useTranslation } from "react-i18next"; - -interface StepProps { - isEdit: boolean; -} - -export default function Step1({ isEdit }: StepProps) { - const { t } = useTranslation() - const { control } = useFormContext(); - - return ( - <> -

{t('accounts.emailAccountRegistration')}

-

- {t('accounts.emailAccountRegistrationDesc')} -

-
- ( - - - {t('accounts.emailAddress')}: - - - - - - {isEdit && ( - - {t('accounts.emailCannotBeModified')} - - )} - - )} - /> - ( - - - {t('accounts.name')}: - - - - - {t('accounts.optional')} - - - )} - /> -
- - ); -} \ No newline at end of file diff --git a/web/src/features/accounts/components/step2.tsx b/web/src/features/accounts/components/step2.tsx deleted file mode 100644 index edc92429..00000000 --- a/web/src/features/accounts/components/step2.tsx +++ /dev/null @@ -1,236 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -import { - FormField, - FormItem, - FormLabel, - FormMessage, - FormControl, - FormDescription, -} from "@/components/ui/form"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Input } from "@/components/ui/input"; -import { useFormContext, useWatch } from "react-hook-form"; -import { Account } from "./action-dialog"; -import { PasswordInput } from "@/components/password-input"; -import useProxyList from "@/hooks/use-proxy"; -import { useTranslation } from "react-i18next"; -import { Checkbox } from "@/components/ui/checkbox"; - -interface StepProps { - isEdit: boolean; -} - -export default function Step2({ isEdit }: StepProps) { - const { t } = useTranslation() - const { control } = useFormContext(); - const { proxyOptions } = useProxyList(); - - const imapAuthMethod = useWatch({ - control, - name: "imap.auth.auth_type", - }); - - return ( - <> -
- ( - - - {t('accounts.imapHost')}: - - - - - - - )} - /> - ( - - - {t('accounts.imapPort')}: - - - field.onChange(parseInt(e.target.value, 10))} /> - - - - )} - /> - ( - - {t('accounts.imapEncryption')}: - - - {t('accounts.chooseEncryptionMethod')} - - - - )} - /> - ( - - {t('accounts.useDangerous')}: - - - - {t('accounts.useDangerousDescription')} - - )} - /> - ( - - - {t('accounts.login_name')}: - - - - - {t('accounts.nameDescription')} - - - )} - /> - ( - - {t('accounts.imapAuthMethod')}: - - - {t('accounts.chooseAuthMethod')} - - - - )} - /> - {imapAuthMethod === "Password" && ( - ( - - - {t('accounts.imapPassword')}: - - - - - - {isEdit && ( - - {t('accounts.leaveEmptyToKeepExisting')} - - )} - - )} - /> - )} - ( - - {t('accounts.useProxy')} ({t('accounts.optional')}): - - - - - {t('accounts.imapProxy')} - - - - )} - /> -
- - ); -} \ No newline at end of file diff --git a/web/src/features/accounts/components/step3.tsx b/web/src/features/accounts/components/step3.tsx deleted file mode 100644 index bd44a464..00000000 --- a/web/src/features/accounts/components/step3.tsx +++ /dev/null @@ -1,535 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -import { - FormField, - FormItem, - FormLabel, - FormMessage, - FormControl, - FormDescription, -} from "@/components/ui/form"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Input } from "@/components/ui/input"; -import { useFormContext } from "react-hook-form"; -import { Account } from "./action-dialog"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; -import { Button } from "@/components/ui/button"; -import { format } from "date-fns"; -import { CalendarIcon } from "lucide-react"; -import { Calendar } from "@/components/ui/calendar"; -import { cn, dateFnsLocaleMap } from "@/lib/utils"; -import { useState } from "react"; -import { Checkbox } from "@/components/ui/checkbox"; -import { useTranslation } from "react-i18next"; -import { enUS } from "date-fns/locale"; -import i18n from "@/i18n"; - - -type SyncMode = 'all' | 'since_fixed' | 'since_relative' | 'before_relative'; -type ScheduleMode = 'interval' | 'cron'; -type CronMode = 'simple' | 'advanced'; -type CronFrequency = 'daily' | 'weekly' | 'monthly'; - -interface CronSimpleState { - frequency: CronFrequency; - hour: number; - minute: number; - dayOfWeek: number; - dayOfMonth: number; -} - -const DEFAULT_CRON_SIMPLE: CronSimpleState = { - frequency: 'daily', - hour: 0, - minute: 0, - dayOfWeek: 1, - dayOfMonth: 1, -}; - -function buildCronFromSimple(s: CronSimpleState): string { - switch (s.frequency) { - case 'daily': - return `0 ${s.minute} ${s.hour} * * *`; - case 'weekly': - return `0 ${s.minute} ${s.hour} * * ${s.dayOfWeek}`; - case 'monthly': - return `0 ${s.minute} ${s.hour} ${s.dayOfMonth} * *`; - } -} - -function tryParseCronToSimple(cron: string): CronSimpleState | null { - const fields = cron.trim().split(/\s+/); - if (fields.length < 6) return null; - - const sec = fields[0]; - const min = fields[1]; - const hour = fields[2]; - const dom = fields[3]; - const month = fields[4]; - const dow = fields[5]; - - if (sec !== '0') return null; - if (month !== '*') return null; - - const minuteVal = parseInt(min, 10); - const hourVal = parseInt(hour, 10); - if (isNaN(minuteVal) || isNaN(hourVal)) return null; - - if (dom === '*' && dow === '*') { - return { frequency: 'daily', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: 1 }; - } - if (dom === '*') { - const dowVal = parseInt(dow, 10); - if (!isNaN(dowVal)) { - return { frequency: 'weekly', hour: hourVal, minute: minuteVal, dayOfWeek: dowVal, dayOfMonth: 1 }; - } - } - if (dow === '*') { - const domVal = parseInt(dom, 10); - if (!isNaN(domVal)) { - return { frequency: 'monthly', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: domVal }; - } - } - return null; -} - -export default function Step3() { - const { t } = useTranslation(); - const { control, getValues, setValue } = useFormContext(); - const current = getValues(); - - const [syncMode, setSyncMode] = useState(() => { - if (current.date_before) return 'before_relative'; - if (current.date_since?.fixed) return 'since_fixed'; - if (current.date_since?.relative) return 'since_relative'; - return 'all'; - }); - - const [scheduleMode, setScheduleMode] = useState(() => { - if (current.download_schedule) return 'cron'; - return 'interval'; - }); - - const [cronMode, setCronMode] = useState(() => { - if (current.download_schedule && tryParseCronToSimple(current.download_schedule)) { - return 'simple'; - } - if (current.download_schedule) return 'advanced'; - return 'simple'; - }); - - const [cronSimple, setCronSimple] = useState(() => { - if (current.download_schedule) { - return tryParseCronToSimple(current.download_schedule) ?? DEFAULT_CRON_SIMPLE; - } - return DEFAULT_CRON_SIMPLE; - }); - - const updateCronFromSimple = (partial: Partial) => { - const next = { ...cronSimple, ...partial }; - setCronSimple(next); - setValue('download_schedule', buildCronFromSimple(next)); - }; - - const handleModeChange = (mode: SyncMode) => { - setSyncMode(mode); - - setValue("date_since", undefined); - setValue("date_before", undefined); - - if (mode === 'since_fixed') { - setValue("date_since.fixed", undefined); - } else if (mode === 'since_relative') { - setValue("date_since.relative", { value: 1, unit: 'Months' }); - } else if (mode === 'before_relative') { - setValue("date_before", { value: 1, unit: 'Years' }); - } - }; - - const handleScheduleModeChange = (mode: ScheduleMode) => { - setScheduleMode(mode); - if (mode === 'interval') { - setValue("download_schedule", undefined); - } else { - setValue("download_interval_min", 60); - if (cronMode === 'simple') { - updateCronFromSimple(cronSimple); - } - } - }; - - return ( -
-
- - {t('accounts.scheduleMode')} - - {t('accounts.scheduleModeDescription')} - - - -
- {scheduleMode === 'interval' ? ( - ( - - {t('accounts.downloadInterval')} - - field.onChange(parseInt(e.target.value, 10))} /> - - - - {t('accounts.downloadIntervalPlaceholder')} - - - )} - /> - ) : ( -
-
- {/* {t('accounts.downloadSchedule')} */} -
- - -
-
- - {cronMode === 'simple' ? ( -
- - {t('accounts.cronFrequency')} - - - - - {t('accounts.cronHour')} - - - - : - - - {t('accounts.cronMinute')} - - - - {cronSimple.frequency === 'weekly' && ( - - {t('accounts.cronDayOfWeek')} - - - )} - - {cronSimple.frequency === 'monthly' && ( - - {t('accounts.cronDayOfMonth')} - - - )} - -
- = {buildCronFromSimple(cronSimple)} -
-
- ) : ( - ( - - - - - - {t('accounts.downloadScheduleDescription')} - - - - )} - /> - )} - - {cronMode === 'simple' && ( - {t('accounts.downloadScheduleDescription')} - )} - -
- {t('accounts.cronTimezoneNote')} -
-
- )} - ( - - {t('accounts.downloadBatchSize')} - - field.onChange(parseInt(e.target.value, 10))} /> - - - - {t('accounts.downloadBatchSizeDescription')} - - - )} - /> -
-
- - ( - - - - -
- {t('accounts.enabled')} - {t('accounts.enabledDescription')} -
- - )} - /> - -
-
- - {t('accounts.downloadScope')} - - {t('accounts.downloadScopeDescription')} - - - -
- {syncMode === 'since_fixed' && ( - { - const currentLang = i18n.language.toLowerCase().replace('_', '-'); - const dateLocale = dateFnsLocaleMap[currentLang] || enUS; - return - {t('accounts.selectDate')} - - - - - - - - field.onChange(date?.toLocaleDateString('en-CA'))} - disabled={(date) => date > new Date() || date < new Date("1900-01-01")} - locale={dateLocale} - /> - - - - ; - - }} - /> - )} - - {(syncMode === 'since_relative' || syncMode === 'before_relative') && ( -
- ( - - {t('accounts.duration', 'Duration')} - - field.onChange(parseInt(e.target.value, 10))} /> - - - - )} - /> - ( - - {t('accounts.unit', 'Unit')} - - - - )} - /> -
- )} -
-
- -
- - ( - - - - -
- {t('accounts.autoDownloadNewMailboxes')} - {t('accounts.autoDownloadNewMailboxesDescription')} -
- - )} - /> - -
-
- ); -} \ No newline at end of file diff --git a/web/src/features/accounts/components/step4.tsx b/web/src/features/accounts/components/step4.tsx deleted file mode 100644 index 07276a41..00000000 --- a/web/src/features/accounts/components/step4.tsx +++ /dev/null @@ -1,179 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -import { useFormContext } from "react-hook-form"; -import { Account } from "./action-dialog"; -import { Accordion, AccordionItem, AccordionTrigger, AccordionContent } from "@/components/ui/accordion"; -import { useTranslation } from "react-i18next"; -import useProxyList from "@/hooks/use-proxy"; - -export default function Step4() { - const { t } = useTranslation(); - const { getValues } = useFormContext(); - const { getUrlById } = useProxyList(); - const summaryData = getValues(); - - - const sinceText = (() => { - if (summaryData.date_since?.fixed) { - return summaryData.date_since.fixed; - } - - if (summaryData.date_since?.relative?.value) { - return `${t('accounts.sinceRelativeValue', { - value: summaryData.date_since!.relative!.value, - unit: t(`accounts.${summaryData.date_since!.relative!.unit!.toLowerCase()}`) - })}`; - } - - return t('accounts.syncAll'); - })(); - - const hasSince = !!summaryData.date_since; - const hasBefore = !!summaryData.date_before?.value; - - return ( -
- - - {t('accounts.email')}: - {summaryData.email} - - - - {t('accounts.name')}: - {summaryData.account_name ?? t('accounts.notAvailable')} - - - - {t('accounts.login_name')}: - {summaryData.login_name ?? t('accounts.notAvailable')} - - - - {t('accounts.imap')}: - -
- - - - - - - - - - - - - - - - - - - - - - - {summaryData.imap.auth.auth_type === 'Password' && ( - - - - - )} - - - - - -
{t('accounts.host')}:{summaryData.imap.host}
{t('accounts.port')}:{summaryData.imap.port}
{t('accounts.encryption')}:{summaryData.imap.encryption}
{t('accounts.useDangerous')}:{`${summaryData.use_dangerous}`}
{t('accounts.authType')}:{summaryData.imap.auth.auth_type}
{t('accounts.password')}:{summaryData.imap.auth.password}
{t('accounts.useProxyField')}: - {(() => { - if (!summaryData.imap.use_proxy) { - return t('accounts.useNoProxy'); - } - const proxyUrl = getUrlById(summaryData.imap.use_proxy); - return proxyUrl || `${t('common.yes')} (${summaryData.imap.use_proxy})`; - })()} -
-
- - - - - - {t('accounts.downloadScope')}: - - - - {hasSince && ( -
- - {t('accounts.sinceFixed')}: - - {sinceText} -
- )} - - {hasBefore && ( -
- - {t('accounts.beforeRelative')}: - - - {t('accounts.beforeRelativeValue', { - value: summaryData.date_before!.value, - unit: t(`accounts.${summaryData.date_before!.unit!.toLowerCase()}`) - })} - -
- )} - - {!hasSince && !hasBefore && ( - - {t('accounts.downloadAll')} - - )} - - - - - - {t('accounts.downloadInterval')}: - {summaryData.download_interval_min} {t('accounts.minutes')} - - - - {t('accounts.downloadBatchSize')}: - {summaryData.download_batch_size} - - - - {t('accounts.downloadSchedule')}: - {summaryData.download_schedule || t('accounts.notAvailable')} - - - - {t('accounts.autoDownloadNewMailboxes')}: - {summaryData.auto_download_new_mailboxes ? t('common.yes') : t('common.no')} - - -
- ); -} diff --git a/web/src/features/accounts/components/tab-download.tsx b/web/src/features/accounts/components/tab-download.tsx new file mode 100644 index 00000000..eea22900 --- /dev/null +++ b/web/src/features/accounts/components/tab-download.tsx @@ -0,0 +1,564 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useState } from "react"; +import { useFormContext } from "react-hook-form"; +import { useTranslation } from "react-i18next"; +import { format } from "date-fns"; +import { CalendarIcon } from "lucide-react"; +import { enUS } from "date-fns/locale"; +import { + FormField, + FormItem, + FormLabel, + FormMessage, + FormControl, + FormDescription, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Button } from "@/components/ui/button"; +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; +import { Calendar } from "@/components/ui/calendar"; +import { cn, dateFnsLocaleMap } from "@/lib/utils"; +import i18n from "@/i18n"; +import { AccountFormValues } from "./schema"; + +type SyncMode = 'all' | 'since_fixed' | 'since_relative' | 'before_relative'; +type ScheduleMode = 'interval' | 'cron'; +type CronMode = 'simple' | 'advanced'; +type CronFrequency = 'daily' | 'weekly' | 'monthly'; + +interface CronSimpleState { + frequency: CronFrequency; + hour: number; + minute: number; + dayOfWeek: number; + dayOfMonth: number; +} + +const DEFAULT_CRON_SIMPLE: CronSimpleState = { + frequency: 'daily', + hour: 0, + minute: 0, + dayOfWeek: 1, + dayOfMonth: 1, +}; + +function buildCronFromSimple(s: CronSimpleState): string { + switch (s.frequency) { + case 'daily': + return `0 ${s.minute} ${s.hour} * * *`; + case 'weekly': + return `0 ${s.minute} ${s.hour} * * ${s.dayOfWeek}`; + case 'monthly': + return `0 ${s.minute} ${s.hour} ${s.dayOfMonth} * *`; + } +} + +function tryParseCronToSimple(cron: string): CronSimpleState | null { + const fields = cron.trim().split(/\s+/); + if (fields.length < 6) return null; + const sec = fields[0]; + const min = fields[1]; + const hour = fields[2]; + const dom = fields[3]; + const month = fields[4]; + const dow = fields[5]; + if (sec !== '0') return null; + if (month !== '*') return null; + const minuteVal = parseInt(min, 10); + const hourVal = parseInt(hour, 10); + if (isNaN(minuteVal) || isNaN(hourVal)) return null; + if (dom === '*' && dow === '*') { + return { frequency: 'daily', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: 1 }; + } + if (dom === '*') { + const dowVal = parseInt(dow, 10); + if (!isNaN(dowVal)) { + return { frequency: 'weekly', hour: hourVal, minute: minuteVal, dayOfWeek: dowVal, dayOfMonth: 1 }; + } + } + if (dow === '*') { + const domVal = parseInt(dom, 10); + if (!isNaN(domVal)) { + return { frequency: 'monthly', hour: hourVal, minute: minuteVal, dayOfWeek: 1, dayOfMonth: domVal }; + } + } + return null; +} + +export function TabDownload() { + const { t } = useTranslation(); + const { control, getValues, setValue } = useFormContext(); + const current = getValues(); + + const [syncMode, setSyncMode] = useState(() => { + if (current.date_before) return 'before_relative'; + if (current.date_since?.fixed) return 'since_fixed'; + if (current.date_since?.relative) return 'since_relative'; + return 'all'; + }); + + const [scheduleMode, setScheduleMode] = useState(() => { + if (current.download_schedule) return 'cron'; + return 'interval'; + }); + + const [cronMode, setCronMode] = useState(() => { + if (current.download_schedule && tryParseCronToSimple(current.download_schedule)) { + return 'simple'; + } + if (current.download_schedule) return 'advanced'; + return 'simple'; + }); + + const [cronSimple, setCronSimple] = useState(() => { + if (current.download_schedule) { + return tryParseCronToSimple(current.download_schedule) ?? DEFAULT_CRON_SIMPLE; + } + return DEFAULT_CRON_SIMPLE; + }); + + const updateCronFromSimple = (partial: Partial) => { + const next = { ...cronSimple, ...partial }; + setCronSimple(next); + setValue('download_schedule', buildCronFromSimple(next)); + }; + + const handleModeChange = (mode: SyncMode) => { + setSyncMode(mode); + setValue("date_since", undefined); + setValue("date_before", undefined); + if (mode === 'since_fixed') { + setValue("date_since.fixed", undefined); + } else if (mode === 'since_relative') { + setValue("date_since.relative", { value: 1, unit: 'Months' }); + } else if (mode === 'before_relative') { + setValue("date_before", { value: 1, unit: 'Years' }); + } + }; + + const handleScheduleModeChange = (mode: ScheduleMode) => { + setScheduleMode(mode); + if (mode === 'interval') { + setValue("download_schedule", undefined); + } else { + setValue("download_interval_min", 60); + if (cronMode === 'simple') { + updateCronFromSimple(cronSimple); + } + } + }; + + const BYTES_PER_MB = 1024 * 1024; + + return ( +
+ {/* Schedule */} +
+

+ {t('accounts.settings.schedule')} +

+ + + {t('accounts.scheduleMode')} + {t('accounts.scheduleModeDescription')} + + + +
+ {scheduleMode === 'interval' && ( + ( + + {t('accounts.downloadInterval')}* + + field.onChange(parseInt(e.target.value, 10))} /> + + {t('accounts.downloadIntervalPlaceholder')} + + + )} + /> + )} +
+ + {scheduleMode === 'cron' && ( +
+
+
+ + +
+
+ + {cronMode === 'simple' ? ( + <> +
+ + {t('accounts.cronFrequency')} + + + + + {t('accounts.cronHour')} + + + + : + + + {t('accounts.cronMinute')} + + + + {cronSimple.frequency === 'weekly' && ( + + {t('accounts.cronDayOfWeek')} + + + )} + + {cronSimple.frequency === 'monthly' && ( + + {t('accounts.cronDayOfMonth')} + + + )} +
+ +

+ {cronSimple.frequency === 'daily' && t('accounts.cronSummaryDaily', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0') })} + {cronSimple.frequency === 'weekly' && (() => { + const dayNames = ['cronSunday', 'cronMonday', 'cronTuesday', 'cronWednesday', 'cronThursday', 'cronFriday', 'cronSaturday']; + return t('accounts.cronSummaryWeekly', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0'), day: t(`accounts.${dayNames[cronSimple.dayOfWeek]}`) }); + })()} + {cronSimple.frequency === 'monthly' && t('accounts.cronSummaryMonthly', { hour: String(cronSimple.hour).padStart(2, '0'), minute: String(cronSimple.minute).padStart(2, '0'), day: cronSimple.dayOfMonth })} + {cronSimple.frequency === 'monthly' && cronSimple.dayOfMonth > 28 && ( + {t('accounts.cronMonthAlignNote')} + )} +

+ + ) : ( + ( + + + + + {t('accounts.downloadScheduleDescription')} + + + )} + /> + )} + +
+ {t('accounts.cronTimezoneNote')} +
+
+ )} +
+ +
+ + {/* Batch & Size */} +
+

+ {t('accounts.settings.performance')} +

+
+ ( + + {t('accounts.downloadBatchSize')}* + + field.onChange(parseInt(e.target.value, 10))} /> + + {t('accounts.downloadBatchSizeDescription')} + + + )} + /> + ( + + {t('accounts.maxEmailSizeBytes')}* + +
+ { + const parsed = parseInt(e.target.value, 10); + field.onChange(isNaN(parsed) ? parsed : parsed * BYTES_PER_MB); + }} + /> + MB +
+ + {t('accounts.maxEmailSizeBytesDescription')} + + + )} + /> +
+
+ +
+ + {/* Download Scope */} +
+

+ {t('accounts.settings.scope')} +

+ + + {t('accounts.downloadScope')} + {t('accounts.downloadScopeDescription')} + + + {syncMode === 'all' && t('accounts.downloadAllDesc')} + {syncMode === 'since_fixed' && t('accounts.sinceFixedDesc')} + {syncMode === 'since_relative' && t('accounts.sinceRelativeDesc')} + {syncMode === 'before_relative' && t('accounts.beforeRelativeDesc')} + + + +
+ {syncMode === 'since_fixed' && ( + { + const currentLang = i18n.language.toLowerCase().replace('_', '-'); + const dateLocale = dateFnsLocaleMap[currentLang] || enUS; + return ( + + {t('accounts.selectDate')}* + + + + + + + + field.onChange(date?.toLocaleDateString('en-CA'))} + disabled={(date) => date > new Date() || date < new Date("1900-01-01")} + locale={dateLocale} + /> + + + + + ); + }} + /> + )} + + {(syncMode === 'since_relative' || syncMode === 'before_relative') && ( +
+ ( + + {t('accounts.duration')}* + + field.onChange(parseInt(e.target.value, 10))} /> + + + + )} + /> + ( + + {t('accounts.unit', 'Unit')}* + + + + )} + /> +
+ )} +
+
+ +
+ + ( + + + + +
+ {t('accounts.autoDownloadNewMailboxes')} + {t('accounts.autoDownloadNewMailboxesDescription')} +
+ + )} + /> +
+ ); +} diff --git a/web/src/features/accounts/components/tab-filters.tsx b/web/src/features/accounts/components/tab-filters.tsx new file mode 100644 index 00000000..7d756730 --- /dev/null +++ b/web/src/features/accounts/components/tab-filters.tsx @@ -0,0 +1,514 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useFormContext, useWatch } from "react-hook-form"; +import { useTranslation } from "react-i18next"; +import { + FormField, + FormItem, + FormLabel, + FormMessage, + FormControl, + FormDescription, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Button } from "@/components/ui/button"; +import { Plus, HelpCircle } from "lucide-react"; +import { PatternInput } from "./pattern-input"; +import type { PatternEntry } from "@/lib/pattern-utils"; +import { newPatternId, simplePatternToRegex } from "@/lib/pattern-utils"; +import { useState } from "react"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { AccountFormValues } from "./schema"; + +const SUGGESTED_SPAM_HEADERS = [ + 'X-Spam-Flag', + 'X-Spam', + 'X-Spam-Status', + 'X-Barracuda-Spam-Status', + 'X-Barracuda-Spam-Flag', + 'X-MS-Exchange-Organization-SCL', +]; + +function toPatternEntries(patterns: string[]): PatternEntry[] { + return patterns.map((p) => ({ + id: newPatternId(), + matchType: 'regex' as const, + value: p, + })); +} + +function patternsToRegexList(entries: PatternEntry[]): string[] { + return entries + .filter((e) => e.value.trim() !== '') + .map((e) => simplePatternToRegex(e.matchType, e.value)); +} + +export function TabFilters() { + const { t } = useTranslation(); + const { control, setValue } = useFormContext(); + const archiveRules = useWatch({ control, name: 'archive_rules' }); + + const enabled = archiveRules?.enabled ?? false; + const sendersInclude = archiveRules?.senders?.include ?? []; + const sendersExclude = archiveRules?.senders?.exclude ?? []; + const subjectsInclude = archiveRules?.subjects?.include ?? []; + const subjectsExclude = archiveRules?.subjects?.exclude ?? []; + const spamHeaders = archiveRules?.spam_headers ?? []; + + const [senderIncludeEntries, setSenderIncludeEntries] = useState( + () => toPatternEntries(sendersInclude) + ); + const [senderExcludeEntries, setSenderExcludeEntries] = useState( + () => toPatternEntries(sendersExclude) + ); + const [subjectIncludeEntries, setSubjectIncludeEntries] = useState( + () => toPatternEntries(subjectsInclude) + ); + const [subjectExcludeEntries, setSubjectExcludeEntries] = useState( + () => toPatternEntries(subjectsExclude) + ); + + // Resync local state when form values change externally (e.g. after form.reset) + const [lastSyncKey, setLastSyncKey] = useState(''); + const syncKey = JSON.stringify({ sendersInclude, sendersExclude, subjectsInclude, subjectsExclude }); + if (syncKey !== lastSyncKey) { + setLastSyncKey(syncKey); + setSenderIncludeEntries(toPatternEntries(sendersInclude)); + setSenderExcludeEntries(toPatternEntries(sendersExclude)); + setSubjectIncludeEntries(toPatternEntries(subjectsInclude)); + setSubjectExcludeEntries(toPatternEntries(subjectsExclude)); + } + + const syncToForm = ( + includeEntries: PatternEntry[], + excludeEntries: PatternEntry[], + fieldPrefix: string + ) => { + const include = patternsToRegexList(includeEntries); + const exclude = patternsToRegexList(excludeEntries); + setValue(`${fieldPrefix}.include` as any, include); + setValue(`${fieldPrefix}.exclude` as any, exclude); + }; + + const addEntry = ( + side: 'include' | 'exclude', + includeEntries: PatternEntry[], + excludeEntries: PatternEntry[], + setIncludeEntries: React.Dispatch>, + setExcludeEntries: React.Dispatch>, + fieldPrefix: string + ) => { + const newEntry: PatternEntry = { id: newPatternId(), matchType: 'contains', value: '' }; + const newInclude = side === 'include' ? [...includeEntries, newEntry] : includeEntries; + const newExclude = side === 'exclude' ? [...excludeEntries, newEntry] : excludeEntries; + setIncludeEntries(newInclude); + setExcludeEntries(newExclude); + syncToForm(newInclude, newExclude, fieldPrefix); + }; + + const updateEntry = ( + id: string, + partial: Partial, + side: 'include' | 'exclude', + includeEntries: PatternEntry[], + excludeEntries: PatternEntry[], + setIncludeEntries: React.Dispatch>, + setExcludeEntries: React.Dispatch>, + fieldPrefix: string + ) => { + if (side === 'include') { + const updated = includeEntries.map((e) => (e.id === id ? { ...e, ...partial } : e)); + setIncludeEntries(updated); + syncToForm(updated, excludeEntries, fieldPrefix); + } else { + const updated = excludeEntries.map((e) => (e.id === id ? { ...e, ...partial } : e)); + setExcludeEntries(updated); + syncToForm(includeEntries, updated, fieldPrefix); + } + }; + + const removeEntry = ( + id: string, + side: 'include' | 'exclude', + includeEntries: PatternEntry[], + excludeEntries: PatternEntry[], + setIncludeEntries: React.Dispatch>, + setExcludeEntries: React.Dispatch>, + fieldPrefix: string + ) => { + if (side === 'include') { + const filtered = includeEntries.filter((e) => e.id !== id); + setIncludeEntries(filtered); + syncToForm(filtered, excludeEntries, fieldPrefix); + } else { + const filtered = excludeEntries.filter((e) => e.id !== id); + setExcludeEntries(filtered); + syncToForm(includeEntries, filtered, fieldPrefix); + } + }; + + const handleEnableChange = (checked: boolean) => { + if (checked) { + setValue('archive_rules', { + enabled: true, + senders: { include: [], exclude: [] }, + subjects: { include: [], exclude: [] }, + skip_larger_than: undefined, + spam_headers: [], + }); + } else { + setValue('archive_rules', undefined); + } + }; + + const addSpamHeader = (header: string) => { + if (!spamHeaders.includes(header)) { + setValue('archive_rules.spam_headers', [...spamHeaders, header]); + } + }; + + const removeSpamHeader = (header: string) => { + setValue('archive_rules.spam_headers', spamHeaders.filter((h) => h !== header)); + }; + + const [newSpamHeader, setNewSpamHeader] = useState(''); + + const handleAddCustomSpamHeader = () => { + const trimmed = newSpamHeader.trim(); + if (trimmed && !spamHeaders.includes(trimmed)) { + setValue('archive_rules.spam_headers', [...spamHeaders, trimmed]); + setNewSpamHeader(''); + } + }; + + const BYTES_PER_MB = 1024 * 1024; + + return ( +
+ {/* Master Switch */} +
+ + + + +
+ {t('accounts.filters.enableFiltering')} + + {t('accounts.filters.enableFilteringDesc')} + +
+ +
+ + {enabled && ( + <> + {/* Sender Filters */} +
+
+

{t('accounts.filters.senderFilter')}

+ + + + + + {t('accounts.filters.senderFilterHelp')} + + +
+ +
+
+

+ {t('accounts.filters.include')} +

+ {senderIncludeEntries.length === 0 ? ( +

+ {t('accounts.filters.noIncludePatterns')} +

+ ) : ( +
+ {senderIncludeEntries.map((entry) => ( + + updateEntry(id, partial, 'include', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders') + } + onRemove={(id) => + removeEntry(id, 'include', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders') + } + /> + ))} +
+ )} + +
+ +
+

+ {t('accounts.filters.exclude')} +

+ {senderExcludeEntries.length === 0 ? ( +

+ {t('accounts.filters.noExcludePatterns')} +

+ ) : ( +
+ {senderExcludeEntries.map((entry) => ( + + updateEntry(id, partial, 'exclude', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders') + } + onRemove={(id) => + removeEntry(id, 'exclude', senderIncludeEntries, senderExcludeEntries, setSenderIncludeEntries, setSenderExcludeEntries, 'archive_rules.senders') + } + /> + ))} +
+ )} + +
+
+
+ + {/* Subject Filters */} +
+
+

{t('accounts.filters.subjectFilter')}

+ + + + + + {t('accounts.filters.subjectFilterHelp')} + + +
+
+
+

+ {t('accounts.filters.include')} +

+ {subjectIncludeEntries.length === 0 ? ( +

+ {t('accounts.filters.noIncludePatterns')} +

+ ) : ( +
+ {subjectIncludeEntries.map((entry) => ( + + updateEntry(id, partial, 'include', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects') + } + onRemove={(id) => + removeEntry(id, 'include', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects') + } + /> + ))} +
+ )} + +
+ +
+

+ {t('accounts.filters.exclude')} +

+ {subjectExcludeEntries.length === 0 ? ( +

+ {t('accounts.filters.noExcludePatterns')} +

+ ) : ( +
+ {subjectExcludeEntries.map((entry) => ( + + updateEntry(id, partial, 'exclude', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects') + } + onRemove={(id) => + removeEntry(id, 'exclude', subjectIncludeEntries, subjectExcludeEntries, setSubjectIncludeEntries, setSubjectExcludeEntries, 'archive_rules.subjects') + } + /> + ))} +
+ )} + +
+
+
+ + {/* Size Limit */} +
+
+

{t('accounts.filters.sizeLimit')}

+
+ ( + + {t('accounts.filters.skipLargerThan')} + +
+ { + const parsed = parseInt(e.target.value, 10); + field.onChange(isNaN(parsed) ? undefined : parsed * BYTES_PER_MB); + }} + /> + MB +
+ + {t('accounts.filters.sizeLimitDesc')} + + + )} + /> +
+ + {/* Spam Headers */} +
+
+

{t('accounts.filters.spamHeaders')}

+ + + + + + {t('accounts.filters.spamHeadersHelp')} + + +
+
+ {spamHeaders.length > 0 ? ( + spamHeaders.map((header) => ( +
+
+ {header} +
+ +
+ )) + ) : ( +

+ {t('accounts.filters.noSpamHeaders')} +

+ )} +
+ +
+ setNewSpamHeader(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') { + e.preventDefault(); + handleAddCustomSpamHeader(); + } + }} + /> + +
+ +
+

{t('accounts.filters.suggestions')}

+
+ {SUGGESTED_SPAM_HEADERS.filter((h) => !spamHeaders.includes(h)).map((header) => ( + + ))} +
+
+
+ + )} +
+ ); +} diff --git a/web/src/features/accounts/components/tab-general.tsx b/web/src/features/accounts/components/tab-general.tsx new file mode 100644 index 00000000..2741d0d5 --- /dev/null +++ b/web/src/features/accounts/components/tab-general.tsx @@ -0,0 +1,91 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useFormContext } from "react-hook-form"; +import { useTranslation } from "react-i18next"; +import { + FormField, + FormItem, + FormLabel, + FormMessage, + FormControl, + FormDescription, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { AccountFormValues } from "./schema"; + +interface TabGeneralProps { + isEdit?: boolean; +} + +export function TabGeneral({ isEdit }: TabGeneralProps) { + const { t } = useTranslation(); + const { control } = useFormContext(); + + return ( +
+ ( + + {t('accounts.email')}* + + + + {isEdit && ( + {t('accounts.emailCannotBeModified')} + )} + + + )} + /> + + ( + + {t('accounts.name')} + + + + + + )} + /> + + ( + + + + +
+ {t('accounts.enabled')} + {t('accounts.enabledDescription')} +
+ + )} + /> +
+ ); +} diff --git a/web/src/features/accounts/components/tab-server.tsx b/web/src/features/accounts/components/tab-server.tsx new file mode 100644 index 00000000..44f9121b --- /dev/null +++ b/web/src/features/accounts/components/tab-server.tsx @@ -0,0 +1,180 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useFormContext } from "react-hook-form"; +import { useTranslation } from "react-i18next"; +import { + FormField, + FormItem, + FormLabel, + FormMessage, + FormControl, + FormDescription, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { PasswordInput } from "@/components/password-input"; +import { AccountFormValues } from "./schema"; + +interface TabServerProps { + isEdit?: boolean; +} + +export function TabServer({ isEdit }: TabServerProps) { + const { t } = useTranslation(); + const { control, watch } = useFormContext(); + const authType = watch('imap.auth.auth_type'); + + return ( +
+ ( + + {t('accounts.imapHost')}* + + + + + + )} + /> + +
+ ( + + {t('accounts.imapPort')}* + + field.onChange(parseInt(e.target.value, 10))} + placeholder={t('accounts.imapPortPlaceholder')} + /> + + + + )} + /> + + ( + + {t('accounts.imapEncryption')}* + + + + )} + /> +
+ + ( + + {t('accounts.login_name')} + + + + {t('accounts.nameDescription')} + + + )} + /> + + ( + + {t('accounts.imapAuthMethod')}* + + + + )} + /> + + {authType === 'Password' && ( + ( + + {t('accounts.imapPassword')}{!isEdit && *} + + + + {isEdit && {t('accounts.leaveEmptyToKeepPassword')}} + + + )} + /> + )} + + ( + + + + +
+ {t('accounts.useDangerous')} +
+ + )} + /> +
+ ); +} diff --git a/web/src/features/accounts/components/table.tsx b/web/src/features/accounts/components/table.tsx index ef094fc0..3ff597ea 100644 --- a/web/src/features/accounts/components/table.tsx +++ b/web/src/features/accounts/components/table.tsx @@ -17,7 +17,7 @@ // along with this program. If not, see . -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { ColumnDef, ColumnFiltersState, @@ -64,7 +64,19 @@ export function AccountTable({ columns, data }: DataTableProps) { const [rowSelection, setRowSelection] = useState({}) const [columnVisibility, setColumnVisibility] = useState({}) const [columnFilters, setColumnFilters] = useState([]) - const [sorting, setSorting] = useState([]) + const [sorting, setSorting] = useState(() => { + const saved = localStorage.getItem('bichon_accounts_sorting'); + return saved ? JSON.parse(saved) : []; + }) + + // Persist sorting state to localStorage + const prevSortingRef = useRef(sorting); + useEffect(() => { + if (prevSortingRef.current !== sorting) { + localStorage.setItem('bichon_accounts_sorting', JSON.stringify(sorting)); + prevSortingRef.current = sorting; + } + }, [sorting]); const table = useReactTable({ data, @@ -127,7 +139,7 @@ export function AccountTable({ columns, data }: DataTableProps) { {row.getVisibleCells().map((cell) => ( (null) @@ -55,6 +55,10 @@ export default function Accounts() { const { data: accountList, isLoading } = useQuery({ queryKey: ['account-list'], queryFn: list_accounts, + refetchInterval: (query) => { + const items = (query.state.data as { items?: { deleting?: boolean }[] })?.items; + return items?.some((item) => item.deleting) ? 5000 : false; + }, }) const hasAccounts = accountList != null && accountList.items.length > 0; @@ -73,15 +77,14 @@ export default function Accounts() {

{require_any_permission(['system:root', 'account:create']) &&
-
- -
+ +
} @@ -103,8 +106,13 @@ export default function Accounts() { {t('accounts.noAccountConfigurationsDesc')}

- +
@@ -113,16 +121,6 @@ export default function Accounts() { - setOpen('add')} /> - - setOpen('add-imap')} - /> - { - setOpen('edit-imap') - setTimeout(() => { - setCurrentRow(null) - }, 500) - }} - currentRow={currentRow} - /> . + +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useMutation } from '@tanstack/react-query'; +import { + Download, FileIcon, ZoomIn, ZoomOut, RotateCcw, + ChevronLeft, ChevronRight, X, +} from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { Dialog, DialogContent } from '@/components/ui/dialog'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; +import { Separator } from '@/components/ui/separator'; +import { Skeleton } from '@/components/ui/skeleton'; +import { toast } from '@/hooks/use-toast'; +import { useTranslation } from 'react-i18next'; + +import { preview_attachment, download_attachment } from '@/api/mailbox/envelope/api'; +import { getFileConfig } from './mail-message-view'; + +const PREVIEWABLE_IMAGE = /^image\/(png|jpeg|gif|webp|svg\+xml)$/; +const PREVIEWABLE_TEXT = /^(text\/(plain|csv|html|xml|css|javascript|markdown)|application\/(json|xml|javascript|x-httpd-php|x-sh|x-perl|x-python|x-ruby))$/; + +const EXT_TO_MIME: Record = { + '.pdf': 'application/pdf', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.gif': 'image/gif', + '.webp': 'image/webp', + '.svg': 'image/svg+xml', + '.txt': 'text/plain', + '.csv': 'text/csv', + '.html': 'text/html', + '.htm': 'text/html', + '.xml': 'application/xml', + '.css': 'text/css', + '.js': 'application/javascript', + '.mjs': 'application/javascript', + '.md': 'text/markdown', + '.json': 'application/json', + '.php': 'application/x-httpd-php', + '.sh': 'application/x-sh', + '.pl': 'application/x-perl', + '.py': 'application/x-python', + '.rb': 'application/x-ruby', +}; + +function getExtensionMime(filename: string): string | null { + const dot = filename.lastIndexOf('.'); + if (dot < 0) return null; + const ext = filename.slice(dot).toLowerCase(); + return EXT_TO_MIME[ext] ?? null; +} + +function resolveContentType(contentType: string, fileName: string): string { + if (contentType && contentType !== 'application/octet-stream') return contentType; + return getExtensionMime(fileName) ?? contentType; +} + +export interface PreviewAttachment { + content_hash: string; + file_type: string; + filename: string; +} + +interface AttachmentPreviewProps { + open: boolean; + onOpenChange: (open: boolean) => void; + accountId: number; + envelopeId: string; + contentHash: string; + contentType: string; + fileName: string; + /** Full attachment list for gallery navigation (optional). */ + attachments?: PreviewAttachment[]; + /** Index of the current attachment within `attachments`. */ + attachmentIndex?: number; +} + +function isImagePreview(contentType: string, fileName?: string) { + return PREVIEWABLE_IMAGE.test(resolveContentType(contentType, fileName ?? '')); +} + +function isPdfPreview(contentType: string, fileName?: string) { + return resolveContentType(contentType, fileName ?? '') === 'application/pdf'; +} + +function isTextPreview(contentType: string, fileName?: string) { + return PREVIEWABLE_TEXT.test(resolveContentType(contentType, fileName ?? '')); +} + +export default function AttachmentPreview({ + open, + onOpenChange, + accountId, + envelopeId, + contentHash, + contentType, + fileName, + attachments, + attachmentIndex, +}: AttachmentPreviewProps) { + const { t } = useTranslation(); + const [blobUrl, setBlobUrl] = useState(null); + const [textContent, setTextContent] = useState(null); + const [imageZoom, setImageZoom] = useState(1); + + // ── Gallery state ────────────────────────────────────────────── + // When attachments list is provided, compute image-only indices for navigation. + const imageIndices = useMemo(() => { + if (!attachments) return []; + return attachments + .map((a, i) => (isImagePreview(a.file_type, a.filename) ? i : -1)) + .filter((i) => i >= 0); + }, [attachments]); + + const [currentIndex, setCurrentIndex] = useState(attachmentIndex ?? 0); + + // Reset to the clicked attachment every time the dialog opens. + useEffect(() => { + if (open) { + setCurrentIndex(attachmentIndex ?? 0); + } + }, [open, attachmentIndex]); + + // Resolve which attachment to display. + const resolved = useMemo(() => { + if (attachments && currentIndex < attachments.length) { + const a = attachments[currentIndex]; + return { + contentHash: a.content_hash, + contentType: a.file_type, + fileName: a.filename, + }; + } + return { contentHash, contentType, fileName }; + }, [attachments, currentIndex, contentHash, contentType, fileName]); + + // Position within image-only list (for "3 / 12" counter). + const imagePos = imageIndices.indexOf(currentIndex); // -1 if not an image + const imageTotal = imageIndices.length; + + const goPrev = useCallback(() => { + if (imagePos > 0) setCurrentIndex(imageIndices[imagePos - 1]); + }, [imagePos, imageIndices]); + + const goNext = useCallback(() => { + if (imagePos < imageTotal - 1) setCurrentIndex(imageIndices[imagePos + 1]); + }, [imagePos, imageTotal, imageIndices]); + + // Keyboard navigation + useEffect(() => { + if (!open) return; + const handler = (e: KeyboardEvent) => { + if (e.key === 'ArrowLeft') goPrev(); + else if (e.key === 'ArrowRight') goNext(); + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [open, goPrev, goNext]); + + const effectiveType = resolveContentType(resolved.contentType, resolved.fileName); + + // ── Fetch preview blob ───────────────────────────────────────── + const previewMutation = useMutation({ + mutationFn: () => preview_attachment(accountId, envelopeId, resolved.contentHash), + onSuccess: (blob) => { + if (isTextPreview(resolved.contentType, resolved.fileName)) { + blob.text().then(setTextContent); + } else { + const typedBlob = new Blob([blob], { type: effectiveType }); + setBlobUrl(URL.createObjectURL(typedBlob)); + } + }, + onError: (error: any) => { + toast({ + title: t('attachment_preview.failedToLoad'), + description: error.message, + variant: 'destructive', + }); + }, + }); + + useEffect(() => { + if (open) { + setBlobUrl(null); + setTextContent(null); + setImageZoom(1); + previewMutation.mutate(); + } + }, [open, resolved.contentHash]); + + useEffect(() => { + return () => { + if (blobUrl) URL.revokeObjectURL(blobUrl); + }; + }, [blobUrl]); + + const handleDownload = () => { + download_attachment(accountId, envelopeId, resolved.contentHash, resolved.fileName); + }; + + const { icon } = useMemo(() => getFileConfig(resolved.contentType), [resolved.contentType]); + + const isImage = isImagePreview(resolved.contentType, resolved.fileName); + const isPdf = isPdfPreview(resolved.contentType, resolved.fileName); + const isText = isTextPreview(resolved.contentType, resolved.fileName); + const showArrows = imageTotal > 1 && isImage; + + return ( + + { + if (isPdf) e.preventDefault(); + }} + > + {/* Toolbar — hidden for PDF (browser's native viewer has its own controls) */} + {!isPdf && ( +
+
+ {icon} + + {resolved.fileName} + + {imagePos >= 0 && imageTotal > 1 && ( + + {imagePos + 1} / {imageTotal} + + )} +
+
+ {isImage && blobUrl && ( + <> + + + + + {t('attachment_preview.zoomIn')} + + + + + + {t('attachment_preview.zoomOut')} + + + + + + {t('attachment_preview.resetZoom')} + + + + )} + + + + + {t('attachment.download')} + +
+
+ )} + + {/* Close button — positioned below browser PDF toolbar */} + + + {/* Navigation arrows */} + {showArrows && ( + <> + + + + )} + + {/* Preview body */} +
+ {previewMutation.isPending ? ( +
+ + + +
+ ) : isImage && blobUrl ? ( +
+ {resolved.fileName} +
+ ) : isPdf && blobUrl ? ( +