From c0a63a1e3cfff5caeb85b2ec2bd9c637c28d98ee Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 15:19:39 +0800 Subject: [PATCH 01/66] feat: enhance autoconfig detection --- crates/core/src/autoconfig/client.rs | 116 ++++++++++++-- crates/core/src/autoconfig/guess.rs | 105 ++++++++++++ crates/core/src/autoconfig/load.rs | 11 +- crates/core/src/autoconfig/mod.rs | 2 + .../core/src/autoconfig/oauth2_providers.rs | 151 ++++++++++++++++++ crates/core/src/autoconfig/tests.rs | 60 +++++++ 6 files changed, 428 insertions(+), 17 deletions(-) create mode 100644 crates/core/src/autoconfig/guess.rs create mode 100644 crates/core/src/autoconfig/oauth2_providers.rs diff --git a/crates/core/src/autoconfig/client.rs b/crates/core/src/autoconfig/client.rs index dcc1d1a..1bc5656 100644 --- a/crates/core/src/autoconfig/client.rs +++ b/crates/core/src/autoconfig/client.rs @@ -45,6 +45,10 @@ pub struct IncomingServer { #[serde(rename = "socketType")] pub socket_type: String, pub username: String, + /// Authentication method from the XML, e.g. "OAuth2", "password-cleartext", + /// "password-encrypted", "GSSAPI", "NTLM". Absent in DNS SRV fallback. + #[serde(default)] + pub authentication: String, } #[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)] @@ -137,6 +141,7 @@ async fn lookup_srv(domain: &str) -> Option { port: imap_port, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![OutgoingServer { protocol: "smtp".to_string(), @@ -153,60 +158,139 @@ async fn lookup_srv(domain: &str) -> Option { // --------------------------------------------------------------------------- /// Discover mail server configuration for a domain using the Thunderbird -/// autoconfig protocol (ISPDB) and DNS SRV fallback. +/// autoconfig protocol (ISPDB), DNS SRV, MX fallback, and finally guessing. /// /// Probe order: /// 1. `https://autoconfig.{domain}/mail/config-v1.1.xml` -/// 2. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` -/// 3. DNS SRV records (`_imaps._tcp` / `_submission._tcp`) -/// 4. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`) +/// 2. `http://autoconfig.{domain}/mail/config-v1.1.xml` +/// 3. `https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` +/// 4. `http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml` +/// 5. DNS SRV records (`_imaps._tcp` / `_submission._tcp`) +/// 6. Thunderbird central ISPDB (`https://autoconfig.thunderbird.net/v1.1/{domain}`) +/// 7. MX lookup → ISPDB for MX domain +/// 8. MX lookup → ISP autoconfig for MX domain +/// 9. GuessConfig — probe common hostnames + ports pub async fn fetch(domain: &str) -> BichonResult { let client = Client::builder() .timeout(std::time::Duration::from_secs(10)) .build() .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - // 1. Try autoconfig subdomain + // ── ISP autoconfig (HTTPS, then HTTP) ────────────────────────── + if let Some(config) = + fetch_xml(&client, &format!("https://autoconfig.{domain}/mail/config-v1.1.xml")).await + { + return Ok(config); + } + if let Some(config) = + fetch_xml(&client, &format!("http://autoconfig.{domain}/mail/config-v1.1.xml")).await + { + return Ok(config); + } + + // ── Well-known path (HTTPS, then HTTP) ───────────────────────── if let Some(config) = fetch_xml( &client, - &format!("https://autoconfig.{domain}/mail/config-v1.1.xml"), + &format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), ) .await { return Ok(config); } - - // 2. Try well-known path if let Some(config) = fetch_xml( &client, - &format!("https://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), + &format!("http://{domain}/.well-known/autoconfig/mail/config-v1.1.xml"), ) .await { return Ok(config); } - // 3. Try DNS SRV records + // ── DNS SRV records ──────────────────────────────────────────── if let Some(config) = lookup_srv(domain).await { return Ok(config); } - // 4. Fall back to Thunderbird central database - if let Some(config) = fetch_xml( - &client, - &format!("https://autoconfig.thunderbird.net/v1.1/{domain}"), - ) - .await + // ── Thunderbird central ISPDB ────────────────────────────────── + if let Some(config) = + fetch_xml(&client, &format!("https://autoconfig.thunderbird.net/v1.1/{domain}")).await { return Ok(config); } + // ── MX fallback ──────────────────────────────────────────────── + if let Some(config) = fetch_for_mx(&client, domain).await { + return Ok(config); + } + + // ── GuessConfig ──────────────────────────────────────────────── + if let Some(config) = crate::autoconfig::guess::guess_config(domain).await { + return Ok(config); + } + Err(raise_error!( format!("No autoconfig found for domain: {domain}"), ErrorCode::InternalError )) } +/// DNS MX lookup → retry ISPDB and ISP autoconfig for the MX domain. +/// +/// Many self-hosted domains have their MX pointed at Google, Microsoft, etc. +/// The MX domain's ISPDB entry covers the original domain. +async fn fetch_for_mx(client: &Client, domain: &str) -> Option { + let mx_domain = lookup_mx_domain(domain).await?; + if mx_domain == domain.to_ascii_lowercase() { + return None; // same domain, already tried above + } + + // Try ISPDB for the MX domain + if let Some(config) = + fetch_xml(client, &format!("https://autoconfig.thunderbird.net/v1.1/{mx_domain}")).await + { + return Some(config); + } + + // Try ISP autoconfig for the MX domain (HTTPS then HTTP) + if let Some(config) = + fetch_xml(client, &format!("https://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await + { + return Some(config); + } + if let Some(config) = + fetch_xml(client, &format!("http://autoconfig.{mx_domain}/mail/config-v1.1.xml")).await + { + return Some(config); + } + + None +} + +/// DNS MX lookup → extract the second-level domain of the first MX hostname. +async fn lookup_mx_domain(domain: &str) -> Option { + let resolver = TokioResolver::builder(TokioConnectionProvider::default()) + .ok()? + .build(); + let lookup = resolver.mx_lookup(domain).await.ok()?; + let record = lookup.iter().next()?; + let mx_host = record.to_string().trim_end_matches('.').to_string(); + + // Extract a reasonable base domain from the MX hostname. + // E.g., "aspmx.l.google.com" → "google.com" + // "company.mail.protection.outlook.com" → "outlook.com" + extract_base_domain(&mx_host) +} + +/// Extract the top two labels from a hostname as a rough base domain. +fn extract_base_domain(host: &str) -> Option { + let parts: Vec<&str> = host.split('.').collect(); + if parts.len() >= 2 { + Some(parts[parts.len() - 2..].join(".")) + } else { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/core/src/autoconfig/guess.rs b/crates/core/src/autoconfig/guess.rs new file mode 100644 index 0000000..9acd4cc --- /dev/null +++ b/crates/core/src/autoconfig/guess.rs @@ -0,0 +1,105 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::account::entity::Encryption; +use crate::autoconfig::client::{IncomingServer, MailConfig}; +use crate::imap::client::Client; +use tracing::{debug, info}; + +/// A single host:port:encryption combination to probe. +struct Guess { + hostname: String, + port: u16, + encryption: Encryption, + socket_type: &'static str, +} + +/// Generate candidates in the same order Thunderbird uses: +/// 1. imap.{domain} — most common +/// 2. mail.{domain} — fallback +/// 3. {domain} — bare domain (rare) +fn make_guesses(domain: &str) -> Vec { + let hosts = [ + format!("imap.{domain}"), + format!("mail.{domain}"), + domain.to_string(), + ]; + + let mut guesses = Vec::with_capacity(hosts.len() * 2); + for host in &hosts { + guesses.push(Guess { + hostname: host.clone(), + port: 993, + encryption: Encryption::Ssl, + socket_type: "SSL", + }); + guesses.push(Guess { + hostname: host.clone(), + port: 143, + encryption: Encryption::StartTls, + socket_type: "STARTTLS", + }); + } + guesses +} + +/// Try to open a connection, read the IMAP banner, and close. +/// Returns `true` if the server responds with an IMAP greeting. +async fn probe(hostname: &str, port: u16, encryption: &Encryption) -> bool { + match Client::connection(hostname, encryption, port, None, true).await { + Ok(_) => { + debug!("GuessConfig probe succeeded: {hostname}:{port} ({encryption:?})"); + true + } + Err(e) => { + debug!("GuessConfig probe failed for {hostname}:{port}: {e:?}"); + false + } + } +} + +/// Thunderbird-style guessing: try common hostnames and ports, probing +/// each with a real TCP connection. +/// +/// Returns the first working `MailConfig`, or `None` if nothing works. +pub async fn guess_config(domain: &str) -> Option { + let guesses = make_guesses(domain); + info!("GuessConfig: trying {} candidates for {domain}", guesses.len()); + + for g in &guesses { + if probe(&g.hostname, g.port, &g.encryption).await { + info!( + "GuessConfig: found working IMAP at {}:{} ({})", + g.hostname, g.port, g.socket_type + ); + return Some(MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: g.hostname.clone(), + port: g.port, + socket_type: g.socket_type.to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), + }], + outgoing: vec![], + }); + } + } + + None +} diff --git a/crates/core/src/autoconfig/load.rs b/crates/core/src/autoconfig/load.rs index 9cf89ef..1408681 100644 --- a/crates/core/src/autoconfig/load.rs +++ b/crates/core/src/autoconfig/load.rs @@ -19,6 +19,7 @@ use crate::account::entity::Encryption; use crate::autoconfig::client::{self, MailConfig}; use crate::autoconfig::entity::{MailServerConfig, ServerConfig}; +use crate::autoconfig::oauth2_providers::lookup_oauth2; use crate::autoconfig::CachedMailSettings; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -54,9 +55,17 @@ pub(crate) fn mail_config_to_server_config(config: &MailConfig) -> Option field and a known + // hostname → issuer mapping determine whether the provider supports OAuth2. + let oauth2 = if imap.authentication.eq_ignore_ascii_case("OAuth2") { + lookup_oauth2(&imap.hostname) + } else { + None + }; + Some(MailServerConfig { imap: ServerConfig::new(imap.hostname.clone(), port, encryption), - oauth2: None, + oauth2, }) } diff --git a/crates/core/src/autoconfig/mod.rs b/crates/core/src/autoconfig/mod.rs index 3980974..4f8afc7 100644 --- a/crates/core/src/autoconfig/mod.rs +++ b/crates/core/src/autoconfig/mod.rs @@ -24,7 +24,9 @@ use serde::{Deserialize, Serialize}; pub mod client; pub mod entity; +pub mod guess; pub mod load; +mod oauth2_providers; #[cfg(test)] mod tests; diff --git a/crates/core/src/autoconfig/oauth2_providers.rs b/crates/core/src/autoconfig/oauth2_providers.rs new file mode 100644 index 0000000..71b8227 --- /dev/null +++ b/crates/core/src/autoconfig/oauth2_providers.rs @@ -0,0 +1,151 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::autoconfig::entity::OAuth2Config; + +/// Per-provider OAuth2 metadata, mirroring Thunderbird's `OAuth2Providers.sys.mjs`. +/// +/// Each entry maps one or more IMAP hostname suffixes to a well-known OIDC issuer +/// and the IMAP-specific OAuth2 scopes. +struct Provider { + /// Suffixes matched case-insensitively against the end of the IMAP hostname. + host_suffixes: &'static [&'static str], + /// The OIDC issuer URL used by the provider. + issuer: &'static str, + /// OAuth2 scope(s) required for IMAP access. + scopes: &'static [&'static str], +} + +const PROVIDERS: &[Provider] = &[ + // Google + Provider { + host_suffixes: &["imap.gmail.com", ".gmail.com", ".googlemail.com"], + issuer: "https://accounts.google.com", + scopes: &["https://mail.google.com/"], + }, + // Microsoft (Outlook / Office 365 / Hotmail / Live) + Provider { + host_suffixes: &[ + "outlook.office365.com", + ".outlook.com", + ".hotmail.com", + ".live.com", + ".office365.com", + ], + issuer: "https://login.microsoftonline.com/common/v2.0", + scopes: &[ + "https://outlook.office365.com/IMAP.AccessAsUser.All", + "offline_access", + ], + }, + // Yahoo / AOL / ATT / Verizon + Provider { + host_suffixes: &[ + "imap.mail.yahoo.com", + ".yahoo.com", + ".yahoodns.net", + ".aol.com", + "imap.aol.com", + ], + issuer: "https://login.yahoo.com", + scopes: &["mail-w"], + }, + // Yandex + Provider { + host_suffixes: &["imap.yandex.ru", "imap.yandex.com", ".yandex.ru"], + issuer: "https://oauth.yandex.com", + scopes: &["imap:all"], + }, + // Mail.ru + Provider { + host_suffixes: &["imap.mail.ru", ".mail.ru", ".bk.ru", ".list.ru", ".inbox.ru"], + issuer: "https://o2.mail.ru", + scopes: &["imap"], + }, + // Fastmail + Provider { + host_suffixes: &["imap.fastmail.com", ".fastmail.com"], + issuer: "https://www.fastmail.com", + scopes: &[ + "https://www.fastmail.com/dev/imap", + "offline_access", + ], + }, + // Comcast + Provider { + host_suffixes: &["imap.comcast.net", ".comcast.net"], + issuer: "https://oauth.xfinity.com", + scopes: &["https://email.comcast.net/"], + }, +]; + +/// Try to find an OAuth2 provider that matches the given IMAP hostname. +/// +/// Matching is case-insensitive and done by suffix: a hostname "imap.gmail.com" +/// matches the suffix ".gmail.com". +pub fn lookup_oauth2(hostname: &str) -> Option { + let host = hostname.to_ascii_lowercase(); + for provider in PROVIDERS { + if provider + .host_suffixes + .iter() + .any(|suffix| host.ends_with(&suffix.to_ascii_lowercase())) + { + return Some(OAuth2Config { + issuer: provider.issuer.to_string(), + scope: provider.scopes.iter().map(|s| s.to_string()).collect(), + auth_url: String::new(), + token_url: String::new(), + }); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_known_providers() { + let cases = [ + ("imap.gmail.com", Some("https://accounts.google.com")), + ("imap.gmail.com", Some("https://accounts.google.com")), + ("outlook.office365.com", Some("https://login.microsoftonline.com/common/v2.0")), + ("imap.mail.yahoo.com", Some("https://login.yahoo.com")), + ("imap.aol.com", Some("https://login.yahoo.com")), + ("imap.yandex.ru", Some("https://oauth.yandex.com")), + ("imap.mail.ru", Some("https://o2.mail.ru")), + ("imap.fastmail.com", Some("https://www.fastmail.com")), + ("imap.comcast.net", Some("https://oauth.xfinity.com")), + ]; + for (hostname, expected_issuer) in &cases { + let result = lookup_oauth2(hostname); + assert_eq!( + result.map(|c| c.issuer), + expected_issuer.map(|s| s.to_string()), + "failed for hostname: {hostname}" + ); + } + } + + #[test] + fn test_unknown_provider() { + assert!(lookup_oauth2("mail.my-company.example").is_none()); + } +} diff --git a/crates/core/src/autoconfig/tests.rs b/crates/core/src/autoconfig/tests.rs index a721116..b9d3670 100644 --- a/crates/core/src/autoconfig/tests.rs +++ b/crates/core/src/autoconfig/tests.rs @@ -195,6 +195,7 @@ fn make_imap_server(host: &str, port: u16, socket_type: &str) -> IncomingServer port, socket_type: socket_type.to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), } } @@ -241,6 +242,7 @@ fn convert_no_imap_only_pop3() { port: 995, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; @@ -266,6 +268,7 @@ fn convert_picks_imap_over_pop3() { port: 995, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }, make_imap_server("imap.example.com", 993, "SSL"), ], @@ -284,6 +287,7 @@ fn convert_imaps_protocol_variant() { port: 993, socket_type: "SSL".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; @@ -300,9 +304,65 @@ fn convert_case_insensitive_protocol() { port: 143, socket_type: "STARTTLS".to_string(), username: "%EMAILADDRESS%".to_string(), + authentication: String::new(), }], outgoing: vec![], }; let result = mail_config_to_server_config(&config).expect("should recognize 'IMAP'"); assert_eq!(result.imap.host, "imap.example.com"); } + +#[test] +fn convert_gmail_oauth2() { + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "imap.gmail.com".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + let oauth2 = result.oauth2.expect("Gmail should have OAuth2"); + assert_eq!(oauth2.issuer, "https://accounts.google.com"); + assert!(oauth2.scope.contains(&"https://mail.google.com/".to_string())); +} + +#[test] +fn convert_outlook_oauth2() { + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "outlook.office365.com".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + let oauth2 = result.oauth2.expect("Outlook should have OAuth2"); + assert!(oauth2.issuer.contains("microsoftonline")); +} + +#[test] +fn convert_unknown_host_no_oauth2() { + // OAuth2 auth flag on an unknown hostname → no OAuth2 returned + let config = MailConfig { + incoming: vec![IncomingServer { + protocol: "imap".to_string(), + hostname: "mail.random-isp.example".to_string(), + port: 993, + socket_type: "SSL".to_string(), + username: "%EMAILADDRESS%".to_string(), + authentication: "OAuth2".to_string(), + }], + outgoing: vec![], + }; + let result = mail_config_to_server_config(&config).expect("should convert"); + assert!(result.oauth2.is_none(), "unknown hostname → no OAuth2 mapping"); +} From 4bd714a67013f16bd26c104fd78436aba994b28f Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 15:20:59 +0800 Subject: [PATCH 02/66] chore: remove custom global allocator --- Cargo.lock | 24 ++---------------------- Cargo.toml | 3 +-- README.md | 1 - crates/admin/Cargo.toml | 3 +-- crates/admin/src/main.rs | 3 --- crates/server/Cargo.toml | 1 - crates/server/src/main.rs | 5 ----- 7 files changed, 4 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4c4f640..e858601 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -301,7 +301,6 @@ dependencies = [ "indicatif", "itertools", "memdb", - "mimalloc", "native_db", "native_model", "serde", @@ -405,7 +404,6 @@ dependencies = [ "email_address", "governor", "http", - "mimalloc", "poem", "poem-derive", "poem-openapi", @@ -1791,9 +1789,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] name = "http" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" dependencies = [ "bytes 1.11.1", "itoa", @@ -2264,15 +2262,6 @@ version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2477,15 +2466,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 20c64da..8184c8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,6 @@ edition = "2021" [workspace.dependencies] chrono = "0.4.44" clap = { version = "4.6.1", features = ["derive", "env"] } -mimalloc = "0.1.52" memdb = { path = "crates/memdb" } itertools = "0.14.0" ring = { version = "0.17.14", features = ["std"] } @@ -39,7 +38,7 @@ reqwest = { version = "0.12.24", default-features = false, features = [ "socks", ] } tokio-socks = "0.5.2" -http = "1.4.0" +http = "1.4.1" regex = "1.12.3" email_address = "0.2.9" futures = "0.3.32" diff --git a/README.md b/README.md index 7fa54d4..a7465b1 100644 --- a/README.md +++ b/README.md @@ -656,7 +656,6 @@ Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or joi | **Frontend** | React 18, TypeScript, Vite 6, ShadCN UI, TanStack Router/Query/Table | | **Charts** | Recharts | | **i18n** | i18next (18 languages) | -| **Allocator** | mimalloc | | **Container** | Ubuntu 24.04, Docker | ## License diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index 0d8ba94..eddbfb2 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -17,5 +17,4 @@ serde_json.workspace = true itertools.workspace = true snafu.workspace = true -memdb.workspace = true -mimalloc = "0.1.50" \ No newline at end of file +memdb.workspace = true \ No newline at end of file diff --git a/crates/admin/src/main.rs b/crates/admin/src/main.rs index 10e7752..15655a1 100644 --- a/crates/admin/src/main.rs +++ b/crates/admin/src/main.rs @@ -18,7 +18,6 @@ use console::style; use dialoguer::{theme::ColorfulTheme, Select}; -use mimalloc::MiMalloc; use crate::{migrate::handle_migration, reset::handle_reset_password}; @@ -26,8 +25,6 @@ pub mod meta; pub mod migrate; pub mod reset; -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; fn main() { run_interactive(); diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 0e814f6..26d6eef 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -33,7 +33,6 @@ tracing.workspace = true tokio.workspace = true http.workspace = true urlencoding.workspace = true -mimalloc.workspace = true [dev-dependencies] poem = { version = "3.1.12", features = ["test"] } diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index e3802de..3cfd511 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -16,12 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . - use bichon_core::error::BichonResult; -use mimalloc::MiMalloc; - -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; #[tokio::main] async fn main() -> BichonResult<()> { From f30cd66e00b62779d9dd7869b037885c80deb37a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 15:22:14 +0800 Subject: [PATCH 03/66] feat: remove folder limit --- crates/admin/src/meta.rs | 6 ++---- crates/core/src/account/migration.rs | 12 ------------ crates/core/src/account/payload.rs | 15 --------------- crates/core/src/account/view.rs | 2 -- web/src/api/account/api.ts | 1 - web/src/locales/ar.json | 7 +------ web/src/locales/da.json | 7 +------ web/src/locales/de.json | 7 +------ web/src/locales/en.json | 7 +------ web/src/locales/es.json | 7 +------ web/src/locales/fi.json | 7 +------ web/src/locales/fr.json | 7 +------ web/src/locales/it.json | 7 +------ web/src/locales/jp.json | 7 +------ web/src/locales/ko.json | 7 +------ web/src/locales/nl.json | 7 +------ web/src/locales/no.json | 7 +------ web/src/locales/pl.json | 7 +------ web/src/locales/pt.json | 7 +------ web/src/locales/ru.json | 7 +------ web/src/locales/sv.json | 7 +------ web/src/locales/zh-tw.json | 7 +------ web/src/locales/zh.json | 7 +------ 23 files changed, 20 insertions(+), 142 deletions(-) diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index ac9eb12..f1a427d 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -117,7 +117,6 @@ pub struct AccountV3 { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, - pub folder_limit: Option, pub sync_folders: Option>, pub account_type: AccountType, pub sync_interval_min: Option, @@ -193,7 +192,7 @@ impl From for AccountV2 { name: value.name, capabilities: value.capabilities, date_since: value.date_since, - folder_limit: value.folder_limit, + folder_limit: None, sync_folders: value.sync_folders, account_type: value.account_type, sync_interval_min: value.sync_interval_min, @@ -217,7 +216,6 @@ impl From for AccountV3 { name: value.name, capabilities: value.capabilities, date_since: value.date_since, - folder_limit: value.folder_limit, sync_folders: value.sync_folders, account_type: value.account_type, sync_interval_min: value.sync_interval_min, @@ -246,7 +244,6 @@ impl From for AccountModel { capabilities: value.capabilities, date_since: value.date_since, date_before: value.date_before, - folder_limit: value.folder_limit, download_folders: value.sync_folders, account_type: value.account_type, download_interval_min: value.sync_interval_min, @@ -661,6 +658,7 @@ impl From for bichon_core::cache::imap::mailbox::MailBox { unseen: value.unseen, uid_next: value.uid_next, uid_validity: value.uid_validity, + highest_uid: None, } } } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index e0250c7..0f743e6 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -80,7 +80,6 @@ pub struct Account { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, - pub folder_limit: Option, pub download_folders: Option>, pub account_type: AccountType, pub download_interval_min: Option, @@ -125,7 +124,6 @@ impl Account { created_at: utc_now!(), updated_at: utc_now!(), use_proxy: request.use_proxy, - folder_limit: request.folder_limit, use_dangerous: request.use_dangerous, pgp_key: request.pgp_key, created_by: user_id, @@ -367,20 +365,10 @@ impl Account { } } - if let Some(folder_limit) = request.folder_limit { - new.folder_limit = Some(folder_limit); - } - if let Some(account_name) = request.account_name { new.account_name = Some(account_name); } - if let Some(clear_folder_limit) = request.clear_folder_limit { - if clear_folder_limit { - new.folder_limit = None; - } - } - if matches!(old.account_type, AccountType::IMAP) { if let Some(imap) = &request.imap { if let Some(current_imap) = &mut new.imap { diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index 34256d1..5fd0e6d 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -41,8 +41,6 @@ pub struct AccountCreateRequest { pub date_since: Option, pub date_before: Option, pub account_type: AccountType, - #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))] - pub folder_limit: Option, #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "10"))))] pub download_interval_min: Option, #[cfg_attr( @@ -145,12 +143,6 @@ pub struct AccountUpdateRequest { pub date_since: Option, pub date_before: Option, pub clear_date_range: Option, - /// Max emails to sync for this folder. - /// If not set, sync all emails. - /// otherwise sync up to `n` most recent emails (min 10). - #[cfg_attr(feature = "web-api", oai(validator(minimum(value = "100"))))] - pub folder_limit: Option, - pub clear_folder_limit: Option, /// Configuration for selective folder (mailbox/label) synchronization /// /// - For IMAP/SMTP accounts: @@ -205,13 +197,6 @@ impl AccountUpdateRequest { )); } - if self.clear_folder_limit == Some(true) && self.folder_limit.is_some() { - return Err(raise_error!( - "clear_folder_limit cannot be combined with folder_limit".into(), - ErrorCode::InvalidParameter - )); - } - if self.clear_date_range == Some(true) && (self.date_since.is_some() || self.date_before.is_some()) { diff --git a/crates/core/src/account/view.rs b/crates/core/src/account/view.rs index 6f9e34f..e3a568c 100644 --- a/crates/core/src/account/view.rs +++ b/crates/core/src/account/view.rs @@ -40,7 +40,6 @@ pub struct AccountResp { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, - pub folder_limit: Option, pub download_folders: Option>, pub account_type: AccountType, pub download_interval_min: Option, @@ -73,7 +72,6 @@ impl AccountResp { capabilities: account.capabilities, date_since: account.date_since, date_before: account.date_before, - folder_limit: account.folder_limit, download_folders: account.download_folders, account_type: account.account_type, download_interval_min: account.download_interval_min, diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index e7bb249..baa6794 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -128,7 +128,6 @@ export interface AccountModel { capabilities?: string[]; date_since?: DateSelection; date_before?: RelativeDate; - folder_limit?: number, download_folders: string[]; download_interval_min?: number; download_batch_size?: number; diff --git a/web/src/locales/ar.json b/web/src/locales/ar.json index 48a7cc9..0eb5f8f 100644 --- a/web/src/locales/ar.json +++ b/web/src/locales/ar.json @@ -177,9 +177,6 @@ "everyMinutes": "كل {{minutes}} دقيقة", "field": "الحقل", "fixed": "ثابت", - "folderLimit": "حد المجلد", - "folderLimitDescription": "حدد عدد رسائل البريد الإلكتروني للمزامنة لكل مجلد (الحد الأدنى 100). اتركه فارغًا بلا حدود.", - "folderLimitPlaceholder": "مثال: 1000", "folderSync": { "autoSelectDescendants": "تحديد العناصر الفرعية تلقائيًا", "autoSelectParents": "تحديد العناصر الأصلية تلقائيًا", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "البريد الإلكتروني مطلوب", - "folderLimitMustBeAtLeast100": "يجب أن يكون حد المجلد 100 على الأقل", - "folderLimitMustBeNumber": "يجب أن يكون حد المجلد رقمًا", "imapHostCannotBeEmpty": "لا يمكن أن يكون مُضيف IMAP فارغًا", "imapHostRequired": "مُضيف IMAP مطلوب", "imapPortMustBeLessThan65536": "يجب أن يكون منفذ IMAP أقل من 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "يجب أن يكون حجم الدُفعة على الأكثر 200", "singleRequestBatchSizeTooSmall": "يجب أن يكون حجم الدُفعة على الأقل 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/da.json b/web/src/locales/da.json index b68990a..e46bc43 100644 --- a/web/src/locales/da.json +++ b/web/src/locales/da.json @@ -177,9 +177,6 @@ "everyMinutes": "hvert {{minutes}} minut", "field": "Felt", "fixed": "Fast", - "folderLimit": "Mappegrænse", - "folderLimitDescription": "Begræns antallet af e-mails, der skal synkroniseres pr. mappe (minimum 100). Lad stå tomt for ingen grænse.", - "folderLimitPlaceholder": "f.eks. 1000", "folderSync": { "autoSelectDescendants": "Vælg efterkommere automatisk", "autoSelectParents": "Vælg forældre automatisk", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "E-mail er påkrævet", - "folderLimitMustBeAtLeast100": "Mappegrænse skal være mindst 100", - "folderLimitMustBeNumber": "Mappegrænse skal være et tal", "imapHostCannotBeEmpty": "IMAP-vært kan ikke være tom", "imapHostRequired": "IMAP-vært er påkrævet", "imapPortMustBeLessThan65536": "IMAP-port skal være mindre end 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Batch‑størrelse skal være højst 200", "singleRequestBatchSizeTooSmall": "Batch‑størrelse skal være mindst 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 44e1396..8f40d77 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -177,9 +177,6 @@ "everyMinutes": "alle {{minutes}} Minuten", "field": "Feld", "fixed": "Fest", - "folderLimit": "Ordnerlimit", - "folderLimitDescription": "Begrenzen Sie die Anzahl der pro Ordner zu synchronisierenden E-Mails (mindestens 100). Lassen Sie das Feld leer für keine Begrenzung.", - "folderLimitPlaceholder": "z.B. 1000", "folderSync": { "autoSelectDescendants": "Unterelemente automatisch auswählen", "autoSelectParents": "Elternelemente automatisch auswählen", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "E-Mail ist erforderlich", - "folderLimitMustBeAtLeast100": "Das Ordnerlimit muss mindestens 100 betragen", - "folderLimitMustBeNumber": "Das Ordnerlimit muss eine Zahl sein", "imapHostCannotBeEmpty": "IMAP-Host darf nicht leer sein", "imapHostRequired": "IMAP-Host ist erforderlich", "imapPortMustBeLessThan65536": "Der IMAP-Port muss kleiner als 65536 sein", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Die Stapelgröße darf höchstens 200 sein", "singleRequestBatchSizeTooSmall": "Die Stapelgröße muss mindestens 10 sein" } -} \ No newline at end of file +} diff --git a/web/src/locales/en.json b/web/src/locales/en.json index dce0cde..cad305d 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -177,9 +177,6 @@ "everyMinutes": "every {{minutes}} minutes", "field": "Field", "fixed": "Fixed", - "folderLimit": "Folder Limit", - "folderLimitDescription": "Limit the number of emails to sync per folder (minimum 100). Leave empty for no limit.", - "folderLimitPlaceholder": "e.g. 1000", "folderSync": { "autoSelectDescendants": "Auto select descendants", "autoSelectParents": "Auto select parents", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "Email is required", - "folderLimitMustBeAtLeast100": "Folder limit must be at least 100", - "folderLimitMustBeNumber": "Folder limit must be a number", "imapHostCannotBeEmpty": "IMAP host cannot be empty", "imapHostRequired": "IMAP host is required", "imapPortMustBeLessThan65536": "IMAP port must be less than 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Batch size must be at most 200", "singleRequestBatchSizeTooSmall": "Batch size must be at least 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/es.json b/web/src/locales/es.json index 6608cdb..d2db893 100644 --- a/web/src/locales/es.json +++ b/web/src/locales/es.json @@ -177,9 +177,6 @@ "everyMinutes": "cada {{minutes}} minutos", "field": "Campo", "fixed": "Fija", - "folderLimit": "Límite de carpetas", - "folderLimitDescription": "Limita el número de correos a sincronizar por carpeta (mínimo 100). Deja vacío para no tener límite.", - "folderLimitPlaceholder": "ej. 1000", "folderSync": { "autoSelectDescendants": "Seleccionar automáticamente los descendientes", "autoSelectParents": "Seleccionar automáticamente los padres", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "El correo electrónico es obligatorio", - "folderLimitMustBeAtLeast100": "El límite de carpetas debe ser de al menos 100", - "folderLimitMustBeNumber": "El límite de carpetas debe ser un número", "imapHostCannotBeEmpty": "El host IMAP no puede estar vacío", "imapHostRequired": "El host IMAP es obligatorio", "imapPortMustBeLessThan65536": "El puerto IMAP debe ser menor que 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "El tamaño del lote debe ser como máximo 200", "singleRequestBatchSizeTooSmall": "El tamaño del lote debe ser al menos 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/fi.json b/web/src/locales/fi.json index b5faf51..edca7ae 100644 --- a/web/src/locales/fi.json +++ b/web/src/locales/fi.json @@ -177,9 +177,6 @@ "everyMinutes": "joka {{minutes}} minuutti", "field": "Kenttä", "fixed": "Kiinteä", - "folderLimit": "Kansioraja", - "folderLimitDescription": "Rajoittaa synkronoitavien sähköpostien määrän kansiota kohden (vähintään 100). Jätä tyhjäksi ilman rajoitusta.", - "folderLimitPlaceholder": "esim. 1000", "folderSync": { "autoSelectDescendants": "Valitse alisolmut automaattisesti", "autoSelectParents": "Valitse yläsolmut automaattisesti", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "Sähköposti on pakollinen", - "folderLimitMustBeAtLeast100": "Kansiorajan on oltava vähintään 100", - "folderLimitMustBeNumber": "Kansiorajan on oltava numero", "imapHostCannotBeEmpty": "IMAP-isäntä ei voi olla tyhjä", "imapHostRequired": "IMAP-isäntä on pakollinen", "imapPortMustBeLessThan65536": "IMAP-portin on oltava pienempi kuin 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Eräkoko tulee olla enintään 200", "singleRequestBatchSizeTooSmall": "Eräkoko tulee olla vähintään 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/fr.json b/web/src/locales/fr.json index 630b7fe..1b9f590 100644 --- a/web/src/locales/fr.json +++ b/web/src/locales/fr.json @@ -177,9 +177,6 @@ "everyMinutes": "toutes les {{minutes}} minutes", "field": "Champ", "fixed": "Fixe", - "folderLimit": "Limite de Dossiers", - "folderLimitDescription": "Limite le nombre d'e-mails à synchroniser par dossier (minimum 100). Laissez vide pour aucune limite.", - "folderLimitPlaceholder": "ex. 1000", "folderSync": { "autoSelectDescendants": "Sélectionner automatiquement les descendants", "autoSelectParents": "Sélectionner automatiquement les parents", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "L'adresse e-mail est obligatoire", - "folderLimitMustBeAtLeast100": "La limite de dossiers doit être d'au moins 100", - "folderLimitMustBeNumber": "La limite de dossiers doit être un nombre", "imapHostCannotBeEmpty": "L'hôte IMAP ne peut pas être vide", "imapHostRequired": "L'hôte IMAP est obligatoire", "imapPortMustBeLessThan65536": "Le port IMAP doit être inférieur à 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "La taille du lot doit être au plus 200", "singleRequestBatchSizeTooSmall": "La taille du lot doit être au moins 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/it.json b/web/src/locales/it.json index d689de6..fc6522d 100644 --- a/web/src/locales/it.json +++ b/web/src/locales/it.json @@ -177,9 +177,6 @@ "everyMinutes": "ogni {{minutes}} minuti", "field": "Campo", "fixed": "Fissa", - "folderLimit": "Limite Cartelle", - "folderLimitDescription": "Limita il numero di email da sincronizzare per cartella (minimo 100). Lascia vuoto per nessun limite.", - "folderLimitPlaceholder": "es. 1000", "folderSync": { "autoSelectDescendants": "Seleziona automaticamente i discendenti", "autoSelectParents": "Seleziona automaticamente i genitori", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "L'email è obbligatoria", - "folderLimitMustBeAtLeast100": "Il limite di cartelle deve essere almeno 100", - "folderLimitMustBeNumber": "Il limite di cartelle deve essere un numero", "imapHostCannotBeEmpty": "L'host IMAP non può essere vuoto", "imapHostRequired": "L'host IMAP è obbligatorio", "imapPortMustBeLessThan65536": "La porta IMAP deve essere inferiore a 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "La dimensione del batch deve essere al massimo 200", "singleRequestBatchSizeTooSmall": "La dimensione del batch deve essere almeno 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/jp.json b/web/src/locales/jp.json index c2827a3..0864ff5 100644 --- a/web/src/locales/jp.json +++ b/web/src/locales/jp.json @@ -177,9 +177,6 @@ "everyMinutes": "{{minutes}}分ごと", "field": "フィールド", "fixed": "固定", - "folderLimit": "フォルダーの制限", - "folderLimitDescription": "フォルダーごとに同期するメールの数を制限します（最小100）。制限なしにする場合は空欄にしてください。", - "folderLimitPlaceholder": "例: 1000", "folderSync": { "autoSelectDescendants": "子項目を自動選択", "autoSelectParents": "親項目を自動選択", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "メールアドレスは必須です", - "folderLimitMustBeAtLeast100": "フォルダーの制限は100以上である必要があります", - "folderLimitMustBeNumber": "フォルダーの制限は数値である必要があります", "imapHostCannotBeEmpty": "IMAPホストは空にできません", "imapHostRequired": "IMAPホストは必須です", "imapPortMustBeLessThan65536": "IMAPポートは65536未満である必要があります", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "バッチサイズは最大でも200でなければなりません", "singleRequestBatchSizeTooSmall": "バッチサイズは最低でも10でなければなりません" } -} \ No newline at end of file +} diff --git a/web/src/locales/ko.json b/web/src/locales/ko.json index 9525994..69685d6 100644 --- a/web/src/locales/ko.json +++ b/web/src/locales/ko.json @@ -177,9 +177,6 @@ "everyMinutes": "매 {{minutes}}분", "field": "필드", "fixed": "고정", - "folderLimit": "폴더 제한", - "folderLimitDescription": "폴더당 동기화할 이메일 수를 제한합니다(최소 100). 제한이 없으면 비워 두십시오.", - "folderLimitPlaceholder": "예: 1000", "folderSync": { "autoSelectDescendants": "하위 항목 자동 선택", "autoSelectParents": "상위 항목 자동 선택", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "이메일 주소는 필수입니다", - "folderLimitMustBeAtLeast100": "폴더 제한은 최소 100 이상이어야 합니다", - "folderLimitMustBeNumber": "폴더 제한은 숫자여야 합니다", "imapHostCannotBeEmpty": "IMAP 호스트는 비워 둘 수 없습니다", "imapHostRequired": "IMAP 호스트는 필수입니다", "imapPortMustBeLessThan65536": "IMAP 포트는 65536보다 작아야 합니다", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "배치 크기는 최대 200이어야 합니다", "singleRequestBatchSizeTooSmall": "배치 크기는 최소 10이어야 합니다" } -} \ No newline at end of file +} diff --git a/web/src/locales/nl.json b/web/src/locales/nl.json index 22a5ff5..b69509c 100644 --- a/web/src/locales/nl.json +++ b/web/src/locales/nl.json @@ -177,9 +177,6 @@ "everyMinutes": "elke {{minutes}} minuten", "field": "Veld", "fixed": "Vast", - "folderLimit": "Mappenlimiet", - "folderLimitDescription": "Beperk het aantal te synchroniseren e-mails per map (minimaal 100). Laat leeg voor geen limiet.", - "folderLimitPlaceholder": "bv. 1000", "folderSync": { "autoSelectDescendants": "Automatisch onderliggende items selecteren", "autoSelectParents": "Automatisch bovenliggende items selecteren", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "E-mail is vereist", - "folderLimitMustBeAtLeast100": "Mappenlimiet moet ten minste 100 zijn", - "folderLimitMustBeNumber": "Mappenlimiet moet een nummer zijn", "imapHostCannotBeEmpty": "IMAP host mag niet leeg zijn", "imapHostRequired": "IMAP host is vereist", "imapPortMustBeLessThan65536": "IMAP poort moet kleiner zijn dan 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Batch‑grootte moet hoogstens 200 zijn", "singleRequestBatchSizeTooSmall": "Batch‑grootte moet ten minste 10 zijn" } -} \ No newline at end of file +} diff --git a/web/src/locales/no.json b/web/src/locales/no.json index b756b80..9078ff1 100644 --- a/web/src/locales/no.json +++ b/web/src/locales/no.json @@ -177,9 +177,6 @@ "everyMinutes": "hvert {{minutes}} minutt", "field": "Felt", "fixed": "Fast", - "folderLimit": "Mappegrense", - "folderLimitDescription": "Begrens antall e-poster som skal synkroniseres per mappe (minimum 100). La stå tomt for ingen grense.", - "folderLimitPlaceholder": "f.eks. 1000", "folderSync": { "autoSelectDescendants": "Velg etterkommere automatisk", "autoSelectParents": "Velg foreldre automatisk", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "E-post er påkrevd", - "folderLimitMustBeAtLeast100": "Mappegrense må være minst 100", - "folderLimitMustBeNumber": "Mappegrense må være et tall", "imapHostCannotBeEmpty": "IMAP-vert kan ikke være tom", "imapHostRequired": "IMAP-vert er påkrevd", "imapPortMustBeLessThan65536": "IMAP-port må være mindre enn 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Batchstørrelse må være maksimalt 200", "singleRequestBatchSizeTooSmall": "Batchstørrelse må være minst 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/pl.json b/web/src/locales/pl.json index 1b9c5ac..62d9794 100644 --- a/web/src/locales/pl.json +++ b/web/src/locales/pl.json @@ -177,9 +177,6 @@ "everyMinutes": "co {{minutes}} minut", "field": "Pole", "fixed": "Dokładnie", - "folderLimit": "Limit folderu", - "folderLimitDescription": "Ogranicz liczbę wiadomości email, które należy synchronizować w jednym folderze (minimum 100). Pozostaw puste, aby nie było limitu.", - "folderLimitPlaceholder": "np. 1000", "folderSync": { "autoSelectDescendants": "Automatyczny wybór dzieci (podrzędnych)", "autoSelectParents": "Automatyczny wybór rodziców (nadrzędnych)", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "Email jest wymagany", - "folderLimitMustBeAtLeast100": "Limit folderów musi być liczbą nie mniejszą niż 100", - "folderLimitMustBeNumber": "Limit folderów musi być liczbą", "imapHostCannotBeEmpty": "Host IMAP nie może być pusty", "imapHostRequired": "Host IMAP jest wymagany", "imapPortMustBeLessThan65536": "Port IMAP musi być liczbą w przedziale 0-65535", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Rozmiar partii może wynosić maksymalnie 200", "singleRequestBatchSizeTooSmall": "Rozmiar partii musi wynosić co najmniej 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/pt.json b/web/src/locales/pt.json index 161197a..ec3f3db 100644 --- a/web/src/locales/pt.json +++ b/web/src/locales/pt.json @@ -177,9 +177,6 @@ "everyMinutes": "A cada {{minutes}} minutos", "field": "Campo", "fixed": "Fixo", - "folderLimit": "Limite de Pasta", - "folderLimitDescription": "Limita o número de emails a sincronizar por pasta (mínimo 100). Deixe vazio para nenhum limite.", - "folderLimitPlaceholder": "Ex: 1000", "folderSync": { "autoSelectDescendants": "Selecionar automaticamente os descendentes", "autoSelectParents": "Selecionar automaticamente os pais", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "O endereço de email é obrigatório", - "folderLimitMustBeAtLeast100": "O Limite de Pasta deve ser pelo menos 100", - "folderLimitMustBeNumber": "O Limite de Pasta deve ser um número", "imapHostCannotBeEmpty": "O Host IMAP não pode estar vazio", "imapHostRequired": "O Host IMAP é obrigatório", "imapPortMustBeLessThan65536": "A Porta IMAP deve ser menor que 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "O tamanho do lote deve ser no máximo 200", "singleRequestBatchSizeTooSmall": "O tamanho do lote deve ser pelo menos 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json index 5acaf3f..a0b852e 100644 --- a/web/src/locales/ru.json +++ b/web/src/locales/ru.json @@ -177,9 +177,6 @@ "everyMinutes": "каждые {{minutes}} мин.", "field": "Поле", "fixed": "Фиксированная", - "folderLimit": "Лимит папки", - "folderLimitDescription": "Ограничить количество писем для синхронизации в папке (минимум 100). Оставьте пустым для снятия ограничений.", - "folderLimitPlaceholder": "например, 1000", "folderSync": { "autoSelectDescendants": "Автоматически выбирать дочерние элементы", "autoSelectParents": "Автоматически выбирать родительские элементы", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "Email обязателен", - "folderLimitMustBeAtLeast100": "Лимит папки должен быть не менее 100", - "folderLimitMustBeNumber": "Лимит папки должен быть числом", "imapHostCannotBeEmpty": "IMAP хост не может быть пустым", "imapHostRequired": "IMAP хост обязателен", "imapPortMustBeLessThan65536": "IMAP порт должен быть меньше 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Размер пакета должен быть не более 200", "singleRequestBatchSizeTooSmall": "Размер пакета должен быть не менее 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/sv.json b/web/src/locales/sv.json index bc0ebb5..92c6518 100644 --- a/web/src/locales/sv.json +++ b/web/src/locales/sv.json @@ -177,9 +177,6 @@ "everyMinutes": "varje {{minutes}} minut", "field": "Fält", "fixed": "Fast", - "folderLimit": "Mappgräns", - "folderLimitDescription": "Begränsa antalet e-postmeddelanden att synkronisera per mapp (minst 100). Lämna tomt för ingen gräns.", - "folderLimitPlaceholder": "t.ex. 1000", "folderSync": { "autoSelectDescendants": "Välj underordnade automatiskt", "autoSelectParents": "Välj överordnade automatiskt", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "E-post krävs", - "folderLimitMustBeAtLeast100": "Mappgräns måste vara minst 100", - "folderLimitMustBeNumber": "Mappgräns måste vara ett tal", "imapHostCannotBeEmpty": "IMAP-värd kan inte vara tom", "imapHostRequired": "IMAP-värd krävs", "imapPortMustBeLessThan65536": "IMAP-port måste vara mindre än 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "Batchstorlek måste vara högst 200", "singleRequestBatchSizeTooSmall": "Batchstorlek måste vara minst 10" } -} \ No newline at end of file +} diff --git a/web/src/locales/zh-tw.json b/web/src/locales/zh-tw.json index fbc5d33..c71fc73 100644 --- a/web/src/locales/zh-tw.json +++ b/web/src/locales/zh-tw.json @@ -177,9 +177,6 @@ "everyMinutes": "每 {{minutes}} 分鐘", "field": "欄位", "fixed": "固定", - "folderLimit": "資料夾限制", - "folderLimitDescription": "限制每個資料夾要同步的郵件數量（最小 100）。若無限制請留空。", - "folderLimitPlaceholder": "例如：1000", "folderSync": { "autoSelectDescendants": "自動選取子項", "autoSelectParents": "自動選取父項", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "電子郵件地址為必填項", - "folderLimitMustBeAtLeast100": "資料夾限制必須至少為 100", - "folderLimitMustBeNumber": "資料夾限制必須是數字", "imapHostCannotBeEmpty": "IMAP 主機不可為空", "imapHostRequired": "IMAP 主機為必填項", "imapPortMustBeLessThan65536": "IMAP 連接埠必須小於 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "批次大小必須最多為200", "singleRequestBatchSizeTooSmall": "批次大小必須至少為10" } -} \ No newline at end of file +} diff --git a/web/src/locales/zh.json b/web/src/locales/zh.json index ca170d5..eb4201c 100644 --- a/web/src/locales/zh.json +++ b/web/src/locales/zh.json @@ -177,9 +177,6 @@ "everyMinutes": "每 {{minutes}} 分钟", "field": "字段", "fixed": "固定", - "folderLimit": "文件夹限制", - "folderLimitDescription": "限制每个文件夹同步的邮件数量（最少 100）。留空表示不限制。", - "folderLimitPlaceholder": "例如 1000", "folderSync": { "autoSelectDescendants": "自动选择子项", "autoSelectParents": "自动选择父项", @@ -1675,8 +1672,6 @@ }, "validation": { "emailRequired": "邮箱为必填项", - "folderLimitMustBeAtLeast100": "文件夹限制必须至少为 100", - "folderLimitMustBeNumber": "文件夹限制必须是数字", "imapHostCannotBeEmpty": "IMAP 主机不能为空", "imapHostRequired": "IMAP 主机为必填项", "imapPortMustBeLessThan65536": "IMAP 端口必须小于 65536", @@ -1695,4 +1690,4 @@ "singleRequestBatchSizeTooLarge": "批大小必须最多为200", "singleRequestBatchSizeTooSmall": "批大小必须至少为10" } -} \ No newline at end of file +} From 83dd9cdd6be029d7c079cf49373e991f013c5365 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 15:23:34 +0800 Subject: [PATCH 04/66] perf: optimize IMAP account fetch flow --- crates/core/src/cache/imap/download/flow.rs | 208 +++++++++--------- .../core/src/cache/imap/download/rebuild.rs | 28 ++- crates/core/src/cache/imap/mailbox.rs | 4 + crates/core/src/imap/executor.rs | 199 +++++++---------- crates/core/src/import/mod.rs | 1 + crates/core/src/mailbox/list.rs | 14 +- crates/smtp/src/server.rs | 1 + 7 files changed, 221 insertions(+), 234 deletions(-) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 025a6f3..1da97f9 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -54,7 +54,7 @@ pub async fn fetch_and_save_by_date( mailbox: &MailBox, direction: FetchDirection, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { let account_id = account.id; let mut session = match ImapExecutor::create_connection(account_id).await { Ok(session) => session, @@ -108,32 +108,18 @@ pub async fn fetch_and_save_by_date( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - let folder_limit = account.folder_limit; // sort small -> bigger let mut uid_vec: Vec = uid_list.into_iter().collect(); uid_vec.sort(); - if let Some(limit) = folder_limit { - let limit = limit.max(100) as usize; - if len > limit { - uid_vec = match direction { - FetchDirection::Since => uid_vec.split_off(len - limit), - FetchDirection::Before => { - uid_vec.truncate(limit); - uid_vec - } - }; - } - } - + let max_uid = uid_vec.last().copied(); let planned = uid_vec.len() as u64; let uid_batches = generate_uid_sequence_hashset( uid_vec, account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize, - false, ); DownloadState::update_folder_progress( account_id, @@ -212,14 +198,16 @@ pub async fn fetch_and_save_by_date( )?; } session.logout().await.ok(); - Ok(()) + Ok(max_uid) } +/// Fetches all messages from a mailbox. +/// Returns `Ok(Some(max_uid))` with the highest UID stored, or `Ok(None)` if empty. pub async fn fetch_and_save_full_mailbox( account: &AccountModel, mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { let mailbox_id = mailbox.id; let account_id = account.id; @@ -262,33 +250,17 @@ pub async fn fetch_and_save_full_mailbox( } }; - let folder_limit = account.folder_limit; - let total_to_fetch = match folder_limit { - Some(limit) if (limit as u64) < total => { - let limit64 = limit as u64; - total.min(limit64.max(100)) - } - _ => total, - }; - - let page_size = if let Some(limit) = folder_limit { - limit - .max(100) - .min(account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE)) - } else { - account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) - }; - - let total_batches = total_to_fetch.div_ceil(page_size as u64); - let desc = folder_limit.is_some(); + let page_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE); + let total_batches = total.div_ceil(page_size as u64); info!( - "Starting full mailbox download for '{}', total={}, limit={:?}, batches={}, desc={}", - mailbox.name, total, folder_limit, total_batches, desc + "Starting full mailbox download for '{}', total={}, batches={}", + mailbox.name, total, total_batches ); let mut current_processed = 0u64; let mut has_error_or_cancel = false; + let mut max_uid: Option = None; for page in 1..=total_batches { if token.is_cancelled() { @@ -300,7 +272,7 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Cancelled, None, @@ -313,12 +285,12 @@ pub async fn fetch_and_save_full_mailbox( &mut session, account_id, mailbox_id, - total_to_fetch, + total, page as u64, page_size as u64, &mailbox.encoded_name(), - desc, token.clone(), + &mut max_uid, ) .await { @@ -327,7 +299,7 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Downloading, None, @@ -339,7 +311,7 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Failed, Some(err_msg), @@ -354,28 +326,24 @@ pub async fn fetch_and_save_full_mailbox( DownloadState::update_folder_progress( account_id, mailbox.name.clone(), - total_to_fetch, + total, current_processed, FolderStatus::Success, None, )?; } session.logout().await.ok(); - Ok(()) + Ok(max_uid) } pub fn generate_uid_sequence_hashset( unique_nums: Vec, chunk_size: usize, - desc: bool, ) -> Vec<(String, u64)> { assert!(!unique_nums.is_empty()); - let mut nums = unique_nums; - if desc { - nums.reverse(); - } let mut result = Vec::new(); + let nums = unique_nums; for chunk in nums.chunks(chunk_size) { let size = chunk.len() as u64; @@ -448,7 +416,7 @@ pub async fn reconcile_mailboxes( break; } - if local_mailbox.uid_validity != remote_mailbox.uid_validity { + let new_highest_uid = if local_mailbox.uid_validity != remote_mailbox.uid_validity { if remote_mailbox.uid_validity.is_none() { let err_msg = format!( "Mailbox '{}' logic error: Server did not provide UIDVALIDITY.", @@ -493,7 +461,7 @@ pub async fn reconcile_mailboxes( FetchDirection::Since, token.clone(), ) - .await?; + .await? } None => match &account.date_before { Some(r) => { @@ -505,7 +473,7 @@ pub async fn reconcile_mailboxes( FetchDirection::Before, token.clone(), ) - .await?; + .await? } None => { rebuild_mailbox_cache( @@ -520,10 +488,12 @@ pub async fn reconcile_mailboxes( } } else { perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone()) - .await?; - } + .await? + }; - mailboxes_to_update.push(remote_mailbox.clone()); + let mut updated = remote_mailbox.clone(); + updated.highest_uid = new_highest_uid; + mailboxes_to_update.push(updated); } //The metadata of this mailbox must only be updated after a successful synchronization; //otherwise, it may cause synchronization errors and result in missing emails in the local sync results. @@ -598,7 +568,11 @@ pub async fn reconcile_mailboxes( }; match result { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -622,64 +596,88 @@ pub async fn reconcile_mailboxes( } //only check new emails and sync +/// Incrementally syncs a mailbox. +/// Returns the new highest UID after sync, or `None` if nothing changed. async fn perform_incremental_sync( account: &AccountModel, local_mailbox: &MailBox, remote_mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { if remote_mailbox.exists > 0 { - let local_max_uid = ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?; - tracing::info!( - "[account {}][mailbox {}] perform_incremental_sync: local_max_uid={:?}, remote.exists={}", - account.id, - local_mailbox.name, - local_max_uid, - remote_mailbox.exists - ); - match local_max_uid { - Some(max_uid) => { - let mut session = ImapExecutor::create_connection(account.id).await?; - let before_date = account - .date_before - .as_ref() - .map(|r| r.calculate_date()) - .transpose()?; - - ImapExecutor::fetch_new_mail( - &mut session, - account, - local_mailbox, - max_uid + 1, - before_date.as_deref(), - token, - ) - .await?; - session.logout().await.ok(); + // Use stored highest_uid if available; otherwise fall back to Tantivy + // query once (backward compatibility with pre-existing databases). + let start_uid = match local_mailbox.highest_uid { + Some(uid) => { + tracing::info!( + "[account {}][mailbox {}] perform_incremental_sync: stored highest_uid={}, remote.exists={}", + account.id, + local_mailbox.name, + uid, + remote_mailbox.exists + ); + uid as u64 + 1 } None => { - info!( - "No maximum UID found in index for mailbox, assuming local cache is missing." + let local_max_uid = + ENVELOPE_MANAGER.get_max_uid(account.id, local_mailbox.id)?; + tracing::info!( + "[account {}][mailbox {}] perform_incremental_sync: highest_uid unset, Tantivy max_uid={:?}, remote.exists={}", + account.id, + local_mailbox.name, + local_max_uid, + remote_mailbox.exists ); - - match &account.date_since { - Some(date_since) => { - fetch_and_save_by_date( - account, - date_since.since_date()?.as_str(), - remote_mailbox, - FetchDirection::Since, - token, - ) - .await?; - } + match local_max_uid { + Some(uid) => uid + 1, None => { - fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; + info!( + "No maximum UID found in index for mailbox, assuming local cache is missing." + ); + + let result = match &account.date_since { + Some(date_since) => { + fetch_and_save_by_date( + account, + date_since.since_date()?.as_str(), + remote_mailbox, + FetchDirection::Since, + token, + ) + .await? + } + None => { + fetch_and_save_full_mailbox(account, remote_mailbox, token) + .await? + } + }; + return Ok(result); } } } - } - } + }; - Ok(()) + let mut session = ImapExecutor::create_connection(account.id).await?; + let before_date = account + .date_before + .as_ref() + .map(|r| r.calculate_date()) + .transpose()?; + + let new_max_uid = ImapExecutor::fetch_new_mail( + &mut session, + account, + local_mailbox, + start_uid, + before_date.as_deref(), + token, + ) + .await?; + session.logout().await.ok(); + + // Keep existing highest_uid if no new mail was fetched. + Ok(new_max_uid.or(local_mailbox.highest_uid)) + } else { + Ok(local_mailbox.highest_uid) + } } diff --git a/crates/core/src/cache/imap/download/rebuild.rs b/crates/core/src/cache/imap/download/rebuild.rs index 85c175d..96d31d9 100644 --- a/crates/core/src/cache/imap/download/rebuild.rs +++ b/crates/core/src/cache/imap/download/rebuild.rs @@ -89,7 +89,11 @@ pub async fn rebuild_cache( }; match fetch_and_save_full_mailbox(&account, &mailbox, token.clone()).await { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -169,7 +173,11 @@ pub async fn rebuild_cache_by_date( match fetch_and_save_by_date(&account, date.as_str(), &mailbox, direction, token.clone()) .await { - Ok(_) => {} + Ok(new_highest_uid) => { + let mut updated = mailbox.clone(); + updated.highest_uid = new_highest_uid; + MailBox::batch_upsert(&[updated])?; + } Err(err) => { has_error = true; tracing::error!("Folder sync task failed: {:#?}", err); @@ -196,7 +204,7 @@ pub async fn rebuild_mailbox_cache( local_mailbox: &MailBox, remote_mailbox: &MailBox, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { ENVELOPE_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox.id]) .await?; @@ -217,11 +225,11 @@ pub async fn rebuild_mailbox_cache( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; - Ok(()) + let result = fetch_and_save_full_mailbox(account, remote_mailbox, token).await?; + Ok(result) } pub async fn rebuild_mailbox_cache_by_date( @@ -231,7 +239,7 @@ pub async fn rebuild_mailbox_cache_by_date( remote: &MailBox, direction: FetchDirection, token: CancellationToken, -) -> BichonResult<()> { +) -> BichonResult> { ENVELOPE_MANAGER .delete_mailbox_envelopes(account.id, vec![local_mailbox_id]) .await?; @@ -252,9 +260,9 @@ pub async fn rebuild_mailbox_cache_by_date( FolderStatus::Success, None, )?; - return Ok(()); + return Ok(None); } - fetch_and_save_by_date(account, date, remote, direction, token).await?; - Ok(()) + let result = fetch_and_save_by_date(account, date, remote, direction, token).await?; + Ok(result) } diff --git a/crates/core/src/cache/imap/mailbox.rs b/crates/core/src/cache/imap/mailbox.rs index b85e712..db11e4e 100644 --- a/crates/core/src/cache/imap/mailbox.rs +++ b/crates/core/src/cache/imap/mailbox.rs @@ -56,6 +56,10 @@ pub struct MailBox { /// The validity identifier for UIDs in this mailbox, used to ensure UID consistency across sessions. /// If `None`, the IMAP server has not provided this information. pub uid_validity: Option, + /// The highest UID that has been successfully downloaded and stored locally. + /// Used for incremental sync: next fetch starts from `highest_uid + 1`. + /// If `None`, a fallback query against the Tantivy index will be performed once. + pub highest_uid: Option, } impl MemDbModel for MailBox { diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 422dfed..561fc56 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -17,8 +17,7 @@ // along with this program. If not, see . use crate::account::migration::AccountModel; -use crate::account::state::{DownloadState, FolderStatus}; -use crate::cache::imap::download::flow::{generate_uid_sequence_hashset, DEFAULT_BATCH_SIZE}; +use crate::account::state::{DownloadState, DownloadStatus, FolderStatus}; use crate::cache::imap::mailbox::MailBox; use crate::envelope::extractor::extract_envelope_and_store_it; use crate::error::code::ErrorCode; @@ -80,6 +79,14 @@ impl ImapExecutor { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)) } + /// Fetches new mail for a mailbox by directly issuing a ranged UID FETCH. + /// + /// Unlike the old two-step approach (UID SEARCH → batch UID FETCH), + /// this sends a single `UID FETCH {start}:*` and streams the results, + /// eliminating one IMAP round-trip. + /// + /// Returns `Ok(Some(max_uid))` with the highest UID fetched, or `Ok(None)` + /// if no new mail was found. pub async fn fetch_new_mail( session: &mut Session>, account: &AccountModel, @@ -87,122 +94,90 @@ impl ImapExecutor { start_uid: u64, before: Option<&str>, token: CancellationToken, - ) -> BichonResult<()> { + ) -> BichonResult> { assert!(start_uid > 0, "start_uid must be greater than 0"); - let query = match before { - Some(date) => format!("UID {start_uid}:* BEFORE {date}"), - None => format!("UID {start_uid}:*"), + // Select the mailbox (read-only) so UID FETCH works. + session + .examine(&mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + + // Build the UID range. IMAP UID FETCH accepts the same range syntax + // as UID SEARCH, so we can skip the separate SEARCH round-trip. + let uid_range = match before { + Some(date) => format!("{start_uid}:* BEFORE {date}"), + None => format!("{start_uid}:*"), }; - let uid_list = match Self::uid_search(session, &mailbox.encoded_name(), &query).await { - Ok(uid_list) => uid_list, - Err(e) => { - let err_msg = format!("UID search failed in [{}]: {:#?}", mailbox.name, e); - DownloadState::update_folder_progress( + info!( + "[account {}][mailbox {}] fetch_new_mail: direct UID FETCH {}", + account.id, mailbox.name, uid_range + ); + + let mut stream = session + .uid_fetch(&uid_range, BODY_FETCH_COMMAND) + .await + .map_err(|e| { + let err_msg = format!( + "UID FETCH failed in [{}]: {:#?}", + mailbox.name, e + ); + let _ = DownloadState::append_session_error(account.id, err_msg); + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })?; + + let mut count = 0u64; + let mut max_uid: Option = None; + while let Some(fetch) = stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + { + if token.is_cancelled() { + tracing::info!( + "Account {}: fetch_new_mail stream interrupted.", + account.id + ); + DownloadState::update_session_status( account.id, - mailbox.name.clone(), - 0, - 0, - FolderStatus::Failed, - Some(err_msg.clone()), + DownloadStatus::Cancelled, + Some("User stopped or system shutdown".to_string()), )?; - DownloadState::append_session_error(account.id, err_msg)?; - return Err(e); + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); } - }; - let len = uid_list.len(); - if len == 0 { - let msg = match before { - Some(date) => format!("No emails found before {}.", date), - None => "No new emails found.".into(), - }; + if let Some(uid) = fetch.uid { + max_uid = Some(max_uid.unwrap_or(0).max(uid)); + } + extract_envelope_and_store_it(fetch, account.id, mailbox.id).await?; + count += 1; + } + + if count == 0 { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), 0, 0, FolderStatus::Success, - Some(msg), + Some("No new emails found.".into()), )?; - return Ok(()); - } - info!( - "[account {}][mailbox {}] {} envelopes need to be fetched (start_uid={})", - account.id, mailbox.name, len, start_uid - ); - tracing::debug!( - "[account {}][mailbox {}] fetch_new_mail UID range: {}..{} ({} uids)", - account.id, - mailbox.name, - start_uid, - uid_list.iter().max().unwrap_or(&0), - len - ); - - let mut uid_vec: Vec = uid_list.into_iter().collect(); - uid_vec.sort(); - let uid_batches = generate_uid_sequence_hashset( - uid_vec, - account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize, - false, - ); - let mut current_processed = 0u64; - let mut has_error_or_cancel = false; - for (index, batch) in uid_batches.into_iter().enumerate() { - if token.is_cancelled() { - break; - } - match Self::uid_batch_retrieve_emails( - session, - account.id, - mailbox.id, - &batch.0, - token.clone(), - ) - .await - { - Ok(_) => { - current_processed += batch.1; - DownloadState::update_folder_progress( - account.id, - mailbox.name.clone(), - len as u64, - current_processed, - FolderStatus::Downloading, - None, - )?; - } - Err(e) => { - let err_msg = format!("Batch {} failed: {:#?}", index, e); - DownloadState::append_session_error(account.id, err_msg.clone())?; - DownloadState::update_folder_progress( - account.id, - mailbox.name.clone(), - len as u64, - current_processed, - FolderStatus::Failed, - Some(err_msg), - )?; - has_error_or_cancel = true; - break; - } - } - } - - if !has_error_or_cancel { + } else { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), - len as u64, - current_processed, + count, + count, FolderStatus::Success, None, )?; } - Ok(()) + Ok(max_uid) } pub async fn batch_retrieve_emails( @@ -213,36 +188,23 @@ impl ImapExecutor { page: u64, page_size: u64, encoded_mailbox_name: &str, - desc: bool, token: CancellationToken, + max_uid: &mut Option, ) -> BichonResult { assert!(page > 0, "Page number must be greater than 0"); assert!(page_size > 0, "Page size must be greater than 0"); - let (start, end) = if desc { - // Fetch messages starting from the newest (descending order) - let end = total.saturating_sub((page - 1) * page_size); - if end == 0 { - return Ok(0); - } - // Calculate start as end - page_size + 1 to avoid off-by-one errors - let start = end.saturating_sub(page_size - 1).max(1); - (start, end) - } else { - // Fetch messages starting from the oldest (ascending order) - let start = (page - 1) * page_size + 1; - if start > total { - return Ok(0); - } - // Calculate end, capped by the total number of messages - let end = (start + page_size - 1).min(total); - (start, end) - }; + // Fetch messages starting from the oldest (ascending order). + let start = (page - 1) * page_size + 1; + if start > total { + return Ok(0); + } + let end = (start + page_size - 1).min(total); let sequence_set = format!("{}:{}", start, end); info!( - "Fetching mailbox '{}' messages: sequence {} (page {}, page_size {}, desc={})", - encoded_mailbox_name, sequence_set, page, page_size, desc + "Fetching mailbox '{}' messages: sequence {} (page {}, page_size {})", + encoded_mailbox_name, sequence_set, page, page_size ); let mut stream = session @@ -263,6 +225,9 @@ impl ImapExecutor { ErrorCode::InternalError )); } + if let Some(uid) = fetch.uid { + *max_uid = Some((*max_uid).unwrap_or(0).max(uid)); + } extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?; count += 1; } diff --git a/crates/core/src/import/mod.rs b/crates/core/src/import/mod.rs index 8cd533e..13d85b0 100644 --- a/crates/core/src/import/mod.rs +++ b/crates/core/src/import/mod.rs @@ -105,6 +105,7 @@ impl ImportEmls { unseen: None, uid_next: None, uid_validity: None, + highest_uid: None, }; let mailbox_id = mailbox.id; // Upsert the mailbox, creating it if it doesn't exist diff --git a/crates/core/src/mailbox/list.rs b/crates/core/src/mailbox/list.rs index 667e220..5fb1068 100644 --- a/crates/core/src/mailbox/list.rs +++ b/crates/core/src/mailbox/list.rs @@ -157,8 +157,13 @@ async fn fetch_remote_with_progress(account_id: u64) -> BichonResult BichonResult<()> { unseen: None, uid_next: None, uid_validity: None, + highest_uid: None, }; let mailbox_id = mailbox.id; From 1a615e1c45046acdad7d9b839a73d948ee5479ba Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 15:27:34 +0800 Subject: [PATCH 05/66] bump to v1.4.0 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e858601..80b873b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.3.0" +version = "1.4.0" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.3.0" +version = "1.4.0" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.3.0" +version = "1.4.0" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.3.0" +version = "1.4.0" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.3.0" +version = "1.4.0" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index 8184c8f..8d62f41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.3.0" +version = "1.4.0" edition = "2021" [workspace.dependencies] From 8e46c7a162951a32a1bec621373412ad64b74e62 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 19:42:26 +0800 Subject: [PATCH 06/66] fix: use valid IMAP UID SEARCH instead of BEFORE in UID FETCH for incremental sync --- .gitignore | 3 +- crates/core/src/account/since.rs | 39 +++ .../src/cache/imap/download/download_type.rs | 19 +- crates/core/src/cache/imap/download/flow.rs | 80 ++---- crates/core/src/imap/executor.rs | 263 ++++++++++++++++-- 5 files changed, 311 insertions(+), 93 deletions(-) diff --git a/.gitignore b/.gitignore index e33b326..f48f1f4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .vscode .idea config.toml -node_modules \ No newline at end of file +node_modules +dedup_report.txt \ No newline at end of file diff --git a/crates/core/src/account/since.rs b/crates/core/src/account/since.rs index 8b018d3..5080960 100644 --- a/crates/core/src/account/since.rs +++ b/crates/core/src/account/since.rs @@ -364,4 +364,43 @@ mod test { }; assert!(e.validate().is_err()); } + + // ── Sliding window tests ────────────────────────────────────── + + #[test] + fn relative_date_calculate_returns_valid_format() { + let r = RelativeDate { + unit: Unit::Years, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + // Expect format like "26-May-2025" + assert!(date_str.len() > 5); + assert!(date_str.contains('-')); + } + + #[test] + fn relative_date_one_year_ago_is_before_now() { + let r = RelativeDate { + unit: Unit::Years, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap(); + let today = chrono::Local::now().date_naive(); + assert!(parsed < today, "1 year ago ({parsed}) should be before today ({today})"); + } + + #[test] + fn relative_date_one_day_ago_is_yesterday() { + let r = RelativeDate { + unit: Unit::Days, + value: 1, + }; + let date_str = r.calculate_date().unwrap(); + let parsed = chrono::NaiveDate::parse_from_str(&date_str, "%d-%b-%Y").unwrap(); + let today = chrono::Local::now().date_naive(); + let yesterday = today - chrono::Duration::days(1); + assert_eq!(parsed, yesterday, "1 day ago should be yesterday"); + } } diff --git a/crates/core/src/cache/imap/download/download_type.rs b/crates/core/src/cache/imap/download/download_type.rs index 2c27d73..1bf691f 100644 --- a/crates/core/src/cache/imap/download/download_type.rs +++ b/crates/core/src/cache/imap/download/download_type.rs @@ -77,10 +77,7 @@ pub async fn decide_next_download_task( } } -fn should_trigger_next_download( - last_trigger_at: i64, - sync_interval_min: i64, -) -> bool { +fn should_trigger_next_download(last_trigger_at: i64, sync_interval_min: i64) -> bool { let now = utc_now!(); now - last_trigger_at > (sync_interval_min * 60 * 1000) } @@ -107,7 +104,10 @@ fn should_trigger_scheduled(schedule_str: &str, last_trigger_at: i64) -> bool { }; let last_dt: DateTime = last_utc.with_timezone(&Local); let now = Local::now(); - schedule.after(&last_dt).next().map_or(false, |next| next <= now) + schedule + .after(&last_dt) + .next() + .map_or(false, |next| next <= now) } #[cfg(test)] @@ -153,13 +153,4 @@ mod test { let last_trigger = now.timestamp_millis() - 61 * 60 * 1000; assert!(should_trigger_scheduled("0 0 * * * *", last_trigger)); } - - #[test] - fn cron_every_hour_no_trigger_if_recent() { - // "0 0 * * * *" = every hour at minute 0, second 0 - // last_trigger was just 1 minute ago → should NOT trigger (except at :00/:01 boundary) - let now = Local::now(); - let last_trigger = now.timestamp_millis() - 60_000; - assert!(!should_trigger_scheduled("0 0 * * * *", last_trigger)); - } } diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 1da97f9..f74ba57 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -32,7 +32,9 @@ use crate::{ SEMAPHORE, }, error::{code::ErrorCode, BichonResult}, - imap::executor::ImapExecutor, + imap::executor::{ + generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, + }, store::tantivy::envelope::ENVELOPE_MANAGER, }, }; @@ -40,7 +42,6 @@ use std::time::Instant; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; -pub const DEFAULT_BATCH_SIZE: u32 = 30; #[derive(Clone, Debug, Eq, PartialEq)] pub enum FetchDirection { @@ -336,59 +337,6 @@ pub async fn fetch_and_save_full_mailbox( Ok(max_uid) } -pub fn generate_uid_sequence_hashset( - unique_nums: Vec, - chunk_size: usize, -) -> Vec<(String, u64)> { - assert!(!unique_nums.is_empty()); - - let mut result = Vec::new(); - let nums = unique_nums; - - for chunk in nums.chunks(chunk_size) { - let size = chunk.len() as u64; - let compressed = compress_uid_list(chunk.to_vec()); - result.push((compressed, size)); - } - - result -} - -pub fn compress_uid_list(nums: Vec) -> String { - if nums.is_empty() { - return String::new(); - } - - let mut sorted_nums = nums; - sorted_nums.sort(); - - let mut result = Vec::new(); - let mut current_range_start = sorted_nums[0]; - let mut current_range_end = sorted_nums[0]; - - for &n in sorted_nums.iter().skip(1) { - if n == current_range_end + 1 { - current_range_end = n; - } else { - if current_range_start == current_range_end { - result.push(current_range_start.to_string()); - } else { - result.push(format!("{}:{}", current_range_start, current_range_end)); - } - current_range_start = n; - current_range_end = n; - } - } - - if current_range_start == current_range_end { - result.push(current_range_start.to_string()); - } else { - result.push(format!("{}:{}", current_range_start, current_range_end)); - } - - result.join(",") -} - pub async fn reconcile_mailboxes( account: &AccountModel, remote_mailboxes: &[MailBox], @@ -632,7 +580,7 @@ async fn perform_incremental_sync( Some(uid) => uid + 1, None => { info!( - "No maximum UID found in index for mailbox, assuming local cache is missing." + "No maximum UID found in index for mailbox, assuming local storage is missing." ); let result = match &account.date_since { @@ -646,10 +594,24 @@ async fn perform_incremental_sync( ) .await? } - None => { - fetch_and_save_full_mailbox(account, remote_mailbox, token) + None => match &account.date_before { + Some(r) => { + fetch_and_save_by_date( + account, + &r.calculate_date()?, + remote_mailbox, + FetchDirection::Before, + token, + ) .await? - } + } + None => { + fetch_and_save_full_mailbox( + account, remote_mailbox, token, + ) + .await? + } + }, }; return Ok(result); } diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 561fc56..1b86c3e 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -79,11 +79,12 @@ impl ImapExecutor { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)) } - /// Fetches new mail for a mailbox by directly issuing a ranged UID FETCH. + /// Fetches new mail for a mailbox. /// - /// Unlike the old two-step approach (UID SEARCH → batch UID FETCH), - /// this sends a single `UID FETCH {start}:*` and streams the results, - /// eliminating one IMAP round-trip. + /// When `before` is `Some(date)`, a two-step approach is used: + /// `UID SEARCH` to find matching UIDs (standard IMAP), then batch `UID FETCH` + /// for the specific UIDs. When `before` is `None`, a direct ranged + /// `UID FETCH {start}:*` is issued and results are streamed. /// /// Returns `Ok(Some(max_uid))` with the highest UID fetched, or `Ok(None)` /// if no new mail was found. @@ -97,19 +98,132 @@ impl ImapExecutor { ) -> BichonResult> { assert!(start_uid > 0, "start_uid must be greater than 0"); - // Select the mailbox (read-only) so UID FETCH works. session .examine(&mailbox.encoded_name()) .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; - // Build the UID range. IMAP UID FETCH accepts the same range syntax - // as UID SEARCH, so we can skip the separate SEARCH round-trip. - let uid_range = match before { - Some(date) => format!("{start_uid}:* BEFORE {date}"), - None => format!("{start_uid}:*"), - }; + match before { + Some(date) => { + Self::fetch_new_mail_with_before(session, account, mailbox, start_uid, date, token) + .await + } + None => Self::fetch_new_mail_range(session, account, mailbox, start_uid, token).await, + } + } + + /// Two-step approach for date-filtered incremental fetch: UID SEARCH first, + /// then batch UID FETCH for matching UIDs. Uses standard IMAP syntax that + /// works across all compliant servers. + async fn fetch_new_mail_with_before( + session: &mut Session>, + account: &AccountModel, + mailbox: &MailBox, + start_uid: u64, + date: &str, + token: CancellationToken, + ) -> BichonResult> { + let query = format!("UID {start_uid}:* BEFORE {date}"); + info!( + "[account {}][mailbox {}] fetch_new_mail: UID SEARCH {}", + account.id, mailbox.name, query + ); + let results = session.uid_search(&query).await.map_err(|e| { + let err_msg = format!("UID SEARCH failed in [{}]: {:#?}", mailbox.name, e); + let _ = DownloadState::append_session_error(account.id, err_msg); + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })?; + + if results.is_empty() { + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + Some("No new emails found.".into()), + )?; + return Ok(None); + } + + let mut uid_vec: Vec = results.into_iter().collect(); + uid_vec.sort(); + let max_uid = uid_vec.last().copied(); + let planned = uid_vec.len() as u64; + let batch_size = account.download_batch_size.unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let uid_batches = generate_uid_sequence_hashset(uid_vec, batch_size); + + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + 0, + FolderStatus::Pending, + None, + )?; + + let mut count = 0u64; + for batch in uid_batches { + if token.is_cancelled() { + DownloadState::update_session_status( + account.id, + DownloadStatus::Cancelled, + Some("User stopped or system shutdown".to_string()), + )?; + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + count, + FolderStatus::Cancelled, + None, + )?; + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); + } + Self::uid_batch_retrieve_emails( + session, + account.id, + mailbox.id, + &batch.0, + token.clone(), + ) + .await?; + count += batch.1; + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + planned, + count, + FolderStatus::Downloading, + None, + )?; + } + + DownloadState::update_folder_progress( + account.id, + mailbox.name.clone(), + count, + count, + FolderStatus::Success, + None, + )?; + Ok(max_uid) + } + + /// Direct ranged UID FETCH without date filtering. Streams results from + /// the server in a single IMAP round-trip. + async fn fetch_new_mail_range( + session: &mut Session>, + account: &AccountModel, + mailbox: &MailBox, + start_uid: u64, + token: CancellationToken, + ) -> BichonResult> { + let uid_range = format!("{start_uid}:*"); info!( "[account {}][mailbox {}] fetch_new_mail: direct UID FETCH {}", account.id, mailbox.name, uid_range @@ -119,10 +233,7 @@ impl ImapExecutor { .uid_fetch(&uid_range, BODY_FETCH_COMMAND) .await .map_err(|e| { - let err_msg = format!( - "UID FETCH failed in [{}]: {:#?}", - mailbox.name, e - ); + let err_msg = format!("UID FETCH failed in [{}]: {:#?}", mailbox.name, e); let _ = DownloadState::append_session_error(account.id, err_msg); raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) })?; @@ -135,10 +246,7 @@ impl ImapExecutor { .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? { if token.is_cancelled() { - tracing::info!( - "Account {}: fetch_new_mail stream interrupted.", - account.id - ); + tracing::info!("Account {}: fetch_new_mail stream interrupted.", account.id); DownloadState::update_session_status( account.id, DownloadStatus::Cancelled, @@ -320,3 +428,120 @@ impl ImapExecutor { ImapConnectionManager::build(account_id).await } } + +pub const DEFAULT_BATCH_SIZE: u32 = 30; + +/// Compresses a sorted list of UIDs into an IMAP sequence-set string. +/// Consecutive UIDs become ranges (e.g. `1:5`), non-consecutive are +/// comma-separated (e.g. `1:5,10,12:15`). +pub fn compress_uid_list(nums: Vec) -> String { + if nums.is_empty() { + return String::new(); + } + + let mut sorted_nums = nums; + sorted_nums.sort(); + + let mut result = Vec::new(); + let mut current_range_start = sorted_nums[0]; + let mut current_range_end = sorted_nums[0]; + + for &n in sorted_nums.iter().skip(1) { + if n == current_range_end + 1 { + current_range_end = n; + } else { + if current_range_start == current_range_end { + result.push(current_range_start.to_string()); + } else { + result.push(format!("{}:{}", current_range_start, current_range_end)); + } + current_range_start = n; + current_range_end = n; + } + } + + if current_range_start == current_range_end { + result.push(current_range_start.to_string()); + } else { + result.push(format!("{}:{}", current_range_start, current_range_end)); + } + + result.join(",") +} + +/// Splits a sorted list of unique UIDs into compressed sequence-set batches. +/// Returns `Vec<(sequence_set_string, batch_count)>`. +pub fn generate_uid_sequence_hashset( + unique_nums: Vec, + chunk_size: usize, +) -> Vec<(String, u64)> { + assert!(!unique_nums.is_empty()); + + let mut result = Vec::new(); + let nums = unique_nums; + + for chunk in nums.chunks(chunk_size) { + let size = chunk.len() as u64; + let compressed = compress_uid_list(chunk.to_vec()); + result.push((compressed, size)); + } + + result +} + +#[cfg(test)] +mod test { + use super::*; + + // ── compress_uid_list ────────────────────────────────────────── + + #[test] + fn compress_empty() { + assert_eq!(compress_uid_list(vec![]), ""); + } + + #[test] + fn compress_single_uid() { + assert_eq!(compress_uid_list(vec![42]), "42"); + } + + #[test] + fn compress_consecutive_range() { + assert_eq!(compress_uid_list(vec![1, 2, 3, 4, 5]), "1:5"); + } + + #[test] + fn compress_mixed_ranges() { + assert_eq!( + compress_uid_list(vec![1, 2, 3, 5, 7, 8, 9, 10]), + "1:3,5,7:10" + ); + } + + #[test] + fn compress_gap_at_boundary() { + assert_eq!(compress_uid_list(vec![1, 2, 4, 5]), "1:2,4:5"); + } + + // ── generate_uid_sequence_hashset ────────────────────────────── + + #[test] + fn batch_single_chunk() { + let batches = generate_uid_sequence_hashset(vec![1, 2, 3], 10); + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].0, "1:3"); + assert_eq!(batches[0].1, 3); + } + + #[test] + fn batch_multiple_chunks() { + let batches = generate_uid_sequence_hashset(vec![1, 2, 3, 4, 5], 2); + assert_eq!(batches.len(), 3); + assert_eq!(batches[0].0, "1:2"); + assert_eq!(batches[0].1, 2); + assert_eq!(batches[1].0, "3:4"); + assert_eq!(batches[1].1, 2); + assert_eq!(batches[2].0, "5"); + assert_eq!(batches[2].1, 1); + } +} From a60b2c7dc409dcabb2533286e8e63ed207f453be Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 20:02:55 +0800 Subject: [PATCH 07/66] Update release.yml --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ab8768..dd7ad7d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release Bichon on: push: tags: - - '[0-9]+.[0-9]+.[0-9]+' + - '*' env: BINARY_NAME: bichon-server BINARY_CLI: bichon-cli From 38453accc6a1355474c471dbc8553b5dc8cce28f Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 20:07:40 +0800 Subject: [PATCH 08/66] Update release.yml --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dd7ad7d..324f9c2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ on: push: tags: - '*' + workflow_dispatch: env: BINARY_NAME: bichon-server BINARY_CLI: bichon-cli From 86869ac8484c9cc935346798764a9db2fa9b0c8c Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 26 May 2026 20:11:49 +0800 Subject: [PATCH 09/66] Update release.yml --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 324f9c2..5ab8768 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,8 +3,7 @@ name: Release Bichon on: push: tags: - - '*' - workflow_dispatch: + - '[0-9]+.[0-9]+.[0-9]+' env: BINARY_NAME: bichon-server BINARY_CLI: bichon-cli From 8fcb55320fdfeecbb57d506e1a54564714dc80b0 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 28 May 2026 02:09:58 +0800 Subject: [PATCH 10/66] Update README.md --- README.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a7465b1..a4b725a 100644 --- a/README.md +++ b/README.md @@ -449,7 +449,7 @@ Storage Layer │ └──────────────┘ └──────────────┘ └──────────────┘ ``` -- **memdb**: Key-value metadata store. Houses accounts, users, roles, OAuth2 configs, proxy settings, and system configuration. All operations wrapped in `tokio::spawn_blocking`. +- **memdb**: Key-value metadata store. Houses accounts, users, roles, OAuth2 configs, proxy settings, and system configuration. - **Tantivy**: Full-text search indices with Zstd compression support. Two separate indices: envelope (email metadata + body text) and attachment (file metadata + extracted text). Batch-committed every 1,000 documents or 60 seconds. - **Fjall**: LZ4-compressed LSM tree key-value store. Two keyspaces — `email_keyspace` and `attachments_keyspace`. Content-hash addressed (BLAKE3) with insert-time deduplication. Values larger than 1 KB stored as separate files (KV separation). @@ -485,6 +485,79 @@ Tantivy Fjall memdb - Manual sync via `POST /api/v1/accounts/:id/start-download`; cancel with `cancel-download` - Busy-check prevents overlapping manual and automatic syncs on the same account +### Content Deduplication & Attachment Storage + +``` + ┌──────────────────────────────────────────┐ + │ Raw EML bytes │ + └────────────────┬─────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────┐ + │ BLAKE3 → email_content_hash │ + └────────────────┬─────────────────────────┘ + │ + ▼ + ┌──────────────────────────────────────────┐ + │ MIME parse → Message │ + └───────┬──────────────────┬──────────────┘ + │ │ + │ ┌────────────┘ + │ │ detach attachments + │ │ + ▼ ▼ + ┌─────────────────┐ ┌──────────────────────────────┐ + │ EMAIL BODY │ │ EACH ATTACHMENT │ + │ │ │ │ + │ Replace raw │ │ BLAKE3(decoded content) │ + │ attachment │ │ → attachment_content_hash │ + │ bytes with │ │ │ + │ placeholder: │ │ Store raw undecoded bytes │ + │ │ │ in Fjall attachments_ks │ + │ <> │ │ Extract text for indexing │ + │ │ │ (PDF, DOCX, etc.) │ + └───────┬─────────┘ └──────────────┬───────────────┘ + │ │ + ▼ │ + ┌──────────────────────────────┐ │ + │ Stripped EML stored in │ │ + │ Fjall email_keyspace │ │ + │ keyed by email_content_hash │ │ + │ (skip if hash exists) │ │ + └──────────────┬───────────────┘ │ + │ │ + ▼ ▼ + ┌─────────────────────────────────────────────────┐ + │ Tantivy full-text index │ + │ envelope index · attachment index │ + └─────────────────────────────────────────────────┘ + + ═══════════════════════════════════════════════════════════════ + + Dedup layers + ┌─────────────────────────────────────────────────────────────────┐ + │ Fjall (insert-time) │ + │ contains_key(hash)? → skip : store with LZ4 compression │ + │ │ + │ Tantivy (periodic, every 12 h) │ + │ Group by (account, mailbox, content_hash) │ + │ Keep latest ingest_at → soft-delete older copies │ + │ Cascade-delete orphaned attachment index entries │ + └─────────────────────────────────────────────────────────────────┘ + + Reconstruction + ┌─────────────────────────────────────────────────────────────────┐ + │ Fetch stripped EML by content_hash from Fjall │ + │ Find <> placeholders │ + │ Replace each with raw attachment blob from Fjall │ + │ Result → byte-identical original EML │ + └─────────────────────────────────────────────────────────────────┘ +``` + +Every ingested email is hashed with BLAKE3. Attachments are detached from the MIME tree, hashed independently (decoded content), and stored as raw undecoded bytes in Fjall's `attachments_keyspace`. The email body is patched with hash-based placeholders and stored in `email_keyspace`. Both keyspaces check for existing hashes before writing — identical content is never stored twice, regardless of which account or folder it arrives in. A periodic index dedup task (every 12 hours) scans Tantivy for duplicate `(account, mailbox, content_hash)` tuples, keeps the most recently ingested copy, and cascade-deletes orphaned attachment entries so UID-based incremental sync remains accurate. The original EML reconstructs byte-for-byte by swapping placeholders back with their attachment blobs. + ## Storage & Backup ### Data Directory Layout @@ -532,21 +605,21 @@ The WebUI is available in **18 languages**: Language preference and UI theme are saved to your user profile and can be changed anytime from the WebUI settings. -## Data Migration (v0.3.7 → v1.0) +## Data Migration (v0.3.7 → v1.x) -Bichon v1.0 introduced a redesigned storage architecture: +Bichon v1.x introduced a redesigned storage architecture: -| Layer | v0.3.7 (Legacy) | v1.0 | -|-------|---------------|------| -| **Index** | Tantivy (shared) | Tantivy (separate envelope + attachment indices) | -| **Raw data** | Tantivy (inline) | Fjall (LZ4-compressed key-value store) | -| **Metadata** | Tantivy (shared) | memdb (dedicated embedded DB) | +| Layer | v0.3.7 (Legacy) | v1.x | +| :--- | :--- | :--- | +| **Index** | Tantivy (shared instance, no full attachments) | Tantivy (separate envelope + attachment indices) | +| **Raw data** | Tantivy (inline, stored in another Tantivy instance) | Fjall (LZ4-compressed LSM-tree key-value store) | +| **Metadata** | Native_DB (shared, disk-based DB powered by redb) | memdb (dedicated, in-house in-memory DB) | -If you ran Bichon prior to v1.0, migrate your data: +If you ran Bichon prior to v1.x, migrate your data: ```bash ./bichon-admin -# Select "Migrate Legacy v0.3.7 Storage to v1.0" +# Select "Migrate Legacy v0.3.7 Storage to v1.x" ``` > [!NOTE] From 4a3c42c1ebdc2de8cfb3389e001ad140e53de0bf Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 28 May 2026 17:47:10 +0800 Subject: [PATCH 11/66] fix: HTTP Error 500 Internal Server Error: Failed for 58344335-2e86-4009-979d-6da0331bff63 - Failed to export an email. Aborting process... #275 --- crates/core/src/message/content.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/core/src/message/content.rs b/crates/core/src/message/content.rs index 02d4277..e35f40b 100644 --- a/crates/core/src/message/content.rs +++ b/crates/core/src/message/content.rs @@ -53,6 +53,7 @@ pub struct AttachmentInfo { /// Page count reported by the extractor, if any. pub extracted_page_count: Option, /// Whether the extracted text came from OCR. + #[serde(default)] pub extracted_is_ocr: bool, } From f4be4a2e8c32096f94e0bc5e25080a10eaf61671 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 28 May 2026 17:49:18 +0800 Subject: [PATCH 12/66] bump to 1.4.1 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80b873b..fc081fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.4.0" +version = "1.4.1" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.4.0" +version = "1.4.1" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.4.0" +version = "1.4.1" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.4.0" +version = "1.4.1" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.4.0" +version = "1.4.1" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index 8d62f41..40ccd8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.4.0" +version = "1.4.1" edition = "2021" [workspace.dependencies] From 4597df515a65e2b2b8b41636b86dbc0ec6340fc5 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 28 May 2026 17:55:13 +0800 Subject: [PATCH 13/66] Update content.rs --- crates/core/src/message/content.rs | 99 ++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/crates/core/src/message/content.rs b/crates/core/src/message/content.rs index e35f40b..7932a48 100644 --- a/crates/core/src/message/content.rs +++ b/crates/core/src/message/content.rs @@ -362,3 +362,102 @@ pub fn retrieve_nested_eml_content( has_remote_content, }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Simulates JSON written by a version before `extracted_text`, `extracted_page_count`, + /// and `extracted_is_ocr` were added to [`AttachmentInfo`]. Deserialization must + /// succeed and fill the missing fields with their defaults. + #[test] + fn attachment_info_backward_compat_no_extracted_fields() { + let old_json = r#"[ + { + "file_type": "application/pdf", + "inline": false, + "filename": "report.pdf", + "size": 12345, + "content_id": null, + "content_hash": "abc123", + "is_message": false + }, + { + "file_type": "image/png", + "inline": true, + "filename": "logo.png", + "size": 6789, + "content_id": "cid:logo@example.com", + "content_hash": "def456", + "is_message": false + } + ]"#; + + let attachments: Vec = + serde_json::from_str(old_json).expect("should deserialize legacy JSON"); + + assert_eq!(attachments.len(), 2); + + // First attachment (regular file) + assert_eq!(attachments[0].file_type, "application/pdf"); + assert!(!attachments[0].inline); + assert_eq!(attachments[0].filename.as_deref(), Some("report.pdf")); + assert_eq!(attachments[0].size, 12345); + assert_eq!(attachments[0].content_id, None); + assert_eq!(attachments[0].content_hash, "abc123"); + assert!(!attachments[0].is_message); + // Fields added after the legacy format — must default correctly + assert_eq!(attachments[0].extracted_text, None); + assert_eq!(attachments[0].extracted_page_count, None); + assert!(!attachments[0].extracted_is_ocr); + + // Second attachment (inline image with content-id) + assert_eq!(attachments[1].file_type, "image/png"); + assert!(attachments[1].inline); + assert_eq!(attachments[1].filename.as_deref(), Some("logo.png")); + assert_eq!(attachments[1].size, 6789); + assert_eq!(attachments[1].content_id.as_deref(), Some("cid:logo@example.com")); + assert_eq!(attachments[1].content_hash, "def456"); + assert!(!attachments[1].is_message); + assert_eq!(attachments[1].extracted_text, None); + assert_eq!(attachments[1].extracted_page_count, None); + assert!(!attachments[1].extracted_is_ocr); + } + + /// Current struct must round-trip through serde_json without data loss. + #[test] + fn attachment_info_round_trip() { + let attachments = vec![ + AttachmentInfo { + file_type: "text/html".into(), + inline: false, + filename: Some("page.html".into()), + size: 42, + content_id: None, + content_hash: "hash1".into(), + is_message: true, + extracted_text: Some("hello world".into()), + extracted_page_count: Some(1), + extracted_is_ocr: false, + }, + AttachmentInfo { + file_type: "application/zip".into(), + inline: false, + filename: Some("archive.zip".into()), + size: 99999, + content_id: None, + content_hash: "hash2".into(), + is_message: false, + extracted_text: None, + extracted_page_count: None, + extracted_is_ocr: true, + }, + ]; + + let json = serde_json::to_string(&attachments).expect("serialize"); + let round_tripped: Vec = + serde_json::from_str(&json).expect("deserialize"); + + assert_eq!(attachments, round_tripped); + } +} From 048d5f361c3827ebba754c01394778db8180e481 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 28 May 2026 22:49:24 +0800 Subject: [PATCH 14/66] fix: Cant migrate with version >= 1.4.0 #277 --- crates/admin/src/meta.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index f1a427d..4cf242c 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -117,6 +117,7 @@ pub struct AccountV3 { pub capabilities: Option>, pub date_since: Option, pub date_before: Option, + pub folder_limit: Option, pub sync_folders: Option>, pub account_type: AccountType, pub sync_interval_min: Option, @@ -192,7 +193,7 @@ impl From for AccountV2 { name: value.name, capabilities: value.capabilities, date_since: value.date_since, - folder_limit: None, + folder_limit: value.folder_limit, sync_folders: value.sync_folders, account_type: value.account_type, sync_interval_min: value.sync_interval_min, @@ -216,6 +217,7 @@ impl From for AccountV3 { name: value.name, capabilities: value.capabilities, date_since: value.date_since, + folder_limit: value.folder_limit, sync_folders: value.sync_folders, account_type: value.account_type, sync_interval_min: value.sync_interval_min, From d40ba90b54ddbcf3b8e24079ea365f614dc6a608 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Fri, 29 May 2026 16:13:19 +0800 Subject: [PATCH 15/66] fix: inline attachment detection and account-scoped export - Treat MIME parts with Content-ID but no Content-Disposition as inline - Add account_ids filter to CLI export search to avoid pulling all accounts - Skip failed emails during export instead of aborting the entire batch --- Cargo.lock | 10 ++-- Cargo.toml | 2 +- config.toml | 2 +- crates/cli/src/api/search.rs | 6 +- crates/cli/src/export/mod.rs | 13 ++-- crates/core/src/envelope/extractor.rs | 85 +++++++++++++++++++++++---- crates/core/src/message/content.rs | 8 ++- crates/core/src/migrate/store.rs | 23 +++++--- 8 files changed, 117 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc081fe..5630146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.4.1" +version = "1.4.2" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.4.1" +version = "1.4.2" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.4.1" +version = "1.4.2" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.4.1" +version = "1.4.2" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.4.1" +version = "1.4.2" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index 40ccd8a..267d925 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.4.1" +version = "1.4.2" edition = "2021" [workspace.dependencies] diff --git a/config.toml b/config.toml index 2bab9bc..e7fe0ca 100644 --- a/config.toml +++ b/config.toml @@ -1,2 +1,2 @@ base_url = "http://localhost:15630" -api_token = "WuqNC0g8yNle7CVnxcvjUwjN" +api_token = "eErI7WN3PtKeLwWAbIfSXCP6" diff --git a/crates/cli/src/api/search.rs b/crates/cli/src/api/search.rs index 0729d59..2a348a5 100644 --- a/crates/cli/src/api/search.rs +++ b/crates/cli/src/api/search.rs @@ -10,13 +10,17 @@ use crate::BichonCliConfig; pub async fn search_messages( client: &Client, config: &BichonCliConfig, + account_ids: Option>, page: u64, page_size: u64, ) -> Option> { let url = format!("{}/api/v1/search-messages", config.base_url); let payload = EmailSearchRequest { - filter: EmailSearchFilter::default(), + filter: EmailSearchFilter { + account_ids, + ..Default::default() + }, page, page_size, sort_by: Some(SortBy::DATE), diff --git a/crates/cli/src/export/mod.rs b/crates/cli/src/export/mod.rs index ef5fd7d..bf63200 100644 --- a/crates/cli/src/export/mod.rs +++ b/crates/cli/src/export/mod.rs @@ -146,20 +146,23 @@ pub async fn handle_account_export( let mut total_pages; loop { - if let Some(batch) = search_messages(&client, config, current_page, page_size).await { + let account_ids = Some(std::collections::HashSet::from([account.id])); + if let Some(batch) = search_messages(&client, config, account_ids, current_page, page_size).await { total_pages = batch.total_pages.unwrap(); pb.set_message(format!("Page {}/{}", current_page, total_pages)); for envelope in batch.items { let success = - download_and_export_with_json_header(&client, config, envelope, &mut file) + download_and_export_with_json_header(&client, config, envelope.clone(), &mut file) .await; if !success { - pb.finish_with_message("Failed"); - eprintln!(" ✘ Failed to export an email. Aborting process..."); - return; + eprintln!( + " ✘ Failed to export email {}, skipping...", + envelope.id + ); + continue; } pb.inc(1); } diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 0839bf7..978d0d6 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -412,23 +412,36 @@ pub async fn detach_and_store_attachments( let mut text_candidates: Vec = Vec::new(); for (raw_start, raw_end, att) in ranges { - // Step 2: Extract raw bytes and store them as standalone documents - let raw_bytes = &original_body[raw_start..raw_end]; - //This is the content hash of the decoded attachment, not the undecoded one. + // mail-parser may report attachment offsets past the body end for + // malformed messages; clamp the range to avoid a slice panic. + let body_len = original_body.len(); + let raw_start = raw_start.min(body_len); + let raw_end = raw_end.min(body_len); + let range_valid = raw_start < raw_end; + + // content hash is computed from the decoded attachment contents, + // which is always available regardless of raw offset validity. let content_hash = compute_content_hash(att.contents()); - //"The actual content stored in the blob is the raw undecoded data, to avoid the reconstructed EML differing from the original due to decoding and re-encoding. - attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes)));// + if range_valid { + let raw_bytes = &original_body[raw_start..raw_end]; + // The actual content stored in the blob is the raw undecoded data. + attachments.push((content_hash.clone(), Bytes::copy_from_slice(raw_bytes))); - // Step 3: Replace raw attachment content with a hash-based placeholder - let placeholder = format!("<>", &content_hash); - let p_bytes = placeholder.as_bytes(); - stripped_eml.splice(raw_start..raw_end, p_bytes.iter().cloned()); + // Replace raw attachment content with a hash-based placeholder + let placeholder = format!("<>", &content_hash); + stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + } else { + // Invalid range: store a zero-length blob so the consistency + // check passes; reattachment will log a warning for the missing + // blob data but won't panic. + attachments.push((content_hash.clone(), Bytes::new())); + } let inline = att .content_disposition() .map(|d| d.is_inline()) - .unwrap_or(false); + .unwrap_or_else(|| att.content_id().is_some()); let file_type = att .content_type() .map(|ct| { @@ -755,4 +768,56 @@ mod test { } } } + + /// Verifies that [`super::detach_and_store_attachments`] does not panic + /// when mail-parser reports attachment offsets past the raw body length. + /// + /// Regression test for: "range end index X out of range for slice of + /// length Y" panic caused by a malformed email whose attachment + /// `raw_end_offset` exceeded the actual body size. + #[tokio::test] + async fn detach_attachments_bounds_check() { + let raw = concat!( + "From: sender@example.com\r\n", + "To: recipient@example.com\r\n", + "Subject: Test\r\n", + "MIME-Version: 1.0\r\n", + "Content-Type: multipart/mixed; boundary=\"bnd\"\r\n", + "\r\n", + "--bnd\r\n", + "Content-Type: text/plain\r\n", + "\r\n", + "Hello\r\n", + "--bnd\r\n", + "Content-Type: application/octet-stream\r\n", + "Content-Disposition: attachment; filename=\"test.bin\"\r\n", + "\r\n", + "AAAAABBBBBCCCCCDDDDDEEEEEAAAAABBBBBCCCCCDDDDDEEEEE\r\n", + "--bnd--\r\n", + ) + .as_bytes() + .to_vec(); + + let message = mail_parser::MessageParser::new() + .parse(&raw) + .expect("parse valid MIME message"); + assert_eq!(message.attachment_count(), 1); + + // Truncate the raw body so the attachment's raw_end_offset lies + // past the body end — exactly the scenario reported by users. + let truncated = &raw[..raw.len() - 20]; + assert!(truncated.len() < raw.len()); + + // Must not panic. + let infos = super::detach_and_store_attachments( + truncated, + &message, + "test_content_hash", + ) + .await; + + // The attachment count must still match so the consistency check + // in reattach_eml_content doesn't fail later. + assert_eq!(infos.len(), 1); + } } diff --git a/crates/core/src/message/content.rs b/crates/core/src/message/content.rs index 7932a48..1f592bb 100644 --- a/crates/core/src/message/content.rs +++ b/crates/core/src/message/content.rs @@ -206,7 +206,9 @@ pub fn retrieve_email_content( content_type.c_subtype.as_deref().unwrap_or("") ); - let inline = disposition.map(|d| d.is_inline()).unwrap_or(false); + let inline = disposition + .map(|d| d.is_inline()) + .unwrap_or_else(|| attachment.content_id().is_some()); if inline { if let Some(html1) = html.as_deref() { @@ -302,7 +304,9 @@ pub fn retrieve_nested_eml_content( for attachment in nested_message.attachments() { let cid = attachment.content_id(); let disposition = attachment.content_disposition(); - let is_inline = disposition.map(|d| d.is_inline()).unwrap_or(false); + let is_inline = disposition + .map(|d| d.is_inline()) + .unwrap_or_else(|| cid.is_some()); if has_html && is_inline && cid.is_some() { let content_id = cid.unwrap(); diff --git a/crates/core/src/migrate/store.rs b/crates/core/src/migrate/store.rs index c453e9b..73d7bdb 100644 --- a/crates/core/src/migrate/store.rs +++ b/crates/core/src/migrate/store.rs @@ -86,13 +86,22 @@ pub fn detach_attachments_standalone( for (raw_start, raw_end, att) in ranges { let content_hash = compute_content_hash(att.contents()); - blobs.push(( - content_hash.clone(), - Bytes::copy_from_slice(&original_body[raw_start..raw_end]), - )); + let body_len = original_body.len(); + let raw_start = raw_start.min(body_len); + let raw_end = raw_end.min(body_len); + let range_valid = raw_start < raw_end; + + if range_valid { + blobs.push(( + content_hash.clone(), + Bytes::copy_from_slice(&original_body[raw_start..raw_end]), + )); + } - let placeholder = format!("<>", &content_hash); - stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + if range_valid { + let placeholder = format!("<>", &content_hash); + stripped_eml.splice(raw_start..raw_end, placeholder.as_bytes().iter().cloned()); + } infos.push(AttachmentInfo { filename: att.attachment_name().map(|n| n.to_string()), @@ -100,7 +109,7 @@ pub fn detach_attachments_standalone( inline: att .content_disposition() .map(|d| d.is_inline()) - .unwrap_or(false), + .unwrap_or_else(|| att.content_id().is_some()), file_type: att .content_type() .map(|ct| { From 1346dd216a7392278ad46750e7544db9a38623af Mon Sep 17 00:00:00 2001 From: rustmailer Date: Fri, 29 May 2026 16:43:38 +0800 Subject: [PATCH 16/66] fix: "No body available" #262 When the request returns an empty body, choose to skip it and print the account ID and UID information, leaving it for the user to investigate themselves. Otherwise, the IMAP download process will be blocked by this. --- crates/core/src/envelope/extractor.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index 978d0d6..a9b30b0 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -50,9 +50,17 @@ pub async fn extract_envelope_and_store_it( .map(|d| d.timestamp_millis()) .unwrap_or(0); let uid = fetch.uid.unwrap_or(0); - let body = fetch - .body() - .ok_or_else(|| raise_error!("No body available".into(), ErrorCode::InternalError))?; + let body = match fetch.body() { + Some(b) => b, + None => { + tracing::warn!( + account_id, + uid = fetch.uid, + "FETCH response has no body, skipping message" + ); + return Ok(()); + } + }; let size = fetch.size.unwrap_or(body.len() as u32); extract_envelope_core(body, uid, size, internal_date, account_id, mailbox_id).await } From e8469da3bc15d7040ae8c51ca2ee8a9cac78899a Mon Sep 17 00:00:00 2001 From: Lei Zhu Date: Sat, 30 May 2026 16:32:06 +0800 Subject: [PATCH 17/66] fix: Add fallback UIDVALIDITY support for non-compliant IMAP servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit adds support for IMAP servers that don't provide UIDVALIDITY, such as Tencent Enterprise Mail (腾讯企业邮箱). Changes: - Added `generate_synthetic_uidvalidity()` function that creates a stable hash-based UIDVALIDITY from the mailbox name - Modified `reconcile_mailboxes()` to use synthetic UIDVALIDITY when the server doesn't provide one - Servers without UIDVALIDITY can now sync all mailboxes including system folders (Sent Messages, Drafts, Deleted Messages) - Incremental sync is supported via the synthetic UIDVALIDITY - Added warning logs to indicate when synthetic UIDVALIDITY is in use - Updated mailbox metadata to store the resolved UIDVALIDITY Fixes issues with: - Tencent Enterprise Mail (腾讯企业邮箱) - Other non-compliant IMAP servers - Mailboxes that don't properly support UIDVALIDITY" --- crates/core/src/cache/imap/download/flow.rs | 53 +++++++++++++-------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index f74ba57..1f9beac 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -337,6 +337,17 @@ pub async fn fetch_and_save_full_mailbox( Ok(max_uid) } +/// Generates a synthetic UIDVALIDITY for IMAP servers that don't provide it. +/// Uses a stable hash of the mailbox name to ensure consistent IDs across sessions. +fn generate_synthetic_uidvalidity(mailbox_name: &str) -> u32 { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + mailbox_name.hash(&mut hasher); + (hasher.finish() as u32).wrapping_add(1) // Avoid 0, which might be reserved +} + pub async fn reconcile_mailboxes( account: &AccountModel, remote_mailboxes: &[MailBox], @@ -364,30 +375,30 @@ pub async fn reconcile_mailboxes( break; } - let new_highest_uid = if local_mailbox.uid_validity != remote_mailbox.uid_validity { - if remote_mailbox.uid_validity.is_none() { - let err_msg = format!( - "Mailbox '{}' logic error: Server did not provide UIDVALIDITY.", - local_mailbox.name + // Handle missing UIDVALIDITY from non-compliant IMAP servers + // (e.g., Tencent Enterprise Mail, etc.) + let remote_uid_validity = match remote_mailbox.uid_validity { + Some(uid) => uid, + None => { + // Generate a synthetic UIDVALIDITY based on mailbox name + let synthetic_uid = generate_synthetic_uidvalidity(&remote_mailbox.name); + + warn!( + "Account {}: Mailbox '{}' - Server did not provide UIDVALIDITY. \ + Using synthetic UIDVALIDITY {} based on mailbox name. \ + This mailbox will be synced but may require periodic rebuilds if the server's mailbox structure changes.", + account_id, remote_mailbox.name, synthetic_uid ); - - warn!("Account {}: {}", account_id, err_msg); - - DownloadState::update_folder_progress( - account_id, - remote_mailbox.name.clone(), - 0, - 0, - FolderStatus::Failed, - Some(err_msg.clone()), - )?; - DownloadState::append_session_error(account_id, err_msg)?; - continue; + + synthetic_uid } + }; + + let new_highest_uid = if local_mailbox.uid_validity != Some(remote_uid_validity) { info!( "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \ The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.", - account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_mailbox.uid_validity + account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_uid_validity ); DownloadState::update_folder_progress( @@ -441,6 +452,10 @@ pub async fn reconcile_mailboxes( let mut updated = remote_mailbox.clone(); updated.highest_uid = new_highest_uid; + // Update uid_validity with the resolved value (either from server or synthetic) + if updated.uid_validity.is_none() { + updated.uid_validity = Some(remote_uid_validity); + } mailboxes_to_update.push(updated); } //The metadata of this mailbox must only be updated after a successful synchronization; From 257736a47bfa10ecbbfb47091170d26809ed4039 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 31 May 2026 06:04:58 +0800 Subject: [PATCH 18/66] fix: add in-memory dedup cache to prevent duplicate emails before indexing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use (account_id, mailbox_id, content_hash) as dedup key with time-based eviction to bound memory at ~50MB. Check happens before mail parsing, so duplicate emails skip all expensive work entirely. - Populate cache from Tantivy FAST columns on startup (7-day window) - Evict oldest 1/4 of entries when exceeding 300K capacity - Graceful degradation: populate failure → empty cache, still works --- crates/core/src/envelope/extractor.rs | 9 +- crates/core/src/store/tantivy/dedup_cache.rs | 504 +++++++++++++++++++ crates/core/src/store/tantivy/mod.rs | 1 + 3 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 crates/core/src/store/tantivy/dedup_cache.rs diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index a9b30b0..db21626 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -26,6 +26,7 @@ use crate::imap::executor::ImapExecutor; use crate::message::content::AttachmentInfo; use crate::store::blob::{DetachedEmail, BLOB_MANAGER}; use crate::store::tantivy::attachment::ATTACHMENT_MANAGER; +use crate::store::tantivy::dedup_cache::DEDUP_CACHE; use crate::store::tantivy::envelope::ENVELOPE_MANAGER; use crate::store::tantivy::model::{AttachmentModel, EnvelopeWithAttachments}; use crate::utils::html::extract_text; @@ -99,6 +100,11 @@ async fn extract_envelope_core( ) -> BichonResult<()> { //The content hash of the original raw EML let email_content_hash = compute_content_hash(body); + if DEDUP_CACHE.contains(account_id, mailbox_id, &email_content_hash) { + tracing::debug!("Duplicate email detected"); + //println!("Duplicate email detected"); + return Ok(()); + } let message: Message<'_> = MessageParser::new().parse(body).ok_or_else(|| { raise_error!( "Email header parse result is not available".into(), @@ -260,7 +266,7 @@ async fn extract_envelope_core( tags: (!final_tags.is_empty()).then_some(final_tags), account_email: None, mailbox_name: None, - content_hash: email_content_hash, + content_hash: email_content_hash.clone(), }; // 'attachments' contains both regular and inline attachments let ea = EnvelopeWithAttachments { @@ -277,6 +283,7 @@ async fn extract_envelope_core( &ea.envelope.content_hash, ); ENVELOPE_MANAGER.queue(doc).await; + DEDUP_CACHE.insert(account_id, mailbox_id, &email_content_hash); for doc in attachment_docs { ATTACHMENT_MANAGER.queue(doc).await; } diff --git a/crates/core/src/store/tantivy/dedup_cache.rs b/crates/core/src/store/tantivy/dedup_cache.rs new file mode 100644 index 0000000..99641a7 --- /dev/null +++ b/crates/core/src/store/tantivy/dedup_cache.rs @@ -0,0 +1,504 @@ +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{LazyLock, Mutex}; + +use crate::store::tantivy::envelope::ENVELOPE_MANAGER; +use crate::store::tantivy::fields::{F_ACCOUNT_ID, F_CONTENT_HASH, F_INGEST_AT, F_MAILBOX_ID}; +use crate::utc_now; + +/// Max entries before evicting the oldest. +/// At ~152 bytes/entry, 300_000 ≈ 45 MB, within the 50 MB budget. +const MAX_ENTRIES: usize = 300_000; + +/// Fraction of entries to keep when evicting (newest 3/4). +const KEEP_FRACTION_NUM: usize = 3; +const KEEP_FRACTION_DEN: usize = 4; + +/// Populate only loads entries ingested within this window. +const POPULATE_WINDOW_MS: i64 = 7 * 24 * 60 * 60 * 1000; // 7 days + +pub static DEDUP_CACHE: LazyLock = LazyLock::new(DedupCache::new); + +pub struct DedupCache { + entries: Mutex>, + max_entries: usize, + populated: AtomicBool, +} + +impl DedupCache { + fn new() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries: MAX_ENTRIES, + populated: AtomicBool::new(false), + } + } + + #[cfg(test)] + fn new_for_test() -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries: MAX_ENTRIES, + populated: AtomicBool::new(true), + } + } + + #[cfg(test)] + fn new_for_test_small(max_entries: usize) -> Self { + Self { + entries: Mutex::new(HashMap::new()), + max_entries, + populated: AtomicBool::new(true), + } + } + + /// Returns true if this `(account_id, mailbox_id, content_hash)` triple + /// has already been seen. + /// + /// On the very first call the cache is populated from the Tantivy index + /// FAST columns (only entries ingested within [`POPULATE_WINDOW_MS`]). + /// If that scan fails the cache starts empty and still operates correctly + /// for newly-arriving emails. + pub fn contains(&self, account_id: u64, mailbox_id: u64, hash: &str) -> bool { + self.ensure_populated(); + + let entries = self.entries.lock().unwrap(); + entries.contains_key(&(account_id, mailbox_id, hash.to_string())) + } + + /// Insert a triple into the cache after it has been queued for indexing. + /// + /// Each entry is stamped with the current time. When the cache exceeds + /// [`MAX_ENTRIES`], the oldest entries are evicted, keeping the newest + /// `MAX_ENTRIES * 3/4`. + pub fn insert(&self, account_id: u64, mailbox_id: u64, hash: &str) { + let mut entries = self.entries.lock().unwrap(); + let now = utc_now!(); + entries.insert((account_id, mailbox_id, hash.to_string()), now); + + if entries.len() > self.max_entries { + let keep = self.max_entries * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN; + let mut vec: Vec<_> = entries.drain().collect(); + // Sort descending by timestamp (newest first) + vec.sort_by(|a, b| b.1.cmp(&a.1)); + for (k, v) in vec.into_iter().take(keep) { + entries.insert(k, v); + } + tracing::warn!( + "DedupCache evicted oldest entries, kept {}/{}", + entries.len(), + keep + ); + } + } + + // ── private ────────────────────────────────────────────────────────────── + + fn ensure_populated(&self) { + if self.populated.load(Ordering::Acquire) { + return; + } + self.do_populate(); + } + + fn do_populate(&self) { + if self + .populated + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Relaxed) + .is_err() + { + return; + } + + let reader = match ENVELOPE_MANAGER.create_reader() { + Ok(r) => r, + Err(e) => { + tracing::warn!("DedupCache: failed to create reader for populate: {e}"); + return; + } + }; + + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = self.entries.lock().unwrap(); + + for segment_reader in searcher.segment_readers() { + let account_col = match segment_reader.fast_fields().u64(F_ACCOUNT_ID) { + Ok(c) => c, + Err(_) => continue, + }; + let mailbox_col = match segment_reader.fast_fields().u64(F_MAILBOX_ID) { + Ok(c) => c, + Err(_) => continue, + }; + let hash_col = match segment_reader.fast_fields().str(F_CONTENT_HASH) { + Ok(Some(c)) => c, + _ => continue, + }; + let ingest_col = match segment_reader.fast_fields().i64(F_INGEST_AT) { + Ok(c) => c, + Err(_) => continue, + }; + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); + let mut hash_buf = String::new(); + if hash_col.ord_to_str(hash_ord, &mut hash_buf).is_err() { + continue; + } + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + + tracing::info!( + "DedupCache populated with {} entries from index (cutoff {}d ago)", + entries.len(), + POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000), + ); + } +} + +// ── Tests ───────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::tantivy::fields::EmailFields; + use crate::store::tantivy::schema::SchemaTools; + use crate::store::tantivy::tokenizers::EuroTokenizer; + use std::fs; + use tantivy::{Index, TantivyDocument}; + + fn temp_dir(name: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir() + .join("bichon-dedup-cache-test") + .join(name) + .join(uuid::Uuid::new_v4().to_string()); + fs::create_dir_all(&dir).unwrap(); + dir + } + + // ── basic contains / insert ───────────────────────────────────────────── + + #[test] + fn contains_after_insert() { + let cache = DedupCache::new_for_test(); + assert!(!cache.contains(1, 10, "hash-aaa")); + cache.insert(1, 10, "hash-aaa"); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + #[test] + fn different_hash_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(1, 10, "hash-bbb")); + } + + #[test] + fn different_account_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(2, 10, "hash-aaa")); + } + + #[test] + fn different_mailbox_not_matched() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + assert!(!cache.contains(1, 20, "hash-aaa")); + } + + #[test] + fn cross_account_allowed() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + cache.insert(2, 10, "hash-aaa"); + assert!(cache.contains(2, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + #[test] + fn cross_mailbox_allowed() { + let cache = DedupCache::new_for_test(); + cache.insert(1, 10, "hash-aaa"); + cache.insert(1, 20, "hash-aaa"); + assert!(cache.contains(1, 20, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-aaa")); + } + + // ── time-based eviction ───────────────────────────────────────────────── + + #[test] + fn eviction_keeps_newest() { + let cap = 100; + let cache = DedupCache::new_for_test_small(cap); + + // Fill to exact capacity. Entry hash-0 is oldest. + for i in 0..cap { + cache.insert(1, 1, &format!("hash-{}", i)); + std::thread::sleep(std::time::Duration::from_micros(100)); + } + assert!(cache.contains(1, 1, "hash-0")); + assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1))); + + // One more triggers eviction + cache.insert(1, 1, "hash-overflow"); + + // Newest survives, oldest evicted + assert!(cache.contains(1, 1, "hash-overflow")); + assert!(cache.contains(1, 1, &format!("hash-{}", cap - 1))); + assert!(!cache.contains(1, 1, "hash-0")); + + let keep = cap * KEEP_FRACTION_NUM / KEEP_FRACTION_DEN; + assert!(cache.entries.lock().unwrap().len() <= keep); + } + + // ── memory bound ──────────────────────────────────────────────────────── + + #[test] + fn memory_bound_within_budget() { + let cache = DedupCache::new_for_test(); + + for i in 0..MAX_ENTRIES { + cache.insert(1, 1, &format!("{:064x}", i)); + } + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), MAX_ENTRIES); + + let capacity = entries.capacity(); + // HashMap with (u64,u64,String) key + i64 value ≈ 112 + map overhead + let approx_bytes = capacity * (104 + 8 + 8); + let approx_mb = approx_bytes as f64 / (1024.0 * 1024.0); + println!( + "DedupCache: {} entries, {} buckets, ~{:.1} MB", + MAX_ENTRIES, capacity, approx_mb + ); + assert!( + approx_mb < 55.0, + "memory estimate {:.1} MB exceeds 55 MB buffer", + approx_mb + ); + } + + // ── populate guard ────────────────────────────────────────────────────── + + #[test] + fn populate_cas_is_idempotent() { + let cache = DedupCache::new_for_test(); + assert!(cache.populated.load(Ordering::Acquire)); + cache.ensure_populated(); + assert!(cache.populated.load(Ordering::Acquire)); + cache.do_populate(); + } + + // ── populate from test index ──────────────────────────────────────────── + + fn build_test_index() -> (Index, &'static EmailFields) { + let dir = temp_dir("populate"); + let schema = SchemaTools::email_schema(); + let fields = SchemaTools::email_fields(); + let index = Index::create_in_dir(&dir, schema).unwrap(); + index.tokenizers().register("euro", EuroTokenizer::new()); + (index, fields) + } + + fn add_email_doc( + fields: &EmailFields, + writer: &mut tantivy::IndexWriter, + account: u64, + mailbox: u64, + hash: &str, + ingest_at: i64, + ) { + let mut doc = TantivyDocument::new(); + doc.add_u64(fields.f_account_id, account); + doc.add_u64(fields.f_mailbox_id, mailbox); + doc.add_text(fields.f_content_hash, hash); + doc.add_i64(fields.f_ingest_at, ingest_at); + doc.add_text(fields.f_id, &uuid::Uuid::new_v4().to_string()); + doc.add_text(fields.f_subject, "test"); + doc.add_text(fields.f_body, "test body"); + doc.add_u64(fields.f_uid, 1); + doc.add_i64(fields.f_date, 1); + doc.add_i64(fields.f_internal_date, 1); + doc.add_u64(fields.f_size, 100); + writer.add_document(doc).unwrap(); + } + + #[test] + fn populate_reads_all_docs_in_window() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 2, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 1, 20, "hash-recent", recent); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert_eq!(cache.entries.lock().unwrap().len(), 3); + assert!(cache.contains(1, 10, "hash-recent")); + assert!(cache.contains(2, 10, "hash-recent")); + assert!(cache.contains(1, 20, "hash-recent")); + } + + #[test] + fn populate_skips_old_entries() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + let old = recent - POPULATE_WINDOW_MS - 60_000; // 1 minute past the window + add_email_doc(&fields, &mut writer, 1, 10, "hash-recent", recent); + add_email_doc(&fields, &mut writer, 1, 10, "hash-old", old); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert!(cache.contains(1, 10, "hash-recent")); + assert!(!cache.contains(1, 10, "hash-old")); + assert_eq!(cache.entries.lock().unwrap().len(), 1); + } + + #[test] + fn populate_skips_deleted_docs() { + let (index, fields) = build_test_index(); + let mut writer = index.writer_with_num_threads(1, 50_000_000).unwrap(); + + let recent = utc_now!(); + add_email_doc(&fields, &mut writer, 1, 10, "hash-keep", recent); + add_email_doc(&fields, &mut writer, 1, 10, "hash-delete", recent); + writer.commit().unwrap(); + + let term = tantivy::Term::from_field_text(fields.f_content_hash, "hash-delete"); + writer.delete_term(term); + writer.commit().unwrap(); + drop(writer); + + let reader = index.reader().unwrap(); + let cache = DedupCache::new_for_test(); + { + let searcher = reader.searcher(); + let cutoff = utc_now!() - POPULATE_WINDOW_MS; + let mut entries = cache.entries.lock().unwrap(); + entries.clear(); + + for segment_reader in searcher.segment_readers() { + let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); + let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); + let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); + + let max_doc = segment_reader.max_doc(); + for doc_id in 0..max_doc { + if segment_reader.is_deleted(doc_id) { + continue; + } + let ingest_at = ingest_col.values.get_val(doc_id); + if ingest_at < cutoff { + continue; + } + let account_id = account_col.values.get_val(doc_id); + let mailbox_id = mailbox_col.values.get_val(doc_id); + + let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let mut hash_buf = String::new(); + hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); + + entries.insert((account_id, mailbox_id, hash_buf), ingest_at); + } + } + } + + assert!(cache.contains(1, 10, "hash-keep")); + assert!(!cache.contains(1, 10, "hash-delete")); + } +} diff --git a/crates/core/src/store/tantivy/mod.rs b/crates/core/src/store/tantivy/mod.rs index 0e1460f..ce95719 100644 --- a/crates/core/src/store/tantivy/mod.rs +++ b/crates/core/src/store/tantivy/mod.rs @@ -25,6 +25,7 @@ use crate::{ pub mod attachment; pub mod dedup; +pub mod dedup_cache; pub mod envelope; pub mod fields; pub mod filter; From 4172f11f0070440449b56dc93ad4c7e8cf79d3ec Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 31 May 2026 06:05:41 +0800 Subject: [PATCH 19/66] Update Cargo.toml --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 267d925..ff5a365 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.4.2" +version = "1.4.3" edition = "2021" [workspace.dependencies] From 4d783d5301b512e268c781c84f44a76b869fecec Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 31 May 2026 06:05:50 +0800 Subject: [PATCH 20/66] Update Cargo.lock --- Cargo.lock | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5630146..62890f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.4.2" +version = "1.4.3" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.4.2" +version = "1.4.3" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.4.2" +version = "1.4.3" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.4.2" +version = "1.4.3" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.4.2" +version = "1.4.3" dependencies = [ "base64 0.22.1", "bichon-core", From a2a51a203791a3c9b620cd6df0345daac16acc25 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Wed, 3 Jun 2026 21:17:35 +0800 Subject: [PATCH 21/66] feat(imap): add message size check before download --- crates/admin/src/meta.rs | 1 + crates/core/src/account/migration.rs | 7 + crates/core/src/account/payload.rs | 2 + crates/core/src/account/view.rs | 2 + crates/core/src/cache/imap/download/flow.rs | 6 +- crates/core/src/imap/executor.rs | 133 ++++++++++++++++-- web/src/api/account/api.ts | 1 + web/src/components/layout/github.tsx | 40 +++++- .../accounts/components/account-detail.tsx | 4 + .../accounts/components/action-dialog.tsx | 5 +- .../features/accounts/components/schema.ts | 7 + .../features/accounts/components/step3.tsx | 32 +++++ .../features/accounts/components/step4.tsx | 11 +- web/src/locales/ar.json | 9 +- web/src/locales/da.json | 9 +- web/src/locales/de.json | 9 +- web/src/locales/en.json | 9 +- web/src/locales/es.json | 9 +- web/src/locales/fi.json | 9 +- web/src/locales/fr.json | 9 +- web/src/locales/it.json | 9 +- web/src/locales/jp.json | 9 +- web/src/locales/ko.json | 9 +- web/src/locales/nl.json | 9 +- web/src/locales/no.json | 9 +- web/src/locales/pl.json | 9 +- web/src/locales/pt.json | 9 +- web/src/locales/ru.json | 9 +- web/src/locales/sv.json | 9 +- web/src/locales/zh-tw.json | 9 +- web/src/locales/zh.json | 9 +- 31 files changed, 376 insertions(+), 37 deletions(-) diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index 4cf242c..f05496d 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -250,6 +250,7 @@ impl From for AccountModel { account_type: value.account_type, download_interval_min: value.sync_interval_min, download_batch_size: value.sync_batch_size, + max_email_size_bytes: None, known_folders: value.known_folders, created_at: value.created_at, updated_at: value.updated_at, diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 0f743e6..27c8bb0 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -84,6 +84,8 @@ pub struct Account { pub account_type: AccountType, pub download_interval_min: Option, pub download_batch_size: Option, + #[serde(default)] + pub max_email_size_bytes: Option, pub known_folders: Option>, pub created_at: i64, pub updated_at: i64, @@ -128,6 +130,7 @@ impl Account { pgp_key: request.pgp_key, created_by: user_id, download_batch_size: request.download_batch_size, + max_email_size_bytes: request.max_email_size_bytes, date_before: request.date_before, auto_download_new_mailboxes: request.auto_download_new_mailboxes, imap_quota_bytes: request.imap_quota_bytes, @@ -395,6 +398,10 @@ impl Account { new.download_batch_size = Some(*download_batch_size); } + if let Some(max_email_size_bytes) = request.max_email_size_bytes { + new.max_email_size_bytes = Some(max_email_size_bytes); + } + if let Some(use_proxy) = request.use_proxy { new.use_proxy = Some(use_proxy); } diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index 5fd0e6d..e796900 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -48,6 +48,7 @@ pub struct AccountCreateRequest { oai(validator(minimum(value = "10"), maximum(value = "200"))) )] pub download_batch_size: Option, + pub max_email_size_bytes: Option, pub use_proxy: Option, pub use_dangerous: bool, pub pgp_key: Option, @@ -165,6 +166,7 @@ pub struct AccountUpdateRequest { oai(validator(minimum(value = "10"), maximum(value = "200"))) )] pub download_batch_size: Option, + pub max_email_size_bytes: Option, /// Optional proxy ID for establishing the connection to external APIs (e.g., Gmail, Outlook). /// - If `None` or not provided, the client will connect directly to the API server. /// - If `Some(proxy_id)`, the client will use the pre-configured proxy with the given ID for API requests. diff --git a/crates/core/src/account/view.rs b/crates/core/src/account/view.rs index e3a568c..36e4f6e 100644 --- a/crates/core/src/account/view.rs +++ b/crates/core/src/account/view.rs @@ -44,6 +44,7 @@ pub struct AccountResp { pub account_type: AccountType, pub download_interval_min: Option, pub download_batch_size: Option, + pub max_email_size_bytes: Option, pub known_folders: Option>, pub created_at: i64, pub updated_at: i64, @@ -76,6 +77,7 @@ impl AccountResp { account_type: account.account_type, download_interval_min: account.download_interval_min, download_batch_size: account.download_batch_size, + max_email_size_bytes: account.max_email_size_bytes, known_folders: account.known_folders, created_at: account.created_at, updated_at: account.updated_at, diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index f74ba57..cdb3842 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -157,12 +157,13 @@ pub async fn fetch_and_save_by_date( account_id, mailbox.id, &batch.0, + account.max_email_size_bytes, token.clone(), ) .await { - Ok(_) => { - current_processed += batch.1; + Ok(processed) => { + current_processed += processed; DownloadState::update_folder_progress( account_id, mailbox.name.clone(), @@ -290,6 +291,7 @@ pub async fn fetch_and_save_full_mailbox( page as u64, page_size as u64, &mailbox.encoded_name(), + account.max_email_size_bytes, token.clone(), &mut max_uid, ) diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 1b86c3e..08ef14f 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -32,6 +32,7 @@ use tokio_util::sync::CancellationToken; use tracing::info; const BODY_FETCH_COMMAND: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[])"; +const SIZE_ONLY_FETCH: &str = "(UID RFC822.SIZE)"; pub struct ImapExecutor; @@ -183,15 +184,16 @@ impl ImapExecutor { ErrorCode::InternalError )); } - Self::uid_batch_retrieve_emails( + let processed = Self::uid_batch_retrieve_emails( session, account.id, mailbox.id, &batch.0, + account.max_email_size_bytes, token.clone(), ) .await?; - count += batch.1; + count += processed; DownloadState::update_folder_progress( account.id, mailbox.name.clone(), @@ -239,7 +241,9 @@ impl ImapExecutor { })?; let mut count = 0u64; + let mut skipped = 0u64; let mut max_uid: Option = None; + let size_limit = account.max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); while let Some(fetch) = stream .try_next() .await @@ -258,6 +262,20 @@ impl ImapExecutor { )); } + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size > 0 && msg_size > size_limit { + tracing::warn!( + account_id = account.id, + mailbox_id = mailbox.id, + uid = fetch.uid, + size = msg_size, + limit = size_limit, + "Skipping oversized email (streaming mode)" + ); + skipped += 1; + continue; + } + if let Some(uid) = fetch.uid { max_uid = Some(max_uid.unwrap_or(0).max(uid)); } @@ -265,7 +283,8 @@ impl ImapExecutor { count += 1; } - if count == 0 { + let total = count + skipped; + if total == 0 { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), @@ -278,10 +297,14 @@ impl ImapExecutor { DownloadState::update_folder_progress( account.id, mailbox.name.clone(), - count, + total, count, FolderStatus::Success, - None, + if skipped > 0 { + Some(format!("{skipped} email(s) skipped due to size limit")) + } else { + None + }, )?; } @@ -296,6 +319,7 @@ impl ImapExecutor { page: u64, page_size: u64, encoded_mailbox_name: &str, + max_email_size_bytes: Option, token: CancellationToken, max_uid: &mut Option, ) -> BichonResult { @@ -315,13 +339,52 @@ impl ImapExecutor { encoded_mailbox_name, sequence_set, page, page_size ); - let mut stream = session - .fetch(sequence_set.as_str(), BODY_FETCH_COMMAND) + let limit = max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + + // PASS 1: fetch only SIZE to identify oversized messages + let acceptable_uids = { + let mut size_stream = session + .fetch(sequence_set.as_str(), SIZE_ONLY_FETCH) + .await + .map_err(|e| { + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })?; + + let mut uids: Vec = Vec::new(); + while let Some(fetch) = size_stream.try_next().await.map_err(|e| { + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })? { + let uid = fetch.uid.unwrap_or(0); + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size == 0 || msg_size <= limit { + uids.push(uid); + } else { + tracing::warn!( + account_id, + mailbox_id, + uid, + size = msg_size, + limit, + "Skipping oversized email" + ); + } + } + uids + }; + + if acceptable_uids.is_empty() { + return Ok(0); + } + + // PASS 2: fetch bodies only for acceptable UIDs + let filtered = compress_uid_list(acceptable_uids); + let mut body_stream = session + .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; let mut count = 0; - while let Some(fetch) = stream + while let Some(fetch) = body_stream .try_next() .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? @@ -347,13 +410,55 @@ impl ImapExecutor { account_id: u64, mailbox_id: u64, uid_set: &str, + max_email_size_bytes: Option, token: CancellationToken, - ) -> BichonResult<()> { - let mut stream = session - .uid_fetch(uid_set, BODY_FETCH_COMMAND) + ) -> BichonResult { + let limit = max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + + // PASS 1: fetch only SIZE to identify oversized messages + let acceptable_uids = { + let mut size_stream = session + .uid_fetch(uid_set, SIZE_ONLY_FETCH) + .await + .map_err(|e| { + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })?; + + let mut uids: Vec = Vec::new(); + while let Some(fetch) = size_stream.try_next().await.map_err(|e| { + raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + })? { + let uid = fetch.uid.unwrap_or(0); + let msg_size = fetch.size.unwrap_or(0) as u64; + if msg_size == 0 || msg_size <= limit { + uids.push(uid); + } else { + tracing::warn!( + account_id, + mailbox_id, + uid, + size = msg_size, + limit, + "Skipping oversized email" + ); + } + } + uids + }; + + if acceptable_uids.is_empty() { + return Ok(0); + } + + // PASS 2: fetch bodies only for acceptable UIDs + let filtered = compress_uid_list(acceptable_uids); + let mut body_stream = session + .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; - while let Some(fetch) = stream + + let mut count = 0u64; + while let Some(fetch) = body_stream .try_next() .await .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? @@ -366,8 +471,9 @@ impl ImapExecutor { )); } extract_envelope_and_store_it(fetch, account_id, mailbox_id).await?; + count += 1; } - Ok(()) + Ok(count) } /// Fetches the raw RFC822 body of a single message by UID. @@ -430,6 +536,7 @@ impl ImapExecutor { } pub const DEFAULT_BATCH_SIZE: u32 = 30; +pub const DEFAULT_MAX_EMAIL_SIZE: u64 = 100 * 1024 * 1024; /// Compresses a sorted list of UIDs into an IMAP sequence-set string. /// Consecutive UIDs become ranges (e.g. `1:5`), non-consecutive are diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index baa6794..cd1f4bb 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -131,6 +131,7 @@ export interface AccountModel { download_folders: string[]; download_interval_min?: number; download_batch_size?: number; + max_email_size_bytes?: number; created_by: number; created_user_name: string; created_user_email: string; diff --git a/web/src/components/layout/github.tsx b/web/src/components/layout/github.tsx index b308493..cec10d1 100644 --- a/web/src/components/layout/github.tsx +++ b/web/src/components/layout/github.tsx @@ -28,18 +28,54 @@ interface GithubLinkButtonProps { title?: string; } +const CACHE_KEY = "github_stars_cache"; +const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours + +interface StarsCache { + stars: number; + fetchedAt: number; +} + +function getCachedStars(repo: string): number | null { + try { + const raw = localStorage.getItem(`${CACHE_KEY}_${repo}`); + if (!raw) return null; + const cache: StarsCache = JSON.parse(raw); + if (Date.now() - cache.fetchedAt > CACHE_TTL) return null; + return cache.stars; + } catch { + return null; + } +} + +function setCachedStars(repo: string, stars: number) { + try { + localStorage.setItem( + `${CACHE_KEY}_${repo}`, + JSON.stringify({ stars, fetchedAt: Date.now() }) + ); + } catch { } +} + export const GithubLinkButton: React.FC = ({ href = "https://github.com/rustmailer/bichon", repo = "rustmailer/bichon", size = 18, title = "View on GitHub", }) => { - const [stars, setStars] = useState(null); + const [stars, setStars] = useState(() => getCachedStars(repo)); useEffect(() => { + if (stars !== null) return; // already have cached value, skip fetch fetch(`https://api.github.com/repos/${repo}`) .then(res => res.json()) - .then(data => setStars(data.stargazers_count)) + .then(data => { + const count = data.stargazers_count; + if (typeof count === "number") { + setStars(count); + setCachedStars(repo, count); + } + }) .catch(() => { }); }, [repo]); diff --git a/web/src/features/accounts/components/account-detail.tsx b/web/src/features/accounts/components/account-detail.tsx index 918ab11..302ee9f 100644 --- a/web/src/features/accounts/components/account-detail.tsx +++ b/web/src/features/accounts/components/account-detail.tsx @@ -97,6 +97,10 @@ export function AccountDetailDrawer({ open, onOpenChange, currentRow }: Props) { {t('accounts.downloadBatchSize')}: {currentRow.download_batch_size} +
+ {t('accounts.maxEmailSizeBytes')}: + {currentRow.max_email_size_bytes ? `${(currentRow.max_email_size_bytes / 1024 / 1024).toFixed(0)} MB` : t('accounts.maxEmailSizeBytesUnlimited')} +
{t('accounts.capabilities')}: diff --git a/web/src/features/accounts/components/action-dialog.tsx b/web/src/features/accounts/components/action-dialog.tsx index ac7ef2f..619a292 100644 --- a/web/src/features/accounts/components/action-dialog.tsx +++ b/web/src/features/accounts/components/action-dialog.tsx @@ -49,7 +49,7 @@ export type Steps = [...Step[]]; const getSteps = (t: (key: string) => string): Steps => [ { id: "step-1", name: t('accounts.steps.emailAddress'), fields: ["email", "account_name"] }, { id: "step-2", name: t('accounts.steps.imap'), fields: ["imap", "use_dangerous", "login_name"] }, - { id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "download_interval_min", "download_batch_size", "auto_download_new_mailboxes", "download_schedule"] }, + { id: "step-3", name: t('accounts.steps.syncPreferences'), fields: ["enabled", "date_since", "date_before", "download_interval_min", "download_batch_size", "max_email_size_bytes", "auto_download_new_mailboxes", "download_schedule"] }, { id: "step-4", name: t('accounts.steps.summary'), fields: [] }, ]; @@ -81,6 +81,7 @@ const defaultValues: Account = { date_before: undefined, download_interval_min: 60, download_batch_size: 30, + max_email_size_bytes: 100 * 1024 * 1024, auto_download_new_mailboxes: true, download_schedule: undefined, }; @@ -111,6 +112,7 @@ const mapCurrentRowToFormValues = (currentRow: AccountModel): Account => { date_before: currentRow.date_before ?? undefined, download_interval_min: currentRow.download_interval_min ?? 60, download_batch_size: currentRow.download_batch_size ?? 30, + max_email_size_bytes: currentRow.max_email_size_bytes ?? 100 * 1024 * 1024, auto_download_new_mailboxes: currentRow.auto_download_new_mailboxes ?? true, download_schedule: currentRow.download_schedule ?? undefined, }; @@ -193,6 +195,7 @@ export function AccountActionDialog({ currentRow, open, onOpenChange }: Props) { date_before: data.date_before, download_interval_min: data.download_interval_min, download_batch_size: data.download_batch_size, + max_email_size_bytes: data.max_email_size_bytes, auto_download_new_mailboxes: data.auto_download_new_mailboxes, download_schedule: data.download_schedule || null, }; diff --git a/web/src/features/accounts/components/schema.ts b/web/src/features/accounts/components/schema.ts index 50d6cdf..80dc935 100644 --- a/web/src/features/accounts/components/schema.ts +++ b/web/src/features/accounts/components/schema.ts @@ -100,6 +100,13 @@ export const getAccountSchema = (isEdit: boolean, t: (key: string) => string) => .max(200, { message: t('validation.singleRequestBatchSizeTooLarge'), }), + max_email_size_bytes: z + .number({ + invalid_type_error: t('validation.maxEmailSizeMustBeNumber'), + }) + .int() + .min(1 * 1024 * 1024, { message: t('validation.maxEmailSizeTooSmall') }) + .max(100 * 1024 * 1024, { message: t('validation.maxEmailSizeTooLarge') }), auto_download_new_mailboxes: z.boolean(), download_schedule: z .string() diff --git a/web/src/features/accounts/components/step3.tsx b/web/src/features/accounts/components/step3.tsx index bd44a46..558ce8a 100644 --- a/web/src/features/accounts/components/step3.tsx +++ b/web/src/features/accounts/components/step3.tsx @@ -392,6 +392,38 @@ export default function Step3() { )} /> + { + const BYTES_PER_MB = 1024 * 1024; + + return ( + + {t('accounts.maxEmailSizeBytes')} + +
+ { + const parsed = parseInt(e.target.value, 10); + field.onChange(isNaN(parsed) ? parsed : parsed * BYTES_PER_MB); + }} + /> + MB +
+ + + + {t('accounts.maxEmailSizeBytesDescription')} + + + ); + }} + />
diff --git a/web/src/features/accounts/components/step4.tsx b/web/src/features/accounts/components/step4.tsx index 07276a4..adcb128 100644 --- a/web/src/features/accounts/components/step4.tsx +++ b/web/src/features/accounts/components/step4.tsx @@ -50,7 +50,11 @@ export default function Step4() { return (
- + {t('accounts.email')}: {summaryData.email} @@ -164,6 +168,11 @@ export default function Step4() { {summaryData.download_batch_size} + + {t('accounts.maxEmailSizeBytes')}: + {summaryData.max_email_size_bytes ? `${(summaryData.max_email_size_bytes / 1024 / 1024).toFixed(0)} MB` : t('accounts.maxEmailSizeBytesUnlimited')} + + {t('accounts.downloadSchedule')}: {summaryData.download_schedule || t('accounts.notAvailable')} diff --git a/web/src/locales/ar.json b/web/src/locales/ar.json index 0eb5f8f..931cfb5 100644 --- a/web/src/locales/ar.json +++ b/web/src/locales/ar.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "اترك فارغًا للاحتفاظ بكلمة المرور الحالية، أو أدخل كلمة مرور جديدة لتحديثها.", "leaveEmptyToKeepPassword": "اترك فارغًا للاحتفاظ بكلمة المرور الحالية", "login_name": "اسم الدخول", + "maxEmailSizeBytes": "الحد الأقصى لحجم البريد", + "maxEmailSizeBytesDescription": "سيتم تخطي الرسائل الأكبر من هذا الحجم. اتركه فارغاً لاستخدام الحد الافتراضي (100 ميجابايت).", + "maxEmailSizeBytesPlaceholder": "الافتراضي: 100 ميجابايت", + "maxEmailSizeBytesUnlimited": "الافتراضي: 100 ميجابايت", "minutes": "دقائق", "months": "أشهر", "mustBeAtLeast1": "يجب أن يكون 1 على الأقل", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "عنوان بريد إلكتروني غير صالح", "invalidUrl": "عنوان URL غير صالح", + "maxEmailSizeMustBeNumber": "يجب أن يكون الحد الأقصى لحجم البريد رقماً.", + "maxEmailSizeTooLarge": "يجب ألا يتجاوز الحد الأقصى لحجم البريد 100 ميجابايت.", + "maxEmailSizeTooSmall": "يجب أن يكون الحد الأقصى لحجم البريد 1 ميجابايت على الأقل.", "passwordMinLength": "يجب أن تتكون كلمة المرور من {{min}} أحرف على الأقل", "passwordRequired": "كلمة المرور مطلوبة عندما تكون طريقة المصادقة هي كلمة المرور", "pleaseEnterPassword": "الرجاء إدخال كلمة المرور الخاصة بك", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "يجب أن يكون حجم الدُفعة على الأكثر 200", "singleRequestBatchSizeTooSmall": "يجب أن يكون حجم الدُفعة على الأقل 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/da.json b/web/src/locales/da.json index e46bc43..7fdcd08 100644 --- a/web/src/locales/da.json +++ b/web/src/locales/da.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Lad stå tomt for at beholde den eksisterende adgangskode, eller indtast en ny for at opdatere den.", "leaveEmptyToKeepPassword": "Lad stå tomt for at beholde nuværende adgangskode", "login_name": "Logindnavn", + "maxEmailSizeBytes": "Maks. e-mailstørrelse", + "maxEmailSizeBytesDescription": "E-mails større end dette vil blive oversprunget. Lad være tom for at bruge standarden (100 MB).", + "maxEmailSizeBytesPlaceholder": "Standard: 100 MB", + "maxEmailSizeBytesUnlimited": "Standard: 100 MB", "minutes": "minutter", "months": "Måneder", "mustBeAtLeast1": "Skal være mindst 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Ugyldig e-mailadresse", "invalidUrl": "Ugyldig URL", + "maxEmailSizeMustBeNumber": "Maks. e-mailstørrelse skal være et tal.", + "maxEmailSizeTooLarge": "Maks. e-mailstørrelse må ikke overstige 100 MB.", + "maxEmailSizeTooSmall": "Maks. e-mailstørrelse skal være mindst 1 MB.", "passwordMinLength": "Adgangskoden skal være mindst {{min}} tegn lang", "passwordRequired": "Adgangskode er påkrævet, når godkendelsesmetoden er Adgangskode", "pleaseEnterPassword": "Indtast venligst din adgangskode", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Batch‑størrelse skal være højst 200", "singleRequestBatchSizeTooSmall": "Batch‑størrelse skal være mindst 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 8f40d77..2dab0dc 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Leer lassen, um das bestehende Passwort beizubehalten, oder einen neuen Wert eingeben, um es zu aktualisieren.", "leaveEmptyToKeepPassword": "Leer lassen, um das aktuelle Passwort beizubehalten", "login_name": "Anmeldename", + "maxEmailSizeBytes": "Max. E-Mail-Größe", + "maxEmailSizeBytesDescription": "Größere E-Mails werden übersprungen. Leer lassen, um den Standardwert (100 MB) zu verwenden.", + "maxEmailSizeBytesPlaceholder": "Standard: 100 MB", + "maxEmailSizeBytesUnlimited": "Standard: 100 MB", "minutes": "Minuten", "months": "Monate", "mustBeAtLeast1": "Muss mindestens 1 sein", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Ungültige E-Mail-Adresse", "invalidUrl": "Ungültige URL", + "maxEmailSizeMustBeNumber": "Die maximale E-Mail-Größe muss eine Zahl sein.", + "maxEmailSizeTooLarge": "Die maximale E-Mail-Größe darf 100 MB nicht überschreiten.", + "maxEmailSizeTooSmall": "Die maximale E-Mail-Größe muss mindestens 1 MB betragen.", "passwordMinLength": "Das Passwort muss mindestens {{min}} Zeichen lang sein", "passwordRequired": "Passwort ist erforderlich, wenn die Authentifizierungsmethode Passwort ist", "pleaseEnterPassword": "Bitte geben Sie Ihr Passwort ein", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Die Stapelgröße darf höchstens 200 sein", "singleRequestBatchSizeTooSmall": "Die Stapelgröße muss mindestens 10 sein" } -} +} \ No newline at end of file diff --git a/web/src/locales/en.json b/web/src/locales/en.json index cad305d..a4e1b50 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Leave empty to keep the existing password, or enter a new password to update it.", "leaveEmptyToKeepPassword": "Leave empty to keep current password", "login_name": "Login Name", + "maxEmailSizeBytes": "Max email size", + "maxEmailSizeBytesDescription": "Emails larger than this will be skipped. Leave empty to use the default (100 MB).", + "maxEmailSizeBytesPlaceholder": "Default: 100 MB", + "maxEmailSizeBytesUnlimited": "Default: 100 MB", "minutes": "minutes", "months": "Months", "mustBeAtLeast1": "Must be at least 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Invalid email address", "invalidUrl": "Invalid URL", + "maxEmailSizeMustBeNumber": "Max email size must be a number.", + "maxEmailSizeTooLarge": "Max email size must not exceed 100 MB.", + "maxEmailSizeTooSmall": "Max email size must be at least 1 MB.", "passwordMinLength": "Password must be at least {{min}} characters long", "passwordRequired": "Password is required when auth method is Password", "pleaseEnterPassword": "Please enter your password", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Batch size must be at most 200", "singleRequestBatchSizeTooSmall": "Batch size must be at least 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/es.json b/web/src/locales/es.json index d2db893..47b97f4 100644 --- a/web/src/locales/es.json +++ b/web/src/locales/es.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Deja vacío para mantener la contraseña existente, o introduce un nuevo valor para actualizarla.", "leaveEmptyToKeepPassword": "Deja vacío para mantener la contraseña actual", "login_name": "Nombre de usuario", + "maxEmailSizeBytes": "Tamaño máx. de correo", + "maxEmailSizeBytesDescription": "Se omitirán los correos más grandes. Déjelo vacío para usar el valor predeterminado (100 MB).", + "maxEmailSizeBytesPlaceholder": "Predeterminado: 100 MB", + "maxEmailSizeBytesUnlimited": "Predeterminado: 100 MB", "minutes": "minutos", "months": "Meses", "mustBeAtLeast1": "Debe ser al menos 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Dirección de correo electrónico inválida", "invalidUrl": "URL inválida", + "maxEmailSizeMustBeNumber": "El tamaño máximo de correo debe ser un número.", + "maxEmailSizeTooLarge": "El tamaño máximo de correo no debe superar los 100 MB.", + "maxEmailSizeTooSmall": "El tamaño máximo de correo debe ser de al menos 1 MB.", "passwordMinLength": "La contraseña debe tener al menos {{min}} caracteres", "passwordRequired": "La contraseña es obligatoria cuando el método de autenticación es Contraseña", "pleaseEnterPassword": "Por favor, introduce tu contraseña", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "El tamaño del lote debe ser como máximo 200", "singleRequestBatchSizeTooSmall": "El tamaño del lote debe ser al menos 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/fi.json b/web/src/locales/fi.json index edca7ae..f5e1820 100644 --- a/web/src/locales/fi.json +++ b/web/src/locales/fi.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Jätä tyhjäksi säilyttääksesi olemassa olevan salasanan, tai syötä uusi päivittääksesi sen.", "leaveEmptyToKeepPassword": "Jätä tyhjäksi säilyttääksesi nykyisen salasanan", "login_name": "Kirjautumisnimi", + "maxEmailSizeBytes": "Sähköpostin maksimikoko", + "maxEmailSizeBytesDescription": "Tätä suuremmat sähköpostit ohitetaan. Jätä tyhjäksi käyttääksesi oletusarvoa (100 MB).", + "maxEmailSizeBytesPlaceholder": "Oletus: 100 MB", + "maxEmailSizeBytesUnlimited": "Oletus: 100 MB", "minutes": "minuuttia", "months": "Kuukautta", "mustBeAtLeast1": "Täytyy olla vähintään 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Virheellinen sähköpostiosoite", "invalidUrl": "Virheellinen URL-osoite", + "maxEmailSizeMustBeNumber": "Sähköpostin maksimikoon on oltava numero.", + "maxEmailSizeTooLarge": "Sähköpostin maksimikoko ei saa ylittää 100 megatavua.", + "maxEmailSizeTooSmall": "Sähköpostin maksimikoon on oltava vähintään 1 MB.", "passwordMinLength": "Salasanan on oltava vähintään {{min}} merkkiä pitkä", "passwordRequired": "Salasana on pakollinen, kun todennusmenetelmä on Salasana", "pleaseEnterPassword": "Syötä salasanasi", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Eräkoko tulee olla enintään 200", "singleRequestBatchSizeTooSmall": "Eräkoko tulee olla vähintään 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/fr.json b/web/src/locales/fr.json index 1b9f590..b553036 100644 --- a/web/src/locales/fr.json +++ b/web/src/locales/fr.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Laissez vide pour conserver le mot de passe existant, ou entrez-en un nouveau pour le mettre à jour.", "leaveEmptyToKeepPassword": "Laisser vide pour conserver le mot de passe actuel", "login_name": "Nom de connexion", + "maxEmailSizeBytes": "Taille max. des e-mails", + "maxEmailSizeBytesDescription": "Les e-mails plus grands seront ignorés. Laisser vide pour utiliser la valeur par défaut (100 MB).", + "maxEmailSizeBytesPlaceholder": "Par défaut : 100 MB", + "maxEmailSizeBytesUnlimited": "Par défaut : 100 MB", "minutes": "minutes", "months": "Mois", "mustBeAtLeast1": "Doit être au moins 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Adresse e-mail non valide", "invalidUrl": "URL non valide", + "maxEmailSizeMustBeNumber": "La taille maximale des e-mails doit être un nombre.", + "maxEmailSizeTooLarge": "La taille maximale des e-mails ne doit pas dépasser 100 MB.", + "maxEmailSizeTooSmall": "La taille maximale des e-mails doit être d'au moins 1 MB.", "passwordMinLength": "Le mot de passe doit contenir au moins {{min}} caractères", "passwordRequired": "Le mot de passe est obligatoire lorsque la méthode d'authentification est Mot de passe", "pleaseEnterPassword": "Veuillez entrer votre mot de passe", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "La taille du lot doit être au plus 200", "singleRequestBatchSizeTooSmall": "La taille du lot doit être au moins 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/it.json b/web/src/locales/it.json index fc6522d..42b3c30 100644 --- a/web/src/locales/it.json +++ b/web/src/locales/it.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Lascia vuoto per mantenere la password esistente, o inseriscine una nuova per aggiornarla.", "leaveEmptyToKeepPassword": "Lascia vuoto per mantenere la password attuale", "login_name": "Nome di accesso", + "maxEmailSizeBytes": "Dimensione massima email", + "maxEmailSizeBytesDescription": "Le email più grandi saranno ignorate. Lascia vuoto per utilizzare il valore predefinito (100 MB).", + "maxEmailSizeBytesPlaceholder": "Predefinito: 100 MB", + "maxEmailSizeBytesUnlimited": "Predefinito: 100 MB", "minutes": "minuti", "months": "Mesi", "mustBeAtLeast1": "Deve essere almeno 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Indirizzo email non valido", "invalidUrl": "URL non valido", + "maxEmailSizeMustBeNumber": "La dimensione massima dell'email deve essere un numero.", + "maxEmailSizeTooLarge": "La dimensione massima dell'email non deve superare i 100 MB.", + "maxEmailSizeTooSmall": "La dimensione massima dell'email deve essere di almeno 1 MB.", "passwordMinLength": "La password deve contenere almeno {{min}} caratteri", "passwordRequired": "La password è obbligatoria quando il metodo di autenticazione è Password", "pleaseEnterPassword": "Inserisci la tua password", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "La dimensione del batch deve essere al massimo 200", "singleRequestBatchSizeTooSmall": "La dimensione del batch deve essere almeno 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/jp.json b/web/src/locales/jp.json index 0864ff5..438ddcd 100644 --- a/web/src/locales/jp.json +++ b/web/src/locales/jp.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "既存のパスワードを保持する場合は空欄にしてください。更新する場合は新しいパスワードを入力してください。", "leaveEmptyToKeepPassword": "現在のパスワードを保持する場合は空欄にしてください", "login_name": "ログイン名", + "maxEmailSizeBytes": "最大メールサイズ", + "maxEmailSizeBytesDescription": "これより大きいメールはスキップされます。空欄にするとデフォルト（100 MB）が使用されます。", + "maxEmailSizeBytesPlaceholder": "デフォルト：100 MB", + "maxEmailSizeBytesUnlimited": "デフォルト：100 MB", "minutes": "分", "months": "月", "mustBeAtLeast1": "1以上である必要があります", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "無効なメールアドレスです", "invalidUrl": "無効なURLです", + "maxEmailSizeMustBeNumber": "最大メールサイズは数値で入力してください。", + "maxEmailSizeTooLarge": "最大メールサイズは 100 MB 以下にしてください。", + "maxEmailSizeTooSmall": "最大メールサイズは 1 MB 以上にしてください。", "passwordMinLength": "パスワードは{{min}}文字以上である必要があります", "passwordRequired": "認証方式がパスワードの場合、パスワードは必須です", "pleaseEnterPassword": "パスワードを入力してください", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "バッチサイズは最大でも200でなければなりません", "singleRequestBatchSizeTooSmall": "バッチサイズは最低でも10でなければなりません" } -} +} \ No newline at end of file diff --git a/web/src/locales/ko.json b/web/src/locales/ko.json index 69685d6..77e17ff 100644 --- a/web/src/locales/ko.json +++ b/web/src/locales/ko.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "기존 비밀번호를 유지하려면 비워 두십시오. 업데이트할 경우에만 새 비밀번호를 입력하십시오.", "leaveEmptyToKeepPassword": "현재 비밀번호를 유지하려면 비워 두십시오", "login_name": "로그인 이름", + "maxEmailSizeBytes": "최대 이메일 크기", + "maxEmailSizeBytesDescription": "이보다 큰 이메일은 건너뜁니다. 기본값(100 MB)을 사용하려면 비워두세요.", + "maxEmailSizeBytesPlaceholder": "기본값: 100 MB", + "maxEmailSizeBytesUnlimited": "기본값: 100 MB", "minutes": "분", "months": "개월", "mustBeAtLeast1": "최소 1 이상이어야 합니다", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "유효하지 않은 이메일 주소", "invalidUrl": "유효하지 않은 URL", + "maxEmailSizeMustBeNumber": "최대 이메일 크기는 숫자여야 합니다.", + "maxEmailSizeTooLarge": "최대 이메일 크기는 100 MB를 초과할 수 없습니다.", + "maxEmailSizeTooSmall": "최대 이메일 크기는 최소 1 MB여야 합니다.", "passwordMinLength": "비밀번호는 {{min}}자 이상이어야 합니다", "passwordRequired": "인증 방법이 비밀번호인 경우 비밀번호는 필수입니다", "pleaseEnterPassword": "비밀번호를 입력하십시오", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "배치 크기는 최대 200이어야 합니다", "singleRequestBatchSizeTooSmall": "배치 크기는 최소 10이어야 합니다" } -} +} \ No newline at end of file diff --git a/web/src/locales/nl.json b/web/src/locales/nl.json index b69509c..396e2e1 100644 --- a/web/src/locales/nl.json +++ b/web/src/locales/nl.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Laat leeg om het bestaande wachtwoord te behouden, of voer een nieuw wachtwoord in om het bij te werken.", "leaveEmptyToKeepPassword": "Laat leeg om huidig wachtwoord te behouden", "login_name": "Inlognaam", + "maxEmailSizeBytes": "Max. e-mailgrootte", + "maxEmailSizeBytesDescription": "E-mails groter dan dit worden overgeslagen. Laat leeg om de standaard (100 MB) te gebruiken.", + "maxEmailSizeBytesPlaceholder": "Standaard: 100 MB", + "maxEmailSizeBytesUnlimited": "Standaard: 100 MB", "minutes": "minuten", "months": "Maanden", "mustBeAtLeast1": "Moet ten minste 1 zijn", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Ongeldig e-mailadres", "invalidUrl": "Ongeldige URL", + "maxEmailSizeMustBeNumber": "Maximale e-mailgrootte moet un nummer zijn.", + "maxEmailSizeTooLarge": "Maximale e-mailgrootte mag niet groter zijn dan 100 MB.", + "maxEmailSizeTooSmall": "Maximale e-mailgrootte moet minstens 1 MB zijn.", "passwordMinLength": "Wachtwoord moet ten minste {{min}} tekens lang zijn", "passwordRequired": "Wachtwoord is vereist wanneer de authenticatiemethode Wachtwoord is", "pleaseEnterPassword": "Voer uw wachtwoord in", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Batch‑grootte moet hoogstens 200 zijn", "singleRequestBatchSizeTooSmall": "Batch‑grootte moet ten minste 10 zijn" } -} +} \ No newline at end of file diff --git a/web/src/locales/no.json b/web/src/locales/no.json index 9078ff1..6e7d619 100644 --- a/web/src/locales/no.json +++ b/web/src/locales/no.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "La stå tomt for å beholde det eksisterende passordet, eller skriv inn et nytt passord for å oppdatere det.", "leaveEmptyToKeepPassword": "La stå tomt for å beholde nåværende passord", "login_name": "Påloggingsnavn", + "maxEmailSizeBytes": "Maks. e-poststørrelse", + "maxEmailSizeBytesDescription": "E-poster større enn dette vil bli hoppet over. La stå tom for å bruke standarden (100 MB).", + "maxEmailSizeBytesPlaceholder": "Standard: 100 MB", + "maxEmailSizeBytesUnlimited": "Standard: 100 MB", "minutes": "minutter", "months": "Måneder", "mustBeAtLeast1": "Må være minst 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Ugyldig e-postadresse", "invalidUrl": "Ugyldig URL", + "maxEmailSizeMustBeNumber": "Maks. e-poststørrelse må være et tall.", + "maxEmailSizeTooLarge": "Maks. e-poststørrelse må ikke overstige 100 MB.", + "maxEmailSizeTooSmall": "Maks. e-poststørrelse må være minst 1 MB.", "passwordMinLength": "Passordet må være minst {{min}} tegn langt", "passwordRequired": "Passord er påkrevd når autentiseringsmetoden er Passord", "pleaseEnterPassword": "Vennligst skriv inn passordet ditt", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Batchstørrelse må være maksimalt 200", "singleRequestBatchSizeTooSmall": "Batchstørrelse må være minst 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/pl.json b/web/src/locales/pl.json index 62d9794..65999c9 100644 --- a/web/src/locales/pl.json +++ b/web/src/locales/pl.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Pozostaw to pole puste, jeśli chcesz zachować dotychczasowe hasło lub wpisz nowe, aby zaktualizować.", "leaveEmptyToKeepPassword": "Pozostaw to pole puste, jeśli chcesz zachować dotychczasowe hasło", "login_name": "Login", + "maxEmailSizeBytes": "Maks. rozmiar e-maila", + "maxEmailSizeBytesDescription": "Większe wiadomości zostaną pominięte. Pozostaw puste, aby użyć domyślnego limitu (100 MB).", + "maxEmailSizeBytesPlaceholder": "Domyślnie: 100 MB", + "maxEmailSizeBytesUnlimited": "Domyślnie: 100 MB", "minutes": "minut", "months": "Miesiące", "mustBeAtLeast1": "Nie mniej jak 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Niewłaściwy adres email", "invalidUrl": "Niewłaściwy URL", + "maxEmailSizeMustBeNumber": "Maksymalny rozmiar e-maila musi być liczbą.", + "maxEmailSizeTooLarge": "Maksymalny rozmiar e-maila nie może przekraczać 100 MB.", + "maxEmailSizeTooSmall": "Maksymalny rozmiar e-maila musi wynosić co najmniej 1 MB.", "passwordMinLength": "Hasło musi posiadać conajmniej {{min}} znaków", "passwordRequired": "Hasło jest wymagane, gdy metodą uwierzytelniania jest hasło", "pleaseEnterPassword": "Proszę podać hasło", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Rozmiar partii może wynosić maksymalnie 200", "singleRequestBatchSizeTooSmall": "Rozmiar partii musi wynosić co najmniej 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/pt.json b/web/src/locales/pt.json index ec3f3db..c47dac7 100644 --- a/web/src/locales/pt.json +++ b/web/src/locales/pt.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Deixe vazio para manter a senha existente. Insira a nova senha apenas se estiver atualizando.", "leaveEmptyToKeepPassword": "Deixe vazio para manter a senha atual", "login_name": "Nome de login", + "maxEmailSizeBytes": "Tamanho máx. do email", + "maxEmailSizeBytesDescription": "Emails maiores do que isso serão ignorados. Deixe vazio para usar o padrão (100 MB).", + "maxEmailSizeBytesPlaceholder": "Padrão: 100 MB", + "maxEmailSizeBytesUnlimited": "Padrão: 100 MB", "minutes": "minutos", "months": "Meses", "mustBeAtLeast1": "Deve ser pelo menos 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Endereço de email inválido", "invalidUrl": "URL inválido", + "maxEmailSizeMustBeNumber": "O tamanho máximo do email deve ser um número.", + "maxEmailSizeTooLarge": "O tamanho máximo do email não deve exceder 100 MB.", + "maxEmailSizeTooSmall": "O tamanho máximo do email deve ser de pelo menos 1 MB.", "passwordMinLength": "A senha deve ter pelo menos {{min}} caracteres", "passwordRequired": "A senha é obrigatória se o método de autenticação for Senha", "pleaseEnterPassword": "Por favor, insira a senha", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "O tamanho do lote deve ser no máximo 200", "singleRequestBatchSizeTooSmall": "O tamanho do lote deve ser pelo menos 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json index a0b852e..d22f2af 100644 --- a/web/src/locales/ru.json +++ b/web/src/locales/ru.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Оставьте пустым, чтобы сохранить существующий пароль, или введите новый для обновления.", "leaveEmptyToKeepPassword": "Оставьте пустым, чтобы сохранить текущий пароль", "login_name": "Имя для входа", + "maxEmailSizeBytes": "Макс. размер письма", + "maxEmailSizeBytesDescription": "Письма больше этого размера будут пропущены. Оставьте пустым для использования значения по умолчанию (100 МБ).", + "maxEmailSizeBytesPlaceholder": "По умолчанию: 100 МБ", + "maxEmailSizeBytesUnlimited": "По умолчанию: 100 МБ", "minutes": "минут", "months": "Месяцы", "mustBeAtLeast1": "Должно быть не менее 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Неверный адрес электронной почты", "invalidUrl": "Неверный URL", + "maxEmailSizeMustBeNumber": "Максимальный размер письма должен быть числом.", + "maxEmailSizeTooLarge": "Максимальный размер письма не должен превышать 100 МБ.", + "maxEmailSizeTooSmall": "Максимальный размер письма должен быть не менее 1 МБ.", "passwordMinLength": "Пароль должен быть не менее {{min}} символов", "passwordRequired": "Пароль обязателен, когда метод авторизации - Пароль", "pleaseEnterPassword": "Пожалуйста, введите ваш пароль", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Размер пакета должен быть не более 200", "singleRequestBatchSizeTooSmall": "Размер пакета должен быть не менее 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/sv.json b/web/src/locales/sv.json index 92c6518..7a70218 100644 --- a/web/src/locales/sv.json +++ b/web/src/locales/sv.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "Lämna tomt för att behålla det befintliga lösenordet, eller ange ett nytt för att uppdatera det.", "leaveEmptyToKeepPassword": "Lämna tomt för att behålla nuvarande lösenord", "login_name": "Inloggningsnamn", + "maxEmailSizeBytes": "Max e-poststorlek", + "maxEmailSizeBytesDescription": "E-post större än detta kommer att hoppas över. Lämna tomt för att använda standard (100 MB).", + "maxEmailSizeBytesPlaceholder": "Standard: 100 MB", + "maxEmailSizeBytesUnlimited": "Standard: 100 MB", "minutes": "minuter", "months": "Månader", "mustBeAtLeast1": "Måste vara minst 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "Ogiltig e-postadress", "invalidUrl": "Ogiltig URL", + "maxEmailSizeMustBeNumber": "Max e-poststorlek måste vara ett nummer.", + "maxEmailSizeTooLarge": "Max e-poststorlek får inte överstiga 100 MB.", + "maxEmailSizeTooSmall": "Max e-poststorlek måste vara minst 1 MB.", "passwordMinLength": "Lösenordet måste vara minst {{min}} tecken långt", "passwordRequired": "Lösenord krävs när autentiseringsmetoden är Lösenord", "pleaseEnterPassword": "Vänligen ange ditt lösenord", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "Batchstorlek måste vara högst 200", "singleRequestBatchSizeTooSmall": "Batchstorlek måste vara minst 10" } -} +} \ No newline at end of file diff --git a/web/src/locales/zh-tw.json b/web/src/locales/zh-tw.json index c71fc73..db11b24 100644 --- a/web/src/locales/zh-tw.json +++ b/web/src/locales/zh-tw.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "保留現有密碼請留空。若要更新，請輸入新密碼。", "leaveEmptyToKeepPassword": "保留現有密碼請留空", "login_name": "登入名稱", + "maxEmailSizeBytes": "最大郵件大小", + "maxEmailSizeBytesDescription": "超出此大小的郵件將被跳過。留空則使用預設值（100 MB）。", + "maxEmailSizeBytesPlaceholder": "預設：100 MB", + "maxEmailSizeBytesUnlimited": "預設：100 MB", "minutes": "分鐘", "months": "月", "mustBeAtLeast1": "必須大於或等於 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "無效的電子郵件地址", "invalidUrl": "無效的網址", + "maxEmailSizeMustBeNumber": "最大郵件大小必須是數字。", + "maxEmailSizeTooLarge": "最大郵件大小不能超過 100 MB。", + "maxEmailSizeTooSmall": "最大郵件大小不能小於 1 MB。", "passwordMinLength": "密碼長度必須至少 {{min}} 個字元", "passwordRequired": "如果驗證方法是密碼，則密碼為必填項", "pleaseEnterPassword": "請輸入密碼", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "批次大小必須最多為200", "singleRequestBatchSizeTooSmall": "批次大小必須至少為10" } -} +} \ No newline at end of file diff --git a/web/src/locales/zh.json b/web/src/locales/zh.json index eb4201c..e3045eb 100644 --- a/web/src/locales/zh.json +++ b/web/src/locales/zh.json @@ -208,6 +208,10 @@ "leaveEmptyToKeepExisting": "留空以保持现有密码，或输入新密码进行更新。", "leaveEmptyToKeepPassword": "留空以保持当前密码", "login_name": "登录名", + "maxEmailSizeBytes": "最大邮件大小", + "maxEmailSizeBytesDescription": "超出此大小的邮件将被跳过。留空则使用默认值（100 MB）。", + "maxEmailSizeBytesPlaceholder": "默认：100 MB", + "maxEmailSizeBytesUnlimited": "默认：100 MB", "minutes": "分钟", "months": "月", "mustBeAtLeast1": "必须至少为 1", @@ -1681,6 +1685,9 @@ "invalidCronExpression": "Invalid cron expression. Must be 6 fields: second minute hour day-of-month month day-of-week (e.g. '0 0 0 * * *')", "invalidEmail": "无效的电子邮件地址", "invalidUrl": "无效的 URL", + "maxEmailSizeMustBeNumber": "最大邮件大小必须是数字。", + "maxEmailSizeTooLarge": "最大邮件大小不能超过 100 MB。", + "maxEmailSizeTooSmall": "最大邮件大小不能小于 1 MB。", "passwordMinLength": "密码长度至少为 {{min}} 个字符", "passwordRequired": "当认证方法为密码时，密码为必填项", "pleaseEnterPassword": "请输入您的密码", @@ -1690,4 +1697,4 @@ "singleRequestBatchSizeTooLarge": "批大小必须最多为200", "singleRequestBatchSizeTooSmall": "批大小必须至少为10" } -} +} \ No newline at end of file From 427f7248d2005416102566f439e924b93efdbcbc Mon Sep 17 00:00:00 2001 From: fama <720793+fama@users.noreply.github.com> Date: Wed, 3 Jun 2026 05:06:08 +0000 Subject: [PATCH 22/66] fix: open NewIndexWriter once across all migration segments Previously, do_migrate_segment created a fresh NewIndexWriter (and therefore a new Fjall Database) on every call, meaning the Fjall database at bichon-storage/ was opened and closed once per segment. This caused the migration to fail mid-way through (observed at segment 9/16) with: Storage(InvalidTag(("ChecksumType", 171))) Root cause: after segment N writes email blobs via Fjall's ingestion API (start_ingestion / write / finish), those SSTables and KV-separated blob files are flushed to disk and the Database is dropped. When segment N+1 calls Database::builder(storage_dir).open(), Fjall must discover and catalog all on-disk files produced by the previous segments. During that discovery it reads SSTable or blob-file block headers and encounters a ChecksumType discriminant byte (171 / 0xAB) that lsm-tree 3.1.4 does not recognise, causing the fatal error. The first N segments succeed because the cumulative set of ingested SSTables stays small enough that Fjall does not need to read the offending headers during reopen. Once enough data has accumulated the reopen triggers a manifest or compaction read that exposes the mismatch. Fix: open NewIndexWriter once, before the segment loop, and pass a &mut reference into each do_migrate_segment call. finish_writers() is called a single time after all segments complete. The Fjall Database stays open for the entire migration and is never closed and reopened, eliminating the incompatible-reopen path entirely. --- crates/admin/src/migrate.rs | 23 +++++++++++++++++++++-- crates/core/src/migrate/mod.rs | 5 +---- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/admin/src/migrate.rs b/crates/admin/src/migrate.rs index 4de0896..0005e5f 100644 --- a/crates/admin/src/migrate.rs +++ b/crates/admin/src/migrate.rs @@ -2,7 +2,7 @@ use std::path::{Path, PathBuf}; use bichon_core::migrate::{ count_eml_segments, do_migrate_segment, is_tantivy_index_dir, - store::{LegacyDirs, NewDirs}, + store::{LegacyDirs, NewDirs, NewIndexWriter}, }; use console::style; use dialoguer::{theme::ColorfulTheme, Confirm, Input}; @@ -326,6 +326,18 @@ pub fn handle_migration(theme: &ColorfulTheme) { .progress_chars("#>-"), ); + let mut writer = match NewIndexWriter::open(NewDirs::new( + new_index_path.clone(), + new_data_path.clone(), + )) { + Ok(w) => w, + Err(e) => { + pb.finish_with_message(format!("{}", style("Migration failed.").red())); + eprintln!("\n{} {:?}", style("✘").red().bold(), e); + return; + } + }; + let mut grand_total_migrated: usize = 0; let mut grand_total_skipped: usize = 0; @@ -337,7 +349,7 @@ pub fn handle_migration(theme: &ColorfulTheme) { match do_migrate_segment( batch_size, legacy, - NewDirs::new(new_index_path.clone(), new_data_path.clone()), + &mut writer, seg_idx, |msg| { if let Some(data) = msg.strip_prefix("TOTAL:") { @@ -407,6 +419,13 @@ pub fn handle_migration(theme: &ColorfulTheme) { pb.set_position((seg_idx + 1) as u64); } + pb.set_message(style("Finalizing indexes...").dim().to_string()); + if let Err(e) = writer.finish_writers() { + pb.finish_with_message(format!("{}", style("Migration failed.").red())); + eprintln!("\n{} {:?}", style("✘").red().bold(), e); + return; + } + pb.finish_with_message(format!( "Migration finished. Total: {}, Skipped: {}", grand_total_migrated, grand_total_skipped diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs index 02f15aa..80255cd 100644 --- a/crates/core/src/migrate/mod.rs +++ b/crates/core/src/migrate/mod.rs @@ -121,7 +121,7 @@ fn is_dir_not_empty(path: &PathBuf) -> std::io::Result { pub fn do_migrate_segment( batch_size: u32, legacy: LegacyDirs, - new_dirs: NewDirs, + writer: &mut NewIndexWriter, segment_index: usize, mut on_progress: F, ) -> BichonResult<()> @@ -226,8 +226,6 @@ where drop(envelope_index); // ── Phase 2: process EML docs, streaming one at a time ───────────── - let mut writer = NewIndexWriter::open(new_dirs)?; - let mut total_migrated = 0usize; let mut total_skipped = 0usize; @@ -308,7 +306,6 @@ where chunk_start = chunk_end; } - writer.finish_writers()?; on_progress(&format!("DONE:{}:{}", total_migrated, total_skipped)); Ok(()) } From 368b18c45f3167c8c3352056d666246849a41063 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 4 Jun 2026 23:46:25 +0800 Subject: [PATCH 23/66] bump to v1.5.0 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62890f4..a500552 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.4.3" +version = "1.5.0" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.4.3" +version = "1.5.0" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.4.3" +version = "1.5.0" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.4.3" +version = "1.5.0" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.4.3" +version = "1.5.0" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index ff5a365..c6c6197 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.4.3" +version = "1.5.0" edition = "2021" [workspace.dependencies] From aebb94ee4ebbd12a001d98bc97087c8176f4e76d Mon Sep 17 00:00:00 2001 From: rustmailer Date: Fri, 5 Jun 2026 09:13:25 +0800 Subject: [PATCH 24/66] fix: preserve non-stored search fields when updating envelope tags update_envelope_tags lost f_body, f_from_text, f_to_text, f_cc_text, f_bcc_text, f_attachment_name_text and f_attachment_name_exact because they are not STORED and field_values() skipped them during delete+add. Rebuild these from stored counterparts and the blob-store EML. --- crates/core/src/migrate/mod.rs | 2 +- crates/core/src/store/tantivy/envelope.rs | 499 +++++++++++++++++++++- 2 files changed, 495 insertions(+), 6 deletions(-) diff --git a/crates/core/src/migrate/mod.rs b/crates/core/src/migrate/mod.rs index 80255cd..05308ef 100644 --- a/crates/core/src/migrate/mod.rs +++ b/crates/core/src/migrate/mod.rs @@ -4,7 +4,7 @@ use crate::{ error::{code::ErrorCode, BichonResult}, migrate::{ legacy::schema::SchemaTools, - store::{LegacyDirs, NewDirs, NewIndexWriter}, + store::{LegacyDirs, NewIndexWriter}, }, raise_error, settings::cli::SETTINGS, diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index d5af7d2..8c7e17b 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -50,10 +50,12 @@ use crate::{ tokenizers::EuroTokenizer, }, }, + utils::html::extract_text, utc_now, }; use chrono::Utc; +use mail_parser::MessageParser; use serde_json::json; use tantivy::{ aggregation::{ @@ -1073,8 +1075,9 @@ impl IndexManager { let searcher = self.create_searcher()?; let mut writer = self.index_writer.lock().await; - let f_tags = SchemaTools::email_fields().f_tags; - let f_id = SchemaTools::email_fields().f_id; + let f = SchemaTools::email_fields(); + let f_tags = f.f_tags; + let f_id = f.f_id; let deduplicated_updates: HashMap> = request .updates .into_iter() @@ -1119,13 +1122,124 @@ impl IndexManager { let mut new_doc = TantivyDocument::new(); + // Copy stored fields, excluding f_tags (handled separately). for (field, value) in old_doc.field_values() { if field != f_tags { new_doc.add_field_value(field, value); } } - for tag in current_tags { - new_doc.add_facet(f_tags, &tag); + + // Reconstruct non-stored text-search fields from their + // stored counterparts. f_from_text / f_to_text / f_cc_text / + // f_bcc_text carry the same content as f_from / f_to / f_cc / f_bcc. + for val in old_doc.get_all(f.f_from) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_from_text, s); + } + } + for val in old_doc.get_all(f.f_to) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_to_text, s); + } + } + for val in old_doc.get_all(f.f_cc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_cc_text, s); + } + } + for val in old_doc.get_all(f.f_bcc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_bcc_text, s); + } + } + + // Reconstruct attachment-name fields from the stored + // f_attachments JSON blob. + if let Some(attrs_val) = old_doc.get_first(f.f_attachments) { + if let Some(json_str) = attrs_val.as_str() { + if let Ok(parsed) = + serde_json::from_str::(json_str) + { + if let Some(arr) = parsed.as_array() { + for att in arr { + let is_inline = att + .get("inline") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let has_cid = att + .get("content_id") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if is_inline && has_cid { + continue; + } + if let Some(filename) = att + .get("filename") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { + new_doc.add_text( + f.f_attachment_name_text, + filename, + ); + new_doc.add_text( + f.f_attachment_name_exact, + filename, + ); + } + } + } + } + } + } + + // Reconstruct body text from the original EML stored in the + // blob store, referenced by f_content_hash. + if let Some(hash_val) = old_doc.get_first(f.f_content_hash) { + if let Some(content_hash) = hash_val.as_str() { + match BLOB_MANAGER.get_email(content_hash) { + Ok(Some(eml_bytes)) => { + if let Some(message) = + MessageParser::new().parse(&eml_bytes) + { + let text = message + .body_text(0) + .map(|cow| cow.into_owned()) + .or_else(|| { + message.body_html(0).map(|cow| { + extract_text(cow.into_owned()) + }) + }) + .unwrap_or_default(); + let body_text = text + .split_whitespace() + .collect::>() + .join(" "); + if !body_text.is_empty() { + new_doc.add_text(f.f_body, &body_text); + } + } + } + Ok(None) => { + tracing::warn!( + content_hash, + "EML not found in blob store during tag update" + ); + } + Err(e) => { + tracing::warn!( + content_hash, + error = %e, + "Failed to fetch EML during tag update" + ); + } + } + } + } + + for tag in ¤t_tags { + new_doc.add_facet(f_tags, tag); } let delete_term = Term::from_field_text(f_id, eid); @@ -1190,7 +1304,7 @@ impl IndexManager { let mailbox_docs: Vec; match sort_by { - SortBy::DATE => { + SortBy::DATE => { let date_docs: Vec<(Option, DocAddress)> = searcher .search( &query, @@ -1568,3 +1682,378 @@ impl IndexManager { Ok(stats) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::tantivy::tokenizers::EuroTokenizer; + use serde_json::json; + use tantivy::{ + collector::Count, + query::{QueryParser, TermQuery}, + schema::IndexRecordOption, + Index, Term, + }; + + /// Build a complete test document with known values across all + /// stored and non-stored fields so we can verify reconstruction. + fn build_test_doc() -> TantivyDocument { + let f = SchemaTools::email_fields(); + let mut doc = TantivyDocument::new(); + + doc.add_text(f.f_id, "test-eid-001"); + doc.add_text(f.f_message_id, ""); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_u64(f.f_uid, 100); + doc.add_text(f.f_subject, "Test Subject Line"); + doc.add_text(f.f_body, "the quick brown fox jumps over the lazy dog"); + doc.add_text(f.f_preview, "the quick brown fox..."); + doc.add_text(f.f_content_hash, "test-content-hash-001"); + // f_from / f_from_text carry the same data + doc.add_text(f.f_from, "alice@example.com"); + doc.add_text(f.f_from_text, "alice@example.com"); + doc.add_text(f.f_to, "bob@example.com"); + doc.add_text(f.f_to_text, "bob@example.com"); + doc.add_text(f.f_cc, "carol@example.com"); + doc.add_text(f.f_cc_text, "carol@example.com"); + doc.add_text(f.f_bcc, "dave@example.com"); + doc.add_text(f.f_bcc_text, "dave@example.com"); + doc.add_i64(f.f_date, 1_700_000_000_000); + doc.add_i64(f.f_internal_date, 1_700_000_000_000); + doc.add_i64(f.f_ingest_at, 1_700_000_000_000); + doc.add_u64(f.f_size, 999); + doc.add_text(f.f_thread_id, "thread-xyz"); + + // Attachment metadata (stored as JSON). + let atts = json!([{ + "filename": "invoice.pdf", + "file_type": "application/pdf", + "inline": false, + "size": 5000, + "content_id": null, + "content_hash": "att-hash-pdf", + "is_message": false + }]); + doc.add_text(f.f_attachments, atts.to_string()); + doc.add_text(f.f_attachment_name_text, "invoice.pdf"); + doc.add_text(f.f_attachment_name_exact, "invoice.pdf"); + doc.add_text(f.f_attachment_ext, "pdf"); + doc.add_text(f.f_attachment_category, "document"); + doc.add_text(f.f_attachment_content_type, "application/pdf"); + doc.add_text(f.f_attachment_content_hash, "att-hash-pdf"); + doc.add_u64(f.f_attachment_count, 1); + doc.add_u64(f.f_regular_attachment_count, 1); + doc.add_u64(f.f_shard_id, 0); + + // Initial tags. + doc.add_facet(f.f_tags, "/inbox"); + doc.add_facet(f.f_tags, "/unread"); + + doc + } + + /// Reconstruct a new tantivy document from `old_doc`, preserving all + /// fields (including non-stored ones) and replacing tags with + /// `new_tags`. Body text is reconstructed from the supplied `eml_cache` + /// (a stand-in for the blob store) rather than from + /// `old_doc.field_values()` because `f_body` is not STORED. + fn reconstruct_for_test( + old_doc: &TantivyDocument, + new_tags: &HashSet, + eml_cache: &HashMap>, + ) -> TantivyDocument { + let f = SchemaTools::email_fields(); + let mut new_doc = TantivyDocument::new(); + + // ── stored fields (except f_tags) ────────────────────────── + for (field, value) in old_doc.field_values() { + if field != f.f_tags { + new_doc.add_field_value(field, value); + } + } + + // ── non-stored text-search fields ────────────────────────── + for val in old_doc.get_all(f.f_from) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_from_text, s); + } + } + for val in old_doc.get_all(f.f_to) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_to_text, s); + } + } + for val in old_doc.get_all(f.f_cc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_cc_text, s); + } + } + for val in old_doc.get_all(f.f_bcc) { + if let Some(s) = val.as_str() { + new_doc.add_text(f.f_bcc_text, s); + } + } + + // ── attachment-name fields ───────────────────────────────── + if let Some(attrs_val) = old_doc.get_first(f.f_attachments) { + if let Some(json_str) = attrs_val.as_str() { + if let Ok(parsed) = serde_json::from_str::(json_str) { + if let Some(arr) = parsed.as_array() { + for att in arr { + let is_inline = att + .get("inline") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let has_cid = att + .get("content_id") + .and_then(|v| v.as_str()) + .map(|s| !s.is_empty()) + .unwrap_or(false); + if is_inline && has_cid { + continue; + } + if let Some(filename) = att + .get("filename") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + { + new_doc.add_text(f.f_attachment_name_text, filename); + new_doc.add_text(f.f_attachment_name_exact, filename); + } + } + } + } + } + } + + // ── body text (from eml cache – stands in for BLOB_MANAGER) ── + if let Some(hash_val) = old_doc.get_first(f.f_content_hash) { + if let Some(content_hash) = hash_val.as_str() { + if let Some(eml_bytes) = eml_cache.get(content_hash) { + if let Some(message) = MessageParser::new().parse(eml_bytes) { + let text = message + .body_text(0) + .map(|cow| cow.into_owned()) + .or_else(|| { + message + .body_html(0) + .map(|cow| extract_text(cow.into_owned())) + }) + .unwrap_or_default(); + let body_text = + text.split_whitespace().collect::>().join(" "); + if !body_text.is_empty() { + new_doc.add_text(f.f_body, &body_text); + } + } + } + } + } + + // ── updated tags ─────────────────────────────────────────── + for tag in new_tags { + new_doc.add_facet(f.f_tags, tag); + } + + new_doc + } + + #[test] + fn update_tags_preserves_non_stored_fields() { + let f = SchemaTools::email_fields(); + + // ---- setup: in-memory index + document -------------------- + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + let doc = build_test_doc(); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } // drop writer so the next one can acquire the lock + + // ---- build a minimal EML so body reconstruction works ------ + let eml = b"From: alice@example.com\r\n\ + To: bob@example.com\r\n\ + Subject: Test\r\n\ + Date: Thu, 01 Jan 2023 00:00:00 +0000\r\n\ + Message-ID: \r\n\ + \r\n\ + the quick brown fox jumps over the lazy dog\r\n"; + let mut eml_cache = HashMap::new(); + eml_cache.insert("test-content-hash-001".to_string(), eml.to_vec()); + + // ---- read old doc, reconstruct, delete + add -------------- + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + let query = TermQuery::new( + Term::from_field_text(f.f_id, "test-eid-001"), + IndexRecordOption::Basic, + ); + let hits = searcher + .search(&query, &TopDocs::with_limit(1).order_by_score()) + .unwrap(); + assert_eq!(hits.len(), 1); + + let old_doc: TantivyDocument = searcher.doc(hits[0].1).unwrap(); + + let mut new_tags = HashSet::new(); + new_tags.insert("/important".to_string()); + new_tags.insert("/inbox".to_string()); + + let new_doc = reconstruct_for_test(&old_doc, &new_tags, &eml_cache); + + let mut writer2 = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer2"); + writer2 + .delete_term(Term::from_field_text(f.f_id, "test-eid-001")); + writer2.add_document(new_doc).unwrap(); + writer2.commit().unwrap(); + + // ---- verify: search for non-stored fields still works ----- + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // Body text (tokenized via "euro") + let body_parser = + QueryParser::for_index(&index, vec![f.f_body]); + let body_hits = searcher + .search(&body_parser.parse_query("quick brown fox").unwrap(), &Count) + .unwrap(); + assert_eq!(body_hits, 1, "body text should survive tag update"); + + // from_text (tokenized via "euro") + let from_parser = + QueryParser::for_index(&index, vec![f.f_from_text]); + let from_hits = searcher + .search( + &from_parser.parse_query("alice@example.com").unwrap(), + &Count, + ) + .unwrap(); + assert_eq!(from_hits, 1, "from_text should survive tag update"); + + // to_text + let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]); + let to_hits = searcher + .search( + &to_parser.parse_query("bob@example.com").unwrap(), + &Count, + ) + .unwrap(); + assert_eq!(to_hits, 1, "to_text should survive tag update"); + + // attachment_name_exact (STRING — not tokenized) + let att_hits = searcher + .search( + &TermQuery::new( + Term::from_field_text(f.f_attachment_name_exact, "invoice.pdf"), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!( + att_hits, 1, + "attachment_name_exact should survive tag update" + ); + + // Updated tags + let tags_hits = searcher + .search( + &TermQuery::new( + Term::from_facet( + f.f_tags, + &Facet::from_text("/important").unwrap(), + ), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!(tags_hits, 1, "new tag /important should be present"); + + // Old tag /unread should be gone since we overwrote with new_tags + let old_tag_hits = searcher + .search( + &TermQuery::new( + Term::from_facet( + f.f_tags, + &Facet::from_text("/unread").unwrap(), + ), + IndexRecordOption::Basic, + ), + &Count, + ) + .unwrap(); + assert_eq!( + old_tag_hits, 0, + "old tag /unread should have been removed" + ); + } + + #[test] + fn body_reconstruction_from_eml_cache() { + // Verify the EML → body_text extraction used inside + // reconstruct_for_test (and therefore update_envelope_tags). + let eml = b"From: x@y\r\n\ + Subject: testing\r\n\ + Date: Thu, 01 Jan 2023 00:00:00 +0000\r\n\ + \r\n\ + hello world from the test suite\r\n"; + + let mut cache = HashMap::new(); + cache.insert("hash-abc".to_string(), eml.to_vec()); + + let f = SchemaTools::email_fields(); + let mut old = TantivyDocument::new(); + old.add_text(f.f_content_hash, "hash-abc"); + + let reconstructed = reconstruct_for_test(&old, &HashSet::new(), &cache); + + // The body should have been extracted from the EML and added + // back to the document. Search for it. + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + writer.add_document(reconstructed).unwrap(); + writer.commit().unwrap(); + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + let parser = QueryParser::for_index(&index, vec![f.f_body]); + let hits = searcher + .search(&parser.parse_query("hello world").unwrap(), &Count) + .unwrap(); + assert_eq!(hits, 1, "body text should be reconstructed from EML"); + } + + #[test] + fn body_reconstruction_missing_eml_is_graceful() { + // When the EML is not in the cache (simulating a blob-store + // miss), the document should still be produced without body. + let f = SchemaTools::email_fields(); + let mut old = TantivyDocument::new(); + old.add_text(f.f_content_hash, "nonexistent-hash"); + + let cache = HashMap::new(); // empty + let reconstructed = reconstruct_for_test(&old, &HashSet::new(), &cache); + + // The document exists but has no body field. + let body_vals: Vec<_> = reconstructed.get_all(f.f_body).collect(); + assert!( + body_vals.is_empty(), + "body should be absent when EML is missing" + ); + } +} From 62cb5264fdc5772ed96943abecfdcd691500ea4a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sat, 6 Jun 2026 15:56:47 +0800 Subject: [PATCH 25/66] Add funding.json for project funding details --- funding.json | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 funding.json diff --git a/funding.json b/funding.json new file mode 100644 index 0000000..7622d1c --- /dev/null +++ b/funding.json @@ -0,0 +1,58 @@ +{ + "$schema": "https://fundingjson.org/schema/v1.1.0.json", + "version": "v1.0.0", + "entity": { + "type": "individual", + "role": "maintainer", + "name": "rustmailer", + "email": "rustmailer.git@gmail.com", + "phone": "", + "description": "I'm an indie developer and the sole maintainer of Bichon, a lightweight open-source email archiver built in Rust. I believe in privacy, data ownership, and the right to self-host your own digital life.", + "webpageUrl": { + "url": "https://github.com/rustmailer" + } + }, + "projects": [ + { + "guid": "bichon", + "name": "Bichon", + "description": "Bichon is a lightweight, high-performance, self-hosted email archiver built in Rust. It synchronizes emails from IMAP servers, indexes them for full-text search, and provides a clean WebUI and REST API for access.\n\nBichon requires no external database and runs as a single binary — making it easy to deploy and maintain. It supports multiple accounts, OAuth2, SOCKS5 proxy, scheduled sync, bulk import (EML/MBOX/PST), and multi-user RBAC.\n\nAs the sole maintainer, I develop and support Bichon in my personal time. With 1.8k GitHub stars and 327k+ Docker pulls, the project has grown well beyond a personal tool and is actively used by individuals and teams worldwide — including a real-world deployment archiving 1.15 million emails across 28 accounts (800 GB original data, compressed to 421 GB on disk).", + "webpageUrl": { + "url": "https://github.com/rustmailer/bichon" + }, + "repositoryUrl": { + "url": "https://github.com/rustmailer/bichon" + }, + "licenses": ["spdx:AGPL-3.0"], + "tags": ["email", "rust", "self-hosted", "archiver", "imap", "full-text-search", "privacy", "webui"] + } + ], + "funding": { + "channels": [ + { + "guid": "buymeacoffee", + "type": "payment-provider", + "address": "https://buymeacoffee.com/rustmailer", + "description": "Support via Buy Me a Coffee." + }, + { + "guid": "bank", + "type": "bank", + "address": "", + "description": "Direct bank transfer also accepted. Please email rustmailer.git@gmail.com for details." + } + ], + "plans": [ + { + "guid": "maintainer-time", + "status": "active", + "name": "Maintainer Time", + "description": "Cover the cost of dedicated development and maintenance time for Bichon — including bug fixes, feature development, security updates, issue triage, and community support.", + "amount": 10000, + "currency": "USD", + "frequency": "yearly", + "channels": ["bank"] + } + ] + } +} From c736afffb03949c9d7ceeda0db5ac2189de580a9 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 7 Jun 2026 15:34:38 +0800 Subject: [PATCH 26/66] fix: account deletion times out #291 --- crates/admin/src/meta.rs | 1 + crates/core/src/account/migration.rs | 51 ++++++++++++++++--- crates/core/src/account/view.rs | 2 + crates/core/src/cache/imap/task.rs | 7 +++ web/src/api/account/api.ts | 1 + .../components/data-table-row-actions.tsx | 7 ++- .../accounts/components/delete-dialog.tsx | 4 +- .../accounts/components/enable-action.tsx | 2 +- .../components/running-state-action.tsx | 3 ++ .../features/accounts/components/table.tsx | 2 +- web/src/features/accounts/index.tsx | 4 ++ web/src/locales/ar.json | 2 + web/src/locales/da.json | 2 + web/src/locales/de.json | 2 + web/src/locales/en.json | 2 + web/src/locales/es.json | 2 + web/src/locales/fi.json | 2 + web/src/locales/fr.json | 2 + web/src/locales/it.json | 2 + web/src/locales/jp.json | 2 + web/src/locales/ko.json | 2 + web/src/locales/nl.json | 2 + web/src/locales/no.json | 2 + web/src/locales/pl.json | 2 + web/src/locales/pt.json | 2 + web/src/locales/ru.json | 2 + web/src/locales/sv.json | 2 + web/src/locales/zh-tw.json | 2 + web/src/locales/zh.json | 2 + 29 files changed, 106 insertions(+), 14 deletions(-) diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index f05496d..d843d62 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -262,6 +262,7 @@ impl From for AccountModel { imap_quota_bytes: None, auto_download_new_mailboxes: None, download_schedule: None, + deleting: false, } } } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 27c8bb0..170ce91 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -97,6 +97,8 @@ pub struct Account { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + #[serde(default)] + pub deleting: bool, } impl MemDbModel for Account { @@ -136,6 +138,7 @@ impl Account { imap_quota_bytes: request.imap_quota_bytes, imap_quota_window: request.imap_quota_window, download_schedule: request.download_schedule, + deleting: false, }) } @@ -223,14 +226,46 @@ impl Account { pub async fn delete(account_id: u64) -> BichonResult<()> { let account = Self::get(account_id)?; - if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { - tracing::error!( - "[CLEANUP_ACCOUNT_ERROR] Account {}: failed to cleanup resources: {:#?}", - account_id, - error - ); - return Err(error); + + // Immediately stop scheduling to prevent new downloads + if matches!(account.account_type, AccountType::IMAP) { + SYNC_TASKS.stop(account.id).await?; } + + // Mark as deleting and disabled so frontend shows status and download tasks skip it + update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.deleting = true; + updated.enabled = false; + Ok(updated) + }, + )?; + + // Spawn background cleanup — heavy work (Tantivy, attachments) runs off the request path + tokio::spawn(async move { + if let Err(error) = Self::cleanup_account_resources_sequential(&account).await { + tracing::error!( + "[CLEANUP_ACCOUNT_ERROR] Account {}: cleanup failed, reverting deleting flag: {:#?}", + account_id, + error + ); + // Revert deleting flag so the user can retry (only if account record still exists) + let _ = update_impl( + DB_MANAGER.db(), + &account_id.to_string(), + move |current: Account| { + let mut updated = current.clone(); + updated.deleting = false; + updated.enabled = true; + Ok(updated) + }, + ); + } + }); + Ok(()) } @@ -239,8 +274,8 @@ impl Account { } async fn cleanup_account_resources_sequential(account: &AccountModel) -> BichonResult<()> { + // Sync task already stopped in delete() before spawning this background task if matches!(account.account_type, AccountType::IMAP) { - SYNC_TASKS.stop(account.id).await?; DownloadState::delete(account.id)?; } OAuth2AccessToken::try_delete(account.id)?; diff --git a/crates/core/src/account/view.rs b/crates/core/src/account/view.rs index 36e4f6e..5703760 100644 --- a/crates/core/src/account/view.rs +++ b/crates/core/src/account/view.rs @@ -58,6 +58,7 @@ pub struct AccountResp { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + pub deleting: bool, } impl AccountResp { @@ -95,6 +96,7 @@ impl AccountResp { imap_quota_window: account.imap_quota_window, auto_download_new_mailboxes: account.auto_download_new_mailboxes, download_schedule: account.download_schedule, + deleting: account.deleting, } } } diff --git a/crates/core/src/cache/imap/task.rs b/crates/core/src/cache/imap/task.rs index 0bac419..66659ef 100644 --- a/crates/core/src/cache/imap/task.rs +++ b/crates/core/src/cache/imap/task.rs @@ -113,6 +113,9 @@ impl AccountDownTask { let account = AccountModel::get(account_id).ok(); match account { Some(account) => { + if account.deleting { + return Ok(()); + } if !account.enabled { let last = LAST_WARN_TIME.load(Ordering::Relaxed); let now = utc_now!(); @@ -246,6 +249,10 @@ impl AccountDownTask { } }; + if account.deleting { + return; + } + if let Err(e) = process_imap_download(&account, token_clone, TriggerType::Manual).await { error!("Manual download failed for {}: {:?}", account_id, e); diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index cd1f4bb..5fb2218 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -144,6 +144,7 @@ export interface AccountModel { imap_quota_bytes?: number; auto_download_new_mailboxes?: boolean; download_schedule?: string; + deleting?: boolean; } export const download_state = async (account_id: number) => { diff --git a/web/src/features/accounts/components/data-table-row-actions.tsx b/web/src/features/accounts/components/data-table-row-actions.tsx index 5187882..6a7ff92 100644 --- a/web/src/features/accounts/components/data-table-row-actions.tsx +++ b/web/src/features/accounts/components/data-table-row-actions.tsx @@ -50,13 +50,16 @@ export function DataTableRowActions({ row }: DataTableRowActionsProps) { const hasPermission = require_any_permission(['system:root', 'account:manage'], row.original.id); const hasReadPermission = require_any_permission(['system:root', 'account:read_details'], row.original.id); + const isDeleting = row.original.deleting === true; const canShowAnyAction = + !isDeleting && ( (hasPermission) || (account_type === 'IMAP' && hasPermission) || - (account_type === 'IMAP' && hasReadPermission); + (account_type === 'IMAP' && hasReadPermission) + ); - const showDownload = account_type === 'IMAP' && hasPermission; + const showDownload = !isDeleting && account_type === 'IMAP' && hasPermission; const handleStartDownload = async () => { try { diff --git a/web/src/features/accounts/components/delete-dialog.tsx b/web/src/features/accounts/components/delete-dialog.tsx index 41e63dd..77a0880 100644 --- a/web/src/features/accounts/components/delete-dialog.tsx +++ b/web/src/features/accounts/components/delete-dialog.tsx @@ -43,8 +43,8 @@ export function AccountDeleteDialog({ open, onOpenChange, currentRow }: Props) { const queryClient = useQueryClient(); function handleSuccess() { toast({ - title: t('dialogs.accountDeleted'), - description: t('dialogs.accountDeletedDesc'), + title: t('dialogs.accountDeletionStarted'), + description: t('dialogs.accountDeletionStartedDesc'), action: {t('common.close')}, }); diff --git a/web/src/features/accounts/components/enable-action.tsx b/web/src/features/accounts/components/enable-action.tsx index e26aac2..887dfbf 100644 --- a/web/src/features/accounts/components/enable-action.tsx +++ b/web/src/features/accounts/components/enable-action.tsx @@ -77,7 +77,7 @@ export function EnableAction({ row }: DataTableRowActionsProps) { setOpen(true)} - disabled={!hasPermission || updateMutation.isPending} + disabled={!hasPermission || updateMutation.isPending || row.original.deleting} /> Deleting... + } let account_type = row.original.account_type; if (account_type === "NoSync") { return n/a diff --git a/web/src/features/accounts/components/table.tsx b/web/src/features/accounts/components/table.tsx index ef094fc..c2ba385 100644 --- a/web/src/features/accounts/components/table.tsx +++ b/web/src/features/accounts/components/table.tsx @@ -127,7 +127,7 @@ export function AccountTable({ columns, data }: DataTableProps) { {row.getVisibleCells().map((cell) => ( { + const items = (query.state.data as { items?: { deleting?: boolean }[] })?.items; + return items?.some((item) => item.deleting) ? 5000 : false; + }, }) const hasAccounts = accountList != null && accountList.items.length > 0; diff --git a/web/src/locales/ar.json b/web/src/locales/ar.json index 931cfb5..4fb65b7 100644 --- a/web/src/locales/ar.json +++ b/web/src/locales/ar.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "فشل حذف الحساب", "accountDeleted": "تم حذف الحساب", "accountDeletedDesc": "تم حذف حسابك بنجاح.", + "accountDeletionStarted": "بدء حذف الحساب", + "accountDeletionStartedDesc": "جاري حذف الحساب في الخلفية، وسيختفي بعد اكتمال التنظيف.", "allResourcesErased": "سيتم مسح جميع الموارد ذات الصلة نهائيًا.", "cannotBeUndone": "لا يمكن التراجع عن هذا الإجراء!", "confirmDelete": "تأكيد الحذف", diff --git a/web/src/locales/da.json b/web/src/locales/da.json index 7fdcd08..df99490 100644 --- a/web/src/locales/da.json +++ b/web/src/locales/da.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Sletning af konto mislykkedes", "accountDeleted": "Konto slettet", "accountDeletedDesc": "Din konto er blevet slettet.", + "accountDeletionStarted": "Kontoen slettes nu", + "accountDeletionStartedDesc": "Kontoen slettes i baggrunden og forsvinder, når oprydningen er færdig.", "allResourcesErased": "Alle relaterede ressourcer vil blive slettet permanent.", "cannotBeUndone": "Denne handling kan ikke fortrydes!", "confirmDelete": "Bekræft Sletning", diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 2dab0dc..179558c 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Löschen des Kontos fehlgeschlagen", "accountDeleted": "Konto gelöscht", "accountDeletedDesc": "Ihr Konto wurde erfolgreich gelöscht.", + "accountDeletionStarted": "Kontolöschung gestartet", + "accountDeletionStartedDesc": "Konto wird im Hintergrund gelöscht und verschwindet nach der Bereinigung.", "allResourcesErased": "Alle zugehörigen Ressourcen werden dauerhaft gelöscht.", "cannotBeUndone": "Diese Aktion kann nicht rückgängig gemacht werden!", "confirmDelete": "Löschung bestätigen", diff --git a/web/src/locales/en.json b/web/src/locales/en.json index a4e1b50..238bcc7 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Account delete Failed", "accountDeleted": "Account Deleted", "accountDeletedDesc": "Your account has been successfully deleted.", + "accountDeletionStarted": "Account deletion started", + "accountDeletionStartedDesc": "Account is being deleted in the background and will disappear after cleanup.", "allResourcesErased": "All related resources will be permanently erased.", "cannotBeUndone": "This action cannot be undone!", "confirmDelete": "Confirm Delete", diff --git a/web/src/locales/es.json b/web/src/locales/es.json index 47b97f4..0c4c3e9 100644 --- a/web/src/locales/es.json +++ b/web/src/locales/es.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Error al eliminar la cuenta", "accountDeleted": "Cuenta eliminada", "accountDeletedDesc": "Tu cuenta ha sido eliminada con éxito.", + "accountDeletionStarted": "Eliminación de cuenta iniciada", + "accountDeletionStartedDesc": "La cuenta se está eliminando en segundo plano y desaparecerá tras la limpieza.", "allResourcesErased": "Todos los recursos asociados se borrarán permanentemente.", "cannotBeUndone": "¡Esta acción no se puede deshacer!", "confirmDelete": "Confirmar eliminación", diff --git a/web/src/locales/fi.json b/web/src/locales/fi.json index f5e1820..9c78869 100644 --- a/web/src/locales/fi.json +++ b/web/src/locales/fi.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Tilin poistaminen epäonnistui", "accountDeleted": "Tili poistettu", "accountDeletedDesc": "Tilisi on poistettu onnistuneesti.", + "accountDeletionStarted": "Tilin poistaminen aloitettu", + "accountDeletionStartedDesc": "Tiliä poistetaan taustalla. Se katoaa, kun puhdistus on valmis.", "allResourcesErased": "Kaikki liittyvät resurssit poistetaan pysyvästi.", "cannotBeUndone": "Tätä toimenpidettä ei voi kumota!", "confirmDelete": "Vahvista poisto", diff --git a/web/src/locales/fr.json b/web/src/locales/fr.json index b553036..71f8e4d 100644 --- a/web/src/locales/fr.json +++ b/web/src/locales/fr.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Échec de la suppression du compte", "accountDeleted": "Compte Supprimé", "accountDeletedDesc": "Votre compte a été supprimé avec succès.", + "accountDeletionStarted": "Suppression du compte lancée", + "accountDeletionStartedDesc": "Compte en cours de suppression en arrière-plan, disparaîtra après nettoyage.", "allResourcesErased": "Toutes les ressources associées seront effacées définitivement.", "cannotBeUndone": "Cette action ne peut pas être annulée !", "confirmDelete": "Confirmer la Suppression", diff --git a/web/src/locales/it.json b/web/src/locales/it.json index 42b3c30..60496a9 100644 --- a/web/src/locales/it.json +++ b/web/src/locales/it.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Eliminazione account fallita", "accountDeleted": "Account Eliminato", "accountDeletedDesc": "Il tuo account è stato eliminato con successo.", + "accountDeletionStarted": "Eliminazione account avviata", + "accountDeletionStartedDesc": "L'account è in fase di eliminazione in background e scomparirà dopo la pulizia.", "allResourcesErased": "Tutte le risorse correlate verranno cancellate permanentemente.", "cannotBeUndone": "Questa azione non può essere annullata!", "confirmDelete": "Conferma Eliminazione", diff --git a/web/src/locales/jp.json b/web/src/locales/jp.json index 438ddcd..59fe40b 100644 --- a/web/src/locales/jp.json +++ b/web/src/locales/jp.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "アカウントの削除に失敗しました", "accountDeleted": "アカウントが削除されました", "accountDeletedDesc": "アカウントが正常に削除されました。", + "accountDeletionStarted": "アカウントの削除を開始しました", + "accountDeletionStartedDesc": "バックグラウンドで削除中です。完了するとリストから消えます。", "allResourcesErased": "関連するすべてのリソースは完全に消去されます。", "cannotBeUndone": "この操作は元に戻せません！", "confirmDelete": "削除の確認", diff --git a/web/src/locales/ko.json b/web/src/locales/ko.json index 77e17ff..6da8d17 100644 --- a/web/src/locales/ko.json +++ b/web/src/locales/ko.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "계정 삭제 실패", "accountDeleted": "계정 삭제됨", "accountDeletedDesc": "계정이 성공적으로 삭제되었습니다.", + "accountDeletionStarted": "계정 삭제 시작됨", + "accountDeletionStartedDesc": "백그라운드에서 삭제 중이며, 정리가 끝나면 목록에서 사라집니다.", "allResourcesErased": "모든 관련 리소스가 영구적으로 지워집니다.", "cannotBeUndone": "이 작업은 되돌릴 수 없습니다!", "confirmDelete": "삭제 확인", diff --git a/web/src/locales/nl.json b/web/src/locales/nl.json index 396e2e1..85d4066 100644 --- a/web/src/locales/nl.json +++ b/web/src/locales/nl.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Account verwijderen Mislukt", "accountDeleted": "Account Verwijderd", "accountDeletedDesc": "Uw account is succesvol verwijderd.", + "accountDeletionStarted": "Verwijdering account gestart", + "accountDeletionStartedDesc": "Account wordt op de achtergrond verwijderd en verdwijnt na opschonen.", "allResourcesErased": "Alle gerelateerde bronnen worden permanent gewist.", "cannotBeUndone": "Deze actie kan niet ongedaan worden gemaakt!", "confirmDelete": "Verwijdering Bevestigen", diff --git a/web/src/locales/no.json b/web/src/locales/no.json index 6e7d619..0c21ca2 100644 --- a/web/src/locales/no.json +++ b/web/src/locales/no.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Sletting av konto mislyktes", "accountDeleted": "Konto slettet", "accountDeletedDesc": "Kontoen din har blitt slettet.", + "accountDeletionStarted": "Kontosletting startet", + "accountDeletionStartedDesc": "Kontoen slettes i bakgrunnen og forsvinner når opprydningen er ferdig.", "allResourcesErased": "Alle relaterte ressurser vil bli permanent slettet.", "cannotBeUndone": "Denne handlingen kan ikke angres!", "confirmDelete": "Bekreft sletting", diff --git a/web/src/locales/pl.json b/web/src/locales/pl.json index 65999c9..4445656 100644 --- a/web/src/locales/pl.json +++ b/web/src/locales/pl.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Bład podczas usuwania konta", "accountDeleted": "Konto usunięte", "accountDeletedDesc": "Konto zostało usunięte.", + "accountDeletionStarted": "Rozpoczęto usuwanie konta", + "accountDeletionStartedDesc": "Konto jest usuwane w tle i zniknie po zakończeniu czyszczenia.", "allResourcesErased": "Wszystkie powiązane zasoby zostaną trwale usunięte.", "cannotBeUndone": "Tej czynności nie można cofnąć!", "confirmDelete": "Potwierdź usunięcie", diff --git a/web/src/locales/pt.json b/web/src/locales/pt.json index c47dac7..51ce366 100644 --- a/web/src/locales/pt.json +++ b/web/src/locales/pt.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Falha ao Excluir Conta", "accountDeleted": "Conta Excluída", "accountDeletedDesc": "A conta foi excluída com sucesso.", + "accountDeletionStarted": "Exclusão da conta iniciada", + "accountDeletionStartedDesc": "A conta está sendo excluída em segundo plano e desaparecerá após a limpeza.", "allResourcesErased": "Todos os recursos relacionados serão permanentemente apagados.", "cannotBeUndone": "Esta ação não pode ser desfeita!", "confirmDelete": "Confirmar Exclusão", diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json index d22f2af..f18284a 100644 --- a/web/src/locales/ru.json +++ b/web/src/locales/ru.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Ошибка удаления аккаунта", "accountDeleted": "Аккаунт удален", "accountDeletedDesc": "Ваш аккаунт был успешно удален.", + "accountDeletionStarted": "Удаление аккаунта запущено", + "accountDeletionStartedDesc": "Аккаунт удаляется в фоновом режиме и исчезнет после очистки.", "allResourcesErased": "Все связанные ресурсы будут безвозвратно стерты.", "cannotBeUndone": "Это действие нельзя отменить!", "confirmDelete": "Подтвердить удаление", diff --git a/web/src/locales/sv.json b/web/src/locales/sv.json index 7a70218..a021acf 100644 --- a/web/src/locales/sv.json +++ b/web/src/locales/sv.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "Borttagning av konto misslyckades", "accountDeleted": "Konto raderat", "accountDeletedDesc": "Ditt konto har tagits bort.", + "accountDeletionStarted": "Kontoradering har startat", + "accountDeletionStartedDesc": "Kontot raderas i bakgrunden och försvinner när rensningen är klar.", "allResourcesErased": "Alla relaterade resurser kommer att raderas permanent.", "cannotBeUndone": "Denna åtgärd kan inte ångras!", "confirmDelete": "Bekräfta borttagning", diff --git a/web/src/locales/zh-tw.json b/web/src/locales/zh-tw.json index db11b24..a348b89 100644 --- a/web/src/locales/zh-tw.json +++ b/web/src/locales/zh-tw.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "帳號刪除失敗", "accountDeleted": "帳號已刪除", "accountDeletedDesc": "帳號已成功刪除。", + "accountDeletionStarted": "帳戶刪除已開始", + "accountDeletionStartedDesc": "帳戶正在背景刪除，清理完成後將從列表中消失。", "allResourcesErased": "所有相關資源將被永久清除。", "cannotBeUndone": "此操作無法復原！", "confirmDelete": "確認刪除", diff --git a/web/src/locales/zh.json b/web/src/locales/zh.json index e3045eb..4d23a90 100644 --- a/web/src/locales/zh.json +++ b/web/src/locales/zh.json @@ -536,6 +536,8 @@ "accountDeleteFailed": "账户删除失败", "accountDeleted": "账户已删除", "accountDeletedDesc": "您的账户已成功删除。", + "accountDeletionStarted": "账户删除已开始", + "accountDeletionStartedDesc": "账户正在后台删除，清理完成后将从列表中消失。", "allResourcesErased": "所有相关资源将被永久删除。", "cannotBeUndone": "此操作无法撤销！", "confirmDelete": "确认删除", From c3a725770cbd723d0b89bbf207d498207d00da11 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 7 Jun 2026 18:18:37 +0800 Subject: [PATCH 27/66] fix: reconnect and retry IMAP batch on BrokenPipe/network errors --- crates/core/src/cache/imap/download/flow.rs | 139 ++++++++++++++++---- crates/core/src/error/mod.rs | 8 ++ crates/core/src/imap/executor.rs | 59 ++++++--- 3 files changed, 160 insertions(+), 46 deletions(-) diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 36805a7..4ce3eac 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -38,10 +38,12 @@ use crate::{ store::tantivy::envelope::ENVELOPE_MANAGER, }, }; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +const MAX_NETWORK_RETRIES: u32 = 3; + #[derive(Clone, Debug, Eq, PartialEq)] pub enum FetchDirection { @@ -152,16 +154,63 @@ pub async fn fetch_and_save_by_date( break; } // Fetch metadata for the current batch of UIDs - match ImapExecutor::uid_batch_retrieve_emails( - &mut session, - account_id, - mailbox.id, - &batch.0, - account.max_email_size_bytes, - token.clone(), - ) - .await - { + let mut retries = 0u32; + let batch_result = loop { + match ImapExecutor::uid_batch_retrieve_emails( + &mut session, + account_id, + mailbox.id, + &batch.0, + account.max_email_size_bytes, + token.clone(), + ) + .await + { + Ok(processed) => break Ok(processed), + Err(e) + if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => + { + retries += 1; + warn!( + account_id, + mailbox = mailbox.name, + index, + retries, + "Network error on batch, reconnecting ({}/{})", + retries, + MAX_NETWORK_RETRIES + ); + match ImapExecutor::create_connection(account_id).await { + Ok(new_session) => { + session = new_session; + if let Err(e2) = session.examine(&mailbox.encoded_name()).await + { + let err_msg = format!( + "Re-examine failed after reconnect: {:#?}", + e2 + ); + DownloadState::append_session_error( + account_id, + err_msg, + )?; + break Err(e); + } + tokio::time::sleep(Duration::from_secs( + 1 << (retries - 1), + )) + .await; + continue; + } + Err(e2) => { + error!(account_id, "Reconnection failed: {:#?}", e2); + break Err(e); + } + } + } + Err(e) => break Err(e), + } + }; + match batch_result { Ok(processed) => { current_processed += processed; DownloadState::update_folder_progress( @@ -283,20 +332,60 @@ pub async fn fetch_and_save_full_mailbox( break; } - match ImapExecutor::batch_retrieve_emails( - &mut session, - account_id, - mailbox_id, - total, - page as u64, - page_size as u64, - &mailbox.encoded_name(), - account.max_email_size_bytes, - token.clone(), - &mut max_uid, - ) - .await - { + let mut retries = 0u32; + let batch_result = loop { + match ImapExecutor::batch_retrieve_emails( + &mut session, + account_id, + mailbox_id, + total, + page as u64, + page_size as u64, + &mailbox.encoded_name(), + account.max_email_size_bytes, + token.clone(), + &mut max_uid, + ) + .await + { + Ok(count) => break Ok(count), + Err(e) + if retries < MAX_NETWORK_RETRIES && e.code() == ErrorCode::NetworkError => + { + retries += 1; + warn!( + account_id, + mailbox = mailbox.name, + page, + retries, + "Network error on batch, reconnecting ({}/{})", + retries, + MAX_NETWORK_RETRIES + ); + match ImapExecutor::create_connection(account_id).await { + Ok(new_session) => { + session = new_session; + if let Err(e2) = session.examine(&mailbox.encoded_name()).await { + let err_msg = format!( + "Re-examine failed after reconnect: {:#?}", + e2 + ); + DownloadState::append_session_error(account_id, err_msg)?; + break Err(e); + } + tokio::time::sleep(Duration::from_secs(1 << (retries - 1))).await; + continue; + } + Err(e2) => { + error!(account_id, "Reconnection failed: {:#?}", e2); + break Err(e); + } + } + } + Err(e) => break Err(e), + } + }; + match batch_result { Ok(count) => { current_processed += count as u64; DownloadState::update_folder_progress( diff --git a/crates/core/src/error/mod.rs b/crates/core/src/error/mod.rs index edf360c..27fad8e 100644 --- a/crates/core/src/error/mod.rs +++ b/crates/core/src/error/mod.rs @@ -16,4 +16,12 @@ pub enum BichonError { }, } +impl BichonError { + pub fn code(&self) -> ErrorCode { + match self { + BichonError::Generic { code, .. } => *code, + } + } +} + pub type BichonResult = std::result::Result; diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 08ef14f..16e281b 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -34,6 +34,23 @@ use tracing::info; const BODY_FETCH_COMMAND: &str = "(UID INTERNALDATE RFC822.SIZE BODY.PEEK[])"; const SIZE_ONLY_FETCH: &str = "(UID RFC822.SIZE)"; +fn classify_imap_error(e: &async_imap::error::Error) -> ErrorCode { + match e { + async_imap::error::Error::Io(io) => matches!( + io.kind(), + std::io::ErrorKind::BrokenPipe + | std::io::ErrorKind::ConnectionReset + | std::io::ErrorKind::ConnectionAborted + | std::io::ErrorKind::TimedOut + | std::io::ErrorKind::UnexpectedEof + ) + .then_some(ErrorCode::NetworkError) + .unwrap_or(ErrorCode::ImapCommandFailed), + async_imap::error::Error::ConnectionLost => ErrorCode::NetworkError, + _ => ErrorCode::ImapCommandFailed, + } +} + pub struct ImapExecutor; impl ImapExecutor { @@ -43,11 +60,11 @@ impl ImapExecutor { let list = session .list(Some(""), Some("*")) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let result = list .try_collect::>() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; Ok(result) } @@ -59,11 +76,11 @@ impl ImapExecutor { session .examine(mailbox_name) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let result = session .uid_search(query) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; Ok(result) } @@ -77,7 +94,7 @@ impl ImapExecutor { session .append(mailbox_name, flags, internaldate, content) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed)) + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e))) } /// Fetches new mail for a mailbox. @@ -102,7 +119,7 @@ impl ImapExecutor { session .examine(&mailbox.encoded_name()) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; match before { Some(date) => { @@ -132,7 +149,7 @@ impl ImapExecutor { let results = session.uid_search(&query).await.map_err(|e| { let err_msg = format!("UID SEARCH failed in [{}]: {:#?}", mailbox.name, e); let _ = DownloadState::append_session_error(account.id, err_msg); - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })?; if results.is_empty() { @@ -237,7 +254,7 @@ impl ImapExecutor { .map_err(|e| { let err_msg = format!("UID FETCH failed in [{}]: {:#?}", mailbox.name, e); let _ = DownloadState::append_session_error(account.id, err_msg); - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })?; let mut count = 0u64; @@ -247,7 +264,7 @@ impl ImapExecutor { while let Some(fetch) = stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? { if token.is_cancelled() { tracing::info!("Account {}: fetch_new_mail stream interrupted.", account.id); @@ -347,12 +364,12 @@ impl ImapExecutor { .fetch(sequence_set.as_str(), SIZE_ONLY_FETCH) .await .map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })?; let mut uids: Vec = Vec::new(); while let Some(fetch) = size_stream.try_next().await.map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })? { let uid = fetch.uid.unwrap_or(0); let msg_size = fetch.size.unwrap_or(0) as u64; @@ -381,13 +398,13 @@ impl ImapExecutor { let mut body_stream = session .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut count = 0; while let Some(fetch) = body_stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? { if token.is_cancelled() { tracing::info!("Account {}: UID fetch stream interrupted.", account_id); @@ -421,12 +438,12 @@ impl ImapExecutor { .uid_fetch(uid_set, SIZE_ONLY_FETCH) .await .map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })?; let mut uids: Vec = Vec::new(); while let Some(fetch) = size_stream.try_next().await.map_err(|e| { - raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed) + raise_error!(format!("{:#?}", e), classify_imap_error(&e)) })? { let uid = fetch.uid.unwrap_or(0); let msg_size = fetch.size.unwrap_or(0) as u64; @@ -455,13 +472,13 @@ impl ImapExecutor { let mut body_stream = session .uid_fetch(&filtered, BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut count = 0u64; while let Some(fetch) = body_stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? { if token.is_cancelled() { tracing::info!("Account {}: UID fetch stream interrupted.", account_id); @@ -489,17 +506,17 @@ impl ImapExecutor { session .examine(encoded_mailbox_name) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut stream = session .uid_fetch(uid.to_string(), BODY_FETCH_COMMAND) .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let fetch = stream .try_next() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? .ok_or_else(|| { raise_error!( format!("UID {uid} not found on IMAP server"), @@ -521,7 +538,7 @@ impl ImapExecutor { // while stream // .try_next() // .await - // .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + // .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? // .is_some() // {} From 3bfc080258bc66cc3bccb2928c94f158a09d993a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 7 Jun 2026 18:21:07 +0800 Subject: [PATCH 28/66] bump to 1.5.1 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a500552..69e4e71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.5.0" +version = "1.5.1" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.5.0" +version = "1.5.1" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.5.0" +version = "1.5.1" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.5.0" +version = "1.5.1" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.5.0" +version = "1.5.1" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index c6c6197..50cd3ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.5.0" +version = "1.5.1" edition = "2021" [workspace.dependencies] From 6f572b15aebdf258add344437c59605702da9623 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 9 Jun 2026 01:33:17 +0800 Subject: [PATCH 29/66] deps: upgrade fjall to 3.1.5 to address NFS Bad file descriptor error #216 #242 #239 #231 #240 --- Cargo.lock | 42 +++++++++++++++++++++--------------------- Cargo.toml | 14 +++++++------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69e4e71..2e3225a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.5.1" +version = "1.5.2" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.5.1" +version = "1.5.2" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.5.1" +version = "1.5.2" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.5.1" +version = "1.5.2" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.5.1" +version = "1.5.2" dependencies = [ "base64 0.22.1", "bichon-core", @@ -656,9 +656,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "js-sys", @@ -1295,9 +1295,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "fjall" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62b25b4d815ae178d7d9e4aa32ee59f072efd5431c736abede1e6ee13c8c453" +checksum = "038acd422d607e0eca09e093f299f9eccf9bd097554343d93746afff81a45113" dependencies = [ "byteorder-lite", "byteview", @@ -1789,9 +1789,9 @@ checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" [[package]] name = "http" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" dependencies = [ "bytes 1.11.1", "itoa", @@ -2315,9 +2315,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lsm-tree" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e447ac67ff6aef4ec07fc19e507b219336cbba90a697c0dbeb1bf51b91536b67" +checksum = "8ef86c3c797c10eefcc73407c43ae48c19d4df686131a8334b2895a513e91df4" dependencies = [ "byteorder-lite", "bytes 1.11.1", @@ -4115,18 +4115,18 @@ checksum = "97c9f5dd7ec5cc6d743f33fcb96de4eb91bb1cc51c5e0ba40cb285a9012043da" [[package]] name = "snafu" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1d4bced6a69f90b2056c03dcff2c4737f98d6fb9e0853493996e1d253ca29c6" +checksum = "d1a012328be2e3f5d5f6f3218147ca02588cea4cb865e876849ab6debcf36522" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40" +checksum = "5f103c50866b8743da9429b8a581d81a27c2d3a9c4ac7df8f8571c1dd7896eda" dependencies = [ "heck", "proc-macro2", @@ -4639,9 +4639,9 @@ dependencies = [ [[package]] name = "tokio-socks" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +checksum = "a7e2948f60dbe26b35f2c7fb74ac2854c1fddded0fe9d7548fcc674a246f7615" dependencies = [ "either", "futures-util", @@ -5039,9 +5039,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.1" +version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7" dependencies = [ "getrandom 0.4.2", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index 50cd3ce..a9f60f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,11 +12,11 @@ members = [ resolver = "2" [workspace.package] -version = "1.5.1" +version = "1.5.2" edition = "2021" [workspace.dependencies] -chrono = "0.4.44" +chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive", "env"] } memdb = { path = "crates/memdb" } itertools = "0.14.0" @@ -28,7 +28,7 @@ tracing = "0.1.44" tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json"] } base64 = "0.22.1" -snafu = "0.9.0" +snafu = "0.9.1" reqwest = { version = "0.12.24", default-features = false, features = [ "json", "stream", @@ -37,8 +37,8 @@ reqwest = { version = "0.12.24", default-features = false, features = [ "blocking", "socks", ] } -tokio-socks = "0.5.2" -http = "1.4.1" +tokio-socks = "0.5.3" +http = "1.4.2" regex = "1.12.3" email_address = "0.2.9" futures = "0.3.32" @@ -84,8 +84,8 @@ mail-send = "0.6.0" rcgen = "0.14.8" rustls-pemfile = "2.2.0" blake3 = "1.8.5" -uuid = { version = "1.23.1", features = ["v4", "serde"] } -fjall = { version = "3.1.4", features = ["lz4", "metrics", "bytes_1"] } +uuid = { version = "1.23.2", features = ["v4", "serde"] } +fjall = { version = "3.1.5", features = ["lz4", "metrics", "bytes_1"] } tracing-log = "0.2.0" tokio-util = "0.7.18" indicatif = "0.18.4" From ce3f8944a34db4fa45bcf77fac510633a4b4347e Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 10 Jun 2026 11:36:38 -0700 Subject: [PATCH 30/66] fix(smtp): don't clobber the IMAP-owned INBOX uid_validity on journal ingest When SMTP journaling ingests a message, parse_email() upserts the account's INBOX MailBox row so the journaled envelope has a row to attach to. But it built the row with uid_validity/highest_uid/uid_next = None and called batch_upsert, which replaces the WHOLE row. The INBOX row id (create_hash(account_id, "INBOX")) is the same id the IMAP sync maintains, so every journaled delivery reset the IMAP-maintained uid_validity to None. On the next reconcile, local_mailbox.uid_validity != Some(remote) is then true, so the mailbox is treated as invalid and wiped + rebuilt. For a large, UID-sparse INBOX whose rebuild gets interrupted, the local copy is silently lost and never restored (the incremental fetch resumes past all existing UIDs). See #297 for the full diagnosis and DB evidence. Fix: only create the INBOX row when it does not already exist; otherwise leave the IMAP-owned row untouched. The journaling path only needs the row to exist so the envelope can attach to it. Co-Authored-By: Claude Opus 4.8 (1M context) --- crates/smtp/src/server.rs | 52 +++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/crates/smtp/src/server.rs b/crates/smtp/src/server.rs index 9659866..be0b868 100644 --- a/crates/smtp/src/server.rs +++ b/crates/smtp/src/server.rs @@ -615,26 +615,40 @@ async fn parse_email(data: &[u8], session: &Session) -> BichonResult<()> { return Ok(()); } }; - let mailbox = MailBox { - id: create_hash(rcpt.id, "INBOX"), - account_id: rcpt.id, - name: "INBOX".into(), - delimiter: Some("/".to_string()), - attributes: vec![Attribute { - attr: AttributeEnum::Extension, - extension: Some("CreatedByBichon".into()), - }], - exists: 0, - unseen: None, - uid_next: None, - uid_validity: None, - highest_uid: None, - }; - let mailbox_id = mailbox.id; + let mailbox_id = create_hash(rcpt.id, "INBOX"); + + // The INBOX row is owned by the IMAP sync, which maintains `uid_validity`, + // `highest_uid` and `uid_next` on it. `batch_upsert` replaces the *whole* + // row, so blindly upserting here (with those fields = None) clobbers the + // IMAP-maintained state back to None. The next reconcile then sees + // `uid_validity` change from Some -> None, treats the mailbox as invalid, + // and wipes + rebuilds it — silently losing the local copy of a large + // mailbox when that rebuild is interrupted (see #297). + // + // We only need the row to *exist* so the journaled envelope can attach to + // it, so create it only when it is missing and otherwise leave the + // IMAP-owned row untouched. + if MailBox::find_mailbox(rcpt.id, mailbox_id)?.is_none() { + let mailbox = MailBox { + id: mailbox_id, + account_id: rcpt.id, + name: "INBOX".into(), + delimiter: Some("/".to_string()), + attributes: vec![Attribute { + attr: AttributeEnum::Extension, + extension: Some("CreatedByBichon".into()), + }], + exists: 0, + unseen: None, + uid_next: None, + uid_validity: None, + highest_uid: None, + }; - if let Err(e) = MailBox::batch_upsert(&[mailbox]) { - tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e); - return Err(e.into()); + if let Err(e) = MailBox::batch_upsert(&[mailbox]) { + tracing::error!("SMTP: Failed to upsert mailbox for {}: {:?}", rcpt.email, e); + return Err(e.into()); + } } extract_envelope_from_smtp(data, rcpt.id, mailbox_id) From 2f5de48c6a6412ab75345a0f7ac26da4d7e6caca Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 11 Jun 2026 09:22:57 +0800 Subject: [PATCH 31/66] fix(smtp): reject journaling attempts to non-local accounts --- crates/smtp/src/server.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/smtp/src/server.rs b/crates/smtp/src/server.rs index 9659866..8257099 100644 --- a/crates/smtp/src/server.rs +++ b/crates/smtp/src/server.rs @@ -21,6 +21,7 @@ use std::net::SocketAddr; use std::time::Duration; use base64::{prelude::BASE64_STANDARD, Engine as _}; +use bichon_core::account::migration::AccountType; use bichon_core::cache::imap::mailbox::{Attribute, AttributeEnum}; use bichon_core::common::signal::SIGNAL_MANAGER; use bichon_core::envelope::extractor::extract_envelope_from_smtp; @@ -429,8 +430,20 @@ where } if is_allowed { - session.rcpt_to.push(account); - stream.write_all(b"250 OK\r\n").await?; + if !matches!(account.account_type, AccountType::NoSync) { + tracing::warn!( + "SMTP: Rejected journaling attempt to IMAP account <{}>", + addr + ); + let err = format!( + "550 5.7.1 <{}>: Not a Bichon local account, journaling is not supported\r\n", + account.email + ); + stream.write_all(err.as_bytes()).await?; + } else { + session.rcpt_to.push(account); + stream.write_all(b"250 OK\r\n").await?; + } } } Ok(None) => { From 41c3b84e6578cad5b1b358ec6e2f284e8b454593 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Sun, 21 Jun 2026 12:50:21 +0800 Subject: [PATCH 32/66] feat(ui): persist account table sorting to localStorage --- web/src/features/accounts/components/table.tsx | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/web/src/features/accounts/components/table.tsx b/web/src/features/accounts/components/table.tsx index c2ba385..3ff597e 100644 --- a/web/src/features/accounts/components/table.tsx +++ b/web/src/features/accounts/components/table.tsx @@ -17,7 +17,7 @@ // along with this program. If not, see . -import { useState } from 'react' +import { useEffect, useRef, useState } from 'react' import { ColumnDef, ColumnFiltersState, @@ -64,7 +64,19 @@ export function AccountTable({ columns, data }: DataTableProps) { const [rowSelection, setRowSelection] = useState({}) const [columnVisibility, setColumnVisibility] = useState({}) const [columnFilters, setColumnFilters] = useState([]) - const [sorting, setSorting] = useState([]) + const [sorting, setSorting] = useState(() => { + const saved = localStorage.getItem('bichon_accounts_sorting'); + return saved ? JSON.parse(saved) : []; + }) + + // Persist sorting state to localStorage + const prevSortingRef = useRef(sorting); + useEffect(() => { + if (prevSortingRef.current !== sorting) { + localStorage.setItem('bichon_accounts_sorting', JSON.stringify(sorting)); + prevSortingRef.current = sorting; + } + }, [sorting]); const table = useReactTable({ data, From e3fd9d2f29b59cc8312f41d01b5b80e30e857adc Mon Sep 17 00:00:00 2001 From: rustmailer Date: Mon, 22 Jun 2026 17:11:20 +0800 Subject: [PATCH 33/66] fix: purge DedupCache entries on account/mailbox/envelope removal --- crates/core/src/store/tantivy/dedup_cache.rs | 149 ++++++++++++++++++- crates/core/src/store/tantivy/envelope.rs | 129 ++++++++-------- 2 files changed, 214 insertions(+), 64 deletions(-) diff --git a/crates/core/src/store/tantivy/dedup_cache.rs b/crates/core/src/store/tantivy/dedup_cache.rs index 99641a7..836a7ec 100644 --- a/crates/core/src/store/tantivy/dedup_cache.rs +++ b/crates/core/src/store/tantivy/dedup_cache.rs @@ -172,6 +172,52 @@ impl DedupCache { POPULATE_WINDOW_MS / (24 * 60 * 60 * 1000), ); } + + /// Remove all entries for a specific account. + pub fn remove_by_account(&self, account_id: u64) { + let mut entries = self.entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(aid, _, _), _| *aid != account_id); + let removed = before - entries.len(); + if removed > 0 { + tracing::info!( + "DedupCache: removed {} entries for account {}", + removed, + account_id + ); + } + } + + /// Remove all entries for a specific mailbox (across all accounts). + pub fn remove_by_mailbox(&self, mailbox_id: u64) { + let mut entries = self.entries.lock().unwrap(); + let before = entries.len(); + entries.retain(|(_, mid, _), _| *mid != mailbox_id); + let removed = before - entries.len(); + if removed > 0 { + tracing::info!( + "DedupCache: removed {} entries for mailbox {}", + removed, + mailbox_id + ); + } + } + + /// Remove a specific triple (most precise removal). + pub fn remove(&self, account_id: u64, mailbox_id: u64, hash: &str) { + let mut entries = self.entries.lock().unwrap(); + if entries + .remove(&(account_id, mailbox_id, hash.to_string())) + .is_some() + { + tracing::debug!( + "DedupCache: removed specific entry ({}, {}, {})", + account_id, + mailbox_id, + hash + ); + } + } } // ── Tests ───────────────────────────────────────────────────────────────────── @@ -366,7 +412,11 @@ mod tests { for segment_reader in searcher.segment_readers() { let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); - let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); let max_doc = segment_reader.max_doc(); @@ -381,7 +431,11 @@ mod tests { let account_id = account_col.values.get_val(doc_id); let mailbox_id = mailbox_col.values.get_val(doc_id); - let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); let mut hash_buf = String::new(); hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); @@ -419,7 +473,11 @@ mod tests { for segment_reader in searcher.segment_readers() { let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); - let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); let max_doc = segment_reader.max_doc(); @@ -434,7 +492,11 @@ mod tests { let account_id = account_col.values.get_val(doc_id); let mailbox_id = mailbox_col.values.get_val(doc_id); - let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); let mut hash_buf = String::new(); hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); @@ -474,7 +536,11 @@ mod tests { for segment_reader in searcher.segment_readers() { let account_col = segment_reader.fast_fields().u64(F_ACCOUNT_ID).unwrap(); let mailbox_col = segment_reader.fast_fields().u64(F_MAILBOX_ID).unwrap(); - let hash_col = segment_reader.fast_fields().str(F_CONTENT_HASH).unwrap().unwrap(); + let hash_col = segment_reader + .fast_fields() + .str(F_CONTENT_HASH) + .unwrap() + .unwrap(); let ingest_col = segment_reader.fast_fields().i64(F_INGEST_AT).unwrap(); let max_doc = segment_reader.max_doc(); @@ -489,7 +555,11 @@ mod tests { let account_id = account_col.values.get_val(doc_id); let mailbox_id = mailbox_col.values.get_val(doc_id); - let hash_ord = hash_col.ords().values_for_doc(doc_id as u32).next().unwrap_or(0); + let hash_ord = hash_col + .ords() + .values_for_doc(doc_id as u32) + .next() + .unwrap_or(0); let mut hash_buf = String::new(); hash_col.ord_to_str(hash_ord, &mut hash_buf).unwrap(); @@ -501,4 +571,71 @@ mod tests { assert!(cache.contains(1, 10, "hash-keep")); assert!(!cache.contains(1, 10, "hash-delete")); } + + // ── removal methods ───────────────────────────────────────────────────── + + #[test] + fn remove_by_account_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-a1"); + cache.insert(1, 20, "hash-a2"); + cache.insert(2, 10, "hash-b1"); + cache.insert(2, 30, "hash-b2"); + cache.insert(1, 10, "hash-a3"); + + assert_eq!(cache.entries.lock().unwrap().len(), 5); + + cache.remove_by_account(1); + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), 2); + assert!(!entries.contains_key(&(1, 10, "hash-a1".to_string()))); + assert!(!entries.contains_key(&(1, 20, "hash-a2".to_string()))); + assert!(!entries.contains_key(&(1, 10, "hash-a3".to_string()))); + + assert!(entries.contains_key(&(2, 10, "hash-b1".to_string()))); + assert!(entries.contains_key(&(2, 30, "hash-b2".to_string()))); + } + + #[test] + fn remove_by_mailbox_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-1"); + cache.insert(1, 20, "hash-2"); + cache.insert(2, 10, "hash-3"); + cache.insert(3, 20, "hash-4"); + cache.insert(1, 10, "hash-5"); + + cache.remove_by_mailbox(10); + + let entries = cache.entries.lock().unwrap(); + assert_eq!(entries.len(), 2); + + assert!(entries.contains_key(&(1, 20, "hash-2".to_string()))); + assert!(entries.contains_key(&(3, 20, "hash-4".to_string()))); + + assert!(!entries.contains_key(&(1, 10, "hash-1".to_string()))); + assert!(!entries.contains_key(&(2, 10, "hash-3".to_string()))); + } + + #[test] + fn remove_specific_triple_works() { + let cache = DedupCache::new_for_test(); + + cache.insert(1, 10, "hash-aaa"); + cache.insert(1, 10, "hash-bbb"); + cache.insert(2, 20, "hash-aaa"); + + assert!(cache.contains(1, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-bbb")); + assert!(cache.contains(2, 20, "hash-aaa")); + + cache.remove(1, 10, "hash-aaa"); + + assert!(!cache.contains(1, 10, "hash-aaa")); + assert!(cache.contains(1, 10, "hash-bbb")); + assert!(cache.contains(2, 20, "hash-aaa")); + } } diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 8c7e17b..1a5e97a 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -40,6 +40,7 @@ use crate::{ envelope::Envelope, tantivy::{ attachment::ATTACHMENT_MANAGER, + dedup_cache::DEDUP_CACHE, fatal_commit, fields::{ F_ACCOUNT_ID, F_DATE, F_FROM, F_ID, F_INGEST_AT, F_INTERNAL_DATE, @@ -50,8 +51,8 @@ use crate::{ tokenizers::EuroTokenizer, }, }, - utils::html::extract_text, utc_now, + utils::html::extract_text, }; use chrono::Utc; @@ -792,6 +793,8 @@ impl IndexManager { attachments_content_hashes, )?; } + + DEDUP_CACHE.remove_by_account(account_id); Ok(()) } @@ -815,8 +818,8 @@ impl IndexManager { } let mut queries: Vec> = Vec::with_capacity(mailbox_ids.len()); - for mailbox_id in mailbox_ids { - queries.push(self.mailbox_query(account_id, mailbox_id)); + for mailbox_id in &mailbox_ids { + queries.push(self.mailbox_query(account_id, *mailbox_id)); } let mut writer = self.index_writer.lock().await; for query in queries { @@ -835,6 +838,11 @@ impl IndexManager { attachments_content_hashes, )?; } + + for mailbox_id in mailbox_ids { + DEDUP_CACHE.remove_by_mailbox(mailbox_id); + } + Ok(()) } @@ -842,6 +850,21 @@ impl IndexManager { &self, query: Box, ) -> BichonResult<(HashSet, HashSet)> { + let (eml_with_mailbox, attachments_content_hashes) = + self.collect_content_hashes_with_mailbox(query)?; + + let eml_content_hashes = eml_with_mailbox + .into_iter() + .map(|(hash, _mailbox_id)| hash) + .collect(); + + Ok((eml_content_hashes, attachments_content_hashes)) + } + + fn collect_content_hashes_with_mailbox( + &self, + query: Box, + ) -> BichonResult<(HashSet<(String, u64)>, HashSet)> { let mut eml_content_hashes = HashSet::new(); let mut attachments_content_hashes = HashSet::new(); @@ -857,10 +880,14 @@ impl IndexManager { .doc::(doc_address) .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mailbox_id = doc.get_first(fields.f_mailbox_id).and_then(|v| v.as_u64()); + // Extract content_hash if let Some(content_hash_value) = doc.get_first(fields.f_content_hash) { - if let Some(str) = content_hash_value.as_str() { - eml_content_hashes.insert(str.to_string()); + if let (Some(hash_str), Some(mailbox_id)) = + (content_hash_value.as_str(), mailbox_id) + { + eml_content_hashes.insert((hash_str.to_string(), mailbox_id)); } } @@ -933,7 +960,7 @@ impl IndexManager { return Ok(()); } - let mut eml_content_hashes: HashSet = HashSet::new(); + let mut eml_content_hash_triples: HashSet<(u64, u64, String)> = HashSet::new(); let mut attachments_content_hashes: HashSet = HashSet::new(); for (account_id, envelope_ids) in &deletes { @@ -944,8 +971,14 @@ impl IndexManager { for eid in unique_ids { let query = self.envelope_query(*account_id, eid); - let (eml_hashes, attachment_hashes) = self.collect_content_hashes(query)?; - eml_content_hashes.extend(eml_hashes); + let (eml_hashes_with_mailbox, attachment_hashes) = + self.collect_content_hashes_with_mailbox(query)?; + + eml_content_hash_triples.extend( + eml_hashes_with_mailbox + .into_iter() + .map(|(hash, mailbox_id)| (*account_id, mailbox_id, hash)), + ); attachments_content_hashes.extend(attachment_hashes); } } @@ -968,7 +1001,12 @@ impl IndexManager { .commit() .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; - if !eml_content_hashes.is_empty() || !attachments_content_hashes.is_empty() { + if !eml_content_hash_triples.is_empty() || !attachments_content_hashes.is_empty() { + let eml_content_hashes: HashSet = eml_content_hash_triples + .iter() + .map(|(_, _, hash)| hash.clone()) + .collect(); + self.cleanup_unused_content( &mut writer, eml_content_hashes, @@ -976,6 +1014,10 @@ impl IndexManager { )?; } + for (aid, mid, hash) in eml_content_hash_triples { + DEDUP_CACHE.remove(aid, mid, &hash); + } + Ok(()) } @@ -1157,8 +1199,7 @@ impl IndexManager { // f_attachments JSON blob. if let Some(attrs_val) = old_doc.get_first(f.f_attachments) { if let Some(json_str) = attrs_val.as_str() { - if let Ok(parsed) = - serde_json::from_str::(json_str) + if let Ok(parsed) = serde_json::from_str::(json_str) { if let Some(arr) = parsed.as_array() { for att in arr { @@ -1179,14 +1220,8 @@ impl IndexManager { .and_then(|v| v.as_str()) .filter(|s| !s.is_empty()) { - new_doc.add_text( - f.f_attachment_name_text, - filename, - ); - new_doc.add_text( - f.f_attachment_name_exact, - filename, - ); + new_doc.add_text(f.f_attachment_name_text, filename); + new_doc.add_text(f.f_attachment_name_exact, filename); } } } @@ -1200,22 +1235,18 @@ impl IndexManager { if let Some(content_hash) = hash_val.as_str() { match BLOB_MANAGER.get_email(content_hash) { Ok(Some(eml_bytes)) => { - if let Some(message) = - MessageParser::new().parse(&eml_bytes) - { + if let Some(message) = MessageParser::new().parse(&eml_bytes) { let text = message .body_text(0) .map(|cow| cow.into_owned()) .or_else(|| { - message.body_html(0).map(|cow| { - extract_text(cow.into_owned()) - }) + message + .body_html(0) + .map(|cow| extract_text(cow.into_owned())) }) .unwrap_or_default(); - let body_text = text - .split_whitespace() - .collect::>() - .join(" "); + let body_text = + text.split_whitespace().collect::>().join(" "); if !body_text.is_empty() { new_doc.add_text(f.f_body, &body_text); } @@ -1304,7 +1335,7 @@ impl IndexManager { let mailbox_docs: Vec; match sort_by { - SortBy::DATE => { + SortBy::DATE => { let date_docs: Vec<(Option, DocAddress)> = searcher .search( &query, @@ -1801,10 +1832,8 @@ mod tests { if let Ok(parsed) = serde_json::from_str::(json_str) { if let Some(arr) = parsed.as_array() { for att in arr { - let is_inline = att - .get("inline") - .and_then(|v| v.as_bool()) - .unwrap_or(false); + let is_inline = + att.get("inline").and_then(|v| v.as_bool()).unwrap_or(false); let has_cid = att .get("content_id") .and_then(|v| v.as_str()) @@ -1841,8 +1870,7 @@ mod tests { .map(|cow| extract_text(cow.into_owned())) }) .unwrap_or_default(); - let body_text = - text.split_whitespace().collect::>().join(" "); + let body_text = text.split_whitespace().collect::>().join(" "); if !body_text.is_empty() { new_doc.add_text(f.f_body, &body_text); } @@ -1911,8 +1939,7 @@ mod tests { let mut writer2 = index .writer_with_num_threads(1, 15_000_000) .expect("writer2"); - writer2 - .delete_term(Term::from_field_text(f.f_id, "test-eid-001")); + writer2.delete_term(Term::from_field_text(f.f_id, "test-eid-001")); writer2.add_document(new_doc).unwrap(); writer2.commit().unwrap(); @@ -1922,16 +1949,14 @@ mod tests { let searcher = reader.searcher(); // Body text (tokenized via "euro") - let body_parser = - QueryParser::for_index(&index, vec![f.f_body]); + let body_parser = QueryParser::for_index(&index, vec![f.f_body]); let body_hits = searcher .search(&body_parser.parse_query("quick brown fox").unwrap(), &Count) .unwrap(); assert_eq!(body_hits, 1, "body text should survive tag update"); // from_text (tokenized via "euro") - let from_parser = - QueryParser::for_index(&index, vec![f.f_from_text]); + let from_parser = QueryParser::for_index(&index, vec![f.f_from_text]); let from_hits = searcher .search( &from_parser.parse_query("alice@example.com").unwrap(), @@ -1943,10 +1968,7 @@ mod tests { // to_text let to_parser = QueryParser::for_index(&index, vec![f.f_to_text]); let to_hits = searcher - .search( - &to_parser.parse_query("bob@example.com").unwrap(), - &Count, - ) + .search(&to_parser.parse_query("bob@example.com").unwrap(), &Count) .unwrap(); assert_eq!(to_hits, 1, "to_text should survive tag update"); @@ -1969,10 +1991,7 @@ mod tests { let tags_hits = searcher .search( &TermQuery::new( - Term::from_facet( - f.f_tags, - &Facet::from_text("/important").unwrap(), - ), + Term::from_facet(f.f_tags, &Facet::from_text("/important").unwrap()), IndexRecordOption::Basic, ), &Count, @@ -1984,19 +2003,13 @@ mod tests { let old_tag_hits = searcher .search( &TermQuery::new( - Term::from_facet( - f.f_tags, - &Facet::from_text("/unread").unwrap(), - ), + Term::from_facet(f.f_tags, &Facet::from_text("/unread").unwrap()), IndexRecordOption::Basic, ), &Count, ) .unwrap(); - assert_eq!( - old_tag_hits, 0, - "old tag /unread should have been removed" - ); + assert_eq!(old_tag_hits, 0, "old tag /unread should have been removed"); } #[test] From 220aa268c193fc621d4dc652b6685c16f63dc66d Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 10:46:36 +0800 Subject: [PATCH 34/66] feat(imap): handle UIDVALIDITY changes via Message-ID comparison instead of full rebuild --- crates/core/src/cache/imap/download/flow.rs | 1023 +++++++++++++++++-- crates/core/src/imap/client.rs | 25 +- crates/core/src/imap/executor.rs | 158 ++- crates/core/src/imap/manager.rs | 42 +- crates/core/src/imap/mock_server.rs | 538 ++++++++++ crates/core/src/imap/mod.rs | 2 + crates/core/src/store/tantivy/envelope.rs | 432 ++++++++ 7 files changed, 2136 insertions(+), 84 deletions(-) create mode 100644 crates/core/src/imap/mock_server.rs diff --git a/crates/core/src/cache/imap/download/flow.rs b/crates/core/src/cache/imap/download/flow.rs index 4ce3eac..3fc0b7b 100644 --- a/crates/core/src/cache/imap/download/flow.rs +++ b/crates/core/src/cache/imap/download/flow.rs @@ -33,7 +33,7 @@ use crate::{ }, error::{code::ErrorCode, BichonResult}, imap::executor::{ - generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, + compress_uid_list, generate_uid_sequence_hashset, ImapExecutor, DEFAULT_BATCH_SIZE, }, store::tantivy::envelope::ENVELOPE_MANAGER, }, @@ -433,12 +433,279 @@ pub async fn fetch_and_save_full_mailbox( fn generate_synthetic_uidvalidity(mailbox_name: &str) -> u32 { use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; - + let mut hasher = DefaultHasher::new(); mailbox_name.hash(&mut hasher); (hasher.finish() as u32).wrapping_add(1) // Avoid 0, which might be reserved } +/// Retry fetching UIDVALIDITY via STATUS command when the initial listing returned None. +/// A None UIDVALIDITY while other STATUS fields (MESSAGES, UNSEEN, UIDNEXT) are present +/// can be caused by transient network issues corrupting just the UIDVALIDITY portion of +/// the response. Retrying avoids unnecessarily triggering a full reconcile. +async fn fetch_uid_validity_with_retry( + account_id: u64, + mailbox_name: &str, + max_retries: u32, +) -> BichonResult> { + let mailbox_name = mailbox_name.to_string(); + fetch_uid_validity_with_retry_inner(max_retries, move || { + let account_id = account_id; + let mailbox_name = mailbox_name.clone(); + async move { + let mut session = ImapExecutor::create_connection(account_id).await?; + let result = session + .status(&mailbox_name, "(UIDVALIDITY)") + .await + .map(|r| r.uid_validity) + .map_err(|e| { + let msg = format!("STATUS failed during UIDVALIDITY retry: {:#?}", e); + raise_error!(msg, ErrorCode::InternalError) + }); + session.logout().await.ok(); + result + } + }) + .await +} + +/// Generic retry loop: calls `fetch_fn` up to `max_retries` times with +/// backoff (500ms × attempt). Returns the first `Some(uid)`, or `Ok(None)` +/// if all attempts return `None` or error. +async fn fetch_uid_validity_with_retry_inner( + max_retries: u32, + mut fetch_fn: F, +) -> BichonResult> +where + F: FnMut() -> Fut, + Fut: std::future::Future>>, +{ + for attempt in 0..max_retries { + if attempt > 0 { + tokio::time::sleep(std::time::Duration::from_millis(500 * attempt as u64)).await; + } + + match fetch_fn().await { + Ok(Some(uid)) => return Ok(Some(uid)), + Ok(None) => { + warn!( + attempt = attempt + 1, + max_retries, + "STATUS returned no UIDVALIDITY" + ); + } + Err(e) => { + warn!( + attempt = attempt + 1, + max_retries, + "UIDVALIDITY fetch attempt failed: {:#?}", e + ); + } + } + } + + Ok(None) +} + +/// Handle uid_validity change without deleting local data. +/// Compares remote Message-IDs with local Tantivy index, downloads only +/// truly missing emails. DedupCache catches any remaining duplicates. +async fn reconcile_uid_validity_change( + account: &AccountModel, + local_mailbox: &MailBox, + remote_mailbox: &MailBox, + token: CancellationToken, +) -> BichonResult> { + let account_id = account.id; + + // Phase 1: connect + examine + let mut session = ImapExecutor::create_connection(account_id).await?; + session + .examine(&remote_mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + // Phase 2: collect remote UIDs, respecting date constraints + let remote_uid_list: Vec = if let Some(date_since) = &account.date_since { + let date = date_since.since_date()?; + let results = session + .uid_search(&format!("SINCE {date}")) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + } else if let Some(date_before) = &account.date_before { + let date = date_before.calculate_date()?; + let results = session + .uid_search(&format!("BEFORE {date}")) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + } else { + let results = session + .uid_search("ALL") + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + let mut v: Vec = results.into_iter().collect(); + v.sort(); + v + }; + + if remote_uid_list.is_empty() { + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + Some("UIDVALIDITY changed but remote mailbox is empty.".into()), + )?; + session.logout().await.ok(); + return Ok(None); + } + + let max_uid = remote_uid_list.last().copied(); + + // Phase 3: fetch remote Message-IDs (headers only, no bodies) + let uid_set = compress_uid_list(remote_uid_list.clone()); + let remote_msg_ids = + ImapExecutor::fetch_uid_metadata(&mut session, &uid_set, token.clone()).await?; + session.logout().await.ok(); + + // Phase 4: query local Message-IDs from Tantivy. + // For a large mailbox this can allocate 50-100 MB of HashSet. + let local_msg_ids = + ENVELOPE_MANAGER.get_message_ids_for_mailbox(account_id, local_mailbox.id)?; + + // Phase 5: compute missing UIDs + let mut missing_uids: Vec = Vec::new(); + for uid in &remote_uid_list { + if token.is_cancelled() { + return Err(raise_error!("Cancelled".into(), ErrorCode::InternalError)); + } + match remote_msg_ids.get(uid) { + Some(Some(msg_id)) if local_msg_ids.contains(msg_id) => { + // already have this email locally + } + _ => missing_uids.push(*uid), + } + } + + // Phase 6: download missing + if missing_uids.is_empty() { + info!( + account_id, + mailbox = remote_mailbox.name, + "UIDVALIDITY changed but all {} emails already exist locally", + remote_uid_list.len() + ); + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + 0, + 0, + FolderStatus::Success, + None, + )?; + } else { + let planned = missing_uids.len() as u64; + info!( + account_id, + mailbox = remote_mailbox.name, + total = remote_uid_list.len(), + missing = planned, + "UIDVALIDITY changed, downloading missing emails" + ); + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + 0, + FolderStatus::Downloading, + Some("UIDVALIDITY changed, downloading missing emails...".into()), + )?; + + let mut session2 = ImapExecutor::create_connection(account_id).await?; + session2 + .examine(&remote_mailbox.encoded_name()) + .await + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let batch_size = account + .download_batch_size + .unwrap_or(DEFAULT_BATCH_SIZE) as usize; + let batches = generate_uid_sequence_hashset(missing_uids, batch_size); + + let mut downloaded = 0u64; + for (index, batch) in batches.into_iter().enumerate() { + if token.is_cancelled() { + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Cancelled, + None, + )?; + session2.logout().await.ok(); + return Err(raise_error!("Cancelled".into(), ErrorCode::InternalError)); + } + + match ImapExecutor::uid_batch_retrieve_emails( + &mut session2, + account_id, + remote_mailbox.id, + &batch.0, + account.max_email_size_bytes, + token.clone(), + ) + .await + { + Ok(processed) => { + downloaded += processed; + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Downloading, + None, + )?; + } + Err(e) => { + let err_msg = format!("Batch {} failed: {:#?}", index, e); + DownloadState::append_session_error(account_id, err_msg.clone())?; + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Failed, + Some(err_msg), + )?; + session2.logout().await.ok(); + return Err(e); + } + } + } + + DownloadState::update_folder_progress( + account_id, + remote_mailbox.name.clone(), + planned, + downloaded, + FolderStatus::Success, + None, + )?; + session2.logout().await.ok(); + } + + Ok(max_uid) +} + pub async fn reconcile_mailboxes( account: &AccountModel, remote_mailboxes: &[MailBox], @@ -471,71 +738,66 @@ pub async fn reconcile_mailboxes( let remote_uid_validity = match remote_mailbox.uid_validity { Some(uid) => uid, None => { - // Generate a synthetic UIDVALIDITY based on mailbox name - let synthetic_uid = generate_synthetic_uidvalidity(&remote_mailbox.name); - - warn!( - "Account {}: Mailbox '{}' - Server did not provide UIDVALIDITY. \ - Using synthetic UIDVALIDITY {} based on mailbox name. \ - This mailbox will be synced but may require periodic rebuilds if the server's mailbox structure changes.", - account_id, remote_mailbox.name, synthetic_uid - ); - - synthetic_uid + if local_mailbox.uid_validity.is_some() { + // We had a real UIDVALIDITY before; a None now is likely + // transient (network jitter). Retry before falling back. + warn!( + "Account {}: Mailbox '{}' - STATUS returned no UIDVALIDITY, retrying to rule out network jitter...", + account_id, remote_mailbox.name + ); + match fetch_uid_validity_with_retry( + account_id, + &remote_mailbox.encoded_name(), + 3, + ) + .await? + { + Some(uid) => { + info!( + "Account {}: Mailbox '{}' - UIDVALIDITY recovered after retry: {}", + account_id, remote_mailbox.name, uid + ); + uid + } + None => { + // All retries exhausted; keep the local value. + // Safer than synthetic because we know the server had one. + let fallback = local_mailbox.uid_validity.unwrap(); + warn!( + "Account {}: Mailbox '{}' - all retries failed, keeping local UIDVALIDITY {}", + account_id, remote_mailbox.name, fallback + ); + fallback + } + } + } else { + // First sync and server genuinely doesn't provide UIDVALIDITY + let synthetic_uid = generate_synthetic_uidvalidity(&remote_mailbox.name); + warn!( + "Account {}: Mailbox '{}' - Server did not provide UIDVALIDITY. \ + Using synthetic UIDVALIDITY {} based on mailbox name. \ + This mailbox will be synced but may require periodic rebuilds if the server's mailbox structure changes.", + account_id, remote_mailbox.name, synthetic_uid + ); + synthetic_uid + } } }; let new_highest_uid = if local_mailbox.uid_validity != Some(remote_uid_validity) { info!( "Account {}: Mailbox '{}' detected with changed uid_validity (local: {:#?}, remote: {:#?}). \ - The mailbox data may be invalid, resetting its envelopes and rebuilding the cache.", + Comparing by Message-ID to find missing emails.", account_id, local_mailbox.name, &local_mailbox.uid_validity, &remote_uid_validity ); - DownloadState::update_folder_progress( - account_id, - local_mailbox.name.clone(), - remote_mailbox.exists as u64, - 0, - FolderStatus::Downloading, - Some("UID validity changed, rebuilding...".into()), - )?; - - match &account.date_since { - Some(date_since) => { - rebuild_mailbox_cache_by_date( - account, - local_mailbox.id, - &date_since.since_date()?, - remote_mailbox, - FetchDirection::Since, - token.clone(), - ) - .await? - } - None => match &account.date_before { - Some(r) => { - rebuild_mailbox_cache_by_date( - account, - local_mailbox.id, - &r.calculate_date()?, - remote_mailbox, - FetchDirection::Before, - token.clone(), - ) - .await? - } - None => { - rebuild_mailbox_cache( - account, - local_mailbox, - remote_mailbox, - token.clone(), - ) - .await? - } - }, - } + reconcile_uid_validity_change( + account, + local_mailbox, + remote_mailbox, + token.clone(), + ) + .await? } else { perform_incremental_sync(account, local_mailbox, remote_mailbox, token.clone()) .await? @@ -749,3 +1011,648 @@ async fn perform_incremental_sync( Ok(local_mailbox.highest_uid) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::imap::session::SessionStream; + use std::sync::Arc; + use tokio_io_timeout::TimeoutStream; + + // ============================================================ + // Pure unit tests (no network) + // ============================================================ + + #[test] + fn test_generate_synthetic_uidvalidity_deterministic() { + let a = generate_synthetic_uidvalidity("INBOX"); + let b = generate_synthetic_uidvalidity("INBOX"); + assert_eq!(a, b, "same mailbox name must produce same uid_validity"); + } + + #[test] + fn test_generate_synthetic_uidvalidity_different_mailboxes() { + let inbox = generate_synthetic_uidvalidity("INBOX"); + let sent = generate_synthetic_uidvalidity("Sent"); + assert_ne!(inbox, sent, "different mailboxes should have different uid_validity"); + } + + #[test] + fn test_generate_synthetic_uidvalidity_non_zero() { + let uid = generate_synthetic_uidvalidity("INBOX"); + assert_ne!(uid, 0, "UIDVALIDITY should not be 0 (reserved)"); + } + + // ============================================================ + // Integration tests (real IMAP server required) + // ============================================================ + // Fill in your IMAP server details below to run these tests. + // cargo test -p bichon-core -- --ignored + + const TEST_IMAP_HOST: &str = "imap.zoho.com"; + const TEST_IMAP_PORT: u16 = 993; + const TEST_IMAP_USERNAME: &str = ""; + const TEST_IMAP_PASSWORD: &str = ""; + const TEST_MAILBOX: &str = "INBOX"; + + /// Build a direct IMAP session bypassing the database-dependent + /// ImapConnectionManager, for local testing with real credentials. + async fn direct_connect( + host: &str, + port: u16, + username: &str, + password: &str, + ) -> Result>, String> { + use rustls::ClientConfig; + use rustls_pki_types::ServerName; + use tokio::net::TcpStream; + use tokio_rustls::TlsConnector; + + // Ensure a rustls crypto provider is installed (ring). + // May already be installed by production code; ignore duplicate. + rustls::crypto::CryptoProvider::install_default( + rustls::crypto::ring::default_provider(), + ) + .ok(); + + let tcp = TcpStream::connect((host, port)) + .await + .map_err(|e| format!("TCP connect error: {e}"))?; + + // Wrap in Pin>> to satisfy SessionStream, + // matching the production path in establish_tcp_connection_with_timeout. + let timeout_stream = TimeoutStream::new(tcp); + let pinned = Box::pin(timeout_stream); + + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|e| format!("Invalid hostname: {e}"))?; + + let config = ClientConfig::builder() + .with_root_certificates(rustls::RootCertStore { + roots: webpki_roots::TLS_SERVER_ROOTS.into(), + }) + .with_no_client_auth(); + + let connector = TlsConnector::from(Arc::new(config)); + let tls_stream = connector + .connect(server_name, pinned) + .await + .map_err(|e| format!("TLS error: {e}"))?; + + let client = async_imap::Client::new(Box::new(tls_stream) as Box); + let session = client + .login(username, password) + .await + .map_err(|(e, _)| format!("Login error: {e}"))?; + + Ok(session) + } + + /// Test STATUS UIDVALIDITY directly — verifies your server returns + /// a valid UIDVALIDITY for the given mailbox. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_status_uid_validity_present() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(UIDVALIDITY)") + .await + .expect("STATUS command should succeed"); + + session.logout().await.ok(); + + match status.uid_validity { + Some(uid) => { + println!("[OK] Server returned UIDVALIDITY: {uid}"); + assert_ne!(uid, 0); + } + None => { + println!("[INFO] Server returned no UIDVALIDITY in STATUS response"); + println!(" This server may need the synthetic fallback or the retry logic."); + } + } + } + + /// Test STATUS with full attributes (MESSAGES UNSEEN UIDNEXT UIDVALIDITY), + /// mimicking the actual listing flow in mailbox::list::fetch_remote_with_progress. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_status_full_attributes() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(MESSAGES UNSEEN UIDNEXT UIDVALIDITY)") + .await + .expect("STATUS with full attributes should succeed"); + + session.logout().await.ok(); + + println!("MESSAGES: {:?}", status.exists); + println!("UNSEEN: {:?}", status.unseen); + println!("UIDNEXT: {:?}", status.uid_next); + println!("UIDVALIDITY: {:?}", status.uid_validity); + } + + /// Simulate the retry flow: call STATUS multiple times and verify + /// UIDVALIDITY is consistently returned (or consistently absent). + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_uid_validity_consistency_over_multiple_status_calls() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let results: Vec> = Vec::with_capacity(5); + let results = std::cell::RefCell::new(results); + + for i in 0..5 { + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + let status = session + .status(TEST_MAILBOX, "(UIDVALIDITY)") + .await + .expect("STATUS should succeed"); + + session.logout().await.ok(); + + println!( + "Call {}: UIDVALIDITY = {:?}", + i + 1, + status.uid_validity + ); + results.borrow_mut().push(status.uid_validity); + } + + let results = results.into_inner(); + let first = results[0]; + let all_same = results.iter().all(|r| *r == first); + if all_same { + println!("[OK] All 5 STATUS calls returned consistent UIDVALIDITY: {first:?}"); + } else { + println!("[WARN] Inconsistent UIDVALIDITY across calls: {results:?}"); + println!(" Network jitter or server-side changes detected."); + } + } + + /// Test `fetch_uid_metadata` against a real IMAP server. + /// Fetches a few UIDs from the configured mailbox and verifies + /// that every returned UID has a valid, non-empty Message-ID. + #[tokio::test] + #[ignore = "requires real IMAP credentials"] + async fn test_fetch_uid_metadata_real_server() { + if TEST_IMAP_HOST.is_empty() || TEST_IMAP_USERNAME.is_empty() { + eprintln!("SKIP: fill in TEST_IMAP_* constants to run this test"); + return; + } + + let mut session = direct_connect( + TEST_IMAP_HOST, + TEST_IMAP_PORT, + TEST_IMAP_USERNAME, + TEST_IMAP_PASSWORD, + ) + .await + .expect("should connect"); + + // Examine to enter the mailbox + session + .examine(TEST_MAILBOX) + .await + .expect("EXAMINE should succeed"); + + // Find actual UIDs via SEARCH (don't assume contiguous 1..N) + let all_uids: Vec = { + let mut v: Vec = session + .uid_search("ALL") + .await + .expect("UID SEARCH ALL should succeed") + .into_iter() + .collect(); + v.sort(); + v + }; + + if all_uids.is_empty() { + eprintln!("SKIP: mailbox is empty, nothing to fetch"); + session.logout().await.ok(); + return; + } + + // Take up to 5 UIDs for a quick test + let sample: Vec = all_uids.into_iter().take(5).collect(); + let uid_set = compress_uid_list(sample.clone()); + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + &uid_set, + CancellationToken::new(), + ) + .await + .expect("fetch_uid_metadata should succeed"); + + session.logout().await.ok(); + + println!( + "[OK] Fetched {} UIDs from mailbox '{}':", + result.len(), + TEST_MAILBOX + ); + for uid in sample.iter() { + let mid = result + .get(uid) + .map(|o| o.as_deref().unwrap_or("")) + .unwrap_or(""); + println!(" UID {uid}: {mid}"); + } + + // Every UID we requested must be present with a valid Message-ID + for uid in &sample { + let msg_id = result + .get(uid) + .unwrap_or_else(|| panic!("UID {uid} missing from result")); + let mid = msg_id + .as_deref() + .unwrap_or_else(|| panic!("UID {uid} returned None Message-ID")); + assert!(!mid.is_empty(), "UID {uid} returned empty Message-ID"); + assert!( + mid.contains('@'), + "UID {uid}: Message-ID '{mid}' does not look like a valid Message-ID" + ); + } + } + + // ============================================================ + // Unit tests for fetch_uid_validity_with_retry_inner + // (pure logic, no network needed) + // ============================================================ + + /// Mock helper: returns the given results in sequence, then always None. + fn mock_results( + results: Vec>>, + ) -> impl FnMut() -> std::future::Ready>> { + let mut iter = results.into_iter(); + move || std::future::ready(iter.next().unwrap_or(Ok(None))) + } + + #[tokio::test] + async fn test_retry_first_attempt_succeeds() { + let result = fetch_uid_validity_with_retry_inner(3, mock_results(vec![Ok(Some(42))])) + .await; + assert_eq!(result.unwrap(), Some(42)); + } + + #[tokio::test] + async fn test_retry_succeeds_after_two_nones() { + // First two attempts return None, third returns Some + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![Ok(None), Ok(None), Ok(Some(99))]), + ) + .await; + assert_eq!(result.unwrap(), Some(99)); + } + + #[tokio::test] + async fn test_retry_succeeds_after_error_then_none() { + // Error, then None, then success + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![ + Err(raise_error!("boom".into(), ErrorCode::NetworkError)), + Ok(None), + Ok(Some(7)), + ]), + ) + .await; + assert_eq!(result.unwrap(), Some(7)); + } + + #[tokio::test] + async fn test_retry_returns_none_after_all_retries_exhausted() { + // All attempts return None + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![Ok(None), Ok(None), Ok(None)]), + ) + .await; + assert_eq!(result.unwrap(), None); + } + + #[tokio::test] + async fn test_retry_returns_none_when_all_attempts_error() { + // All attempts return Err — still returns Ok(None), not propagating the error + let result = fetch_uid_validity_with_retry_inner( + 3, + mock_results(vec![ + Err(raise_error!("e1".into(), ErrorCode::NetworkError)), + Err(raise_error!("e2".into(), ErrorCode::NetworkError)), + Err(raise_error!("e3".into(), ErrorCode::NetworkError)), + ]), + ) + .await; + assert!(result.is_ok()); + assert_eq!(result.unwrap(), None); + } + + #[tokio::test] + async fn test_retry_respects_max_retries() { + // max_retries=5, success on 5th attempt + let result = fetch_uid_validity_with_retry_inner( + 5, + mock_results(vec![ + Ok(None), + Ok(None), + Ok(None), + Ok(None), + Ok(Some(5)), + ]), + ) + .await; + assert_eq!(result.unwrap(), Some(5)); + } + + #[tokio::test] + async fn test_retry_stops_at_max_retries() { + // max_retries=2, third value is Some but should never be reached + let result = fetch_uid_validity_with_retry_inner( + 2, + mock_results(vec![Ok(None), Ok(None), Ok(Some(42))]), + ) + .await; + assert_eq!(result.unwrap(), None, "should stop after max_retries (2)"); + } + + #[tokio::test] + async fn test_retry_max_retries_zero() { + // max_retries=0 means no attempts at all + let result = fetch_uid_validity_with_retry_inner( + 0, + mock_results(vec![Ok(Some(42))]), + ) + .await; + assert_eq!(result.unwrap(), None); + } + + // ============================================================ + // Mock IMAP server integration tests + // ============================================================ + + use crate::imap::mock_server::{ + examine_response, uid_fetch_metadata_response, uid_fetch_rfc822_response, + minimal_eml, MockImapServer, MockImapServerHandle, + }; + + /// Build an `async_imap::Session` connected to the mock server, + /// authenticated and with the given mailbox examined. + async fn mock_session( + handle: &MockImapServerHandle, + ) -> async_imap::Session> { + let tcp = tokio::net::TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let timeout_stream = TimeoutStream::new(tcp); + let pinned: std::pin::Pin>> = + Box::pin(timeout_stream); + let stream: Box = Box::new(pinned); + let mut client = async_imap::Client::new(stream); + + // Read greeting + client.read_response().await.unwrap(); + + // Login + let mut session = client.login("user", "pass").await.map_err(|(e, _)| { + panic!("Login failed: {e:?}") + }).unwrap(); + + // Examine + session.examine("INBOX").await.unwrap(); + + session + } + + #[tokio::test] + async fn fetch_uid_metadata_with_mock_server() { + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 3, 42, 4)) + .respond("UID FETCH", uid_fetch_metadata_response(&[ + (1, ""), + (2, ""), + (3, ""), + ])) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:3", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert_eq!(result.len(), 3); + assert_eq!( + result.get(&1).unwrap().as_deref(), + Some("msg-a@test.com") + ); + assert_eq!( + result.get(&2).unwrap().as_deref(), + Some("msg-b@test.com") + ); + assert_eq!( + result.get(&3).unwrap().as_deref(), + Some("msg-c@test.com") + ); + + session.logout().await.ok(); + } + + /// Verify basic UID FETCH (FLAGS only, no body) works with the mock server. + #[tokio::test] + async fn fetch_uid_flags_with_mock_server() { + // Response with just UID and FLAGS — no body literal parsing needed. + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 2, 42, 3)) + .respond( + "UID FETCH", + b"* 1 FETCH (UID 1 FLAGS (\\Seen))\r\n\ +* 2 FETCH (UID 2 FLAGS (\\Flagged))\r\n\ +{TAG} OK FETCH completed\r\n" + .to_vec(), + ) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let uids: Vec = { + let mut stream = session + .uid_fetch("1:2", "(UID FLAGS)") + .await + .unwrap(); + + use futures::TryStreamExt; + let mut uids = Vec::new(); + while let Some(fetch) = stream.try_next().await.unwrap() { + uids.push(fetch.uid.unwrap_or(0)); + } + uids + }; + + assert_eq!(uids, vec![1, 2]); + + session.logout().await.ok(); + } + + /// Verify UID FETCH with BODY[] (full RFC822) works with the mock server. + #[tokio::test] + async fn fetch_uid_rfc822_with_mock_server() { + let eml = minimal_eml("Test Subject", "test@example.com"); + let eml_len = eml.len(); + + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 1, 42, 2)) + .respond("UID FETCH", uid_fetch_rfc822_response(1, &eml)) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let bodies: Vec<(u32, Vec)> = { + let mut stream = session + .uid_fetch("1:1", "(UID BODY[])") + .await + .unwrap(); + + use futures::TryStreamExt; + let mut bodies = Vec::new(); + while let Some(fetch) = stream.try_next().await.unwrap() { + let uid = fetch.uid.unwrap_or(0); + let body = fetch.body().map(|b| b.to_vec()).unwrap_or_default(); + bodies.push((uid, body)); + } + bodies + }; + + assert_eq!(bodies.len(), 1); + assert_eq!(bodies[0].0, 1); + assert_eq!(bodies[0].1.len(), eml_len); + assert!(String::from_utf8_lossy(&bodies[0].1).contains("Test Subject")); + + session.logout().await.ok(); + } + + #[tokio::test] + async fn fetch_uid_metadata_empty_mailbox() { + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 0, 42, 1)) + .respond("UID FETCH", b"{TAG} OK FETCH completed\r\n".to_vec()) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:*", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert!( + result.is_empty(), + "empty mailbox should return empty map" + ); + + session.logout().await.ok(); + } + + #[tokio::test] + async fn fetch_uid_metadata_missing_message_id() { + // One entry has a Message-ID, the other has no header at all. + let header_with_msgid = + "From: sender@example.com\r\n\ + Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ + Message-ID: \r\n\r\n"; + let header_without_msgid = "\r\n"; + let len1 = header_with_msgid.len(); + let len2 = header_without_msgid.len(); + + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN done\r\n") + .respond("EXAMINE", examine_response("INBOX", 2, 42, 3)) + .respond( + "UID FETCH", + format!( + "* 1 FETCH (UID 1 BODY[HEADER] {{{len1}}}\r\n\ +{header_with_msgid})\r\n\ +* 2 FETCH (UID 2 BODY[HEADER] {{{len2}}}\r\n\ +{header_without_msgid})\r\n\ +{{TAG}} OK FETCH completed\r\n" + ) + .into_bytes(), + ) + .start() + .await; + + let mut session = mock_session(&handle).await; + + let result = ImapExecutor::fetch_uid_metadata( + &mut session, + "1:2", + CancellationToken::new(), + ) + .await + .unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!( + result.get(&1).unwrap().as_deref(), + Some("ok@test.com") + ); + // UID 2 has no Message-ID header → None + assert_eq!(result.get(&2).unwrap().as_deref(), None); + + session.logout().await.ok(); + } +} diff --git a/crates/core/src/imap/client.rs b/crates/core/src/imap/client.rs index 1fd70b0..85cbcee 100644 --- a/crates/core/src/imap/client.rs +++ b/crates/core/src/imap/client.rs @@ -34,6 +34,25 @@ use std::ops::DerefMut; use tokio::io::BufWriter; use tracing::debug; +/// Classify an `io::Error` (from TLS stream I/O) for IMAP connection errors. +/// `UnexpectedEof` is treated as a network error because many servers skip +/// the TLS `close_notify` alert, causing rustls to emit this error when the +/// TCP connection is dropped normally. +fn classify_io_error(e: &std::io::Error) -> ErrorCode { + use std::io::ErrorKind; + matches!( + e.kind(), + ErrorKind::BrokenPipe + | ErrorKind::ConnectionReset + | ErrorKind::ConnectionAborted + | ErrorKind::TimedOut + | ErrorKind::UnexpectedEof + | ErrorKind::NotConnected + ) + .then_some(ErrorCode::NetworkError) + .unwrap_or(ErrorCode::ImapCommandFailed) +} + #[derive(Debug)] pub(crate) struct Client { inner: ImapClient>, @@ -141,7 +160,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is SSL.".into(), @@ -171,7 +190,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "failed to read greeting".into(), @@ -202,7 +221,7 @@ impl Client { let _greeting = client .read_response() .await - .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::ImapCommandFailed))? + .map_err(|e| raise_error!(format!("{:#?}", e), classify_io_error(&e)))? .ok_or_else(|| { raise_error!( "Failed to read IMAP greeting — this usually indicates an incorrect encryption setting (SSL vs. STARTTLS). Your current setting is STARTTLS.".into(), diff --git a/crates/core/src/imap/executor.rs b/crates/core/src/imap/executor.rs index 16e281b..3f5313e 100644 --- a/crates/core/src/imap/executor.rs +++ b/crates/core/src/imap/executor.rs @@ -27,7 +27,7 @@ use crate::{error::BichonResult, imap::manager::ImapConnectionManager}; use async_imap::types::Name; use async_imap::Session; use futures::TryStreamExt; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use tokio_util::sync::CancellationToken; use tracing::info; @@ -260,7 +260,9 @@ impl ImapExecutor { let mut count = 0u64; let mut skipped = 0u64; let mut max_uid: Option = None; - let size_limit = account.max_email_size_bytes.unwrap_or(DEFAULT_MAX_EMAIL_SIZE); + let size_limit = account + .max_email_size_bytes + .unwrap_or(DEFAULT_MAX_EMAIL_SIZE); while let Some(fetch) = stream .try_next() .await @@ -363,14 +365,14 @@ impl ImapExecutor { let mut size_stream = session .fetch(sequence_set.as_str(), SIZE_ONLY_FETCH) .await - .map_err(|e| { - raise_error!(format!("{:#?}", e), classify_imap_error(&e)) - })?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut uids: Vec = Vec::new(); - while let Some(fetch) = size_stream.try_next().await.map_err(|e| { - raise_error!(format!("{:#?}", e), classify_imap_error(&e)) - })? { + while let Some(fetch) = size_stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { let uid = fetch.uid.unwrap_or(0); let msg_size = fetch.size.unwrap_or(0) as u64; if msg_size == 0 || msg_size <= limit { @@ -437,14 +439,14 @@ impl ImapExecutor { let mut size_stream = session .uid_fetch(uid_set, SIZE_ONLY_FETCH) .await - .map_err(|e| { - raise_error!(format!("{:#?}", e), classify_imap_error(&e)) - })?; + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; let mut uids: Vec = Vec::new(); - while let Some(fetch) = size_stream.try_next().await.map_err(|e| { - raise_error!(format!("{:#?}", e), classify_imap_error(&e)) - })? { + while let Some(fetch) = size_stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { let uid = fetch.uid.unwrap_or(0); let msg_size = fetch.size.unwrap_or(0) as u64; if msg_size == 0 || msg_size <= limit { @@ -550,6 +552,37 @@ impl ImapExecutor { ) -> BichonResult>> { ImapConnectionManager::build(account_id).await } + + /// Fetch UID → Message-ID mapping without downloading bodies. + /// `uid_set` is an IMAP sequence-set string (e.g. "1:100" or "1,3,5"). + pub async fn fetch_uid_metadata( + session: &mut Session>, + uid_set: &str, + token: CancellationToken, + ) -> BichonResult>> { + let mut stream = session + .uid_fetch(uid_set, "(UID BODY.PEEK[HEADER])") + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))?; + + let mut result = HashMap::new(); + while let Some(fetch) = stream + .try_next() + .await + .map_err(|e| raise_error!(format!("{:#?}", e), classify_imap_error(&e)))? + { + if token.is_cancelled() { + return Err(raise_error!( + "Stream cancelled".into(), + ErrorCode::InternalError + )); + } + let uid = fetch.uid.unwrap_or(0); + let msg_id = fetch.header().and_then(parse_message_id_header); + result.insert(uid, msg_id); + } + Ok(result) + } } pub const DEFAULT_BATCH_SIZE: u32 = 30; @@ -613,6 +646,27 @@ pub fn generate_uid_sequence_hashset( result } +fn parse_message_id_header(header_bytes: &[u8]) -> Option { + let header = std::str::from_utf8(header_bytes).ok()?; + for line in header.lines() { + if let Some(value) = line + .strip_prefix("Message-ID:") + .or_else(|| line.strip_prefix("Message-Id:")) + .or_else(|| line.strip_prefix("Message-id:")) + { + // mail_parser strips angle brackets, so we must do the same + // to ensure comparisons against the Tantivy index match. + let trimmed = value.trim(); + let stripped = trimmed.strip_prefix('<').unwrap_or(trimmed); + let stripped = stripped.strip_suffix('>').unwrap_or(stripped); + if !stripped.is_empty() { + return Some(stripped.to_string()); + } + } + } + None +} + #[cfg(test)] mod test { use super::*; @@ -668,4 +722,80 @@ mod test { assert_eq!(batches[2].0, "5"); assert_eq!(batches[2].1, 1); } + + // ── parse_message_id_header ───────────────────────────────────── + + #[test] + fn parse_standard_message_id() { + let header = b"Message-ID: \r\n"; + assert_eq!( + parse_message_id_header(header), + Some("abc123@example.com".into()) + ); + } + + #[test] + fn parse_message_id_lowercase() { + let header = b"Message-Id: \r\n"; + assert_eq!( + parse_message_id_header(header), + Some("foo@bar.com".into()) + ); + } + + #[test] + fn parse_message_id_extra_whitespace() { + let header = b"Message-ID: \r\n"; + assert_eq!( + parse_message_id_header(header), + Some("spaces@test.com".into()) + ); + } + + #[test] + fn parse_empty_message_id_returns_none() { + let header = b"Message-ID: <>\r\n"; + assert_eq!(parse_message_id_header(header), None); + } + + #[test] + fn parse_missing_header_returns_none() { + let header = b"X-Custom: something\r\n"; + assert_eq!(parse_message_id_header(header), None); + } + + #[test] + fn parse_empty_body_returns_none() { + assert_eq!(parse_message_id_header(b""), None); + } + + #[test] + fn parse_message_id_in_full_header() { + // The Message-ID line is in the middle, not at the start. + let header = b"From: sender@example.com\r\n\ +Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ +Subject: test\r\n\ +Message-ID: \r\n\ +To: recipient@example.com\r\n\r\n"; + assert_eq!( + parse_message_id_header(header), + Some("mid@example.com".into()) + ); + } + + #[test] + fn parse_message_id_only_in_full_header() { + // Only a few headers, Message-ID is among them. + let header = b"From: a@b.com\r\nMessage-ID: \r\n\r\n"; + assert_eq!(parse_message_id_header(header), Some("x@y.com".into())); + } + + #[test] + fn parse_message_id_no_brackets_still_works() { + let header = b"Message-ID: plain@example.com\r\n"; + assert_eq!( + parse_message_id_header(header), + Some("plain@example.com".into()) + ); + } } diff --git a/crates/core/src/imap/manager.rs b/crates/core/src/imap/manager.rs index d32f517..e048edd 100644 --- a/crates/core/src/imap/manager.rs +++ b/crates/core/src/imap/manager.rs @@ -95,16 +95,40 @@ impl ImapConnectionManager { pub async fn build(account_id: u64) -> BichonResult>> { let account = AccountModel::get(account_id)?; - let client = match Self::create_client(&account).await { - Ok(client) => client, - Err(error) => { - error!( - "Failed to create IMAP {}'s client: {:#?}", - &account.email, error - ); - return Err(error); + let account_email = account.email.clone(); + + let mut client = None; + for attempt in 0..3u32 { + match Self::create_client(&account).await { + Ok(c) => { + client = Some(c); + break; + } + Err(error) if error.code() == ErrorCode::NetworkError && attempt < 2 => { + warn!( + "IMAP connection attempt {}/3 to {} failed (network error), retrying...", + attempt + 1, + account_email + ); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + continue; + } + Err(error) => { + error!( + "Failed to create IMAP {}'s client: {:#?}", + account_email, error + ); + return Err(error); + } } - }; + } + + let client = client.ok_or_else(|| { + raise_error!( + format!("Failed to create IMAP {}'s client after 3 attempts", account_email), + ErrorCode::NetworkError + ) + })?; let mut session = match Self::authenticate(client, &account).await { Ok(session) => session, diff --git a/crates/core/src/imap/mock_server.rs b/crates/core/src/imap/mock_server.rs new file mode 100644 index 0000000..c37c345 --- /dev/null +++ b/crates/core/src/imap/mock_server.rs @@ -0,0 +1,538 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! A minimal scriptable IMAP server for integration testing. +//! +//! Each instance listens on a random localhost port and responds to a +//! pre-configured script of (expected_command, response) pairs. Commands +//! are matched by substring — the first matching pattern wins. +//! +//! # Example +//! ```ignore +//! let server = MockImapServer::new() +//! .greeting("* OK ready\r\n") +//! .respond("LOGIN", "A0 OK logged in\r\n") +//! .respond("CAPABILITY", "* CAPABILITY IMAP4rev1\r\nA0 OK done\r\n") +//! .respond("STATUS", "* STATUS INBOX (MESSAGES 10 UIDVALIDITY 42)\r\nA0 OK\r\n") +//! .respond("LOGOUT", "* BYE\r\nA0 OK\r\n") +//! .start() +//! .await; +//! +//! let (host, port) = server.addr(); +//! // connect to host:port with Encryption::None +//! ``` + +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::net::{TcpListener, TcpStream}; + +type Response = Vec; + +pub struct MockImapServer { + greeting: Vec, + script: Vec<(String, Response)>, +} + +impl MockImapServer { + pub fn new() -> Self { + Self { + greeting: b"* OK Mock IMAP server ready\r\n".to_vec(), + script: Vec::new(), + } + } + + /// Set the greeting banner sent immediately after connection. + pub fn greeting(mut self, banner: impl Into>) -> Self { + self.greeting = banner.into(); + self + } + + /// Add a script step: when a client command *contains* `pattern` (case-insensitive), + /// respond with `response`. Steps are checked in insertion order. + pub fn respond(mut self, pattern: impl Into, response: impl Into>) -> Self { + self.script.push((pattern.into(), response.into())); + self + } + + /// Start the server on a random port. Returns a handle whose `addr()` gives + /// the `(host, port)` to connect to. + pub async fn start(self) -> MockImapServerHandle { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local_addr"); + + let server = Arc::new(self); + + tokio::spawn(async move { + loop { + match listener.accept().await { + Ok((stream, _)) => { + let srv = server.clone(); + tokio::spawn(async move { + srv.handle_connection(stream).await; + }); + } + Err(_) => break, + } + } + }); + + MockImapServerHandle { addr } + } + + async fn handle_connection(&self, mut stream: TcpStream) { + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Send greeting + if writer.write_all(&self.greeting).await.is_err() { + return; + } + + let mut line = String::new(); + loop { + line.clear(); + match reader.read_line(&mut line).await { + Ok(0) => break, // EOF + Ok(_) => {} + Err(_) => break, + } + + let tag = extract_tag(&line).unwrap_or("A0"); + let matched = self.find_match(&line); + if let Some(response) = matched { + let substituted = substitute_tag(response, tag); + if writer.write_all(&substituted).await.is_err() { + break; + } + } else { + // Default: send tagged OK for commands we don't handle + let fallback = format!("{tag} OK done\r\n"); + if writer.write_all(fallback.as_bytes()).await.is_err() { + break; + } + } + } + } + + fn find_match(&self, line: &str) -> Option<&[u8]> { + let line_lower = line.to_lowercase(); + for (pattern, response) in &self.script { + if line_lower.contains(&pattern.to_lowercase()) { + return Some(response); + } + } + None + } +} + +impl Default for MockImapServer { + fn default() -> Self { + Self::new() + } +} + +/// Handle to a running mock IMAP server. The server stops when this handle +/// is dropped. +pub struct MockImapServerHandle { + addr: SocketAddr, +} + +impl MockImapServerHandle { + pub fn host(&self) -> String { + self.addr.ip().to_string() + } + + pub fn port(&self) -> u16 { + self.addr.port() + } +} + +fn extract_tag(line: &str) -> Option<&str> { + line.split_whitespace().next() +} + +/// Replace `{TAG}` placeholders in `response` with `tag`. +fn substitute_tag(response: &[u8], tag: &str) -> Vec { + let placeholder = b"{TAG}"; + if response.is_empty() || !contains_slice(response, placeholder) { + return response.to_vec(); + } + let tag_bytes = tag.as_bytes(); + let mut result = Vec::with_capacity(response.len()); + let mut pos = 0; + while let Some(idx) = find_slice(&response[pos..], placeholder) { + result.extend_from_slice(&response[pos..pos + idx]); + result.extend_from_slice(tag_bytes); + pos += idx + placeholder.len(); + } + result.extend_from_slice(&response[pos..]); + result +} + +fn contains_slice(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) +} + +fn find_slice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|w| w == needle) +} + +// ============================================================ +// Pre-built response helpers +// ============================================================ + +/// Build a tagged OK response. +pub fn ok(tag: impl AsRef, msg: impl AsRef) -> Vec { + format!("{} OK {}\r\n", tag.as_ref(), msg.as_ref()).into_bytes() +} + +/// Build a STATUS response line. +pub fn status_response( + mailbox: &str, + messages: u32, + unseen: u32, + uid_next: u32, + uid_validity: Option, +) -> Vec { + let uv = uid_validity + .map(|v| format!(" UIDVALIDITY {v}")) + .unwrap_or_default(); + let text = format!( + "* STATUS \"{mailbox}\" (MESSAGES {messages} UNSEEN {unseen} UIDNEXT {uid_next}{uv})\r\n" + ); + // Clients expect a tagged response after the untagged STATUS line. + // We produce a generic OK that works for any tag. + let mut out = text.into_bytes(); + out.extend_from_slice(b"{TAG} OK STATUS completed\r\n"); + out +} + +/// Build an EXAMINE response with mailbox data. +pub fn examine_response( + _mailbox: &str, + exists: u32, + uid_validity: u32, + uid_next: u32, +) -> Vec { + format!( + "* FLAGS (\\Seen \\Answered \\Flagged \\Deleted \\Draft)\r\n\ + * OK [PERMANENTFLAGS ()]\r\n\ + * {exists} EXISTS\r\n\ + * 0 RECENT\r\n\ + * OK [UIDVALIDITY {uid_validity}]\r\n\ + * OK [UIDNEXT {uid_next}]\r\n\ + * OK [HIGHESTMODSEQ 1]\r\n\ + {{TAG}} OK [READ-ONLY] EXAMINE completed\r\n" + ) + .into_bytes() +} + +/// Build a UID SEARCH response for the given UID list. +pub fn uid_search_response(uids: &[u32]) -> Vec { + let uid_str = uids + .iter() + .map(|u| u.to_string()) + .collect::>() + .join(" "); + format!("* SEARCH {uid_str}\r\n{{TAG}} OK SEARCH completed\r\n").into_bytes() +} + +/// Build a UID FETCH response returning full headers (for BODY[HEADER]). +/// Each entry: (uid, message_id) +pub fn uid_fetch_metadata_response(entries: &[(u32, &str)]) -> Vec { + let mut out = Vec::new(); + for (uid, msg_id) in entries { + // Build a minimal header that contains the Message-ID line. + let header_data = format!( + "From: sender@example.com\r\n\ +To: recipient@example.com\r\n\ +Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ +Subject: test\r\n\ +Message-ID: {msg_id}\r\n\r\n" + ); + let header_len = header_data.len(); + let line = format!( + "* {uid} FETCH (UID {uid} BODY[HEADER] {{{header_len}}}\r\n\ +{header_data}\ +)\r\n", + ); + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(b"{TAG} OK FETCH completed\r\n"); + out +} + +/// Build a UID FETCH RFC822 response with a full email body. +pub fn uid_fetch_rfc822_response(uid: u32, eml: &[u8]) -> Vec { + let header = format!( + "* {uid} FETCH (UID {uid} RFC822 {{{len}}}\r\n", + len = eml.len() + ); + let mut out = header.into_bytes(); + out.extend_from_slice(eml); + out.extend_from_slice(b")\r\n{TAG} OK FETCH completed\r\n"); + out +} + +/// A minimal RFC822 email fixture for testing. +pub fn minimal_eml(subject: &str, message_id: &str) -> Vec { + format!( + "From: sender@example.com\r\n\ + To: recipient@example.com\r\n\ + Subject: {subject}\r\n\ + Message-ID: <{message_id}>\r\n\ + Date: Thu, 01 Jan 2025 00:00:00 +0000\r\n\ + MIME-Version: 1.0\r\n\ + Content-Type: text/plain; charset=utf-8\r\n\ + \r\n\ + This is a test email: {subject}.\r\n" + ) + .into_bytes() +} + +// ============================================================ +// Self-tests for the mock server itself +// ============================================================ + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect_and_read_greeting(host: &str, port: u16) -> String { + let mut stream = TcpStream::connect((host, port)).await.unwrap(); + let (reader, _writer) = stream.split(); + let mut reader = BufReader::new(reader); + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + line + } + + async fn send_and_recv(host: &str, port: u16, cmd: &str) -> String { + let mut stream = TcpStream::connect((host, port)).await.unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + + // Send command + writer.write_all(cmd.as_bytes()).await.unwrap(); + writer.write_all(b"\r\n").await.unwrap(); + + // Read response (may be multi-line; read until tagged response) + let mut out = String::new(); + loop { + line.clear(); + reader.read_line(&mut line).await.unwrap(); + out.push_str(&line); + if line.starts_with("A0") || line.starts_with("A1") { + break; + } + } + out + } + + #[tokio::test] + async fn test_mock_greeting() { + let handle = MockImapServer::new().start().await; + let greeting = connect_and_read_greeting(&handle.host(), handle.port()).await; + assert!(greeting.starts_with("* OK")); + } + + #[tokio::test] + async fn test_mock_scripted_response() { + let handle = MockImapServer::new() + .respond( + "LOGIN", + "A0 OK LOGIN completed\r\n", + ) + .start() + .await; + + let resp = send_and_recv(&handle.host(), handle.port(), "A0 LOGIN u p").await; + assert!(resp.contains("LOGIN completed")); + } + + #[tokio::test] + async fn test_mock_fallback_on_unmatched() { + let handle = MockImapServer::new().start().await; + + // Send a command that has no scripted response + let resp = send_and_recv(&handle.host(), handle.port(), "A0 NOOP").await; + assert!(resp.contains("OK done"), "unmatched command should get fallback OK"); + } + + #[tokio::test] + async fn test_status_response_helper() { + let resp = status_response("INBOX", 10, 2, 11, Some(42)); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("MESSAGES 10")); + assert!(text.contains("UNSEEN 2")); + assert!(text.contains("UIDNEXT 11")); + assert!(text.contains("UIDVALIDITY 42")); + } + + #[tokio::test] + async fn test_status_response_without_uidvalidity() { + let resp = status_response("INBOX", 10, 2, 11, None); + let text = String::from_utf8(resp).unwrap(); + assert!(!text.contains("UIDVALIDITY")); + assert!(text.contains("MESSAGES 10")); + } + + #[tokio::test] + async fn test_examine_response() { + let resp = examine_response("INBOX", 10, 42, 11); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("UIDVALIDITY 42")); + assert!(text.contains("10 EXISTS")); + } + + #[tokio::test] + async fn test_uid_search_response() { + let resp = uid_search_response(&[1, 3, 5]); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("SEARCH 1 3 5")); + } + + #[tokio::test] + async fn test_uid_fetch_metadata_response() { + let resp = uid_fetch_metadata_response(&[(1, "msg-a@x.com"), (2, "msg-b@x.com")]); + let text = String::from_utf8(resp).unwrap(); + assert!(text.contains("Message-ID: msg-a@x.com")); + assert!(text.contains("Message-ID: msg-b@x.com")); + } + + #[tokio::test] + async fn test_multiple_commands_in_sequence() { + let handle = MockImapServer::new() + .respond("LOGIN", "A0 OK LOGIN\r\n") + .respond("STATUS", status_response("INBOX", 5, 1, 6, Some(99))) + .respond("LOGOUT", "* BYE\r\nA0 OK\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // LOGIN + writer.write_all(b"A0 LOGIN u p\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + assert!(buf.contains("LOGIN")); + + // STATUS + writer + .write_all(b"A0 STATUS INBOX (MESSAGES UNSEEN UIDNEXT UIDVALIDITY)\r\n") + .await + .unwrap(); + buf.clear(); + // Read multi-line STATUS response (untagged line + tagged OK) + loop { + reader.read_line(&mut buf).await.unwrap(); + if buf.contains("UIDVALIDITY 99") { + // Consume the tagged OK line that follows + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + break; + } + } + + // LOGOUT + writer.write_all(b"A0 LOGOUT\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + assert!(buf.contains("BYE")); + } + + #[tokio::test] + async fn test_tag_substitution_in_response() { + // Use {TAG} placeholder in the response and verify it gets the + // client's actual tag ("A5") substituted in. + let handle = MockImapServer::new() + .respond("LOGIN", "{TAG} OK LOGIN succeeded\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // Send LOGIN with non-standard tag + writer.write_all(b"A5 LOGIN u p\r\n").await.unwrap(); + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + + assert!( + buf.contains("A5 OK LOGIN succeeded"), + "expected 'A5 OK LOGIN succeeded', got '{buf}'" + ); + } + + #[tokio::test] + async fn test_tag_substitution_multiple_placeholders() { + let handle = MockImapServer::new() + .respond("NOOP", "* 0 RECENT\r\n{TAG} OK NOOP done\r\n") + .start() + .await; + + let mut stream = TcpStream::connect((handle.host(), handle.port())) + .await + .unwrap(); + let (reader, mut writer) = stream.split(); + let mut reader = BufReader::new(reader); + + // Read greeting + let mut buf = String::new(); + reader.read_line(&mut buf).await.unwrap(); + + // Send with tag "B99" + writer.write_all(b"B99 NOOP\r\n").await.unwrap(); + + // Read all lines + let mut all = String::new(); + loop { + buf.clear(); + reader.read_line(&mut buf).await.unwrap(); + all.push_str(&buf); + if buf.starts_with("B99") { + break; + } + } + + assert!(all.contains("* 0 RECENT\r\n")); + assert!(all.contains("B99 OK NOOP done\r\n")); + } +} diff --git a/crates/core/src/imap/mod.rs b/crates/core/src/imap/mod.rs index 479c954..0866f1a 100644 --- a/crates/core/src/imap/mod.rs +++ b/crates/core/src/imap/mod.rs @@ -26,3 +26,5 @@ pub mod session; pub mod stats; #[cfg(test)] mod tests; +#[cfg(test)] +pub mod mock_server; diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index 1a5e97a..a7ccac7 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -278,6 +278,80 @@ impl IndexManager { Box::new(boolean_query) } + /// Return all Message-IDs stored in Tantivy for a given mailbox. + /// Prefer `mailbox_contains_message_id` for existence checks on large + /// mailboxes — this method loads everything into a HashSet. + pub fn get_message_ids_for_mailbox( + &self, + account_id: u64, + mailbox_id: u64, + ) -> BichonResult> { + let query = self.mailbox_query(account_id, mailbox_id); + let fields = SchemaTools::email_fields(); + let searcher = self.create_searcher()?; + + let docs = searcher + .search(&query, &DocSetCollector) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + let mut result = HashSet::new(); + for doc_address in docs { + let doc = searcher + .doc::(doc_address) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + + if let Some(v) = doc.get_first(fields.f_message_id) { + if let Some(s) = v.as_str() { + if !s.is_empty() { + result.insert(s.to_string()); + } + } + } + } + Ok(result) + } + + /// Check whether a specific Message-ID exists in a mailbox. + /// Uses a TermQuery — O(1) per call, no allocation proportional to + /// mailbox size. Suitable for large mailboxes where + /// `get_message_ids_for_mailbox` would allocate too much memory. + pub fn mailbox_contains_message_id( + &self, + account_id: u64, + mailbox_id: u64, + message_id: &str, + ) -> BichonResult { + let fields = SchemaTools::email_fields(); + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_account_id, account_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(fields.f_mailbox_id, mailbox_id), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(fields.f_message_id, message_id), + IndexRecordOption::Basic, + )), + ), + ]); + let searcher = self.create_searcher()?; + let count = searcher + .search(&query, &Count) + .map_err(|e| raise_error!(format!("{:#?}", e), ErrorCode::InternalError))?; + Ok(count > 0) + } + fn envelope_query(&self, account_id: u64, eid: &str) -> Box { let account_id_query = TermQuery::new( Term::from_field_u64(SchemaTools::email_fields().f_account_id, account_id), @@ -2069,4 +2143,362 @@ mod tests { "body should be absent when EML is missing" ); } + + // ── get_message_ids_for_mailbox ───────────────────────────────── + + #[test] + fn get_message_ids_returns_stored_ids() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + // Insert two docs for mailbox 10, one for mailbox 20 + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc1 = TantivyDocument::new(); + doc1.add_u64(f.f_account_id, 1); + doc1.add_u64(f.f_mailbox_id, 10); + doc1.add_text(f.f_message_id, ""); + doc1.add_text(f.f_id, "id-a"); + doc1.add_u64(f.f_uid, 1); + doc1.add_text(f.f_content_hash, "hash-a"); + writer.add_document(doc1).unwrap(); + + let mut doc2 = TantivyDocument::new(); + doc2.add_u64(f.f_account_id, 1); + doc2.add_u64(f.f_mailbox_id, 10); + doc2.add_text(f.f_message_id, ""); + doc2.add_text(f.f_id, "id-b"); + doc2.add_u64(f.f_uid, 2); + doc2.add_text(f.f_content_hash, "hash-b"); + writer.add_document(doc2).unwrap(); + + let mut doc3 = TantivyDocument::new(); + doc3.add_u64(f.f_account_id, 1); + doc3.add_u64(f.f_mailbox_id, 20); + doc3.add_text(f.f_message_id, ""); + doc3.add_text(f.f_id, "id-c"); + doc3.add_u64(f.f_uid, 3); + doc3.add_text(f.f_content_hash, "hash-c"); + writer.add_document(doc3).unwrap(); + + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // We can't easily call ENVELOPE_MANAGER.get_message_ids_for_mailbox + // because it reads from ENVELOPE_MANAGER's own index, not our in-memory one. + // Instead, test the query pattern directly. + let query: Box = { + let account_query = TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + ); + let mailbox_query = TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + ); + Box::new(BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(mailbox_query)), + ])) + }; + + let docs = searcher + .search(&query, &DocSetCollector) + .unwrap(); + + let mut ids: Vec = Vec::new(); + for addr in docs { + let doc: TantivyDocument = searcher.doc(addr).unwrap(); + if let Some(v) = doc.get_first(f.f_message_id) { + if let Some(s) = v.as_str() { + ids.push(s.to_string()); + } + } + } + ids.sort(); + + assert_eq!(ids, vec!["", ""]); + } + + #[test] + fn get_message_ids_empty_mailbox_returns_empty() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + // Doc for a different mailbox + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 99); + doc.add_text(f.f_message_id, ""); + doc.add_text(f.f_id, "id-other"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-other"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query: Box = { + let account_query = TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + ); + let mailbox_query = TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + ); + Box::new(BooleanQuery::new(vec![ + (Occur::Must, Box::new(account_query)), + (Occur::Must, Box::new(mailbox_query)), + ])) + }; + + let docs = searcher.search(&query, &DocSetCollector).unwrap(); + assert!(docs.is_empty()); + } + + // ── mailbox_contains_message_id ─────────────────────────────── + + #[test] + fn mailbox_contains_message_id_finds_existing() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_text(f.f_message_id, "abc@example.com"); + doc.add_text(f.f_id, "id-1"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + // We test the query pattern directly (can't call ENVELOPE_MANAGER + // which uses a different index). + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "abc@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + + let count = searcher.search(&query, &Count).unwrap(); + assert_eq!(count, 1); + } + + #[test] + fn mailbox_contains_message_id_returns_zero_for_missing() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + let mut doc = TantivyDocument::new(); + doc.add_u64(f.f_account_id, 1); + doc.add_u64(f.f_mailbox_id, 10); + doc.add_text(f.f_message_id, "existing@example.com"); + doc.add_text(f.f_id, "id-1"); + doc.add_u64(f.f_uid, 1); + doc.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + let query = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "nonexistent@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + + let count = searcher.search(&query, &Count).unwrap(); + assert_eq!(count, 0); + } + + #[test] + fn mailbox_contains_message_id_respects_mailbox_boundary() { + let f = SchemaTools::email_fields(); + let index = Index::create_in_ram(SchemaTools::email_schema()); + index.tokenizers().register("euro", EuroTokenizer::new()); + + { + let mut writer = index + .writer_with_num_threads(1, 15_000_000) + .expect("writer"); + + // Same Message-ID in mailbox 10 + let mut doc1 = TantivyDocument::new(); + doc1.add_u64(f.f_account_id, 1); + doc1.add_u64(f.f_mailbox_id, 10); + doc1.add_text(f.f_message_id, "shared@example.com"); + doc1.add_text(f.f_id, "id-1"); + doc1.add_u64(f.f_uid, 1); + doc1.add_text(f.f_content_hash, "hash-1"); + writer.add_document(doc1).unwrap(); + + // Same Message-ID in mailbox 20 (different mailbox) + let mut doc2 = TantivyDocument::new(); + doc2.add_u64(f.f_account_id, 1); + doc2.add_u64(f.f_mailbox_id, 20); + doc2.add_text(f.f_message_id, "shared@example.com"); + doc2.add_text(f.f_id, "id-2"); + doc2.add_u64(f.f_uid, 2); + doc2.add_text(f.f_content_hash, "hash-2"); + writer.add_document(doc2).unwrap(); + writer.commit().unwrap(); + } + + let reader = index.reader().unwrap(); + reader.reload().unwrap(); + let searcher = reader.searcher(); + + // Query mailbox 10: should find 1 + let q10 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 10), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q10, &Count).unwrap(), 1); + + // Query mailbox 20: should find 1 + let q20 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 20), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q20, &Count).unwrap(), 1); + + // Query mailbox 99 (no docs): should find 0 + let q99 = BooleanQuery::new(vec![ + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_account_id, 1), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_u64(f.f_mailbox_id, 99), + IndexRecordOption::Basic, + )), + ), + ( + Occur::Must, + Box::new(TermQuery::new( + Term::from_field_text(f.f_message_id, "shared@example.com"), + IndexRecordOption::Basic, + )), + ), + ]); + assert_eq!(searcher.search(&q99, &Count).unwrap(), 0); + } } From 6fcc9e0e8e062438152e85dce92db251631d6614 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 17:18:46 +0800 Subject: [PATCH 35/66] bump to v1.5.3 --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e3225a..c907763 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bichon-admin" -version = "1.5.2" +version = "1.5.3" dependencies = [ "bichon-core", "console", @@ -311,7 +311,7 @@ dependencies = [ [[package]] name = "bichon-cli" -version = "1.5.2" +version = "1.5.3" dependencies = [ "base64 0.22.1", "bichon-core", @@ -337,7 +337,7 @@ dependencies = [ [[package]] name = "bichon-core" -version = "1.5.2" +version = "1.5.3" dependencies = [ "async-imap", "base64 0.22.1", @@ -396,7 +396,7 @@ dependencies = [ [[package]] name = "bichon-server" -version = "1.5.2" +version = "1.5.3" dependencies = [ "bichon-core", "bichon-smtp", @@ -420,7 +420,7 @@ dependencies = [ [[package]] name = "bichon-smtp" -version = "1.5.2" +version = "1.5.3" dependencies = [ "base64 0.22.1", "bichon-core", diff --git a/Cargo.toml b/Cargo.toml index a9f60f8..0338994 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ resolver = "2" [workspace.package] -version = "1.5.2" +version = "1.5.3" edition = "2021" [workspace.dependencies] From 04745458bc0048129c705e5b6a303924a721243c Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 21:38:31 +0800 Subject: [PATCH 36/66] feat(blob): add embedded KV storage engine for email archiving --- .gitignore | 3 +- Cargo.lock | 205 ++++- Cargo.toml | 4 +- crates/admin/Cargo.toml | 2 +- crates/admin/src/meta.rs | 2 +- crates/blob/Cargo.lock | 1105 +++++++++++++++++++++++++ crates/blob/Cargo.toml | 23 + crates/blob/benches/benchmark.rs | 316 +++++++ crates/blob/src/account.rs | 242 ++++++ crates/blob/src/bucket.rs | 333 ++++++++ crates/blob/src/cache.rs | 189 +++++ crates/blob/src/checksum.rs | 65 ++ crates/blob/src/compress.rs | 97 +++ crates/blob/src/engine.rs | 399 +++++++++ crates/blob/src/error.rs | 53 ++ crates/blob/src/gc.rs | 267 ++++++ crates/blob/src/lib.rs | 16 + crates/blob/src/meta.rs | 156 ++++ crates/blob/src/recovery.rs | 200 +++++ crates/blob/src/segment.rs | 454 ++++++++++ crates/blob/src/types.rs | 88 ++ crates/blob/tests/acid_test.rs | 498 +++++++++++ crates/blob/tests/integration_test.rs | 273 ++++++ crates/core/Cargo.toml | 2 +- crates/core/src/admin/meta.rs | 2 +- crates/core/src/database/manager.rs | 2 +- crates/core/src/database/mod.rs | 2 +- crates/memdb/Cargo.toml | 2 +- crates/memdb/tests/integration.rs | 6 +- crates/memdb/tests/stress.rs | 8 +- 30 files changed, 4976 insertions(+), 38 deletions(-) create mode 100644 crates/blob/Cargo.lock create mode 100644 crates/blob/Cargo.toml create mode 100644 crates/blob/benches/benchmark.rs create mode 100644 crates/blob/src/account.rs create mode 100644 crates/blob/src/bucket.rs create mode 100644 crates/blob/src/cache.rs create mode 100644 crates/blob/src/checksum.rs create mode 100644 crates/blob/src/compress.rs create mode 100644 crates/blob/src/engine.rs create mode 100644 crates/blob/src/error.rs create mode 100644 crates/blob/src/gc.rs create mode 100644 crates/blob/src/lib.rs create mode 100644 crates/blob/src/meta.rs create mode 100644 crates/blob/src/recovery.rs create mode 100644 crates/blob/src/segment.rs create mode 100644 crates/blob/src/types.rs create mode 100644 crates/blob/tests/acid_test.rs create mode 100644 crates/blob/tests/integration_test.rs diff --git a/.gitignore b/.gitignore index f48f1f4..fc2da8d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .idea config.toml node_modules -dedup_report.txt \ No newline at end of file +dedup_report.txt +crates/*/target \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index c907763..0fc62d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -60,6 +60,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -296,11 +302,11 @@ name = "bichon-admin" version = "1.5.3" dependencies = [ "bichon-core", + "bichon-memdb", "console", "dialoguer", "indicatif", - "itertools", - "memdb", + "itertools 0.14.0", "native_db", "native_model", "serde", @@ -309,6 +315,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "lz4_flex", + "rand 0.10.1", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tracing", + "zstd", +] + [[package]] name = "bichon-cli" version = "1.5.3" @@ -341,6 +363,7 @@ version = "1.5.3" dependencies = [ "async-imap", "base64 0.22.1", + "bichon-memdb", "blake3", "bytes 1.11.1", "chrono", @@ -355,12 +378,11 @@ dependencies = [ "governor", "hickory-resolver", "html2text", - "itertools", + "itertools 0.14.0", "itoa", "lru 0.18.0", "mail-parser", "mail-send", - "memdb", "murmur3", "num_cpus", "oauth2", @@ -394,6 +416,18 @@ dependencies = [ "whichlang", ] +[[package]] +name = "bichon-memdb" +version = "0.1.0" +dependencies = [ + "rand 0.9.2", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "bichon-server" version = "1.5.3" @@ -613,6 +647,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.60" @@ -668,6 +708,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -868,6 +935,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -1602,6 +1702,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -2128,6 +2239,24 @@ dependencies = [ "phf 0.11.3", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2338,9 +2467,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -2445,18 +2574,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "memdb" -version = "0.1.0" -dependencies = [ - "rand 0.9.2", - "serde", - "serde_json", - "tempfile", - "thiserror 2.0.18", - "tokio", -] - [[package]] name = "memmap2" version = "0.9.10" @@ -2858,6 +2975,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl" version = "0.10.78" @@ -3090,6 +3213,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "poem" version = "3.1.12" @@ -3159,7 +3310,7 @@ dependencies = [ "email_address", "futures-util", "indexmap", - "itertools", + "itertools 0.14.0", "mime", "num-traits", "poem", @@ -4320,7 +4471,7 @@ dependencies = [ "futures-channel", "futures-util", "htmlescape", - "itertools", + "itertools 0.14.0", "levenshtein_automata", "log", "lru 0.16.4", @@ -4371,7 +4522,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4423,7 +4574,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ "futures-util", - "itertools", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -4574,6 +4725,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.11.0" diff --git a/Cargo.toml b/Cargo.toml index 0338994..23537dc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/memdb", "crates/core", + "crates/blob", "crates/server", "crates/cli", "crates/admin", @@ -18,7 +19,8 @@ edition = "2021" [workspace.dependencies] chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive", "env"] } -memdb = { path = "crates/memdb" } +bichon-memdb = { path = "crates/memdb" } +bichon-blob = { path = "crates/blob" } itertools = "0.14.0" ring = { version = "0.17.14", features = ["std"] } serde = { version = "1.0.228", features = ["derive"] } diff --git a/crates/admin/Cargo.toml b/crates/admin/Cargo.toml index eddbfb2..5184ed5 100644 --- a/crates/admin/Cargo.toml +++ b/crates/admin/Cargo.toml @@ -17,4 +17,4 @@ serde_json.workspace = true itertools.workspace = true snafu.workspace = true -memdb.workspace = true \ No newline at end of file +bichon-memdb.workspace = true \ No newline at end of file diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index d843d62..d0118be 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -20,7 +20,7 @@ use bichon_core::{ }; use console::style; use itertools::Itertools; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use native_db::*; use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; diff --git a/crates/blob/Cargo.lock b/crates/blob/Cargo.lock new file mode 100644 index 0000000..b31a3bd --- /dev/null +++ b/crates/blob/Cargo.lock @@ -0,0 +1,1105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bichon-blob" +version = "0.1.0" +dependencies = [ + "crc32fast", + "criterion", + "lz4_flex", + "rand", + "serde", + "serde_json", + "tempfile", + "thiserror", + "tracing", + "zstd", +] + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "criterion" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf7af66b0989381bd0be551bd7cc91912a655a58c6918420c9527b1fd8b4679" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasip2", + "wasip3", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" + +[[package]] +name = "lz4_flex" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/crates/blob/Cargo.toml b/crates/blob/Cargo.toml new file mode 100644 index 0000000..559396b --- /dev/null +++ b/crates/blob/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "bichon-blob" +version = "0.1.0" +edition = "2021" +description = "Embedded KV storage engine for email" + +[dependencies] +crc32fast = "1.4" +zstd = "0.13" +lz4_flex = "0.13.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tracing = "0.1" +thiserror = "2" + +[dev-dependencies] +tempfile = "3" +rand = "0.10.1" +criterion = { version = "0.6", features = ["html_reports"] } + +[[bench]] +name = "benchmark" +harness = false diff --git a/crates/blob/benches/benchmark.rs b/crates/blob/benches/benchmark.rs new file mode 100644 index 0000000..278531a --- /dev/null +++ b/crates/blob/benches/benchmark.rs @@ -0,0 +1,316 @@ +use criterion::{criterion_group, criterion_main, BatchSize, Criterion, Throughput}; +use std::time::Duration; +use tempfile::TempDir; + +use bichon_blob::{Codec, Config, Engine}; + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let mut v = Vec::with_capacity(size); + // Fill with somewhat realistic text-like data so compression works + let pattern = b"The quick brown fox jumps over the lazy dog. "; + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +pub fn bench_write_small(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024); // 1 KB + let mut counter = 0u64; + + group.bench_function("1KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_medium(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(64 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(64 * 1024); // 64 KB + let mut counter = 0u64; + + group.bench_function("64KB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_write_large(c: &mut Criterion) { + let mut group = c.benchmark_group("write"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + let mut counter = 0u64; + + group.bench_function("1MB", |b| { + b.iter_batched( + || { + counter += 1; + (make_key(counter), value.clone()) + }, + |(key, val)| { + engine + .write("bench", key, &val, Codec::Zstd) + .unwrap() + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_read_cache_hit(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate: 10 keys, all in same bucket → cache hit after first read + let value = make_value(4096); + for i in 0..10u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 10); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_cache_miss(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 8; // Small cache to force misses + let engine = Engine::open(dir.path(), config).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + // Write 1000 keys spread across all 16 buckets — small LRU will thrash + for i in 0..1000u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("cache_miss", |b| { + b.iter(|| { + let key = make_key(counter % 1000); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_read_large_value(c: &mut Criterion) { + let mut group = c.benchmark_group("read"); + group.throughput(Throughput::Bytes(1024 * 1024)); + group.measurement_time(Duration::from_secs(10)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(1024 * 1024); // 1 MB + for i in 0..5u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter = 0u64; + group.bench_function("1MB_cache_hit", |b| { + b.iter(|| { + let key = make_key(counter % 5); + counter += 1; + std::hint::black_box(engine.read("bench", &key).unwrap()); + }) + }); + group.finish(); +} + +pub fn bench_delete(c: &mut Criterion) { + let mut group = c.benchmark_group("delete"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(10)); + + group.bench_function("delete", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + let value = make_value(4096); + let mut counter = 0u64; + + b.iter_batched( + || { + counter += 1; + let key = make_key(counter); + engine + .write("bench", key, &value, Codec::Zstd) + .unwrap(); + key + }, + |key| { + engine.delete("bench", &key).unwrap(); + }, + BatchSize::SmallInput, + ) + }); + group.finish(); +} + +pub fn bench_mixed_workload(c: &mut Criterion) { + let mut group = c.benchmark_group("mixed"); + group.throughput(Throughput::Elements(1)); + group.measurement_time(Duration::from_secs(15)); + + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Pre-populate with 500 entries + let value = make_value(8192); + for i in 0..500u64 { + engine + .write("bench", make_key(i), &value, Codec::Zstd) + .unwrap(); + } + + let mut counter: u64 = 500; + group.bench_function("80w_15r_5d", |b| { + b.iter(|| { + counter += 1; + let op = counter % 100; + match op { + 0..=79 => { + // 80% writes + let key = make_key(counter); + let val = make_value(4096); + engine.write("bench", key, &val, Codec::Zstd).unwrap(); + } + 80..=94 => { + // 15% reads + std::hint::black_box(engine.read("bench", &make_key(counter % 500)).unwrap()); + } + _ => { + // 5% deletes + if counter % 2 == 0 { + let key = make_key(counter % 500); + let _ = engine.delete("bench", &key); + } + } + } + }) + }); + group.finish(); +} + +pub fn bench_gc(c: &mut Criterion) { + let mut group = c.benchmark_group("gc"); + group.measurement_time(Duration::from_secs(30)); + group.sample_size(10); + + group.bench_function("gc_30pct_deleted", |b| { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("bench").unwrap(); + + // Fill a segment with ~1000 entries, then delete 30% + let value = make_value(200_000); // 200KB each → ~1000 entries to fill 256MB + let n = 1200u64; + for i in 0..n { + engine + .write("bench", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~30% + for i in (0..n).step_by(3) { + engine.delete("bench", &make_key(i)).unwrap(); + } + + b.iter(|| { + engine.gc("bench").unwrap(); + }) + }); + group.finish(); +} + +criterion_group!( + benches, + bench_write_small, + bench_write_medium, + bench_write_large, + bench_read_cache_hit, + bench_read_cache_miss, + bench_read_large_value, + bench_delete, + bench_mixed_workload, + bench_gc, +); +criterion_main!(benches); diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs new file mode 100644 index 0000000..9dc3a14 --- /dev/null +++ b/crates/blob/src/account.rs @@ -0,0 +1,242 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::{Error, Result}; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader, SegmentWriter}; +use crate::types::Codec; + +pub struct Account { + pub id: String, + dir: PathBuf, + meta: AccountMeta, + active_writer: SegmentWriter, + readers: HashMap, + write_lock: Mutex<()>, +} + +impl Account { + /// Open an existing account. + pub fn open(store_root: &Path, account_id: &str) -> Result { + let dir = store_root.join("accounts").join(account_id); + if !dir.exists() { + return Err(Error::AccountNotFound(account_id.to_string())); + } + + let meta = AccountMeta::load(&dir)?; + + // Open active writer + let seg_path = dir + .join("segments") + .join(segment::segment_filename(meta.active_segment_id)); + let active_writer = if seg_path.exists() { + SegmentWriter::open_append(seg_path, meta.active_segment_id)? + } else { + fs::create_dir_all(dir.join("segments"))?; + SegmentWriter::create(seg_path, meta.active_segment_id)? + }; + + // Open readers for existing sealed segments + let mut readers = HashMap::new(); + for (&seg_id, stats) in &meta.segments { + if stats.sealed { + let seg_path = dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if seg_path.exists() { + readers.insert(seg_id, SegmentReader::open(seg_path, seg_id)?); + } + } + } + + Ok(Self { + id: account_id.to_string(), + dir, + meta, + active_writer, + readers, + write_lock: Mutex::new(()), + }) + } + + /// Create a new account. + pub fn create(store_root: &Path, account_id: &str) -> Result { + let dir = store_root.join("accounts").join(account_id); + if dir.exists() { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + + fs::create_dir_all(dir.join("segments"))?; + BucketFile::ensure_dir(&dir)?; + + let meta = AccountMeta::new(account_id.to_string(), 1); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let active_writer = SegmentWriter::create(seg_path, 1)?; + + meta.save(&dir)?; + + Ok(Self { + id: account_id.to_string(), + dir, + meta, + active_writer, + readers: HashMap::new(), + write_lock: Mutex::new(()), + }) + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + pub fn meta(&self) -> &AccountMeta { + &self.meta + } + + /// Lock for writing. Returns a guard. + pub fn lock_write(&self) -> std::sync::MutexGuard<'_, ()> { + self.write_lock.lock().unwrap() + } + + /// Mark the segment as indexed up to the given offset and persist meta. + /// Called after append_index to enable incremental recovery. + pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> { + if let Some(stats) = self.meta.segments.get_mut(&segment_id) { + if indexed_up_to_offset > stats.indexed_up_to_offset { + stats.indexed_up_to_offset = indexed_up_to_offset; + } + } + self.meta.save(&self.dir) + } + + /// Append an entry without fsync. Returns (segment_id, offset, data_size). + /// Caller must hold the write lock and should call `flush_active()` after. + pub fn append_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + if self.active_writer.is_full() { + self.seal_active()?; + } + + use crate::segment::Entry; + let entry = if flags == 1 { + Entry::tombstone(key) + } else { + Entry::new(key, data, flags, codec) + }; + + let data_size = entry.data.len() as u32; + let segment_id = self.active_writer.id(); + let offset = self.active_writer.append(&entry)?; + + let stats = self + .meta + .segments + .entry(segment_id) + .or_insert_with(|| SegmentStats::new(segment_id)); + stats.total_bytes += data_size as u64; + if flags == 1 { + stats.deleted_bytes += entry.raw_size as u64; + } + stats.recompute_ratio(); + + Ok((segment_id, offset, data_size)) + } + + /// Fsync the active segment and persist meta. + pub fn flush_active(&mut self) -> Result<()> { + self.active_writer.fsync()?; + self.meta.save(&self.dir) + } + + /// Write an entry with fsync. Convenience wrapper for single writes. + pub fn write_entry( + &mut self, + key: [u8; 32], + data: &[u8], + flags: u8, + codec: Codec, + ) -> Result<(u32, u64, u32)> { + let result = self.append_entry(key, data, flags, codec)?; + self.flush_active()?; + Ok(result) + } + + fn seal_active(&mut self) -> Result<()> { + let old_id = self.active_writer.id(); + let old_stats = self + .meta + .segments + .entry(old_id) + .or_insert_with(|| SegmentStats::new(old_id)); + old_stats.sealed = true; + + // Open reader for the old segment + let seg_path = self + .dir + .join("segments") + .join(segment::segment_filename(old_id)); + self.readers + .insert(old_id, SegmentReader::open(seg_path, old_id)?); + + // Create new segment + let new_id = old_id + 1; + self.meta.active_segment_id = new_id; + let new_path = self + .dir + .join("segments") + .join(segment::segment_filename(new_id)); + self.active_writer = SegmentWriter::create(new_path, new_id)?; + self.meta.save(&self.dir)?; + + Ok(()) + } + + /// Get the on-disk path for a segment. + pub fn segment_path(&self, segment_id: u32) -> Result { + let filename = segment::segment_filename(segment_id); + let path = self.dir.join("segments").join(&filename); + if path.exists() { + Ok(path) + } else { + Err(Error::SegmentNotFound(segment_id)) + } + } + + /// Append index record to the appropriate bucket file. No fsync. + pub fn append_index(&self, record: &IndexRecord) -> Result<()> { + let bucket_id = bucket::bucket_id(&record.key); + let bf = BucketFile::open(&self.dir, bucket_id); + bf.append(record) + } + + /// Return list of sealed segment IDs for GC consideration. + pub fn sealed_segments(&self) -> Vec { + self.meta + .segments + .iter() + .filter(|(_, s)| s.sealed) + .map(|(id, _)| *id) + .collect() + } + + /// All segment IDs (including active). + pub fn all_segment_ids(&self) -> Vec { + let mut ids: Vec = self.meta.segments.keys().copied().collect(); + if !ids.contains(&self.meta.active_segment_id) { + ids.push(self.meta.active_segment_id); + } + ids.sort_unstable(); + ids + } +} diff --git a/crates/blob/src/bucket.rs b/crates/blob/src/bucket.rs new file mode 100644 index 0000000..7acaed6 --- /dev/null +++ b/crates/blob/src/bucket.rs @@ -0,0 +1,333 @@ +use std::fs::{File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::error::Result; +use crate::types::{BUCKET_COUNT, INDEX_RECORD_SIZE}; + +/// On-disk format: 52 bytes per record. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexRecord { + pub key: [u8; 32], + pub segment_id: u32, + pub offset: u64, + pub data_size: u32, + pub flags: u8, +} + +impl IndexRecord { + pub fn new(key: [u8; 32], segment_id: u32, offset: u64, data_size: u32, flags: u8) -> Self { + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + pub fn encode(&self) -> [u8; INDEX_RECORD_SIZE] { + let mut buf = [0u8; INDEX_RECORD_SIZE]; + buf[0..32].copy_from_slice(&self.key); + buf[32..36].copy_from_slice(&self.segment_id.to_le_bytes()); + buf[36..44].copy_from_slice(&self.offset.to_le_bytes()); + buf[44..48].copy_from_slice(&self.data_size.to_le_bytes()); + buf[48] = self.flags; + // bytes 49..52 are padding (keep zero) + buf + } + + pub fn decode(buf: &[u8; INDEX_RECORD_SIZE]) -> Self { + let mut key = [0u8; 32]; + key.copy_from_slice(&buf[0..32]); + let segment_id = u32::from_le_bytes(buf[32..36].try_into().unwrap()); + let offset = u64::from_le_bytes(buf[36..44].try_into().unwrap()); + let data_size = u32::from_le_bytes(buf[44..48].try_into().unwrap()); + let flags = buf[48]; + Self { + key, + segment_id, + offset, + data_size, + flags, + } + } +} + +/// Represents a loaded and deduplicated bucket in memory. +pub struct BucketIndex { + pub bucket_id: u16, + /// Records sorted by key, deduplicated (one record per key, latest wins). + pub records: Vec, +} + +impl BucketIndex { + /// Build from raw records: sort by key, dedup keeping the one with max offset. + pub fn from_records(mut records: Vec, bucket_id: u16) -> Self { + records.sort_by_key(|a| a.key); + // Dedup: keep last (max offset) for each key + let mut deduped = Vec::with_capacity(records.len()); + let mut i = 0; + while i < records.len() { + let mut best = i; + let mut j = i + 1; + while j < records.len() && records[j].key == records[i].key { + if records[j].offset > records[best].offset { + best = j; + } + j += 1; + } + deduped.push(records[best].clone()); + i = j; + } + Self { + bucket_id, + records: deduped, + } + } + + /// Binary search for a key. Returns the record if found. + pub fn find(&self, key: &[u8; 32]) -> Option<&IndexRecord> { + match self.records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => Some(&self.records[idx]), + Err(_) => None, + } + } + + /// Append a new record and maintain sorted order. + pub fn insert(&mut self, record: IndexRecord) { + match self.records.binary_search_by(|r| r.key.cmp(&record.key)) { + Ok(idx) => { + // Replace if newer (larger offset) + if record.offset > self.records[idx].offset { + self.records[idx] = record; + } + } + Err(idx) => { + self.records.insert(idx, record); + } + } + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } +} + +/// Manages a bucket index file on disk. +pub struct BucketFile { + path: PathBuf, + bucket_id: u16, +} + +impl BucketFile { + pub fn path_for(account_dir: &Path, bucket_id: u16) -> PathBuf { + account_dir.join("buckets").join(format!("{:02x}.idx", bucket_id)) + } + + pub fn open(account_dir: &Path, bucket_id: u16) -> Self { + Self { + path: Self::path_for(account_dir, bucket_id), + bucket_id, + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bucket_id(&self) -> u16 { + self.bucket_id + } + + /// Ensure the buckets directory exists. + pub fn ensure_dir(account_dir: &Path) -> Result<()> { + let dir = account_dir.join("buckets"); + std::fs::create_dir_all(&dir)?; + Ok(()) + } + + /// Append a single record to the bucket file. + pub fn append(&self, record: &IndexRecord) -> Result<()> { + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + file.write_all(&record.encode())?; + Ok(()) + } + + /// Append multiple records at once. + pub fn append_batch(&self, records: &[IndexRecord]) -> Result<()> { + if records.is_empty() { + return Ok(()); + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(&self.path)?; + for r in records { + file.write_all(&r.encode())?; + } + Ok(()) + } + + /// Load all records from the bucket file. + /// If the file size is not a multiple of INDEX_RECORD_SIZE (partial write), + /// the trailing bytes are silently ignored. + pub fn load_all(&self) -> Result> { + if !self.path.exists() { + return Ok(Vec::new()); + } + let data = std::fs::read(&self.path)?; + let remainder = data.len() % INDEX_RECORD_SIZE; + let count = data.len() / INDEX_RECORD_SIZE; + let mut records = Vec::with_capacity(count); + for i in 0..count { + let start = i * INDEX_RECORD_SIZE; + let end = start + INDEX_RECORD_SIZE; + let buf: &[u8; INDEX_RECORD_SIZE] = data[start..end] + .try_into() + .map_err(|_| crate::error::Error::BucketIndexCorrupt { + path: self.path.clone(), + reason: format!("unexpected file size {}, not a multiple of {}", data.len(), INDEX_RECORD_SIZE), + })?; + records.push(IndexRecord::decode(buf)); + } + if remainder > 0 { + tracing::warn!( + "Bucket file {:?} has {} trailing bytes (expected multiple of {}), ignoring", + self.path, remainder, INDEX_RECORD_SIZE + ); + } + Ok(records) + } + + /// Load all records, sort, and deduplicate into a BucketIndex. + pub fn load_index(&self) -> Result { + let records = self.load_all()?; + Ok(BucketIndex::from_records(records, self.bucket_id)) + } + + /// Rewrite the bucket file with a sorted, deduplicated set of records. + pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> { + let temp_path = self.path.with_extension("idx.tmp"); + { + let mut file = File::create(&temp_path)?; + for r in records { + file.write_all(&r.encode())?; + } + file.sync_all()?; + } + std::fs::rename(&temp_path, &self.path)?; + Ok(()) + } + + /// Delete the bucket file. + pub fn delete(&self) -> Result<()> { + if self.path.exists() { + std::fs::remove_file(&self.path)?; + } + Ok(()) + } +} + +/// Compute bucket_id from a key's first 2 bytes. +pub fn bucket_id(key: &[u8; 32]) -> u16 { + u16::from_be_bytes([key[0], key[1]]) % BUCKET_COUNT +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_index_record_encode_decode() { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&[1, 2, 3, 4]); + let rec = IndexRecord::new(key, 5, 12345, 500, 0); + let encoded = rec.encode(); + let decoded = IndexRecord::decode(&encoded); + assert_eq!(rec, decoded); + } + + #[test] + fn test_bucket_id_deterministic() { + let mut key = [0u8; 32]; + key[0] = 0x00; + key[1] = 0x0F; + assert_eq!(bucket_id(&key), 15); + key[0] = 0x00; + key[1] = 0x10; + assert_eq!(bucket_id(&key), 0); + } + + #[test] + fn test_bucket_append_and_load() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + let r2 = IndexRecord::new([2u8; 32], 1, 200, 60, 0); + + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [2u8; 32]); + } + + #[test] + fn test_bucket_index_dedup() { + let recs = vec![ + IndexRecord::new([1u8; 32], 1, 100, 50, 0), + IndexRecord::new([1u8; 32], 2, 200, 50, 0), // newer offset wins + IndexRecord::new([2u8; 32], 1, 300, 60, 0), + ]; + let idx = BucketIndex::from_records(recs, 0); + assert_eq!(idx.len(), 2); + let found = idx.find(&[1u8; 32]).unwrap(); + assert_eq!(found.segment_id, 2); + assert_eq!(found.offset, 200); + } + + #[test] + fn test_bucket_index_find_missing() { + let recs = vec![IndexRecord::new([1u8; 32], 1, 100, 50, 0)]; + let idx = BucketIndex::from_records(recs, 0); + assert!(idx.find(&[99u8; 32]).is_none()); + } + + #[test] + fn test_bucket_rewrite() { + let dir = TempDir::new().unwrap(); + let bucket = BucketFile::open(dir.path(), 0); + BucketFile::ensure_dir(dir.path()).unwrap(); + + let r1 = IndexRecord::new([3u8; 32], 1, 300, 70, 0); + let r2 = IndexRecord::new([1u8; 32], 1, 100, 50, 0); + bucket.append(&r1).unwrap(); + bucket.append(&r2).unwrap(); + + // Rewrite sorted + let sorted = vec![r2.clone(), r1.clone()]; + bucket.rewrite(&sorted).unwrap(); + + let loaded = bucket.load_all().unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded[0].key, [1u8; 32]); + assert_eq!(loaded[1].key, [3u8; 32]); + } +} diff --git a/crates/blob/src/cache.rs b/crates/blob/src/cache.rs new file mode 100644 index 0000000..d33cd55 --- /dev/null +++ b/crates/blob/src/cache.rs @@ -0,0 +1,189 @@ +use std::collections::HashMap; +use std::sync::Mutex; + +use crate::bucket::{BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; + +/// Cache key: (account_name, bucket_id) +type CacheKey = (String, u16); + +/// LRU bucket cache. Thread-safe. +pub struct BucketCache { + max_entries: usize, + entries: Mutex>, + index: Mutex>, +} + +struct CacheEntry { + key: CacheKey, + index: BucketIndex, +} + +impl BucketCache { + pub fn new(max_entries: usize) -> Self { + Self { + max_entries: max_entries.max(1), + entries: Mutex::new(Vec::new()), + index: Mutex::new(HashMap::new()), + } + } + + /// Get or load a bucket index. Returns the sorted, deduplicated records for the bucket. + pub fn get_or_load( + &self, + account: &str, + bucket_id: u16, + account_dir: &std::path::Path, + ) -> Result> { + let key: CacheKey = (account.to_string(), bucket_id); + + // Check cache + { + let index = self.index.lock().unwrap(); + if let Some(&pos) = index.get(&key) { + let entries = self.entries.lock().unwrap(); + return Ok(entries[pos].index.records.clone()); + } + } + + // Load from disk + let bucket_file = BucketFile::open(account_dir, bucket_id); + let bucket_index = bucket_file.load_index()?; + let records = bucket_index.records.clone(); + + // Insert into cache + self.insert(key, bucket_index); + + Ok(records) + } + + fn insert(&self, key: CacheKey, index: BucketIndex) { + let mut idx_map = self.index.lock().unwrap(); + let mut entries = self.entries.lock().unwrap(); + + // If already exists, update and move to front + if let Some(&pos) = idx_map.get(&key) { + entries[pos].index = index; + let entry = entries.remove(pos); + entries.insert(0, entry); + // Rebuild index + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + return; + } + + // Evict if full + if entries.len() >= self.max_entries { + if let Some(evicted) = entries.pop() { + idx_map.remove(&evicted.key); + } + } + + // Insert at front (most recently used) + entries.insert(0, CacheEntry { key: key.clone(), index }); + // Rebuild index (positions shifted) + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + + /// Insert or update a single record in a cached bucket. If bucket not cached, no-op. + pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut idx_map = self.index.lock().unwrap(); + + if let Some(&pos) = idx_map.get(&key) { + let mut entries = self.entries.lock().unwrap(); + entries[pos].index.insert(record); + // Move to front + let entry = entries.remove(pos); + entries.insert(0, entry); + // Rebuild index + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + } + + /// Invalidate a cached bucket. + pub fn invalidate(&self, account: &str, bucket_id: u16) { + let key: CacheKey = (account.to_string(), bucket_id); + let mut idx_map = self.index.lock().unwrap(); + if let Some(&pos) = idx_map.get(&key) { + let mut entries = self.entries.lock().unwrap(); + entries.remove(pos); + idx_map.clear(); + for (i, e) in entries.iter().enumerate() { + idx_map.insert(e.key.clone(), i); + } + } + } + + pub fn len(&self) -> usize { + self.entries.lock().unwrap().len() + } + + pub fn is_empty(&self) -> bool { + self.entries.lock().unwrap().is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bucket::IndexRecord; + use tempfile::TempDir; + + #[test] + fn test_cache_miss_loads_from_disk() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([1u8; 32], 1, 100, 50, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + } + + #[test] + fn test_cache_hit() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + bf.append(&IndexRecord::new([2u8; 32], 1, 200, 60, 0)) + .unwrap(); + + let cache = BucketCache::new(10); + let _ = cache.get_or_load("test", 0, dir.path()).unwrap(); + // Second call should hit cache + let records = cache + .get_or_load("test", 0, dir.path()) + .unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(cache.len(), 1); + } + + #[test] + fn test_cache_eviction() { + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let cache = BucketCache::new(2); + + for b in 0..4 { + let bf = BucketFile::open(dir.path(), b); + bf.append(&IndexRecord::new([b as u8; 32], 1, 100, 50, 0)) + .unwrap(); + let _ = cache.get_or_load("test", b, dir.path()).unwrap(); + } + + assert!(cache.len() <= 2); + } +} diff --git a/crates/blob/src/checksum.rs b/crates/blob/src/checksum.rs new file mode 100644 index 0000000..b773a2c --- /dev/null +++ b/crates/blob/src/checksum.rs @@ -0,0 +1,65 @@ +use crc32fast::Hasher; + +pub fn crc32(data: &[u8]) -> u32 { + let mut h = Hasher::new(); + h.update(data); + h.finalize() +} + +pub struct CrcWriter { + hasher: Hasher, +} + +impl Default for CrcWriter { + fn default() -> Self { + Self::new() + } +} + +impl CrcWriter { + pub fn new() -> Self { + Self { + hasher: Hasher::new(), + } + } + + pub fn update(&mut self, data: &[u8]) { + self.hasher.update(data); + } + + pub fn finalize(self) -> u32 { + self.hasher.finalize() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_crc32_deterministic() { + let a = crc32(b"hello"); + let b = crc32(b"hello"); + assert_eq!(a, b); + } + + #[test] + fn test_crc32_different() { + let a = crc32(b"hello"); + let b = crc32(b"world"); + assert!(a != b); + } + + #[test] + fn test_crc_writer_matches_crc32() { + let mut w = CrcWriter::new(); + w.update(b"hello"); + w.update(b" world"); + assert_eq!(w.finalize(), crc32(b"hello world")); + } + + #[test] + fn test_crc32_empty() { + assert_eq!(crc32(b""), 0); + } +} diff --git a/crates/blob/src/compress.rs b/crates/blob/src/compress.rs new file mode 100644 index 0000000..c6e0b7f --- /dev/null +++ b/crates/blob/src/compress.rs @@ -0,0 +1,97 @@ +use crate::types::Codec; + +pub fn compress(data: &[u8], codec: Codec, threshold: usize, level: i32) -> (Vec, Codec) { + if data.len() < threshold { + return (data.to_vec(), Codec::None); + } + let (compressed, actual_codec) = match codec { + Codec::Zstd => { + match zstd::encode_all(data, level) { + Ok(out) => (out, Codec::Zstd), + Err(e) => { + tracing::warn!("zstd compression failed, storing uncompressed: {}", e); + (data.to_vec(), Codec::None) + } + } + } + Codec::Lz4 => { + let out = lz4_flex::compress(data); + (out, Codec::Lz4) + } + Codec::None => (data.to_vec(), Codec::None), + }; + // If compression made it larger, store uncompressed + if compressed.len() >= data.len() { + (data.to_vec(), Codec::None) + } else { + (compressed, actual_codec) + } +} + +pub fn decompress(data: &[u8], codec: Codec, raw_size: usize) -> crate::error::Result> { + match codec { + Codec::None => Ok(data.to_vec()), + Codec::Zstd => { + zstd::decode_all(data) + .map_err(|e| crate::error::Error::Compression(format!("zstd decompress: {}", e))) + } + Codec::Lz4 => { + lz4_flex::decompress(data, raw_size) + .map_err(|e| crate::error::Error::Compression(format!("lz4 decompress: {}", e))) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_small_data_not_compressed() { + let data = b"hi"; + let (out, codec) = compress(data, Codec::Zstd, 4096, 0); + assert_eq!(out, b"hi"); + assert_eq!(codec, Codec::None); + } + + #[test] + fn test_large_data_compressed_zstd() { + let data = vec![b'A'; 5000]; + let (out, codec) = compress(&data, Codec::Zstd, 4096, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } + + #[test] + fn test_roundtrip_zstd() { + let data = vec![b'B'; 10000]; + let (compressed, codec) = compress(&data, Codec::Zstd, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_lz4() { + let data = vec![b'C'; 10000]; + let (compressed, codec) = compress(&data, Codec::Lz4, 4096, 0); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_roundtrip_none() { + let data = vec![b'D'; 100]; + let (compressed, codec) = compress(&data, Codec::None, 4096, 0); + assert_eq!(codec, Codec::None); + let decompressed = decompress(&compressed, codec, data.len()).unwrap(); + assert_eq!(decompressed, data); + } + + #[test] + fn test_threshold_zero_always_compresses() { + let data = vec![b'E'; 100]; + let (out, codec) = compress(&data, Codec::Zstd, 0, 0); + assert_eq!(codec, Codec::Zstd); + assert!(out.len() < data.len()); + } +} diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs new file mode 100644 index 0000000..f04fc38 --- /dev/null +++ b/crates/blob/src/engine.rs @@ -0,0 +1,399 @@ +use std::collections::HashMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::RwLock; + +use crate::account::Account; +use crate::bucket::{self, IndexRecord}; +use crate::cache::BucketCache; +use crate::compress; +use crate::error::{Error, Result}; +use crate::gc::{self, GcStats}; +use crate::meta::GlobalMeta; +use crate::recovery; +use crate::segment::{self, SegmentReader}; +use crate::types::{Codec, Config, ENTRY_HEADER_SIZE}; + +pub struct Engine { + root: PathBuf, + config: Config, + cache: BucketCache, + accounts: RwLock>, +} + +#[derive(Debug, Clone)] +pub struct AccountStats { + pub account_id: String, + pub total_keys: u64, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub segment_count: usize, +} + +impl Engine { + /// Open or create the store at `path`. Runs recovery on startup. + pub fn open(path: &Path, config: Config) -> Result { + config.validate()?; + fs::create_dir_all(path)?; + fs::create_dir_all(path.join("accounts"))?; + + let mut global = GlobalMeta::load(path)?; + global.save(path)?; + + let cache = BucketCache::new(config.lru_bucket_count); + + // Discover accounts on disk + let accounts_dir = path.join("accounts"); + let mut accounts = HashMap::new(); + + if accounts_dir.exists() { + for entry in fs::read_dir(&accounts_dir)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + let account_name = entry.file_name().to_string_lossy().into_owned(); + + // Clean up temp files from interrupted GC + let _ = recovery::cleanup_temp_files(&entry.path()); + + // Run recovery + match recovery::recover_account(&entry.path()) { + Ok(_meta) => { + match Account::open(path, &account_name) { + Ok(account) => { + accounts.insert(account_name, account); + } + Err(e) => { + tracing::warn!( + "Failed to open account {}: {}", + account_name, + e + ); + } + } + } + Err(e) => { + tracing::warn!( + "Failed to recover account {}: {}", + account_name, + e + ); + } + } + } + } + } + + // Update global account list + global.accounts = accounts.keys().cloned().collect(); + global.save(path)?; + + Ok(Self { + root: path.to_path_buf(), + config, + cache, + accounts: RwLock::new(accounts), + }) + } + + // ── Account management ────────────────────────────────────────────────── + + pub fn create_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + if accounts.contains_key(account_id) { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + let account = Account::create(&self.root, account_id)?; + accounts.insert(account_id.to_string(), account); + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn delete_account(&self, account_id: &str) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .remove(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + let account_dir = account.dir().to_path_buf(); + drop(account); + fs::remove_dir_all(&account_dir)?; + + let mut global = GlobalMeta::load(&self.root)?; + global.accounts = accounts.keys().cloned().collect(); + global.save(&self.root)?; + + Ok(()) + } + + pub fn list_accounts(&self) -> Vec { + let accounts = self.accounts.read().unwrap(); + accounts.keys().cloned().collect() + } + + // ── Read / Write / Delete ─────────────────────────────────────────────── + + pub fn write( + &self, + account_id: &str, + key: [u8; 32], + value: &[u8], + codec: Codec, + ) -> Result<()> { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + // Acquire per-account write lock + { + let _lock = account.lock_write(); + } + + let (data, actual_codec) = + compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + account.write_entry(key, &data, 0, actual_codec)?; + + let record = IndexRecord::new(key, segment_id, offset, data_size, 0); + account.append_index(&record)?; + + // Update indexed_up_to_offset for incremental recovery + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + account.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(&key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result>> { + let bucket_id = bucket::bucket_id(key); + + // Acquire lock only to get bucket records and segment routing info. + let record: Option = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let records = self + .cache + .get_or_load(account_id, bucket_id, account.dir())?; + match records.binary_search_by(|r| r.key.cmp(key)) { + Ok(idx) => Some(records[idx].clone()), + Err(_) => None, + } + }; // accounts read lock dropped here — I/O happens outside the lock + + let record = match record { + Some(r) => r, + None => return Ok(None), + }; + + if record.is_tombstone() { + return Ok(None); + } + + // I/O outside the global lock + let seg_path = self + .root + .join("accounts") + .join(account_id) + .join("segments") + .join(segment::segment_filename(record.segment_id)); + + if !seg_path.exists() { + return Err(Error::SegmentNotFound(record.segment_id)); + } + + let reader = SegmentReader::open(seg_path, record.segment_id)?; + let (entry, _) = reader.read_entry_at(record.offset)?; + + let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?; + + Ok(Some(value)) + } + + pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + // Acquire per-account write lock + { + let _lock = account.lock_write(); + } + + let (segment_id, offset, data_size) = + account.write_entry(*key, &[], 1, Codec::None)?; + + let record = IndexRecord::new(*key, segment_id, offset, data_size, 1); + account.append_index(&record)?; + + // Update indexed_up_to_offset for incremental recovery + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + account.mark_indexed(segment_id, entry_end)?; + + let bucket_id = bucket::bucket_id(key); + self.cache.update_record(account_id, bucket_id, record); + + Ok(()) + } + + // ── Batch write ───────────────────────────────────────────────────────── + + /// Batch-write multiple entries with a single fsync. + /// Each element is (key, value, codec). + pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec, Codec)]) -> Result<()> { + if entries.is_empty() { + return Ok(()); + } + + let mut accounts = self.accounts.write().unwrap(); + let account = accounts + .get_mut(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + { + let _lock = account.lock_write(); + } + + // Phase 1: compress and append all entries without fsync + let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len()); + for (key, value, codec) in entries { + if value.len() > crate::types::MAX_VALUE_SIZE { + return Err(Error::ValueTooLarge { size: value.len() }); + } + let (data, actual_codec) = + compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level); + + let (segment_id, offset, data_size) = + account.append_entry(*key, &data, 0, actual_codec)?; + + let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + let record = IndexRecord::new(*key, segment_id, offset, data_size, 0); + pending.push((record, entry_end)); + } + + // Phase 2: single fsync + account.flush_active()?; + + // Phase 3: append indices and update cache + for (record, entry_end) in &pending { + account.append_index(record)?; + account.mark_indexed(record.segment_id, *entry_end)?; + + let bucket_id = bucket::bucket_id(&record.key); + self.cache.update_record(account_id, bucket_id, record.clone()); + } + + Ok(()) + } + + // ── GC ────────────────────────────────────────────────────────────────── + + pub fn gc(&self, account_id: &str) -> Result> { + // Get account dir while holding read lock, then release before heavy I/O. + let account_dir = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + account.dir().to_path_buf() + }; // read lock released — GC runs without blocking reads + + let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?; + + // Invalidate all cached buckets for this account after GC rewrites them + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(result) + } + + pub fn compact_buckets(&self, account_id: &str) -> Result<()> { + let account_dir = { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + account.dir().to_path_buf() + }; + + gc::compact_buckets(&account_dir)?; + + for bid in 0..crate::types::BUCKET_COUNT { + self.cache.invalidate(account_id, bid); + } + + Ok(()) + } + + // ── Stats / Shutdown ──────────────────────────────────────────────────── + + pub fn stats(&self, account_id: &str) -> Result { + let accounts = self.accounts.read().unwrap(); + let account = accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + + let meta = account.meta(); + let mut total_bytes = 0u64; + let mut deleted_bytes = 0u64; + + for seg in meta.segments.values() { + total_bytes += seg.total_bytes; + deleted_bytes += seg.deleted_bytes; + } + + // Count total live keys from bucket indices + let mut total_keys = 0u64; + for bid in 0..crate::types::BUCKET_COUNT { + if let Ok(records) = + self.cache + .get_or_load(account_id, bid, account.dir()) + { + total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64; + } + } + + Ok(AccountStats { + account_id: account_id.to_string(), + total_keys, + total_bytes, + deleted_bytes, + segment_count: meta.segments.len(), + }) + } + + /// Gracefully shut down: fsync all active segments and persist meta. + pub fn shutdown(&self) -> Result<()> { + let mut accounts = self.accounts.write().unwrap(); + for (_, account) in accounts.iter_mut() { + account.flush_active()?; + } + let global = GlobalMeta::load(&self.root)?; + global.save(&self.root)?; + tracing::info!("bichon-blob shut down cleanly"); + Ok(()) + } +} + +impl Drop for Engine { + fn drop(&mut self) { + if let Err(e) = self.shutdown() { + tracing::error!("bichon-blob shutdown error: {}", e); + } + } +} diff --git a/crates/blob/src/error.rs b/crates/blob/src/error.rs new file mode 100644 index 0000000..a745429 --- /dev/null +++ b/crates/blob/src/error.rs @@ -0,0 +1,53 @@ +use std::{io, path::PathBuf}; + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + #[error("JSON error: {0}")] + Json(#[from] serde_json::Error), + + #[error("CRC32 mismatch at {path}:{offset}")] + CrcMismatch { path: PathBuf, offset: u64 }, + + #[error("Corrupt entry at {path}:{offset}: {reason}")] + CorruptEntry { + path: PathBuf, + offset: u64, + reason: String, + }, + + #[error("Account not found: {0}")] + AccountNotFound(String), + + #[error("Account already exists: {0}")] + AccountAlreadyExists(String), + + #[error("Segment not found: {0}")] + SegmentNotFound(u32), + + #[error("Value too large: {size} bytes (max 100 MB)")] + ValueTooLarge { size: usize }, + + #[error("Compression error: {0}")] + Compression(String), + + #[error("Disk full: {0}")] + DiskFull(String), + + #[error("Invalid config: {0}")] + InvalidConfig(String), + + #[error("Bucket index corrupt at {path}: {reason}")] + BucketIndexCorrupt { path: PathBuf, reason: String }, + + #[error("Segment file truncated at {path}: expected {expected}, got {actual}")] + SegmentTruncated { + path: PathBuf, + expected: u64, + actual: u64, + }, +} diff --git a/crates/blob/src/gc.rs b/crates/blob/src/gc.rs new file mode 100644 index 0000000..619d08f --- /dev/null +++ b/crates/blob/src/gc.rs @@ -0,0 +1,267 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::bucket::{self, BucketFile, BucketIndex, IndexRecord}; +use crate::error::Result; +#[cfg(test)] +use crate::meta::SegmentStats; +use crate::segment::{self, SegmentReader, SegmentWriter}; + +/// Result of a GC run. +#[derive(Debug)] +pub struct GcStats { + pub segment_id: u32, + pub bytes_before: u64, + pub bytes_after: u64, + pub entries_kept: usize, + pub entries_skipped: usize, +} + +/// Run GC on an account: pick the sealed segment with highest deleted_ratio, +/// rewrite it without deleted/overwritten entries, then rebuild all bucket files. +pub fn gc_account( + account_dir: &Path, + deleted_ratio_threshold: f64, +) -> Result> { + let meta = crate::meta::AccountMeta::load(account_dir)?; + + // Find the best candidate + let candidate = meta + .segments + .values() + .filter(|s| s.sealed && s.deleted_ratio >= deleted_ratio_threshold) + .max_by(|a, b| a.deleted_ratio.partial_cmp(&b.deleted_ratio).unwrap()); + + let target = match candidate { + Some(s) => s.clone(), + None => return Ok(None), + }; + + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(target.segment_id)); + let reader = SegmentReader::open(seg_path.clone(), target.segment_id)?; + + // Build a global view: for each key, which entry (segment_id + offset) is the latest? + let mut latest_key: HashMap<[u8; 32], (u32, u64)> = HashMap::new(); + + for &seg_id in meta.segments.keys() { + let rpath = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !rpath.exists() { + continue; + } + let r = SegmentReader::open(rpath, seg_id)?; + let _ = r.scan_entries(0, |entry, offset| { + match latest_key.get(&entry.key) { + Some((existing_seg, existing_off)) => { + if seg_id > *existing_seg + || (seg_id == *existing_seg && offset > *existing_off) + { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + None => { + latest_key.insert(entry.key, (seg_id, offset)); + } + } + Ok(()) + })?; + } + + // Create temp segment with a unique name + let timestamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let temp_name = format!("temp_{:016x}.seg", timestamp); + let temp_path = account_dir.join("segments").join(&temp_name); + let mut writer = SegmentWriter::create(temp_path.clone(), target.segment_id)?; + + let mut bytes_after: u64 = 0; + let mut entries_kept: usize = 0; + let mut entries_skipped: usize = 0; + + reader.scan_entries(0, |entry, offset| { + // Skip tombstones + if entry.is_tombstone() { + entries_skipped += 1; + return Ok(()); + } + // Skip if this key has a newer entry in another segment + if let Some((latest_seg, latest_off)) = latest_key.get(&entry.key) { + if *latest_seg != target.segment_id || *latest_off != offset { + entries_skipped += 1; + return Ok(()); + } + } + // Keep this entry + writer.append(entry)?; + bytes_after += entry.data.len() as u64; + entries_kept += 1; + Ok(()) + })?; + + writer.fsync()?; + + // Atomic rename: replace old segment with new one + fs::rename(&temp_path, &seg_path)?; + + // Rebuild all bucket files + rebuild_buckets(account_dir, &meta)?; + + // Update meta + let mut meta = crate::meta::AccountMeta::load(account_dir)?; + if let Some(stats) = meta.segments.get_mut(&target.segment_id) { + stats.total_bytes = bytes_after; + stats.deleted_bytes = 0; + stats.recompute_ratio(); + } + meta.save(account_dir)?; + + Ok(Some(GcStats { + segment_id: target.segment_id, + bytes_before: target.total_bytes, + bytes_after, + entries_kept, + entries_skipped, + })) +} + +/// Rebuild all 16 bucket files from scratch by scanning all segments. +fn rebuild_buckets(account_dir: &Path, meta: &crate::meta::AccountMeta) -> Result<()> { + let mut bucket_records: HashMap> = HashMap::new(); + for i in 0..crate::types::BUCKET_COUNT { + bucket_records.insert(i, Vec::new()); + } + + for &seg_id in meta.segments.keys() { + let seg_path = account_dir + .join("segments") + .join(segment::segment_filename(seg_id)); + if !seg_path.exists() { + continue; + } + let reader = SegmentReader::open(seg_path, seg_id)?; + reader.scan_entries(0, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + bucket_records.entry(bid).or_default().push(rec); + Ok(()) + })?; + } + + for (bid, records) in &bucket_records { + let index = BucketIndex::from_records(records.clone(), *bid); + let bf = BucketFile::open(account_dir, *bid); + bf.rewrite(&index.records)?; + } + + Ok(()) +} + +/// Compact bucket files: load, dedup, rewrite. +pub fn compact_buckets(account_dir: &Path) -> Result<()> { + for bid in 0..crate::types::BUCKET_COUNT { + let bf = BucketFile::open(account_dir, bid); + if bf.path().exists() { + let index = bf.load_index()?; + bf.rewrite(&index.records)?; + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::segment::Entry; + use crate::types::Codec; + use tempfile::TempDir; + + fn setup_account(dir: &Path) { + fs::create_dir_all(dir.join("segments")).unwrap(); + crate::bucket::BucketFile::ensure_dir(dir).unwrap(); + + let seg_path = dir + .join("segments") + .join(segment::segment_filename(1)); + let mut writer = SegmentWriter::create(seg_path, 1).unwrap(); + + // Write 5 entries + for i in 0..5u8 { + let mut key = [0u8; 32]; + key[0] = i; + let entry = Entry::new(key, &vec![i; 1000], 0, Codec::None); + writer.append(&entry).unwrap(); + } + + // Tombstone entry 2 + let mut key2 = [0u8; 32]; + key2[0] = 2; + let tomb = Entry::tombstone(key2); + writer.append(&tomb).unwrap(); + + writer.fsync().unwrap(); + + // Save meta + let mut meta = crate::meta::AccountMeta::new("test".into(), 2); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 6000, + deleted_bytes: 1000, + deleted_ratio: 1000.0 / 6000.0, + sealed: true, + indexed_up_to_offset: 0, + }, + ); + // Make segment 2 active so segment 1 is sealed + let seg2_path = dir + .join("segments") + .join(segment::segment_filename(2)); + SegmentWriter::create(seg2_path, 2).unwrap(); + meta.save(dir).unwrap(); + } + + #[test] + fn test_gc_removes_tombstones() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + + let result = gc_account(dir.path(), 0.01).unwrap(); + assert!(result.is_some()); + + // Verify segment 1 no longer has the tombstone'd entry + let seg_path = dir + .path() + .join("segments") + .join(segment::segment_filename(1)); + let reader = SegmentReader::open(seg_path, 1).unwrap(); + let mut count = 0; + reader.scan_entries(0, |entry, _offset| { + count += 1; + assert!(entry.key[0] != 2); + Ok(()) + }).unwrap(); + assert_eq!(count, 4); // 5 original - 1 tombstoned + } + + #[test] + fn test_compact_buckets() { + let dir = TempDir::new().unwrap(); + setup_account(dir.path()); + compact_buckets(dir.path()).unwrap(); + // Should not panic + } +} diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs new file mode 100644 index 0000000..c95be02 --- /dev/null +++ b/crates/blob/src/lib.rs @@ -0,0 +1,16 @@ +pub mod account; +pub mod bucket; +pub mod cache; +pub mod checksum; +pub mod compress; +pub mod engine; +pub mod error; +pub mod gc; +pub mod meta; +pub mod recovery; +pub mod segment; +pub mod types; + +pub use engine::{AccountStats, Engine}; +pub use error::{Error, Result}; +pub use types::{Codec, Config}; diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs new file mode 100644 index 0000000..d4636e0 --- /dev/null +++ b/crates/blob/src/meta.rs @@ -0,0 +1,156 @@ +use std::collections::HashMap; +use std::path::Path; + +use crate::error::Result; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GlobalMeta { + pub version: u32, + pub accounts: Vec, +} + +impl Default for GlobalMeta { + fn default() -> Self { + Self { + version: 1, + accounts: Vec::new(), + } + } +} + +impl GlobalMeta { + pub fn load(store_root: &Path) -> Result { + let path = store_root.join("global_meta.json"); + if !path.exists() { + return Ok(Self::default()); + } + let data = std::fs::read_to_string(&path)?; + Ok(serde_json::from_str(&data)?) + } + + pub fn save(&self, store_root: &Path) -> Result<()> { + let path = store_root.join("global_meta.json"); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_string_pretty(self)?; + std::fs::write(&tmp, &data)?; + std::fs::rename(&tmp, &path)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SegmentStats { + pub segment_id: u32, + pub total_bytes: u64, + pub deleted_bytes: u64, + pub deleted_ratio: f64, + pub sealed: bool, + /// Byte offset up to which entries have been indexed in bucket files. + /// Recovery starts scanning from here instead of 0. + pub indexed_up_to_offset: u64, +} + +impl SegmentStats { + pub fn new(segment_id: u32) -> Self { + Self { + segment_id, + total_bytes: 0, + deleted_bytes: 0, + deleted_ratio: 0.0, + sealed: false, + indexed_up_to_offset: 0, + } + } + + pub fn recompute_ratio(&mut self) { + if self.total_bytes > 0 { + self.deleted_ratio = self.deleted_bytes as f64 / self.total_bytes as f64; + } else { + self.deleted_ratio = 0.0; + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AccountMeta { + pub account_id: String, + pub active_segment_id: u32, + pub segments: HashMap, +} + +impl AccountMeta { + pub fn new(account_id: String, active_segment_id: u32) -> Self { + Self { + account_id, + active_segment_id, + segments: HashMap::new(), + } + } + + pub fn load(account_dir: &Path) -> Result { + let path = account_dir.join("meta.json"); + if !path.exists() { + return Err(crate::error::Error::AccountNotFound( + account_dir.to_string_lossy().into(), + )); + } + let data = std::fs::read_to_string(&path)?; + Ok(serde_json::from_str(&data)?) + } + + pub fn save(&self, account_dir: &Path) -> Result<()> { + let path = account_dir.join("meta.json"); + let tmp = path.with_extension("json.tmp"); + let data = serde_json::to_string_pretty(self)?; + std::fs::write(&tmp, &data)?; + std::fs::rename(&tmp, &path)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_global_meta_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = GlobalMeta::default(); + meta.accounts.push("alice".into()); + meta.save(dir.path()).unwrap(); + + let loaded = GlobalMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.accounts, vec!["alice"]); + } + + #[test] + fn test_global_meta_default_when_missing() { + let dir = TempDir::new().unwrap(); + let meta = GlobalMeta::load(dir.path()).unwrap(); + assert!(meta.accounts.is_empty()); + } + + #[test] + fn test_account_meta_roundtrip() { + let dir = TempDir::new().unwrap(); + let mut meta = AccountMeta::new("alice".into(), 1); + meta.segments.insert( + 1, + SegmentStats { + segment_id: 1, + total_bytes: 1000, + deleted_bytes: 300, + deleted_ratio: 0.3, + sealed: false, + indexed_up_to_offset: 0, + }, + ); + meta.save(dir.path()).unwrap(); + + let loaded = AccountMeta::load(dir.path()).unwrap(); + assert_eq!(loaded.active_segment_id, 1); + assert_eq!(loaded.segments[&1].total_bytes, 1000); + } +} diff --git a/crates/blob/src/recovery.rs b/crates/blob/src/recovery.rs new file mode 100644 index 0000000..cffb902 --- /dev/null +++ b/crates/blob/src/recovery.rs @@ -0,0 +1,200 @@ +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +use crate::bucket::{self, BucketFile, IndexRecord}; +use crate::error::Result; +use crate::meta::{AccountMeta, SegmentStats}; +use crate::segment::{self, SegmentReader}; + +/// Recover an account after a crash: scan segments, repair indices, update stats. +pub fn recover_account(account_dir: &Path) -> Result { + let meta_path = account_dir.join("meta.json"); + let mut meta = if meta_path.exists() { + AccountMeta::load(account_dir).unwrap_or_else(|_| { + AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + ) + }) + } else { + return Ok(AccountMeta::new( + account_dir + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into(), + 1, + )); + }; + + // Discover all segment files on disk + let seg_dir = account_dir.join("segments"); + if !seg_dir.exists() { + fs::create_dir_all(&seg_dir)?; + } + + let mut disk_segments: Vec = Vec::new(); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".seg") && !name_str.contains("temp_") { + if let Some(id_str) = name_str.strip_suffix(".seg") { + if let Ok(id) = id_str.parse::() { + disk_segments.push(id); + } + } + } + } + } + disk_segments.sort_unstable(); + + if disk_segments.is_empty() { + meta.active_segment_id = 1; + } else { + let max_id = *disk_segments.last().unwrap(); + meta.active_segment_id = max_id; + } + + // Ensure buckets directory exists + let buckets_dir = account_dir.join("buckets"); + fs::create_dir_all(&buckets_dir)?; + + // For each segment, scan only the unindexed tail and update stats incrementally + for &seg_id in &disk_segments { + let seg_path = seg_dir.join(segment::segment_filename(seg_id)); + let file_size = fs::metadata(&seg_path)?.len(); + + // Preserve existing stats; start fresh if this is a newly discovered segment + let mut stats = meta.segments.remove(&seg_id).unwrap_or_else(|| SegmentStats::new(seg_id)); + let is_sealed = seg_id != meta.active_segment_id; + stats.sealed = is_sealed; + + // Scan start: from last indexed offset. Clamp defensively. + let scan_start = if stats.indexed_up_to_offset <= file_size { + stats.indexed_up_to_offset + } else { + 0 + }; + + // If fully indexed, skip scanning entirely + if scan_start >= file_size { + meta.segments.insert(seg_id, stats); + continue; + } + + let reader = SegmentReader::open(seg_path.clone(), seg_id)?; + let mut new_records: HashMap> = HashMap::new(); + + let truncation_point = reader.scan_entries(scan_start, |entry, offset| { + let bid = bucket::bucket_id(&entry.key); + let rec = IndexRecord::new( + entry.key, + seg_id, + offset, + entry.data.len() as u32, + entry.flags, + ); + new_records.entry(bid).or_default().push(rec); + + stats.total_bytes += entry.data.len() as u64; + if entry.is_tombstone() { + stats.deleted_bytes += entry.raw_size as u64; + } + + Ok(()) + })?; + + // Merge new records into bucket files (only the newly discovered ones) + for (bid, records) in &new_records { + let bf = BucketFile::open(account_dir, *bid); + bf.append_batch(records)?; + } + + // Truncate if tail corruption found + if truncation_point < file_size { + segment::truncate_segment(&seg_path, truncation_point)?; + } + + stats.indexed_up_to_offset = truncation_point; + stats.recompute_ratio(); + meta.segments.insert(seg_id, stats); + } + + meta.save(account_dir)?; + + Ok(meta) +} + +/// Clean up leftover temp files from interrupted GC. +pub fn cleanup_temp_files(account_dir: &Path) -> Result<()> { + let seg_dir = account_dir.join("segments"); + if seg_dir.exists() { + for entry in fs::read_dir(&seg_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.starts_with("temp_") { + let path = entry.path(); + tracing::warn!("Removing leftover temp file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + // Also cleanup temp bucket files + let buckets_dir = account_dir.join("buckets"); + if buckets_dir.exists() { + for entry in fs::read_dir(&buckets_dir)? { + let entry = entry?; + let name = entry.file_name(); + let name_str = name.to_string_lossy(); + if name_str.ends_with(".tmp") { + let path = entry.path(); + tracing::warn!("Removing leftover temp bucket file: {:?}", path); + fs::remove_file(&path)?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_recover_fresh_account() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(&account_dir).unwrap(); + + let meta = recover_account(&account_dir).unwrap(); + assert_eq!(meta.active_segment_id, 1); + assert!(meta.segments.is_empty()); + } + + #[test] + fn test_cleanup_temp_files() { + let dir = TempDir::new().unwrap(); + let account_dir = dir.path().join("test"); + fs::create_dir_all(account_dir.join("segments")).unwrap(); + fs::create_dir_all(account_dir.join("buckets")).unwrap(); + fs::write( + account_dir.join("segments").join("temp_ABC123.seg"), + b"garbage", + ) + .unwrap(); + fs::write(account_dir.join("buckets").join("00.idx.tmp"), b"garbage").unwrap(); + + cleanup_temp_files(&account_dir).unwrap(); + + assert!(!account_dir.join("segments").join("temp_ABC123.seg").exists()); + } +} diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs new file mode 100644 index 0000000..1e3085f --- /dev/null +++ b/crates/blob/src/segment.rs @@ -0,0 +1,454 @@ +use std::fs::{self, File, OpenOptions}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +use crate::checksum; +use crate::error::{Error, Result}; +use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE}; + +/// In-memory representation of a stored entry. +#[derive(Debug, Clone)] +pub struct Entry { + pub flags: u8, + pub codec: Codec, + pub key: [u8; 32], + pub raw_size: u32, + pub data: Vec, +} + +impl Entry { + /// Create a normal data entry. + pub fn new(key: [u8; 32], raw_data: &[u8], flags: u8, codec: Codec) -> Self { + Self { + flags, + codec, + key, + raw_size: raw_data.len() as u32, + data: raw_data.to_vec(), + } + } + + /// Create a tombstone entry. + pub fn tombstone(key: [u8; 32]) -> Self { + Self { + flags: 1, + codec: Codec::None, + key, + raw_size: 0, + data: Vec::new(), + } + } + + pub fn is_tombstone(&self) -> bool { + self.flags == 1 + } + + /// Total on-disk size: header + data + pub fn disk_size(&self) -> usize { + ENTRY_HEADER_SIZE + self.data.len() + } +} + +/// Write entries sequentially to a segment file. +pub struct SegmentWriter { + file: File, + path: PathBuf, + id: u32, + bytes_written: u64, +} + +impl SegmentWriter { + pub fn create(path: PathBuf, id: u32) -> Result { + let file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&path)?; + Ok(Self { + file, + path, + id, + bytes_written: 0, + }) + } + + pub fn open_append(path: PathBuf, id: u32) -> Result { + let mut file = OpenOptions::new().write(true).open(&path)?; + file.seek(SeekFrom::End(0))?; + let bytes_written = file.stream_position()?; + Ok(Self { + file, + path, + id, + bytes_written, + }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn bytes_written(&self) -> u64 { + self.bytes_written + } + + pub fn is_full(&self) -> bool { + self.bytes_written >= SEGMENT_MAX_SIZE + } + + /// Append an entry. Returns the offset where it was written. + pub fn append(&mut self, entry: &Entry) -> Result { + let offset = self.bytes_written; + self.write_entry(entry) + .map_err(|e| map_io_err(e, &self.path))?; + Ok(offset) + } + + fn write_entry(&mut self, entry: &Entry) -> Result<()> { + let data_size = entry.data.len() as u32; + + // Write magic + self.file.write_all(&ENTRY_MAGIC.to_le_bytes())?; + + // CRC32 placeholder: write zeros, remember position + let crc_pos = self.file.stream_position()?; + self.file.write_all(&0u32.to_le_bytes())?; + + // Write flags, codec, key, raw_size, data_size + self.file.write_all(&[entry.flags])?; + self.file.write_all(&[entry.codec as u8])?; + self.file.write_all(&entry.key)?; + self.file.write_all(&entry.raw_size.to_le_bytes())?; + self.file.write_all(&data_size.to_le_bytes())?; + + // Write data + self.file.write_all(&entry.data)?; + + // Calculate CRC32 over everything after the crc32 field + let crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[entry.flags]); + hasher.update(&[entry.codec as u8]); + hasher.update(&entry.key); + hasher.update(&entry.raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&entry.data); + hasher.finalize() + }; + + // Seek back and write the real CRC32 + self.file.seek(SeekFrom::Start(crc_pos))?; + self.file.write_all(&crc.to_le_bytes())?; + + // Seek back to end + self.file.seek(SeekFrom::End(0))?; + + self.bytes_written += entry.disk_size() as u64; + Ok(()) + } + + pub fn fsync(&self) -> Result<()> { + self.file.sync_all().map_err(|e| { + if e.kind() == std::io::ErrorKind::StorageFull { + Error::DiskFull(format!("{}: {}", self.path.display(), e)) + } else { + Error::Io(e) + } + })?; + Ok(()) + } +} + +/// Read entries from a segment file. +pub struct SegmentReader { + path: PathBuf, + id: u32, +} + +impl SegmentReader { + pub fn open(path: PathBuf, id: u32) -> Result { + Ok(Self { path, id }) + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn path(&self) -> &Path { + &self.path + } + + pub fn file_size(&self) -> Result { + Ok(fs::metadata(&self.path)?.len()) + } + + /// Read a single entry at the given offset. Returns the entry and the offset of the next entry. + pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> { + let mut file = File::open(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + + // Read magic + let mut magic_buf = [0u8; 4]; + file.read_exact(&mut magic_buf)?; + let magic = u32::from_le_bytes(magic_buf); + if magic != ENTRY_MAGIC { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("bad magic: 0x{:08X}", magic), + }); + } + + // Read CRC32 + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf)?; + let stored_crc = u32::from_le_bytes(crc_buf); + + // Read flags, codec + let mut flags_buf = [0u8; 1]; + file.read_exact(&mut flags_buf)?; + let flags = flags_buf[0]; + + let mut codec_buf = [0u8; 1]; + file.read_exact(&mut codec_buf)?; + let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("unknown codec: {}", codec_buf[0]), + })?; + + // Read key, raw_size, data_size + let mut key = [0u8; 32]; + file.read_exact(&mut key)?; + + let mut raw_size_buf = [0u8; 4]; + file.read_exact(&mut raw_size_buf)?; + let raw_size = u32::from_le_bytes(raw_size_buf); + + let mut data_size_buf = [0u8; 4]; + file.read_exact(&mut data_size_buf)?; + let data_size = u32::from_le_bytes(data_size_buf); + + // Read data + let mut data = vec![0u8; data_size as usize]; + file.read_exact(&mut data)?; + + // Verify CRC32 (over everything after the crc32 field) + let computed_crc = { + let mut hasher = checksum::CrcWriter::new(); + hasher.update(&[flags]); + hasher.update(&[codec as u8]); + hasher.update(&key); + hasher.update(&raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&data); + hasher.finalize() + }; + + if stored_crc != computed_crc { + return Err(Error::CrcMismatch { + path: self.path.clone(), + offset, + }); + } + + let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + + Ok(( + Entry { + flags, + codec, + key, + raw_size, + data, + }, + next_offset, + )) + } + + /// Read data portion of an entry (for pread-style reads when you already know offset + data_size). + pub fn read_data(&self, offset: u64, data_size: u32) -> Result> { + let mut file = File::open(&self.path)?; + // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes + let data_start = offset + ENTRY_HEADER_SIZE as u64; + file.seek(SeekFrom::Start(data_start))?; + let mut buf = vec![0u8; data_size as usize]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Read the full entry header + data for verification (used by recovery and GC). + pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result> { + let mut file = File::open(&self.path)?; + file.seek(SeekFrom::Start(offset))?; + let total = ENTRY_HEADER_SIZE + data_size as usize; + let mut buf = vec![0u8; total]; + file.read_exact(&mut buf)?; + Ok(buf) + } + + /// Iterate over all valid entries in the segment, calling f for each. + /// Stops when hitting a corrupt/incomplete entry at the tail. + pub fn scan_entries(&self, start_offset: u64, mut f: F) -> Result + where + F: FnMut(&Entry, u64) -> Result<()>, + { + let file_size = self.file_size()?; + let mut offset = start_offset; + + while offset + ENTRY_HEADER_SIZE as u64 <= file_size { + match self.read_entry_at(offset) { + Ok((entry, next)) => { + f(&entry, offset)?; + offset = next; + } + Err(Error::CrcMismatch { .. }) | Err(Error::CorruptEntry { .. }) => { + // If near end of file (within one max entry), truncate + if file_size - offset < ENTRY_HEADER_SIZE as u64 + 100 * 1024 * 1024 { + // Likely a partial write at tail, stop here + break; + } else { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: "mid-file corruption detected".into(), + }); + } + } + Err(e) => return Err(e), + } + } + + Ok(offset) // return the truncation point + } +} + +/// Truncate a segment file to the given size. +pub fn truncate_segment(path: &Path, size: u64) -> Result<()> { + let file = OpenOptions::new().write(true).open(path)?; + file.set_len(size)?; + Ok(()) +} + +/// Map an Error, converting Io(StorageFull) to DiskFull with path context. +fn map_io_err(e: Error, path: &Path) -> Error { + match e { + Error::Io(io) if io.kind() == std::io::ErrorKind::StorageFull => { + Error::DiskFull(format!("{}: {}", path.display(), io)) + } + _ => e, + } +} + +/// Segment file name from id: "00000001.seg" +pub fn segment_filename(id: u32) -> String { + format!("{:08}.seg", id) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn temp_segment_path(dir: &TempDir, id: u32) -> PathBuf { + dir.path().join(segment_filename(id)) + } + + #[test] + fn test_write_and_read_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xAAu8; 32]; + let data = b"hello world".to_vec(); + + let entry = Entry::new(key, &data, 0, Codec::None); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, next) = reader.read_entry_at(0).unwrap(); + + assert_eq!(read_entry.key, key); + assert_eq!(read_entry.data, data); + assert_eq!(read_entry.flags, 0); + assert_eq!(read_entry.raw_size, 11); + assert!(next > 0); + } + + #[test] + fn test_tombstone_entry() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let key = [0xBBu8; 32]; + + let entry = Entry::tombstone(key); + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + writer.append(&entry).unwrap(); + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let (read_entry, _) = reader.read_entry_at(0).unwrap(); + + assert!(read_entry.is_tombstone()); + assert_eq!(read_entry.data.len(), 0); + } + + #[test] + fn test_multiple_entries() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + + let entries: Vec<_> = (0..10) + .map(|i| { + let mut key = [0u8; 32]; + key[0] = i; + Entry::new(key, &vec![i; 100], 0, Codec::None) + }) + .collect(); + + { + let mut writer = SegmentWriter::create(path.clone(), 1).unwrap(); + for e in &entries { + writer.append(e).unwrap(); + } + writer.fsync().unwrap(); + } + + let reader = SegmentReader::open(path, 1).unwrap(); + let mut offset = 0u64; + for (i, expected) in entries.iter().enumerate() { + let (entry, next) = reader.read_entry_at(offset).unwrap(); + assert_eq!(entry.key[0], i as u8); + assert_eq!(entry.data, expected.data); + offset = next; + } + } + + #[test] + fn test_bad_magic_detected() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + // Write garbage + std::fs::write(&path, vec![0xFFu8; 100]).unwrap(); + + let reader = SegmentReader::open(path, 1).unwrap(); + let result = reader.read_entry_at(0); + assert!(result.is_err()); + } + + #[test] + fn test_is_full() { + let dir = TempDir::new().unwrap(); + let path = temp_segment_path(&dir, 1); + let writer = SegmentWriter::create(path, 1).unwrap(); + assert!(!writer.is_full()); + } +} diff --git a/crates/blob/src/types.rs b/crates/blob/src/types.rs new file mode 100644 index 0000000..33c885c --- /dev/null +++ b/crates/blob/src/types.rs @@ -0,0 +1,88 @@ +use serde::{Deserialize, Serialize}; + +/// Magic number for entry identification +pub const ENTRY_MAGIC: u32 = 0xB3DB_0001; + +/// Fixed header size: magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) +pub const ENTRY_HEADER_SIZE: usize = 50; + +/// Index record size: key(32) + segment_id(4) + offset(8) + data_size(4) + flags(1) + _pad(3) +pub const INDEX_RECORD_SIZE: usize = 52; + +/// Maximum segment size (256 MB) +pub const SEGMENT_MAX_SIZE: u64 = 256 * 1024 * 1024; + +/// Number of hash buckets per account +pub const BUCKET_COUNT: u16 = 16; + +/// Maximum value size (100 MB) +pub const MAX_VALUE_SIZE: usize = 100 * 1024 * 1024; + +/// Default compression threshold (4 KB) +pub const DEFAULT_COMPRESS_THRESHOLD: usize = 4096; + +/// Default LRU bucket cache size +pub const DEFAULT_LRU_BUCKET_COUNT: usize = 256; + +/// Default GC deleted ratio threshold +pub const DEFAULT_GC_DELETED_RATIO: f64 = 0.30; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Codec { + None = 0, + Zstd = 1, + Lz4 = 2, +} + +impl Codec { + pub fn from_u8(v: u8) -> Option { + match v { + 0 => Some(Codec::None), + 1 => Some(Codec::Zstd), + 2 => Some(Codec::Lz4), + _ => None, + } + } +} + +#[derive(Debug, Clone)] +pub struct Config { + pub compress_threshold: usize, + pub default_codec: Codec, + pub compression_level: i32, + pub lru_bucket_count: usize, + pub gc_deleted_ratio: f64, +} + +impl Default for Config { + fn default() -> Self { + Self { + compress_threshold: DEFAULT_COMPRESS_THRESHOLD, + default_codec: Codec::Zstd, + compression_level: 0, + lru_bucket_count: DEFAULT_LRU_BUCKET_COUNT, + gc_deleted_ratio: DEFAULT_GC_DELETED_RATIO, + } + } +} + +impl Config { + pub fn validate(&self) -> crate::error::Result<()> { + if self.lru_bucket_count == 0 { + return Err(crate::error::Error::InvalidConfig( + "lru_bucket_count must be > 0".into(), + )); + } + if self.gc_deleted_ratio <= 0.0 || self.gc_deleted_ratio >= 1.0 { + return Err(crate::error::Error::InvalidConfig( + "gc_deleted_ratio must be in (0.0, 1.0)".into(), + )); + } + if self.compression_level < 0 { + return Err(crate::error::Error::InvalidConfig( + "compression_level must be >= 0".into(), + )); + } + Ok(()) + } +} diff --git a/crates/blob/tests/acid_test.rs b/crates/blob/tests/acid_test.rs new file mode 100644 index 0000000..e04dae1 --- /dev/null +++ b/crates/blob/tests/acid_test.rs @@ -0,0 +1,498 @@ +/// Crash-consistency and ACID property tests for bichon-blob. +/// +/// Since we can't kill the process mid-write in an inline test, we simulate crashes +/// by dropping the Engine without calling any cleanup (close/drop is the "crash"), +/// then re-opening and verifying recovery produced consistent state. +/// +/// For true power-loss simulation, each test writes data, drops the engine abruptly, +/// then reopens and verifies: no corruption, no lost committed data, no partial writes. + +use std::fs; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::thread; + +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn make_key(seed: u64) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&seed.to_le_bytes()); + key +} + +fn make_value(size: usize) -> Vec { + let pattern = b"The quick brown fox jumps over the lazy dog. "; + let mut v = Vec::with_capacity(size); + while v.len() < size { + let rem = size - v.len(); + let n = rem.min(pattern.len()); + v.extend_from_slice(&pattern[..n]); + } + v +} + +// --------------------------------------------------------------------------- +// 1. Durability: committed data survives crash +// --------------------------------------------------------------------------- + +#[test] +fn test_durability_single_write_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(42); + let value = make_value(8192); + + // Write + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // <-- Engine dropped = simulated crash + + // Recover + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_durability_many_writes_survive_crash() { + let dir = TempDir::new().unwrap(); + let n = 500; + let value = make_value(2048); + let mut keys = Vec::new(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..n { + let key = make_key(i as u64); + keys.push(key); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + } // crash + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (i, key) in keys.iter().enumerate() { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result, Some(value.clone()), "missing key at index {}", i); + } + } +} + +#[test] +fn test_durability_delete_survives_crash() { + let dir = TempDir::new().unwrap(); + let key = make_key(99); + let value = make_value(4096); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } // crash after write + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.delete("alice", &key).unwrap(); + } // crash after delete + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None, "delete should persist across crash"); + } +} + +// --------------------------------------------------------------------------- +// 2. Atomicity: no partial writes visible after crash +// --------------------------------------------------------------------------- + +#[test] +fn test_atomicity_no_partial_entries_after_crash() { + let dir = TempDir::new().unwrap(); + + // Write enough entries to fill part of a segment, then crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + let value = make_value(50_000); // big enough to notice + for i in 0..200u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + } // crash + + // Recovery should clean up any partial tail entries and all committed + // entries should be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(50_000); + for i in 0..200u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert_eq!( + result, + Some(value.clone()), + "committed key {} should be intact", + i + ); + } + } +} + +#[test] +fn test_atomicity_crash_during_segment_roll() { + let dir = TempDir::new().unwrap(); + let big_value = make_value(2 * 1024 * 1024); // 2 MB each entry + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + // Write enough to cross at least one segment boundary (256 MB) + for i in 0..140u64 { + engine + .write("alice", make_key(i), &big_value, Codec::None) + .unwrap(); + } + } // crash mid-way or after multiple segments + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // All committed writes (that returned Ok) must be readable + for i in 0..140u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!( + result.is_some(), + "key {} should exist after segment roll recovery", + i + ); + } + } +} + +// --------------------------------------------------------------------------- +// 3. Consistency: CRC detects corruption, no silent data loss +// --------------------------------------------------------------------------- + +#[test] +fn test_consistency_crc_detects_corruption() { + let dir = TempDir::new().unwrap(); + let key = make_key(77); + let value = make_value(8192); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Corrupt the segment file by flipping a byte + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + // Flip a byte in the data portion, not the header + let flip_pos = data.len() - 100; + data[flip_pos] ^= 0xFF; + fs::write(&seg_path, &data).unwrap(); + + // Reading should detect CRC mismatch + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key); + // Either error or None is acceptable — never silently wrong data + match result { + Err(_) => {} // CRC mismatch detected — good + Ok(None) => {} // index may point to truncated/removed data + Ok(Some(v)) => { + if v == value { + panic!("CRC corruption was NOT detected — silent data corruption!"); + } + // If value differs, index pointed elsewhere after recovery + } + } + } +} + +#[test] +fn test_consistency_corrupt_magic_truncated_on_recovery() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..10u64 { + engine + .write("alice", make_key(i), &make_value(4096), Codec::Zstd) + .unwrap(); + } + } + + // Append garbage to the segment file (simulating partial write from crash) + let seg_path = find_first_segment(dir.path(), "alice"); + let mut data = fs::read(&seg_path).unwrap(); + let orig_len = data.len(); + // Append garbage that doesn't start with the magic number + data.extend_from_slice(&[0xFF; 200]); + fs::write(&seg_path, &data).unwrap(); + + // Recovery should truncate the garbage + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Verify committed data is still intact + for i in 0..10u64 { + let result = engine.read("alice", &make_key(i)).unwrap(); + assert!(result.is_some(), "committed key {} should survive tail truncation", i); + } + } + + // Verify file was actually truncated + let truncated_len = fs::metadata(&seg_path).unwrap().len(); + assert!(truncated_len <= orig_len as u64, "garbage should have been truncated"); +} + +// --------------------------------------------------------------------------- +// 4. Isolation: concurrent reader sees consistent snapshot +// --------------------------------------------------------------------------- + +#[test] +fn test_isolation_reader_sees_snapshot_not_partial_write() { + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Pre-populate a known key + let original_value = make_value(4096); + let key = make_key(100); + engine + .write("alice", key, &original_value, Codec::Zstd) + .unwrap(); + + let running = Arc::new(AtomicBool::new(true)); + let writer_done = Arc::new(AtomicBool::new(false)); + + // Spawn a writer that continuously overwrites the same key + let writer_engine = engine.clone(); + let writer_running = running.clone(); + let writer_done_flag = writer_done.clone(); + let writer_key = key; + + let writer = thread::spawn(move || { + for i in 0..1000u64 { + if !writer_running.load(Ordering::Relaxed) { + break; + } + let val = make_value(4096 + (i as usize % 100)); + writer_engine + .write("alice", writer_key, &val, Codec::Zstd) + .unwrap(); + thread::yield_now(); + } + writer_done_flag.store(true, Ordering::SeqCst); + }); + + // Concurrent reader: reads should never panic or hang + let reader_engine = engine.clone(); + let reader_running = running.clone(); + let reader = thread::spawn(move || { + let mut reads = 0; + while reads < 500 { + if !reader_running.load(Ordering::Relaxed) && reads > 0 { + break; + } + let result = reader_engine.read("alice", &key); + match result { + Ok(Some(_)) | Ok(None) => {} // OK + Err(e) => { + // Accept transient errors but report them + eprintln!("reader saw error: {:?}", e); + } + } + reads += 1; + thread::yield_now(); + } + }); + + reader.join().unwrap(); + running.store(false, Ordering::SeqCst); + writer.join().unwrap(); + + // Final read should see the last committed value (not partial) + let final_result = engine.read("alice", &key).unwrap(); + assert!(final_result.is_some(), "final read should find a value"); +} + +// --------------------------------------------------------------------------- +// 5. Crash during GC: old data intact, no corruption +// --------------------------------------------------------------------------- + +#[test] +fn test_crash_during_gc_leaves_data_intact() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let value = make_value(500_000); // 500 KB each + // Write enough entries and delete some to create GC candidate + for i in 0..500u64 { + engine + .write("alice", make_key(i), &value, Codec::None) + .unwrap(); + } + // Delete ~40% + for i in (0..500u64).step_by(5) { + engine.delete("alice", &make_key(i)).unwrap(); + } + // Single GC run (may or may not trigger) + let _ = engine.gc("alice"); + } // crash after GC + + // All non-deleted entries must still be readable + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let value = make_value(500_000); + for i in 0..500u64 { + let key = make_key(i); + let result = engine.read("alice", &key).unwrap(); + if i % 5 == 0 { + // Deleted keys + assert_eq!(result, None, "key {} should be deleted", i); + } else { + assert_eq!( + result, + Some(value.clone()), + "key {} should survive GC+crash", + i + ); + } + } + } +} + +// --------------------------------------------------------------------------- +// 6. Multiple crash-reopen cycles (torture test) +// --------------------------------------------------------------------------- + +#[test] +fn test_multiple_crash_reopen_cycles() { + use std::collections::HashSet; + + let dir = TempDir::new().unwrap(); + let value = make_value(4096); + let mut alive: HashSet = HashSet::new(); + + // Populate and crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + for i in 0..50u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, write more, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 100..150u64 { + engine + .write("alice", make_key(i), &value, Codec::Zstd) + .unwrap(); + alive.insert(i); + } + } + + // Reopen, verify all exist, delete some, crash + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some()); + } + for i in 0..10u64 { + engine.delete("alice", &make_key(i)).unwrap(); + alive.remove(&i); + } + } + + // Final reopen: survivors exist, deleted gone + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for &k in &alive { + assert!(engine.read("alice", &make_key(k)).unwrap().is_some(), + "key {} should exist", k); + } + for i in 0..10u64 { + assert_eq!(engine.read("alice", &make_key(i)).unwrap(), None, + "key {} should be deleted", i); + } + } +} + +// --------------------------------------------------------------------------- +// 7. Account-level isolation +// --------------------------------------------------------------------------- + +#[test] +fn test_account_isolation_crash_one_account_does_not_affect_others() { + let dir = TempDir::new().unwrap(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + engine + .write("alice", make_key(1), &make_value(4096), Codec::Zstd) + .unwrap(); + engine + .write("bob", make_key(1), &make_value(8192), Codec::Zstd) + .unwrap(); + } + + // Delete alice's account dir partially to simulate corruption + // Then verify bob is intact + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + // Bob should be fine + let result = engine.read("bob", &make_key(1)).unwrap(); + assert!(result.is_some(), "bob should be unaffected"); + } +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +fn find_first_segment(store_root: &Path, account: &str) -> std::path::PathBuf { + let seg_dir = store_root.join("accounts").join(account).join("segments"); + for entry in fs::read_dir(&seg_dir).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".seg") && !name.contains("temp_") { + return entry.path(); + } + } + panic!("no segment found in {:?}", seg_dir); +} diff --git a/crates/blob/tests/integration_test.rs b/crates/blob/tests/integration_test.rs new file mode 100644 index 0000000..2f6d19c --- /dev/null +++ b/crates/blob/tests/integration_test.rs @@ -0,0 +1,273 @@ +use bichon_blob::{Codec, Config, Engine}; +use tempfile::TempDir; + +#[test] +fn test_create_and_list_accounts() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + + engine.create_account("alice").unwrap(); + engine.create_account("bob").unwrap(); + + let accounts = engine.list_accounts(); + assert!(accounts.contains(&"alice".to_string())); + assert!(accounts.contains(&"bob".to_string())); +} + +#[test] +fn test_write_and_read() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xAA; 32]; + let value = b"Hello, this is a test email!".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_read_missing_key() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xFF; 32]; + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xBB; 32]; + let value = b"Some email content".to_vec(); + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + engine.delete("alice", &key).unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); +} + +#[test] +fn test_delete_account() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.delete_account("alice").unwrap(); + + let accounts = engine.list_accounts(); + assert!(!accounts.contains(&"alice".to_string())); +} + +#[test] +fn test_small_value_not_compressed() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xCC; 32]; + let value = b"hi"; // Smaller than 4KB threshold + + engine + .write("alice", key, value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.to_vec())); +} + +#[test] +fn test_large_value() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0xDD; 32]; + let value = vec![b'X'; 100_000]; // 100KB + + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); +} + +#[test] +fn test_multiple_keys() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 100; + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let value = format!("email number {}", i).into_bytes(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(format!("email number {}", i).into_bytes())); + } +} + +#[test] +fn test_gc() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + // Write many entries + let value = vec![b'Y'; 5000]; + let n = 100; + + for i in 0..n { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine + .write("alice", key, &value, Codec::None) + .unwrap(); + } + + // Delete even-numbered keys + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + engine.delete("alice", &key).unwrap(); + } + + // Run GC + let _result = engine.gc("alice").unwrap(); + + // Verify remaining keys still readable + for i in (1..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value.clone())); + } + + // Deleted keys should not exist + for i in (0..n).step_by(2) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, None); + } +} + +#[test] +fn test_reopen_persistence() { + let dir = TempDir::new().unwrap(); + let key = [0xEE; 32]; + let value = b"persistent data".to_vec(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine + .write("alice", key, &value, Codec::Zstd) + .unwrap(); + } + + // Reopen + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + let result = engine.read("alice", &key).unwrap(); + assert_eq!(result, Some(value)); + } +} + +#[test] +fn test_stats() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + engine + .write("alice", [1u8; 32], b"hello", Codec::None) + .unwrap(); + + let stats = engine.stats("alice").unwrap(); + assert!(stats.total_bytes > 0); +} + +#[test] +fn test_batch_write() { + let dir = TempDir::new().unwrap(); + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let n = 50; + let entries: Vec<_> = (0..n) + .map(|i: u64| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + let value = format!("batch email {}", i).into_bytes(); + (key, value, Codec::Zstd) + }) + .collect(); + + engine.write_batch("alice", &entries).unwrap(); + + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } +} + +#[test] +fn test_batch_write_persistence() { + let dir = TempDir::new().unwrap(); + let entries: Vec<_> = (0..30u64) + .map(|i| { + let mut key = [0u8; 32]; + key[0..8].copy_from_slice(&i.to_le_bytes()); + (key, format!("persist {}", i).into_bytes(), Codec::Zstd) + }) + .collect(); + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + engine.write_batch("alice", &entries).unwrap(); + } + + { + let engine = Engine::open(dir.path(), Config::default()).unwrap(); + for (key, value, _) in &entries { + let result = engine.read("alice", key).unwrap(); + assert_eq!(result.as_ref(), Some(value)); + } + } +} + +#[test] +fn test_invalid_config_rejected() { + let dir = TempDir::new().unwrap(); + let mut config = Config::default(); + config.lru_bucket_count = 0; + assert!(Engine::open(dir.path(), config).is_err()); + + let mut config = Config::default(); + config.gc_deleted_ratio = 1.5; + assert!(Engine::open(dir.path(), config).is_err()); +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 83bfc9c..5e76afd 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -19,7 +19,7 @@ poem-openapi = { version = "5.1.16", features = [ ], optional = true } chrono.workspace = true clap.workspace = true -memdb.workspace = true +bichon-memdb.workspace = true itertools.workspace = true ring.workspace = true serde.workspace = true diff --git a/crates/core/src/admin/meta.rs b/crates/core/src/admin/meta.rs index b7f1439..e1858d0 100644 --- a/crates/core/src/admin/meta.rs +++ b/crates/core/src/admin/meta.rs @@ -18,7 +18,7 @@ use std::path::Path; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use crate::{ database::MemDbModel, diff --git a/crates/core/src/database/manager.rs b/crates/core/src/database/manager.rs index afd0b98..6e6f258 100644 --- a/crates/core/src/database/manager.rs +++ b/crates/core/src/database/manager.rs @@ -17,7 +17,7 @@ // along with this program. If not, see . use crate::settings::dir::DATA_DIR_MANAGER; -use memdb::{Durability, MemDb}; +use bichon_memdb::{Durability, MemDb}; use std::sync::LazyLock; use std::time::Duration; diff --git a/crates/core/src/database/mod.rs b/crates/core/src/database/mod.rs index c7a6cc8..778fca1 100644 --- a/crates/core/src/database/mod.rs +++ b/crates/core/src/database/mod.rs @@ -20,7 +20,7 @@ use crate::common::paginated::Paginated; use crate::error::code::ErrorCode; use crate::error::BichonResult; use crate::raise_error; -use memdb::{MemDb, Transaction}; +use bichon_memdb::{MemDb, Transaction}; use serde::de::DeserializeOwned; use serde::Serialize; diff --git a/crates/memdb/Cargo.toml b/crates/memdb/Cargo.toml index c0700e2..b3c6876 100644 --- a/crates/memdb/Cargo.toml +++ b/crates/memdb/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "memdb" +name = "bichon-memdb" version = "0.1.0" edition = "2021" diff --git a/crates/memdb/tests/integration.rs b/crates/memdb/tests/integration.rs index ef0d586..3058795 100644 --- a/crates/memdb/tests/integration.rs +++ b/crates/memdb/tests/integration.rs @@ -1,4 +1,4 @@ -use memdb::{DbError, MemDb, Page}; +use bichon_memdb::{DbError, MemDb, Page}; use serde::{Deserialize, Serialize}; use tempfile::TempDir; @@ -458,7 +458,7 @@ fn test_wal_seq_skips_already_snapshotted_entries() { // Verify WAL only contains seq=3. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 2).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 2).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].seq, 3); @@ -696,7 +696,7 @@ async fn test_concurrent_writes_wal_seq_monotonic() { // Verify WAL seq is strictly monotonic. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 50); let mut last = 0u64; for e in &entries { diff --git a/crates/memdb/tests/stress.rs b/crates/memdb/tests/stress.rs index 31702c8..803b1a8 100644 --- a/crates/memdb/tests/stress.rs +++ b/crates/memdb/tests/stress.rs @@ -1,4 +1,4 @@ -use memdb::{Durability, MemDb, Page}; +use bichon_memdb::{Durability, MemDb, Page}; use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -548,7 +548,7 @@ fn stress_wal_seq_monotonic_under_load() { } let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), n); let mut last = 0u64; for e in &entries { @@ -842,7 +842,7 @@ async fn wal_concurrent_persistent_writes() { // Verify strict seq ordering in WAL under concurrent load. let wal_path = db_path.join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), total as usize); let mut last = 0u64; for e in &entries { @@ -1153,7 +1153,7 @@ fn wal_large_transaction_batch() { // The entire transaction should be a single WAL entry. let wal_path = dir.path().join("wal.jsonl"); - let entries = memdb::wal::read_after(&wal_path, 0).unwrap(); + let entries = bichon_memdb::wal::read_after(&wal_path, 0).unwrap(); assert_eq!(entries.len(), 1); assert_eq!(entries[0].ops.len(), n as usize); From cc540636694bb9a098c86902ec75645890a5c9ed Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 22:55:13 +0800 Subject: [PATCH 37/66] chore(blob): add bincode dependency and new meta error variants --- crates/blob/Cargo.toml | 1 + crates/blob/src/error.rs | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/crates/blob/Cargo.toml b/crates/blob/Cargo.toml index 559396b..4a94e9b 100644 --- a/crates/blob/Cargo.toml +++ b/crates/blob/Cargo.toml @@ -10,6 +10,7 @@ zstd = "0.13" lz4_flex = "0.13.1" serde = { version = "1", features = ["derive"] } serde_json = "1" +bincode = "1" tracing = "0.1" thiserror = "2" diff --git a/crates/blob/src/error.rs b/crates/blob/src/error.rs index a745429..377c840 100644 --- a/crates/blob/src/error.rs +++ b/crates/blob/src/error.rs @@ -50,4 +50,10 @@ pub enum Error { expected: u64, actual: u64, }, + + #[error("Corrupt metadata file: {0}")] + CorruptMeta(String), + + #[error("Unsupported metadata version {version} in {path}")] + UnsupportedMetaVersion { path: PathBuf, version: u32 }, } From e442db4a2d4c25c18846fd0149760d88ee20d6e6 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:01:32 +0800 Subject: [PATCH 38/66] feat(blob): replace JSON metadata with bincode + CRC32 binary format --- Cargo.lock | 1 + crates/blob/src/meta.rs | 159 +++++++++++++++++++++++++++++------- crates/blob/src/recovery.rs | 6 +- 3 files changed, 134 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0fc62d0..dbd6c42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,6 +319,7 @@ dependencies = [ name = "bichon-blob" version = "0.1.0" dependencies = [ + "bincode", "crc32fast", "criterion", "lz4_flex", diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs index d4636e0..7185d45 100644 --- a/crates/blob/src/meta.rs +++ b/crates/blob/src/meta.rs @@ -1,9 +1,54 @@ -use std::collections::HashMap; +use std::collections::BTreeMap; use std::path::Path; +use crate::checksum; use crate::error::Result; use serde::{Deserialize, Serialize}; +const META_VERSION: u32 = 1; + +// ── Helpers ──────────────────────────────────────────────────────────────── + +fn write_bin(path: &Path, value: &T) -> Result<()> { + let payload = bincode::serialize(value).map_err(|e| { + crate::error::Error::CorruptMeta(format!("{}: bincode encode: {}", path.display(), e)) + })?; + let crc = checksum::crc32(&payload); + let mut buf = Vec::with_capacity(8 + payload.len()); + buf.extend_from_slice(&crc.to_le_bytes()); + buf.extend_from_slice(&META_VERSION.to_le_bytes()); + buf.extend_from_slice(&payload); + + let tmp = path.with_extension("bin.tmp"); + std::fs::write(&tmp, &buf)?; + std::fs::rename(&tmp, path)?; + Ok(()) +} + +fn read_bin Deserialize<'de>>(path: &Path) -> Result { + let data = std::fs::read(path)?; + if data.len() < 8 { + return Err(crate::error::Error::CorruptMeta(path.display().to_string())); + } + let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap()); + let version = u32::from_le_bytes(data[4..8].try_into().unwrap()); + if version != META_VERSION { + return Err(crate::error::Error::UnsupportedMetaVersion { + path: path.to_path_buf(), + version, + }); + } + let computed = checksum::crc32(&data[8..]); + if stored_crc != 0 && stored_crc != computed { + return Err(crate::error::Error::CorruptMeta(path.display().to_string())); + } + bincode::deserialize(&data[8..]).map_err(|e| { + crate::error::Error::CorruptMeta(format!("{}: bincode decode: {}", path.display(), e)) + }) +} + +// ── GlobalMeta ───────────────────────────────────────────────────────────── + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GlobalMeta { pub version: u32, @@ -13,7 +58,7 @@ pub struct GlobalMeta { impl Default for GlobalMeta { fn default() -> Self { Self { - version: 1, + version: META_VERSION, accounts: Vec::new(), } } @@ -21,24 +66,33 @@ impl Default for GlobalMeta { impl GlobalMeta { pub fn load(store_root: &Path) -> Result { - let path = store_root.join("global_meta.json"); - if !path.exists() { - return Ok(Self::default()); + let bin_path = store_root.join("global_meta.bin"); + if bin_path.exists() { + return read_bin(&bin_path); + } + // Migration from JSON + let json_path = store_root.join("global_meta.json"); + if json_path.exists() { + let data = std::fs::read_to_string(&json_path)?; + let mut meta: Self = serde_json::from_str(&data)?; + meta.accounts.sort(); + write_bin(&bin_path, &meta)?; + let _ = std::fs::remove_file(&json_path); + return Ok(meta); } - let data = std::fs::read_to_string(&path)?; - Ok(serde_json::from_str(&data)?) + Ok(Self::default()) } pub fn save(&self, store_root: &Path) -> Result<()> { - let path = store_root.join("global_meta.json"); - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_string_pretty(self)?; - std::fs::write(&tmp, &data)?; - std::fs::rename(&tmp, &path)?; - Ok(()) + let path = store_root.join("global_meta.bin"); + let mut meta = self.clone(); + meta.accounts.sort(); + write_bin(&path, &meta) } } +// ── SegmentStats ─────────────────────────────────────────────────────────── + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SegmentStats { pub segment_id: u32, @@ -72,11 +126,13 @@ impl SegmentStats { } } +// ── AccountMeta ──────────────────────────────────────────────────────────── + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AccountMeta { pub account_id: String, pub active_segment_id: u32, - pub segments: HashMap, + pub segments: BTreeMap, } impl AccountMeta { @@ -84,28 +140,31 @@ impl AccountMeta { Self { account_id, active_segment_id, - segments: HashMap::new(), + segments: BTreeMap::new(), } } pub fn load(account_dir: &Path) -> Result { - let path = account_dir.join("meta.json"); - if !path.exists() { - return Err(crate::error::Error::AccountNotFound( - account_dir.to_string_lossy().into(), - )); + let bin_path = account_dir.join("meta.bin"); + if bin_path.exists() { + return read_bin(&bin_path); } - let data = std::fs::read_to_string(&path)?; - Ok(serde_json::from_str(&data)?) + // Migration from JSON + let json_path = account_dir.join("meta.json"); + if json_path.exists() { + let data = std::fs::read_to_string(&json_path)?; + let meta: Self = serde_json::from_str(&data)?; + write_bin(&bin_path, &meta)?; + let _ = std::fs::remove_file(&json_path); + return Ok(meta); + } + Err(crate::error::Error::AccountNotFound( + account_dir.to_string_lossy().into(), + )) } pub fn save(&self, account_dir: &Path) -> Result<()> { - let path = account_dir.join("meta.json"); - let tmp = path.with_extension("json.tmp"); - let data = serde_json::to_string_pretty(self)?; - std::fs::write(&tmp, &data)?; - std::fs::rename(&tmp, &path)?; - Ok(()) + write_bin(&account_dir.join("meta.bin"), self) } } @@ -115,7 +174,7 @@ mod tests { use tempfile::TempDir; #[test] - fn test_global_meta_roundtrip() { + fn test_global_meta_bin_roundtrip() { let dir = TempDir::new().unwrap(); let mut meta = GlobalMeta::default(); meta.accounts.push("alice".into()); @@ -123,6 +182,8 @@ mod tests { let loaded = GlobalMeta::load(dir.path()).unwrap(); assert_eq!(loaded.accounts, vec!["alice"]); + assert!(!dir.path().join("global_meta.json").exists()); + assert!(dir.path().join("global_meta.bin").exists()); } #[test] @@ -133,7 +194,23 @@ mod tests { } #[test] - fn test_account_meta_roundtrip() { + fn test_json_migration() { + let dir = TempDir::new().unwrap(); + // Write old JSON format + let json = r#"{"version":1,"accounts":["bob","alice"]}"#; + std::fs::write(dir.path().join("global_meta.json"), json).unwrap(); + + let meta = GlobalMeta::load(dir.path()).unwrap(); + // Should be sorted + assert_eq!(meta.accounts, vec!["alice", "bob"]); + // JSON should be removed + assert!(!dir.path().join("global_meta.json").exists()); + // BIN should exist + assert!(dir.path().join("global_meta.bin").exists()); + } + + #[test] + fn test_account_meta_bin_roundtrip() { let dir = TempDir::new().unwrap(); let mut meta = AccountMeta::new("alice".into(), 1); meta.segments.insert( @@ -153,4 +230,26 @@ mod tests { assert_eq!(loaded.active_segment_id, 1); assert_eq!(loaded.segments[&1].total_bytes, 1000); } + + #[test] + fn test_corrupt_bin_detected() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap(); + let result = AccountMeta::load(dir.path()); + assert!(result.is_err()); + } + + #[test] + fn test_bin_sorted_keys() { + let dir = TempDir::new().unwrap(); + let mut meta = AccountMeta::new("test".into(), 1); + meta.segments.insert(3, SegmentStats::new(3)); + meta.segments.insert(1, SegmentStats::new(1)); + meta.segments.insert(2, SegmentStats::new(2)); + meta.save(dir.path()).unwrap(); + + let loaded = AccountMeta::load(dir.path()).unwrap(); + let keys: Vec = loaded.segments.keys().copied().collect(); + assert_eq!(keys, vec![1, 2, 3]); + } } diff --git a/crates/blob/src/recovery.rs b/crates/blob/src/recovery.rs index cffb902..b72f093 100644 --- a/crates/blob/src/recovery.rs +++ b/crates/blob/src/recovery.rs @@ -9,8 +9,10 @@ use crate::segment::{self, SegmentReader}; /// Recover an account after a crash: scan segments, repair indices, update stats. pub fn recover_account(account_dir: &Path) -> Result { - let meta_path = account_dir.join("meta.json"); - let mut meta = if meta_path.exists() { + let meta_bin = account_dir.join("meta.bin"); + let meta_json = account_dir.join("meta.json"); + let meta_exists = meta_bin.exists() || meta_json.exists(); + let mut meta = if meta_exists { AccountMeta::load(account_dir).unwrap_or_else(|_| { AccountMeta::new( account_dir From 495c91b7ae793cf12a7a228fb72da908cc15743a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:10:15 +0800 Subject: [PATCH 39/66] fix(blob): address code review issues - CRC guard, tests, clone --- crates/blob/src/meta.rs | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs index 7185d45..4c1ee87 100644 --- a/crates/blob/src/meta.rs +++ b/crates/blob/src/meta.rs @@ -39,7 +39,7 @@ fn read_bin Deserialize<'de>>(path: &Path) -> Result { }); } let computed = checksum::crc32(&data[8..]); - if stored_crc != 0 && stored_crc != computed { + if stored_crc != computed { return Err(crate::error::Error::CorruptMeta(path.display().to_string())); } bincode::deserialize(&data[8..]).map_err(|e| { @@ -237,6 +237,37 @@ mod tests { std::fs::write(dir.path().join("meta.bin"), vec![0xFFu8; 100]).unwrap(); let result = AccountMeta::load(dir.path()); assert!(result.is_err()); + // 0xFFFFFFFF version triggers UnsupportedMetaVersion + assert!(matches!(result.unwrap_err(), crate::error::Error::UnsupportedMetaVersion { .. })); + } + + #[test] + fn test_crc_corruption_detected() { + let dir = TempDir::new().unwrap(); + // Write a well-formed header (version=1) but with wrong CRC bytes + let mut buf = Vec::new(); + buf.extend_from_slice(&0xDEADBEEFu32.to_le_bytes()); // wrong CRC + buf.extend_from_slice(&1u32.to_le_bytes()); // version = 1 (OK) + buf.extend_from_slice(b"some payload bytes"); // payload + std::fs::write(dir.path().join("meta.bin"), &buf).unwrap(); + let result = AccountMeta::load(dir.path()); + assert!(matches!(result.unwrap_err(), crate::error::Error::CorruptMeta(_))); + } + + #[test] + fn test_account_json_migration() { + let dir = TempDir::new().unwrap(); + // Write old JSON format for AccountMeta + let json = r#"{"account_id":"alice","active_segment_id":5,"segments":{}}"#; + std::fs::write(dir.path().join("meta.json"), json).unwrap(); + + let meta = AccountMeta::load(dir.path()).unwrap(); + assert_eq!(meta.account_id, "alice"); + assert_eq!(meta.active_segment_id, 5); + // JSON should be removed + assert!(!dir.path().join("meta.json").exists()); + // BIN should exist + assert!(dir.path().join("meta.bin").exists()); } #[test] From 4e12ae5457cbe109955fd23adb72916df356da12 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:13:39 +0800 Subject: [PATCH 40/66] refactor(blob): replace global RwLock with per-account Arc --- crates/blob/src/account.rs | 116 +++++++++++++-------- crates/blob/src/engine.rs | 200 +++++++++++++++++-------------------- crates/blob/src/lib.rs | 1 + 3 files changed, 166 insertions(+), 151 deletions(-) diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs index 9dc3a14..f8d2fd1 100644 --- a/crates/blob/src/account.rs +++ b/crates/blob/src/account.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex, RwLock}; use crate::bucket::{self, BucketFile, IndexRecord}; use crate::error::{Error, Result}; @@ -9,26 +9,78 @@ use crate::meta::{AccountMeta, SegmentStats}; use crate::segment::{self, SegmentReader, SegmentWriter}; use crate::types::Codec; -pub struct Account { - pub id: String, +// ── AccountHandle ────────────────────────────────────────────────────────── + +pub struct AccountHandle { + id: String, dir: PathBuf, - meta: AccountMeta, - active_writer: SegmentWriter, - readers: HashMap, - write_lock: Mutex<()>, + inner: RwLock, + pub write_mutex: Mutex<()>, } -impl Account { +impl AccountHandle { + pub fn id(&self) -> &str { + &self.id + } + + pub fn dir(&self) -> &Path { + &self.dir + } + /// Open an existing account. - pub fn open(store_root: &Path, account_id: &str) -> Result { + pub fn open(store_root: &Path, account_id: &str) -> Result> { let dir = store_root.join("accounts").join(account_id); if !dir.exists() { return Err(Error::AccountNotFound(account_id.to_string())); } + let inner = AccountInner::open(&dir, account_id)?; + Ok(Arc::new(Self { + id: account_id.to_string(), + dir, + inner: RwLock::new(inner), + write_mutex: Mutex::new(()), + })) + } + + /// Create a new account. + pub fn create(store_root: &Path, account_id: &str) -> Result> { + let dir = store_root.join("accounts").join(account_id); + if dir.exists() { + return Err(Error::AccountAlreadyExists(account_id.to_string())); + } + let inner = AccountInner::create(&dir, account_id)?; + Ok(Arc::new(Self { + id: account_id.to_string(), + dir, + inner: RwLock::new(inner), + write_mutex: Mutex::new(()), + })) + } + + /// Lock the inner state for reading. + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, AccountInner> { + self.inner.read().unwrap() + } + + /// Lock the inner state for writing. + pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> { + self.inner.write().unwrap() + } +} + +// ── AccountInner ─────────────────────────────────────────────────────────── + +pub struct AccountInner { + dir: PathBuf, + pub meta: AccountMeta, + active_writer: SegmentWriter, + readers: HashMap, +} - let meta = AccountMeta::load(&dir)?; +impl AccountInner { + fn open(dir: &Path, _account_id: &str) -> Result { + let meta = AccountMeta::load(dir)?; - // Open active writer let seg_path = dir .join("segments") .join(segment::segment_filename(meta.active_segment_id)); @@ -39,7 +91,6 @@ impl Account { SegmentWriter::create(seg_path, meta.active_segment_id)? }; - // Open readers for existing sealed segments let mut readers = HashMap::new(); for (&seg_id, stats) in &meta.segments { if stats.sealed { @@ -53,24 +104,16 @@ impl Account { } Ok(Self { - id: account_id.to_string(), - dir, + dir: dir.to_path_buf(), meta, active_writer, readers, - write_lock: Mutex::new(()), }) } - /// Create a new account. - pub fn create(store_root: &Path, account_id: &str) -> Result { - let dir = store_root.join("accounts").join(account_id); - if dir.exists() { - return Err(Error::AccountAlreadyExists(account_id.to_string())); - } - + fn create(dir: &Path, account_id: &str) -> Result { fs::create_dir_all(dir.join("segments"))?; - BucketFile::ensure_dir(&dir)?; + BucketFile::ensure_dir(dir)?; let meta = AccountMeta::new(account_id.to_string(), 1); @@ -79,33 +122,21 @@ impl Account { .join(segment::segment_filename(1)); let active_writer = SegmentWriter::create(seg_path, 1)?; - meta.save(&dir)?; + meta.save(dir)?; Ok(Self { - id: account_id.to_string(), - dir, + dir: dir.to_path_buf(), meta, active_writer, readers: HashMap::new(), - write_lock: Mutex::new(()), }) } - pub fn dir(&self) -> &Path { - &self.dir - } - pub fn meta(&self) -> &AccountMeta { &self.meta } - /// Lock for writing. Returns a guard. - pub fn lock_write(&self) -> std::sync::MutexGuard<'_, ()> { - self.write_lock.lock().unwrap() - } - /// Mark the segment as indexed up to the given offset and persist meta. - /// Called after append_index to enable incremental recovery. pub fn mark_indexed(&mut self, segment_id: u32, indexed_up_to_offset: u64) -> Result<()> { if let Some(stats) = self.meta.segments.get_mut(&segment_id) { if indexed_up_to_offset > stats.indexed_up_to_offset { @@ -115,8 +146,7 @@ impl Account { self.meta.save(&self.dir) } - /// Append an entry without fsync. Returns (segment_id, offset, data_size). - /// Caller must hold the write lock and should call `flush_active()` after. + /// Append an entry without fsync. pub fn append_entry( &mut self, key: [u8; 32], @@ -159,7 +189,7 @@ impl Account { self.meta.save(&self.dir) } - /// Write an entry with fsync. Convenience wrapper for single writes. + /// Write an entry with fsync. pub fn write_entry( &mut self, key: [u8; 32], @@ -181,7 +211,6 @@ impl Account { .or_insert_with(|| SegmentStats::new(old_id)); old_stats.sealed = true; - // Open reader for the old segment let seg_path = self .dir .join("segments") @@ -189,7 +218,6 @@ impl Account { self.readers .insert(old_id, SegmentReader::open(seg_path, old_id)?); - // Create new segment let new_id = old_id + 1; self.meta.active_segment_id = new_id; let new_path = self @@ -213,14 +241,14 @@ impl Account { } } - /// Append index record to the appropriate bucket file. No fsync. + /// Append index record to the appropriate bucket file. pub fn append_index(&self, record: &IndexRecord) -> Result<()> { let bucket_id = bucket::bucket_id(&record.key); let bf = BucketFile::open(&self.dir, bucket_id); bf.append(record) } - /// Return list of sealed segment IDs for GC consideration. + /// Return list of sealed segment IDs. pub fn sealed_segments(&self) -> Vec { self.meta .segments diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs index f04fc38..eaefb7c 100644 --- a/crates/blob/src/engine.rs +++ b/crates/blob/src/engine.rs @@ -1,24 +1,23 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; -use crate::account::Account; +use crate::account::AccountHandle; use crate::bucket::{self, IndexRecord}; use crate::cache::BucketCache; use crate::compress; use crate::error::{Error, Result}; use crate::gc::{self, GcStats}; use crate::meta::GlobalMeta; -use crate::recovery; -use crate::segment::{self, SegmentReader}; +use crate::segment::SegmentReader; use crate::types::{Codec, Config, ENTRY_HEADER_SIZE}; pub struct Engine { root: PathBuf, config: Config, cache: BucketCache, - accounts: RwLock>, + accounts: RwLock>>, } #[derive(Debug, Clone)] @@ -31,7 +30,6 @@ pub struct AccountStats { } impl Engine { - /// Open or create the store at `path`. Runs recovery on startup. pub fn open(path: &Path, config: Config) -> Result { config.validate()?; fs::create_dir_all(path)?; @@ -42,7 +40,6 @@ impl Engine { let cache = BucketCache::new(config.lru_bucket_count); - // Discover accounts on disk let accounts_dir = path.join("accounts"); let mut accounts = HashMap::new(); @@ -52,15 +49,13 @@ impl Engine { if entry.file_type()?.is_dir() { let account_name = entry.file_name().to_string_lossy().into_owned(); - // Clean up temp files from interrupted GC - let _ = recovery::cleanup_temp_files(&entry.path()); + let _ = crate::recovery::cleanup_temp_files(&entry.path()); - // Run recovery - match recovery::recover_account(&entry.path()) { + match crate::recovery::recover_account(&entry.path()) { Ok(_meta) => { - match Account::open(path, &account_name) { - Ok(account) => { - accounts.insert(account_name, account); + match AccountHandle::open(path, &account_name) { + Ok(handle) => { + accounts.insert(account_name, handle); } Err(e) => { tracing::warn!( @@ -83,7 +78,6 @@ impl Engine { } } - // Update global account list global.accounts = accounts.keys().cloned().collect(); global.save(path)?; @@ -95,15 +89,15 @@ impl Engine { }) } - // ── Account management ────────────────────────────────────────────────── + // ── Account management ────────────────────────────────────────────── pub fn create_account(&self, account_id: &str) -> Result<()> { let mut accounts = self.accounts.write().unwrap(); if accounts.contains_key(account_id) { return Err(Error::AccountAlreadyExists(account_id.to_string())); } - let account = Account::create(&self.root, account_id)?; - accounts.insert(account_id.to_string(), account); + let handle = AccountHandle::create(&self.root, account_id)?; + accounts.insert(account_id.to_string(), handle); let mut global = GlobalMeta::load(&self.root)?; global.accounts = accounts.keys().cloned().collect(); @@ -114,12 +108,12 @@ impl Engine { pub fn delete_account(&self, account_id: &str) -> Result<()> { let mut accounts = self.accounts.write().unwrap(); - let account = accounts + let handle = accounts .remove(account_id) .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; - let account_dir = account.dir().to_path_buf(); - drop(account); + let account_dir = handle.dir().to_path_buf(); + drop(handle); fs::remove_dir_all(&account_dir)?; let mut global = GlobalMeta::load(&self.root)?; @@ -134,7 +128,7 @@ impl Engine { accounts.keys().cloned().collect() } - // ── Read / Write / Delete ─────────────────────────────────────────────── + // ── Read / Write / Delete ─────────────────────────────────────────── pub fn write( &self, @@ -147,28 +141,28 @@ impl Engine { return Err(Error::ValueTooLarge { size: value.len() }); } - let mut accounts = self.accounts.write().unwrap(); - let account = accounts - .get_mut(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; - // Acquire per-account write lock - { - let _lock = account.lock_write(); - } + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); let (data, actual_codec) = compress::compress(value, codec, self.config.compress_threshold, self.config.compression_level); let (segment_id, offset, data_size) = - account.write_entry(key, &data, 0, actual_codec)?; + inner.write_entry(key, &data, 0, actual_codec)?; let record = IndexRecord::new(key, segment_id, offset, data_size, 0); - account.append_index(&record)?; + inner.append_index(&record)?; - // Update indexed_up_to_offset for incremental recovery let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; - account.mark_indexed(segment_id, entry_end)?; + inner.mark_indexed(segment_id, entry_end)?; let bucket_id = bucket::bucket_id(&key); self.cache.update_record(account_id, bucket_id, record); @@ -179,38 +173,32 @@ impl Engine { pub fn read(&self, account_id: &str, key: &[u8; 32]) -> Result>> { let bucket_id = bucket::bucket_id(key); - // Acquire lock only to get bucket records and segment routing info. - let record: Option = { + let handle = { let accounts = self.accounts.read().unwrap(); - let account = accounts + accounts .get(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; + + let (record, seg_path): (IndexRecord, PathBuf) = { + let inner = handle.read(); let records = self .cache - .get_or_load(account_id, bucket_id, account.dir())?; + .get_or_load(account_id, bucket_id, handle.dir())?; match records.binary_search_by(|r| r.key.cmp(key)) { - Ok(idx) => Some(records[idx].clone()), - Err(_) => None, + Ok(idx) => { + let r = records[idx].clone(); + if r.is_tombstone() { + return Ok(None); + } + let seg_path = inner.segment_path(r.segment_id)?; + (r, seg_path) + } + Err(_) => return Ok(None), } - }; // accounts read lock dropped here — I/O happens outside the lock - - let record = match record { - Some(r) => r, - None => return Ok(None), }; - if record.is_tombstone() { - return Ok(None); - } - - // I/O outside the global lock - let seg_path = self - .root - .join("accounts") - .join(account_id) - .join("segments") - .join(segment::segment_filename(record.segment_id)); - if !seg_path.exists() { return Err(Error::SegmentNotFound(record.segment_id)); } @@ -224,25 +212,25 @@ impl Engine { } pub fn delete(&self, account_id: &str, key: &[u8; 32]) -> Result<()> { - let mut accounts = self.accounts.write().unwrap(); - let account = accounts - .get_mut(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; - // Acquire per-account write lock - { - let _lock = account.lock_write(); - } + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); let (segment_id, offset, data_size) = - account.write_entry(*key, &[], 1, Codec::None)?; + inner.write_entry(*key, &[], 1, Codec::None)?; let record = IndexRecord::new(*key, segment_id, offset, data_size, 1); - account.append_index(&record)?; + inner.append_index(&record)?; - // Update indexed_up_to_offset for incremental recovery let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; - account.mark_indexed(segment_id, entry_end)?; + inner.mark_indexed(segment_id, entry_end)?; let bucket_id = bucket::bucket_id(key); self.cache.update_record(account_id, bucket_id, record); @@ -250,25 +238,24 @@ impl Engine { Ok(()) } - // ── Batch write ───────────────────────────────────────────────────────── + // ── Batch write ───────────────────────────────────────────────────── - /// Batch-write multiple entries with a single fsync. - /// Each element is (key, value, codec). pub fn write_batch(&self, account_id: &str, entries: &[([u8; 32], Vec, Codec)]) -> Result<()> { if entries.is_empty() { return Ok(()); } - let mut accounts = self.accounts.write().unwrap(); - let account = accounts - .get_mut(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; - { - let _lock = account.lock_write(); - } + let _write_lock = handle.write_mutex.lock().unwrap(); + let mut inner = handle.write(); - // Phase 1: compress and append all entries without fsync let mut pending: Vec<(IndexRecord, u64)> = Vec::with_capacity(entries.len()); for (key, value, codec) in entries { if value.len() > crate::types::MAX_VALUE_SIZE { @@ -278,20 +265,18 @@ impl Engine { compress::compress(value, *codec, self.config.compress_threshold, self.config.compression_level); let (segment_id, offset, data_size) = - account.append_entry(*key, &data, 0, actual_codec)?; + inner.append_entry(*key, &data, 0, actual_codec)?; let entry_end = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; let record = IndexRecord::new(*key, segment_id, offset, data_size, 0); pending.push((record, entry_end)); } - // Phase 2: single fsync - account.flush_active()?; + inner.flush_active()?; - // Phase 3: append indices and update cache for (record, entry_end) in &pending { - account.append_index(record)?; - account.mark_indexed(record.segment_id, *entry_end)?; + inner.append_index(record)?; + inner.mark_indexed(record.segment_id, *entry_end)?; let bucket_id = bucket::bucket_id(&record.key); self.cache.update_record(account_id, bucket_id, record.clone()); @@ -300,21 +285,19 @@ impl Engine { Ok(()) } - // ── GC ────────────────────────────────────────────────────────────────── + // ── GC ────────────────────────────────────────────────────────────── pub fn gc(&self, account_id: &str) -> Result> { - // Get account dir while holding read lock, then release before heavy I/O. let account_dir = { let accounts = self.accounts.read().unwrap(); - let account = accounts + let handle = accounts .get(account_id) .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; - account.dir().to_path_buf() - }; // read lock released — GC runs without blocking reads + handle.dir().to_path_buf() + }; let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?; - // Invalidate all cached buckets for this account after GC rewrites them for bid in 0..crate::types::BUCKET_COUNT { self.cache.invalidate(account_id, bid); } @@ -325,10 +308,10 @@ impl Engine { pub fn compact_buckets(&self, account_id: &str) -> Result<()> { let account_dir = { let accounts = self.accounts.read().unwrap(); - let account = accounts + let handle = accounts .get(account_id) .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; - account.dir().to_path_buf() + handle.dir().to_path_buf() }; gc::compact_buckets(&account_dir)?; @@ -340,15 +323,19 @@ impl Engine { Ok(()) } - // ── Stats / Shutdown ──────────────────────────────────────────────────── + // ── Stats / Shutdown ──────────────────────────────────────────────── pub fn stats(&self, account_id: &str) -> Result { - let accounts = self.accounts.read().unwrap(); - let account = accounts - .get(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; + let handle = { + let accounts = self.accounts.read().unwrap(); + accounts + .get(account_id) + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() + }; - let meta = account.meta(); + let inner = handle.read(); + let meta = inner.meta(); let mut total_bytes = 0u64; let mut deleted_bytes = 0u64; @@ -357,12 +344,11 @@ impl Engine { deleted_bytes += seg.deleted_bytes; } - // Count total live keys from bucket indices let mut total_keys = 0u64; for bid in 0..crate::types::BUCKET_COUNT { if let Ok(records) = self.cache - .get_or_load(account_id, bid, account.dir()) + .get_or_load(account_id, bid, handle.dir()) { total_keys += records.iter().filter(|r| !r.is_tombstone()).count() as u64; } @@ -377,11 +363,11 @@ impl Engine { }) } - /// Gracefully shut down: fsync all active segments and persist meta. pub fn shutdown(&self) -> Result<()> { - let mut accounts = self.accounts.write().unwrap(); - for (_, account) in accounts.iter_mut() { - account.flush_active()?; + let accounts = self.accounts.read().unwrap(); + for (_, handle) in accounts.iter() { + let mut inner = handle.write(); + inner.flush_active()?; } let global = GlobalMeta::load(&self.root)?; global.save(&self.root)?; diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs index c95be02..11bb3b9 100644 --- a/crates/blob/src/lib.rs +++ b/crates/blob/src/lib.rs @@ -11,6 +11,7 @@ pub mod recovery; pub mod segment; pub mod types; +pub use account::AccountHandle; pub use engine::{AccountStats, Engine}; pub use error::{Error, Result}; pub use types::{Codec, Config}; From b0374517e8f5d036240485d0ac7732e4712bc30c Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:20:24 +0800 Subject: [PATCH 41/66] fix(blob): address review issues - visibility, unused param --- crates/blob/src/account.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs index f8d2fd1..2af2717 100644 --- a/crates/blob/src/account.rs +++ b/crates/blob/src/account.rs @@ -15,7 +15,7 @@ pub struct AccountHandle { id: String, dir: PathBuf, inner: RwLock, - pub write_mutex: Mutex<()>, + pub(crate) write_mutex: Mutex<()>, } impl AccountHandle { @@ -33,7 +33,7 @@ impl AccountHandle { if !dir.exists() { return Err(Error::AccountNotFound(account_id.to_string())); } - let inner = AccountInner::open(&dir, account_id)?; + let inner = AccountInner::open(&dir)?; Ok(Arc::new(Self { id: account_id.to_string(), dir, @@ -72,13 +72,13 @@ impl AccountHandle { pub struct AccountInner { dir: PathBuf, - pub meta: AccountMeta, + meta: AccountMeta, active_writer: SegmentWriter, readers: HashMap, } impl AccountInner { - fn open(dir: &Path, _account_id: &str) -> Result { + fn open(dir: &Path) -> Result { let meta = AccountMeta::load(dir)?; let seg_path = dir From 11e40750fe493d043163b3ccefec1ced4e8de39d Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:23:03 +0800 Subject: [PATCH 42/66] feat(blob): add FilePool read cache and fix BucketCache TOCTOU --- crates/blob/src/account.rs | 14 +++ crates/blob/src/cache.rs | 164 +++++++++++++++++++++-------------- crates/blob/src/engine.rs | 5 +- crates/blob/src/file_pool.rs | 53 +++++++++++ crates/blob/src/lib.rs | 1 + crates/blob/src/segment.rs | 89 +++++++++++++++++++ 6 files changed, 257 insertions(+), 69 deletions(-) create mode 100644 crates/blob/src/file_pool.rs diff --git a/crates/blob/src/account.rs b/crates/blob/src/account.rs index 2af2717..b95794d 100644 --- a/crates/blob/src/account.rs +++ b/crates/blob/src/account.rs @@ -5,6 +5,7 @@ use std::sync::{Arc, Mutex, RwLock}; use crate::bucket::{self, BucketFile, IndexRecord}; use crate::error::{Error, Result}; +use crate::file_pool::FilePool; use crate::meta::{AccountMeta, SegmentStats}; use crate::segment::{self, SegmentReader, SegmentWriter}; use crate::types::Codec; @@ -16,6 +17,7 @@ pub struct AccountHandle { dir: PathBuf, inner: RwLock, pub(crate) write_mutex: Mutex<()>, + file_pool: FilePool, } impl AccountHandle { @@ -39,6 +41,7 @@ impl AccountHandle { dir, inner: RwLock::new(inner), write_mutex: Mutex::new(()), + file_pool: FilePool::new(8), })) } @@ -54,6 +57,7 @@ impl AccountHandle { dir, inner: RwLock::new(inner), write_mutex: Mutex::new(()), + file_pool: FilePool::new(8), })) } @@ -66,6 +70,16 @@ impl AccountHandle { pub fn write(&self) -> std::sync::RwLockWriteGuard<'_, AccountInner> { self.inner.write().unwrap() } + + /// Get a cached file handle for a segment. + pub fn get_segment_file(&self, seg_id: u32, path: &Path) -> Result>> { + self.file_pool.get(seg_id, path) + } + + /// Invalidate cached file handles for a segment (after GC). + pub fn invalidate_file_cache(&self, seg_id: u32) { + self.file_pool.invalidate(seg_id); + } } // ── AccountInner ─────────────────────────────────────────────────────────── diff --git a/crates/blob/src/cache.rs b/crates/blob/src/cache.rs index d33cd55..b25830d 100644 --- a/crates/blob/src/cache.rs +++ b/crates/blob/src/cache.rs @@ -1,17 +1,22 @@ use std::collections::HashMap; +use std::path::Path; use std::sync::Mutex; use crate::bucket::{BucketFile, BucketIndex, IndexRecord}; use crate::error::Result; -/// Cache key: (account_name, bucket_id) type CacheKey = (String, u16); -/// LRU bucket cache. Thread-safe. +/// Thread-safe LRU bucket cache with single-lock interior. +/// Eliminates the TOCTOU race in the old two-Mutex design. pub struct BucketCache { + inner: Mutex, +} + +struct CacheInner { max_entries: usize, - entries: Mutex>, - index: Mutex>, + entries: Vec, + index: HashMap, } struct CacheEntry { @@ -22,27 +27,28 @@ struct CacheEntry { impl BucketCache { pub fn new(max_entries: usize) -> Self { Self { - max_entries: max_entries.max(1), - entries: Mutex::new(Vec::new()), - index: Mutex::new(HashMap::new()), + inner: Mutex::new(CacheInner { + max_entries: max_entries.max(1), + entries: Vec::new(), + index: HashMap::new(), + }), } } - /// Get or load a bucket index. Returns the sorted, deduplicated records for the bucket. + /// Get or load a bucket index. Eliminates TOCTOU via double-checked locking. pub fn get_or_load( &self, account: &str, bucket_id: u16, - account_dir: &std::path::Path, + account_dir: &Path, ) -> Result> { let key: CacheKey = (account.to_string(), bucket_id); // Check cache { - let index = self.index.lock().unwrap(); - if let Some(&pos) = index.get(&key) { - let entries = self.entries.lock().unwrap(); - return Ok(entries[pos].index.records.clone()); + let inner = self.inner.lock().unwrap(); + if let Some(&pos) = inner.index.get(&key) { + return Ok(inner.entries[pos].index.records.clone()); } } @@ -51,84 +57,74 @@ impl BucketCache { let bucket_index = bucket_file.load_index()?; let records = bucket_index.records.clone(); - // Insert into cache - self.insert(key, bucket_index); - - Ok(records) - } - - fn insert(&self, key: CacheKey, index: BucketIndex) { - let mut idx_map = self.index.lock().unwrap(); - let mut entries = self.entries.lock().unwrap(); - - // If already exists, update and move to front - if let Some(&pos) = idx_map.get(&key) { - entries[pos].index = index; - let entry = entries.remove(pos); - entries.insert(0, entry); - // Rebuild index - idx_map.clear(); - for (i, e) in entries.iter().enumerate() { - idx_map.insert(e.key.clone(), i); + // Insert with double-check (another thread might have beaten us) + { + let mut inner = self.inner.lock().unwrap(); + if let Some(&pos) = inner.index.get(&key) { + return Ok(inner.entries[pos].index.records.clone()); } - return; - } - - // Evict if full - if entries.len() >= self.max_entries { - if let Some(evicted) = entries.pop() { - idx_map.remove(&evicted.key); + // Evict if full + if inner.entries.len() >= inner.max_entries { + if let Some(evicted) = inner.entries.pop() { + inner.index.remove(&evicted.key); + } + } + // Insert at front + inner.entries.insert(0, CacheEntry { + key: key.clone(), + index: bucket_index, + }); + // Rebuild index + inner.index.clear(); + for i in 0..inner.entries.len() { + let key = inner.entries[i].key.clone(); + inner.index.insert(key, i); } } - // Insert at front (most recently used) - entries.insert(0, CacheEntry { key: key.clone(), index }); - // Rebuild index (positions shifted) - idx_map.clear(); - for (i, e) in entries.iter().enumerate() { - idx_map.insert(e.key.clone(), i); - } + Ok(records) } - /// Insert or update a single record in a cached bucket. If bucket not cached, no-op. + /// Insert or update a single record in a cached bucket. pub fn update_record(&self, account: &str, bucket_id: u16, record: IndexRecord) { let key: CacheKey = (account.to_string(), bucket_id); - let mut idx_map = self.index.lock().unwrap(); + let mut inner = self.inner.lock().unwrap(); - if let Some(&pos) = idx_map.get(&key) { - let mut entries = self.entries.lock().unwrap(); - entries[pos].index.insert(record); + if let Some(&pos) = inner.index.get(&key) { + inner.entries[pos].index.insert(record); // Move to front - let entry = entries.remove(pos); - entries.insert(0, entry); + let entry = inner.entries.remove(pos); + inner.entries.insert(0, entry); // Rebuild index - idx_map.clear(); - for (i, e) in entries.iter().enumerate() { - idx_map.insert(e.key.clone(), i); + inner.index.clear(); + for i in 0..inner.entries.len() { + let entry_key = inner.entries[i].key.clone(); + inner.index.insert(entry_key, i); } } } - /// Invalidate a cached bucket. + /// Invalidate a cached bucket (after GC rewrites bucket files). pub fn invalidate(&self, account: &str, bucket_id: u16) { let key: CacheKey = (account.to_string(), bucket_id); - let mut idx_map = self.index.lock().unwrap(); - if let Some(&pos) = idx_map.get(&key) { - let mut entries = self.entries.lock().unwrap(); - entries.remove(pos); - idx_map.clear(); - for (i, e) in entries.iter().enumerate() { - idx_map.insert(e.key.clone(), i); + let mut inner = self.inner.lock().unwrap(); + + if let Some(&pos) = inner.index.get(&key) { + inner.entries.remove(pos); + inner.index.clear(); + for i in 0..inner.entries.len() { + let entry_key = inner.entries[i].key.clone(); + inner.index.insert(entry_key, i); } } } pub fn len(&self) -> usize { - self.entries.lock().unwrap().len() + self.inner.lock().unwrap().entries.len() } pub fn is_empty(&self) -> bool { - self.entries.lock().unwrap().is_empty() + self.inner.lock().unwrap().entries.is_empty() } } @@ -163,7 +159,6 @@ mod tests { let cache = BucketCache::new(10); let _ = cache.get_or_load("test", 0, dir.path()).unwrap(); - // Second call should hit cache let records = cache .get_or_load("test", 0, dir.path()) .unwrap(); @@ -186,4 +181,39 @@ mod tests { assert!(cache.len() <= 2); } + + #[test] + fn test_concurrent_get_or_load_no_deadlock() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + crate::bucket::BucketFile::ensure_dir(dir.path()).unwrap(); + let bf = BucketFile::open(dir.path(), 0); + for i in 0..10u8 { + bf.append(&IndexRecord::new([i; 32], 1, i as u64 * 100, 50, 0)) + .unwrap(); + } + + let cache = Arc::new(BucketCache::new(10)); + let dir_path = dir.path().to_path_buf(); + + let mut handles = vec![]; + for _ in 0..4 { + let cache = cache.clone(); + let dir_path = dir_path.clone(); + handles.push(thread::spawn(move || { + for _ in 0..100 { + let records = cache + .get_or_load("test", 0, &dir_path) + .unwrap(); + assert_eq!(records.len(), 10); + } + })); + } + + for h in handles { + h.join().unwrap(); + } + } } diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs index eaefb7c..18d13b8 100644 --- a/crates/blob/src/engine.rs +++ b/crates/blob/src/engine.rs @@ -203,8 +203,9 @@ impl Engine { return Err(Error::SegmentNotFound(record.segment_id)); } - let reader = SegmentReader::open(seg_path, record.segment_id)?; - let (entry, _) = reader.read_entry_at(record.offset)?; + let reader = SegmentReader::open(seg_path.clone(), record.segment_id)?; + let file = handle.get_segment_file(record.segment_id, &seg_path)?; + let (entry, _) = reader.read_entry_at_file(record.offset, &file)?; let value = compress::decompress(&entry.data, entry.codec, entry.raw_size as usize)?; diff --git a/crates/blob/src/file_pool.rs b/crates/blob/src/file_pool.rs new file mode 100644 index 0000000..229533b --- /dev/null +++ b/crates/blob/src/file_pool.rs @@ -0,0 +1,53 @@ +use std::collections::VecDeque; +use std::fs::File; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +use crate::error::Result; + +/// Simple LRU pool of open file handles, keyed by segment_id. +/// Uses Arc> to allow safe concurrent reads from the same segment. +pub struct FilePool { + max_entries: usize, + entries: Mutex>)>>, +} + +impl FilePool { + pub fn new(max_entries: usize) -> Self { + Self { + max_entries: max_entries.max(1), + entries: Mutex::new(VecDeque::new()), + } + } + + /// Get an open File for the given segment. Reuses cached handle if available. + pub fn get(&self, seg_id: u32, path: &Path) -> Result>> { + let mut entries = self.entries.lock().unwrap(); + + // Check for existing entry + for (i, (id, _)) in entries.iter().enumerate() { + if *id == seg_id { + let (_, file) = entries.remove(i).unwrap(); + entries.push_front((seg_id, file.clone())); + return Ok(file); + } + } + + // Open new file + let file = Arc::new(Mutex::new(File::open(path)?)); + + // Evict oldest if full + if entries.len() >= self.max_entries { + entries.pop_back(); + } + + entries.push_front((seg_id, file.clone())); + Ok(file) + } + + /// Remove a cached file handle (e.g. after GC rewrites a segment). + pub fn invalidate(&self, seg_id: u32) { + let mut entries = self.entries.lock().unwrap(); + entries.retain(|(id, _)| *id != seg_id); + } +} diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs index 11bb3b9..46feff4 100644 --- a/crates/blob/src/lib.rs +++ b/crates/blob/src/lib.rs @@ -5,6 +5,7 @@ pub mod checksum; pub mod compress; pub mod engine; pub mod error; +pub mod file_pool; pub mod gc; pub mod meta; pub mod recovery; diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs index 1e3085f..0d7b162 100644 --- a/crates/blob/src/segment.rs +++ b/crates/blob/src/segment.rs @@ -1,6 +1,7 @@ use std::fs::{self, File, OpenOptions}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; +use std::sync::Mutex; use crate::checksum; use crate::error::{Error, Result}; @@ -269,6 +270,94 @@ impl SegmentReader { )) } + /// Read a single entry at the given offset using a pre-opened File (via Mutex). + /// This avoids the per-read File::open cost for hot segments. + pub fn read_entry_at_file(&self, offset: u64, file: &Mutex) -> Result<(Entry, u64)> { + use std::io::{Read, Seek, SeekFrom}; + + let mut file = file.lock().unwrap(); + + file.seek(SeekFrom::Start(offset))?; + + // Read magic + let mut magic_buf = [0u8; 4]; + file.read_exact(&mut magic_buf)?; + let magic = u32::from_le_bytes(magic_buf); + if magic != ENTRY_MAGIC { + return Err(Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("bad magic: 0x{:08X}", magic), + }); + } + + // Read CRC32 + let mut crc_buf = [0u8; 4]; + file.read_exact(&mut crc_buf)?; + let stored_crc = u32::from_le_bytes(crc_buf); + + // Read flags, codec + let mut flags_buf = [0u8; 1]; + file.read_exact(&mut flags_buf)?; + let flags = flags_buf[0]; + + let mut codec_buf = [0u8; 1]; + file.read_exact(&mut codec_buf)?; + let codec = Codec::from_u8(codec_buf[0]).ok_or_else(|| Error::CorruptEntry { + path: self.path.clone(), + offset, + reason: format!("unknown codec: {}", codec_buf[0]), + })?; + + // Read key, raw_size, data_size + let mut key = [0u8; 32]; + file.read_exact(&mut key)?; + + let mut raw_size_buf = [0u8; 4]; + file.read_exact(&mut raw_size_buf)?; + let raw_size = u32::from_le_bytes(raw_size_buf); + + let mut data_size_buf = [0u8; 4]; + file.read_exact(&mut data_size_buf)?; + let data_size = u32::from_le_bytes(data_size_buf); + + // Read data + let mut data = vec![0u8; data_size as usize]; + file.read_exact(&mut data)?; + + // Verify CRC32 + let computed_crc = { + let mut hasher = crate::checksum::CrcWriter::new(); + hasher.update(&[flags]); + hasher.update(&[codec as u8]); + hasher.update(&key); + hasher.update(&raw_size.to_le_bytes()); + hasher.update(&data_size.to_le_bytes()); + hasher.update(&data); + hasher.finalize() + }; + + if stored_crc != computed_crc { + return Err(Error::CrcMismatch { + path: self.path.clone(), + offset, + }); + } + + let next_offset = offset + ENTRY_HEADER_SIZE as u64 + data_size as u64; + + Ok(( + Entry { + flags, + codec, + key, + raw_size, + data, + }, + next_offset, + )) + } + /// Read data portion of an entry (for pread-style reads when you already know offset + data_size). pub fn read_data(&self, offset: u64, data_size: u32) -> Result> { let mut file = File::open(&self.path)?; From cd98c050b62a4f2e901d47ec040a1f08818eeca2 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:28:33 +0800 Subject: [PATCH 43/66] fix(blob): invalidate FilePool after GC to prevent stale reads --- crates/blob/src/engine.rs | 8 ++++++++ crates/blob/src/segment.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs index 18d13b8..126dc9b 100644 --- a/crates/blob/src/engine.rs +++ b/crates/blob/src/engine.rs @@ -299,6 +299,14 @@ impl Engine { let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?; + // Invalidate FilePool for GC'd segments (they were rewritten via rename) + if let Some(ref stats) = result { + let accounts = self.accounts.read().unwrap(); + if let Some(handle) = accounts.get(account_id) { + handle.invalidate_file_cache(stats.segment_id); + } + } + for bid in 0..crate::types::BUCKET_COUNT { self.cache.invalidate(account_id, bid); } diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs index 0d7b162..30e4c5a 100644 --- a/crates/blob/src/segment.rs +++ b/crates/blob/src/segment.rs @@ -273,7 +273,6 @@ impl SegmentReader { /// Read a single entry at the given offset using a pre-opened File (via Mutex). /// This avoids the per-read File::open cost for hot segments. pub fn read_entry_at_file(&self, offset: u64, file: &Mutex) -> Result<(Entry, u64)> { - use std::io::{Read, Seek, SeekFrom}; let mut file = file.lock().unwrap(); From b6957c8ceb6a3420d4bb9557023903612cad136d Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:29:48 +0800 Subject: [PATCH 44/66] test(blob): add concurrent access and crash recovery integration tests --- crates/blob/tests/integration_test.rs | 106 ++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/crates/blob/tests/integration_test.rs b/crates/blob/tests/integration_test.rs index 2f6d19c..c1e6501 100644 --- a/crates/blob/tests/integration_test.rs +++ b/crates/blob/tests/integration_test.rs @@ -271,3 +271,109 @@ fn test_invalid_config_rejected() { config.gc_deleted_ratio = 1.5; assert!(Engine::open(dir.path(), config).is_err()); } + +#[test] +fn test_concurrent_reads() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Write some data + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &vec![i as u8; 1024], Codec::None).unwrap(); + } + + // Spawn 4 threads, each reading a different subset + let mut handles = vec![]; + for t in 0..4 { + let engine = engine.clone(); + handles.push(thread::spawn(move || { + for i in (t * 12)..((t + 1) * 12) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&(i as u32).to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "key {} should exist", i); + } + })); + } + for h in handles { + h.join().unwrap(); + } +} + +#[test] +fn test_concurrent_writes_different_accounts() { + use std::sync::Arc; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + + for name in &["alice", "bob", "carol"] { + engine.create_account(name).unwrap(); + } + + let mut handles = vec![]; + for (t, name) in ["alice", "bob", "carol"].iter().enumerate() { + let engine = engine.clone(); + let account_name = name.to_string(); + handles.push(thread::spawn(move || { + for i in 0..20 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes()); + let value = vec![(t * 100 + i) as u8; 512]; + engine.write(&account_name, key, &value, Codec::None).unwrap(); + } + })); + } + for h in handles { + h.join().unwrap(); + } + + // Verify all writes persisted + for (t, name) in ["alice", "bob", "carol"].iter().enumerate() { + for i in 0..20 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&((t * 100 + i) as u32).to_le_bytes()); + let read = engine.read(name, &key).unwrap(); + assert!(read.is_some(), "account {} key {} should exist", name, i); + } + } +} + +#[test] +fn test_crash_recovery() { + let dir = TempDir::new().unwrap(); + let dir_path = dir.path().to_path_buf(); + + // Phase 1: write data, then drop without shutdown (simulates crash) + { + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &vec![i as u8; 512], Codec::None).unwrap(); + } + // Engine dropped here without calling shutdown() + } + + // Phase 2: reopen — recovery should run, data should be intact + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + let stats = engine.stats("alice").unwrap(); + assert!(stats.total_keys > 0, "recovery should preserve data"); + + // Verify reads work + for i in 0..50u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "key {} should survive crash recovery", i); + } +} From ff59d47a0d1d107d004ae3e80f99504d0f5d022a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Tue, 23 Jun 2026 23:57:34 +0800 Subject: [PATCH 45/66] fix(blob): fsync meta before rename and hold write_mutex during GC --- crates/blob/src/engine.rs | 32 ++++--- crates/blob/src/meta.rs | 7 +- crates/blob/tests/integration_test.rs | 118 ++++++++++++++++++++++++++ 3 files changed, 142 insertions(+), 15 deletions(-) diff --git a/crates/blob/src/engine.rs b/crates/blob/src/engine.rs index 126dc9b..0b71a6d 100644 --- a/crates/blob/src/engine.rs +++ b/crates/blob/src/engine.rs @@ -289,22 +289,23 @@ impl Engine { // ── GC ────────────────────────────────────────────────────────────── pub fn gc(&self, account_id: &str) -> Result> { - let account_dir = { + let handle = { let accounts = self.accounts.read().unwrap(); - let handle = accounts + accounts .get(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; - handle.dir().to_path_buf() + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() }; - let result = gc::gc_account(&account_dir, self.config.gc_deleted_ratio)?; + // Hold write_mutex to prevent concurrent writes from racing + // with GC's bucket rebuild phase. + let _write_lock = handle.write_mutex.lock().unwrap(); + + let result = gc::gc_account(handle.dir(), self.config.gc_deleted_ratio)?; // Invalidate FilePool for GC'd segments (they were rewritten via rename) if let Some(ref stats) = result { - let accounts = self.accounts.read().unwrap(); - if let Some(handle) = accounts.get(account_id) { - handle.invalidate_file_cache(stats.segment_id); - } + handle.invalidate_file_cache(stats.segment_id); } for bid in 0..crate::types::BUCKET_COUNT { @@ -315,15 +316,18 @@ impl Engine { } pub fn compact_buckets(&self, account_id: &str) -> Result<()> { - let account_dir = { + let handle = { let accounts = self.accounts.read().unwrap(); - let handle = accounts + accounts .get(account_id) - .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))?; - handle.dir().to_path_buf() + .ok_or_else(|| Error::AccountNotFound(account_id.to_string()))? + .clone() }; - gc::compact_buckets(&account_dir)?; + // Hold write_mutex — compact rewrites all bucket files. + let _write_lock = handle.write_mutex.lock().unwrap(); + + gc::compact_buckets(handle.dir())?; for bid in 0..crate::types::BUCKET_COUNT { self.cache.invalidate(account_id, bid); diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs index 4c1ee87..750b2c9 100644 --- a/crates/blob/src/meta.rs +++ b/crates/blob/src/meta.rs @@ -20,7 +20,12 @@ fn write_bin(path: &Path, value: &T) -> Result<()> { buf.extend_from_slice(&payload); let tmp = path.with_extension("bin.tmp"); - std::fs::write(&tmp, &buf)?; + { + use std::io::Write; + let mut f = std::fs::File::create(&tmp)?; + f.write_all(&buf)?; + f.sync_all()?; + } std::fs::rename(&tmp, path)?; Ok(()) } diff --git a/crates/blob/tests/integration_test.rs b/crates/blob/tests/integration_test.rs index c1e6501..de634f1 100644 --- a/crates/blob/tests/integration_test.rs +++ b/crates/blob/tests/integration_test.rs @@ -377,3 +377,121 @@ fn test_crash_recovery() { assert!(read.is_some(), "key {} should survive crash recovery", i); } } + +#[test] +fn test_meta_bin_durability() { + // Verify meta.bin has valid CRC and can be read after a write cycle. + let dir = TempDir::new().unwrap(); + let dir_path = dir.path().to_path_buf(); + + { + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + engine.create_account("alice").unwrap(); + + let key = [0x42u8; 32]; + engine.write("alice", key, b"durable", Codec::None).unwrap(); + } + // Engine dropped → shutdown() called → meta saved via write_bin (with fsync) + + // Verify meta.bin exists and has valid CRC + let meta_path = dir_path + .join("accounts") + .join("alice") + .join("meta.bin"); + assert!(meta_path.exists(), "meta.bin should exist after clean shutdown"); + + let data = std::fs::read(&meta_path).unwrap(); + assert!(data.len() >= 8, "meta.bin should have at least 8 bytes (crc + version)"); + + let stored_crc = u32::from_le_bytes(data[0..4].try_into().unwrap()); + assert_ne!(stored_crc, 0, "stored CRC should be non-zero"); + + // Reopen and verify data is intact + let engine = Engine::open(&dir_path, Config::default()).unwrap(); + let read = engine.read("alice", &[0x42u8; 32]).unwrap(); + assert_eq!(read, Some(b"durable".to_vec())); +} + +#[test] +fn test_gc_concurrent_with_writes() { + // GC should not lose entries that are written concurrently. + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::thread; + + let dir = TempDir::new().unwrap(); + let engine = Arc::new(Engine::open(dir.path(), Config::default()).unwrap()); + engine.create_account("alice").unwrap(); + + // Pre-fill: write enough to trigger eventual GC + let big_value = vec![b'X'; 8192]; + for i in 0..500u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.write("alice", key, &big_value, Codec::None).unwrap(); + } + + // Delete some to create GC candidates + for i in 0..250u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + engine.delete("alice", &key).unwrap(); + } + + let running = Arc::new(AtomicBool::new(true)); + let engine_gc = engine.clone(); + let running_gc = running.clone(); + + // Thread 1: run GC in a loop + let gc_handle = thread::spawn(move || { + while running_gc.load(Ordering::Relaxed) { + let _ = engine_gc.gc("alice"); + thread::sleep(std::time::Duration::from_millis(10)); + } + }); + + // Thread 2: keep writing new entries + let engine_write = engine.clone(); + let running_write = running.clone(); + let write_handle = thread::spawn(move || { + let mut counter = 10000u32; + while running_write.load(Ordering::Relaxed) { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&counter.to_le_bytes()); + engine_write + .write("alice", key, &vec![counter as u8; 256], Codec::None) + .unwrap(); + counter += 1; + } + counter + }); + + // Let them race for a bit + thread::sleep(std::time::Duration::from_millis(500)); + running.store(false, Ordering::Relaxed); + + gc_handle.join().unwrap(); + let final_counter = write_handle.join().unwrap(); + + // All written entries must be readable + let mut missing = 0; + for i in 0..500u32 { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + if engine.read("alice", &key).unwrap().is_none() { + // Entries 0..250 were deleted, they should be gone + if i >= 250 { + missing += 1; + } + } + } + assert_eq!(missing, 0, "pre-existing entries should survive concurrent GC"); + + // Entries written during the race should be readable + for i in 10000..final_counter { + let mut key = [0u8; 32]; + key[0..4].copy_from_slice(&i.to_le_bytes()); + let read = engine.read("alice", &key).unwrap(); + assert!(read.is_some(), "concurrently written key {} should exist after GC", i); + } +} From b0f229618c16ee5bb20f75eb3e9239c68c5c67af Mon Sep 17 00:00:00 2001 From: rustmailer Date: Wed, 24 Jun 2026 00:03:56 +0800 Subject: [PATCH 46/66] refactor(blob): add NFS-safe file I/O layer --- crates/blob/src/bucket.rs | 16 ++-- crates/blob/src/file_pool.rs | 3 +- crates/blob/src/fs.rs | 142 +++++++++++++++++++++++++++++++++++ crates/blob/src/lib.rs | 1 + crates/blob/src/meta.rs | 9 +-- crates/blob/src/segment.rs | 21 +++--- 6 files changed, 161 insertions(+), 31 deletions(-) create mode 100644 crates/blob/src/fs.rs diff --git a/crates/blob/src/bucket.rs b/crates/blob/src/bucket.rs index 7acaed6..208346f 100644 --- a/crates/blob/src/bucket.rs +++ b/crates/blob/src/bucket.rs @@ -1,4 +1,4 @@ -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::io::Write; use std::path::{Path, PathBuf}; @@ -218,17 +218,13 @@ impl BucketFile { } /// Rewrite the bucket file with a sorted, deduplicated set of records. + /// Uses atomic temp+rename to be safe on NFS. pub fn rewrite(&self, records: &[IndexRecord]) -> Result<()> { - let temp_path = self.path.with_extension("idx.tmp"); - { - let mut file = File::create(&temp_path)?; - for r in records { - file.write_all(&r.encode())?; - } - file.sync_all()?; + let mut buf = Vec::with_capacity(records.len() * INDEX_RECORD_SIZE); + for r in records { + buf.extend_from_slice(&r.encode()); } - std::fs::rename(&temp_path, &self.path)?; - Ok(()) + crate::fs::create_atomic(&self.path, &buf) } /// Delete the bucket file. diff --git a/crates/blob/src/file_pool.rs b/crates/blob/src/file_pool.rs index 229533b..ec8ec5b 100644 --- a/crates/blob/src/file_pool.rs +++ b/crates/blob/src/file_pool.rs @@ -4,6 +4,7 @@ use std::path::Path; use std::sync::{Arc, Mutex}; use crate::error::Result; +use crate::fs as fs_util; /// Simple LRU pool of open file handles, keyed by segment_id. /// Uses Arc> to allow safe concurrent reads from the same segment. @@ -34,7 +35,7 @@ impl FilePool { } // Open new file - let file = Arc::new(Mutex::new(File::open(path)?)); + let file = Arc::new(Mutex::new(fs_util::open_read(path)?)); // Evict oldest if full if entries.len() >= self.max_entries { diff --git a/crates/blob/src/fs.rs b/crates/blob/src/fs.rs new file mode 100644 index 0000000..5e0ffc1 --- /dev/null +++ b/crates/blob/src/fs.rs @@ -0,0 +1,142 @@ +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::path::Path; +use std::time::Duration; + +use crate::error::Result; + +/// Max retries for transient filesystem errors (NFS ESTALE, CIFS sharing violations, etc.) +const MAX_RETRIES: u32 = 5; +const RETRY_DELAY: Duration = Duration::from_millis(20); + +/// Check if an I/O error is transient (retryable). +fn is_transient(err: &io::Error) -> bool { + use std::io::ErrorKind; + matches!( + err.kind(), + ErrorKind::TimedOut + | ErrorKind::Interrupted + | ErrorKind::WouldBlock + | ErrorKind::UnexpectedEof + ) || err.raw_os_error() == Some(116) // ESTALE on Linux +} + +/// Open an existing file for reading, with retry on transient errors (NFS ESTALE etc.). +pub fn open_read(path: &Path) -> Result { + let mut last_err = None; + for attempt in 0..MAX_RETRIES { + match File::open(path) { + Ok(f) => return Ok(f), + Err(e) if is_transient(&e) => { + last_err = Some(e); + if attempt > 0 { + std::thread::sleep(RETRY_DELAY * attempt); + } + continue; + } + Err(e) => return Err(e.into()), + } + } + Err(crate::error::Error::Io(last_err.unwrap())) +} + +/// Open an existing file for writing, with retry on transient errors. +pub fn open_write(path: &Path) -> Result { + let mut last_err = None; + for attempt in 0..MAX_RETRIES { + match OpenOptions::new().write(true).open(path) { + Ok(f) => return Ok(f), + Err(e) if is_transient(&e) => { + last_err = Some(e); + if attempt > 0 { + std::thread::sleep(RETRY_DELAY * attempt); + } + continue; + } + Err(e) => return Err(e.into()), + } + } + Err(crate::error::Error::Io(last_err.unwrap())) +} + +/// Create a new file atomically: write content to a temp file, fsync, then rename. +/// Avoids `create_new(true)` which is racy on NFS. +pub fn create_atomic(path: &Path, content: &[u8]) -> Result<()> { + let tmp = path.with_extension( + path.extension() + .map(|e| format!("{}.tmp", e.to_string_lossy())) + .unwrap_or_else(|| "tmp".to_string()), + ); + + { + let mut f = File::create(&tmp)?; + f.write_all(content)?; + f.sync_all()?; + } + + fs::rename(&tmp, path)?; + Ok(()) +} + +/// Truncate an existing file to the given size, with retry. +pub fn truncate(path: &Path, size: u64) -> Result<()> { + let f = open_write(path)?; + f.set_len(size)?; + f.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn test_open_read_existing() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.txt"); + std::fs::write(&path, b"hello").unwrap(); + + let mut f = open_read(&path).unwrap(); + let mut s = String::new(); + std::io::Read::read_to_string(&mut f, &mut s).unwrap(); + assert_eq!(s, "hello"); + } + + #[test] + fn test_open_read_missing() { + let dir = TempDir::new().unwrap(); + let result = open_read(&dir.path().join("nope.txt")); + assert!(result.is_err()); + } + + #[test] + fn test_create_atomic_success() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("data.bin"); + create_atomic(&path, b"hello world").unwrap(); + + let content = std::fs::read(&path).unwrap(); + assert_eq!(content, b"hello world"); + // Temp file should not exist + assert!(!dir.path().join("data.bin.tmp").exists()); + } + + #[test] + fn test_create_atomic_overwrites() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("data.bin"); + create_atomic(&path, b"first").unwrap(); + create_atomic(&path, b"second").unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"second"); + } + + #[test] + fn test_truncate() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("trunc.bin"); + std::fs::write(&path, b"1234567890").unwrap(); + truncate(&path, 5).unwrap(); + assert_eq!(std::fs::metadata(&path).unwrap().len(), 5); + } +} diff --git a/crates/blob/src/lib.rs b/crates/blob/src/lib.rs index 46feff4..78b9958 100644 --- a/crates/blob/src/lib.rs +++ b/crates/blob/src/lib.rs @@ -6,6 +6,7 @@ pub mod compress; pub mod engine; pub mod error; pub mod file_pool; +pub mod fs; pub mod gc; pub mod meta; pub mod recovery; diff --git a/crates/blob/src/meta.rs b/crates/blob/src/meta.rs index 750b2c9..6238e68 100644 --- a/crates/blob/src/meta.rs +++ b/crates/blob/src/meta.rs @@ -19,14 +19,7 @@ fn write_bin(path: &Path, value: &T) -> Result<()> { buf.extend_from_slice(&META_VERSION.to_le_bytes()); buf.extend_from_slice(&payload); - let tmp = path.with_extension("bin.tmp"); - { - use std::io::Write; - let mut f = std::fs::File::create(&tmp)?; - f.write_all(&buf)?; - f.sync_all()?; - } - std::fs::rename(&tmp, path)?; + crate::fs::create_atomic(path, &buf)?; Ok(()) } diff --git a/crates/blob/src/segment.rs b/crates/blob/src/segment.rs index 30e4c5a..06906bd 100644 --- a/crates/blob/src/segment.rs +++ b/crates/blob/src/segment.rs @@ -1,10 +1,11 @@ -use std::fs::{self, File, OpenOptions}; +use std::fs::{self, File}; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::sync::Mutex; use crate::checksum; use crate::error::{Error, Result}; +use crate::fs as fs_util; use crate::types::{Codec, ENTRY_HEADER_SIZE, ENTRY_MAGIC, SEGMENT_MAX_SIZE}; /// In-memory representation of a stored entry. @@ -60,10 +61,8 @@ pub struct SegmentWriter { impl SegmentWriter { pub fn create(path: PathBuf, id: u32) -> Result { - let file = OpenOptions::new() - .create_new(true) - .write(true) - .open(&path)?; + // Use create+truncate instead of create_new to avoid NFS O_EXCL issues. + let file = File::create(&path)?; Ok(Self { file, path, @@ -73,7 +72,7 @@ impl SegmentWriter { } pub fn open_append(path: PathBuf, id: u32) -> Result { - let mut file = OpenOptions::new().write(true).open(&path)?; + let mut file = fs_util::open_write(&path)?; file.seek(SeekFrom::End(0))?; let bytes_written = file.stream_position()?; Ok(Self { @@ -188,7 +187,7 @@ impl SegmentReader { /// Read a single entry at the given offset. Returns the entry and the offset of the next entry. pub fn read_entry_at(&self, offset: u64) -> Result<(Entry, u64)> { - let mut file = File::open(&self.path)?; + let mut file = fs_util::open_read(&self.path)?; file.seek(SeekFrom::Start(offset))?; // Read magic @@ -359,7 +358,7 @@ impl SegmentReader { /// Read data portion of an entry (for pread-style reads when you already know offset + data_size). pub fn read_data(&self, offset: u64, data_size: u32) -> Result> { - let mut file = File::open(&self.path)?; + let mut file = fs_util::open_read(&self.path)?; // Skip magic(4) + crc32(4) + flags(1) + codec(1) + key(32) + raw_size(4) + data_size(4) = 50 bytes let data_start = offset + ENTRY_HEADER_SIZE as u64; file.seek(SeekFrom::Start(data_start))?; @@ -370,7 +369,7 @@ impl SegmentReader { /// Read the full entry header + data for verification (used by recovery and GC). pub fn read_full_entry(&self, offset: u64, data_size: u32) -> Result> { - let mut file = File::open(&self.path)?; + let mut file = fs_util::open_read(&self.path)?; file.seek(SeekFrom::Start(offset))?; let total = ENTRY_HEADER_SIZE + data_size as usize; let mut buf = vec![0u8; total]; @@ -416,9 +415,7 @@ impl SegmentReader { /// Truncate a segment file to the given size. pub fn truncate_segment(path: &Path, size: u64) -> Result<()> { - let file = OpenOptions::new().write(true).open(path)?; - file.set_len(size)?; - Ok(()) + fs_util::truncate(path, size) } /// Map an Error, converting Io(StorageFull) to DiskFull with path context. From 9e55026f12fd9acb52c7cbff4cca2eac5ac2f432 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Wed, 24 Jun 2026 17:32:14 +0800 Subject: [PATCH 47/66] feat: add SSO/OIDC support to user model and settings --- crates/admin/src/meta.rs | 4 +++- crates/core/src/ext/text_extractor.rs | 14 ++++++++++-- crates/core/src/settings/cli.rs | 25 ++++++++++++++++++--- crates/core/src/settings/mod.rs | 9 ++++++++ crates/core/src/users/mod.rs | 11 ++++++++++ crates/core/src/users/view.rs | 5 +++++ crates/core/src/utils/mod.rs | 2 +- web/src/features/auth/user-auth-form.tsx | 18 ++++++++++++++- web/src/hooks/use-edition.ts | 28 ++++++++++++++++++++++++ web/src/locales/en.json | 1 + 10 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 web/src/hooks/use-edition.ts diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index d0118be..45efed6 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -18,9 +18,9 @@ use bichon_core::{ token::TokenType, users::{acl::AccessControl, role::RoleType}, }; +use bichon_memdb::{Durability, MemDb}; use console::style; use itertools::Itertools; -use bichon_memdb::{Durability, MemDb}; use native_db::*; use native_model::{native_model, Model}; use serde::{Deserialize, Serialize}; @@ -574,6 +574,8 @@ impl From for bichon_core::users::BichonUserV2 { acl: value.acl, theme: value.theme, language: value.language, + sso_id: None, + sso_provider: None, } } } diff --git a/crates/core/src/ext/text_extractor.rs b/crates/core/src/ext/text_extractor.rs index baf1243..3a3a4af 100644 --- a/crates/core/src/ext/text_extractor.rs +++ b/crates/core/src/ext/text_extractor.rs @@ -63,8 +63,18 @@ pub const MAX_EXTRACT_BYTES: usize = 10 * 1024 * 1024; pub fn should_try_extract(content_type: &str, ext: &str) -> bool { matches!( ext, - "pdf" | "doc" | "docx" | "xls" | "xlsx" | "ppt" | "pptx" - | "txt" | "rtf" | "odt" | "ods" | "odp" + "pdf" + | "doc" + | "docx" + | "xls" + | "xlsx" + | "ppt" + | "pptx" + | "txt" + | "rtf" + | "odt" + | "ods" + | "odp" ) || content_type.starts_with("text/") } diff --git a/crates/core/src/settings/cli.rs b/crates/core/src/settings/cli.rs index 3c1f091..627aa5b 100644 --- a/crates/core/src/settings/cli.rs +++ b/crates/core/src/settings/cli.rs @@ -308,6 +308,26 @@ pub struct Settings { help = "Enable SMTP authentication requirement" )] pub bichon_smtp_auth_required: bool, + + /// Enable OIDC-based Single Sign-On (Pro/Enterprise feature). + #[clap(long, default_value = "false", env, help = "Enable OpenID Connect SSO")] + pub bichon_oidc_enabled: bool, + + /// OIDC issuer URL (e.g. https://keycloak.example.com/realms/myorg). + #[clap(long, env, help = "OpenID Connect issuer URL")] + pub bichon_oidc_issuer_url: Option, + + /// OIDC client ID registered with the IdP. + #[clap(long, env, help = "OpenID Connect client ID")] + pub bichon_oidc_client_id: Option, + + /// OIDC client secret registered with the IdP. + #[clap(long, env, help = "OpenID Connect client secret")] + pub bichon_oidc_client_secret: Option, + + /// OIDC redirect URI (must match what's registered with the IdP). + #[clap(long, env, help = "OpenID Connect redirect URI")] + pub bichon_oidc_redirect_uri: Option, } impl Settings { @@ -317,9 +337,8 @@ impl Settings { // rejects it, fall back to parsing with only the binary name so that // the settings come entirely from environment variables. let args: Vec = std::env::args().collect(); - let s = Self::try_parse_from(&args).unwrap_or_else(|_| { - Self::parse_from(std::iter::once(args[0].clone())) - }); + let s = Self::try_parse_from(&args) + .unwrap_or_else(|_| Self::parse_from(std::iter::once(args[0].clone()))); if s.bichon_encrypt_password.is_none() && s.bichon_encrypt_password_file.is_none() { panic!( "One of --bichon_encrypt_password or --bichon_encrypt_password_file has to be set" diff --git a/crates/core/src/settings/mod.rs b/crates/core/src/settings/mod.rs index 827e836..7d4e203 100644 --- a/crates/core/src/settings/mod.rs +++ b/crates/core/src/settings/mod.rs @@ -60,6 +60,11 @@ pub struct SystemConfigurations { pub bichon_smtp_auth_required: bool, pub bichon_smtp_tls_key_path: Option, pub bichon_smtp_tls_cert_path: Option, + + pub bichon_oidc_enabled: bool, + pub bichon_oidc_issuer_url: Option, + pub bichon_oidc_client_id: Option, + pub bichon_oidc_redirect_uri: Option, } impl From<&Settings> for SystemConfigurations { @@ -94,6 +99,10 @@ impl From<&Settings> for SystemConfigurations { bichon_smtp_auth_required: s.bichon_smtp_auth_required, bichon_smtp_tls_key_path: s.bichon_smtp_tls_key_path.clone(), bichon_smtp_tls_cert_path: s.bichon_smtp_tls_cert_path.clone(), + bichon_oidc_enabled: s.bichon_oidc_enabled, + bichon_oidc_issuer_url: s.bichon_oidc_issuer_url.clone(), + bichon_oidc_client_id: s.bichon_oidc_client_id.clone(), + bichon_oidc_redirect_uri: s.bichon_oidc_redirect_uri.clone(), } } } diff --git a/crates/core/src/users/mod.rs b/crates/core/src/users/mod.rs index 9e6d994..84b74d7 100644 --- a/crates/core/src/users/mod.rs +++ b/crates/core/src/users/mod.rs @@ -87,6 +87,11 @@ pub struct BichonUserV2 { pub theme: Option, pub language: Option, + + /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim). + pub sso_id: Option, + /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`. + pub sso_provider: Option, } impl MemDbModel for BichonUserV2 { @@ -192,6 +197,8 @@ impl BichonUserV2 { global_permissions, theme: self.theme, language: self.language, + sso_id: self.sso_id, + sso_provider: self.sso_provider, } } @@ -226,6 +233,8 @@ impl BichonUserV2 { acl: None, theme: None, language: None, + sso_id: None, + sso_provider: None, }; // 3. Generate and insert an initial access token for the first-time setup @@ -382,6 +391,8 @@ impl BichonUserV2 { account_access_map: request.account_access_map, theme: request.theme, language: request.language, + sso_id: None, + sso_provider: None, }; let user_clone = user.clone(); diff --git a/crates/core/src/users/view.rs b/crates/core/src/users/view.rs index 2839635..4ef909d 100644 --- a/crates/core/src/users/view.rs +++ b/crates/core/src/users/view.rs @@ -52,4 +52,9 @@ pub struct UserView { pub acl: Option, pub theme: Option, pub language: Option, + + /// SSO identity: unique subject ID from the external IdP (e.g. OIDC `sub` claim). + pub sso_id: Option, + /// SSO provider identifier: `"oidc"` or future `"saml"` / `"ldap"`. + pub sso_provider: Option, } diff --git a/crates/core/src/utils/mod.rs b/crates/core/src/utils/mod.rs index 35ef0a6..49c463a 100644 --- a/crates/core/src/utils/mod.rs +++ b/crates/core/src/utils/mod.rs @@ -138,7 +138,7 @@ macro_rules! generate_token { }}; } -pub(crate) fn generate_token_impl(bit_strength: usize) -> String { +pub fn generate_token_impl(bit_strength: usize) -> String { let byte_length = (bit_strength + 23) / 24 * 3; let random_bytes: Vec = (0..byte_length).map(|_| rand::random::()).collect(); let mut encoded = general_purpose::URL_SAFE.encode(&random_bytes); diff --git a/web/src/features/auth/user-auth-form.tsx b/web/src/features/auth/user-auth-form.tsx index 4b4bcc7..3b2e340 100644 --- a/web/src/features/auth/user-auth-form.tsx +++ b/web/src/features/auth/user-auth-form.tsx @@ -41,9 +41,10 @@ import { useLocation, useNavigate } from '@tanstack/react-router' import { Button } from '@/components/button' import { useTranslation } from 'react-i18next' import i18n from '@/i18n' -import { Loader2, LogIn } from 'lucide-react' +import { Loader2, LogIn, Shield } from 'lucide-react' import { login } from '@/api/users/api' import { useTheme } from '@/context/theme-context' +import { useEdition } from '@/hooks/use-edition' type UserAuthFormProps = HTMLAttributes @@ -52,6 +53,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) { const { setTheme } = useTheme(); const navigate = useNavigate() const { t } = useTranslation() + const { isPro } = useEdition() const { search } = useLocation(); const redirect = toSearchParams(search).get('redirect') || '/'; @@ -156,6 +158,20 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) { {isLoading ? : } {t('auth.login')} + + {isPro && ( + + )}
diff --git a/web/src/hooks/use-edition.ts b/web/src/hooks/use-edition.ts new file mode 100644 index 0000000..b436fd4 --- /dev/null +++ b/web/src/hooks/use-edition.ts @@ -0,0 +1,28 @@ +import axiosInstance from '@/api/axiosInstance' +import { useQuery } from '@tanstack/react-query' + +export interface EditionInfo { + features: string[] + edition: 'community' | 'pro' | 'enterprise' + version: string +} + +async function fetchEdition(): Promise { + const { data } = await axiosInstance.get('api/v1/features') + return data +} + +export function useEdition() { + const { data } = useQuery({ + queryKey: ['edition'], + queryFn: fetchEdition, + staleTime: Infinity, + retry: 1, + }) + + return { + isPro: data?.edition === 'pro' || data?.edition === 'enterprise', + edition: data?.edition ?? 'community', + features: data?.features ?? [], + } as const +} diff --git a/web/src/locales/en.json b/web/src/locales/en.json index 238bcc7..35eadb6 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -423,6 +423,7 @@ "sessionExpired": "Session expired!", "sessionExpiredDesc": "Your session has ended due to inactivity. Please log in again to continue.", "somethingWentWrong": "Something went wrong", + "ssoLogin": "Sign in with SSO", "username": "Username", "welcome": "Welcome to Bichon", "youWillNeedToLogInAgain": "You will need to log in again to access your account." From bba1ea5cc7b8f1bdd8e23eec6940b1df03571b9d Mon Sep 17 00:00:00 2001 From: rustmailer Date: Wed, 24 Jun 2026 21:45:05 +0800 Subject: [PATCH 48/66] feat: add per-account FilterRule, ExtractionRules and ArchiveRules with regex validation --- crates/admin/src/meta.rs | 2 + crates/core/src/account/migration.rs | 519 ++++++++++++++++++++++++++ crates/core/src/account/payload.rs | 34 +- crates/core/src/envelope/extractor.rs | 74 +++- 4 files changed, 622 insertions(+), 7 deletions(-) diff --git a/crates/admin/src/meta.rs b/crates/admin/src/meta.rs index 45efed6..e23edb6 100644 --- a/crates/admin/src/meta.rs +++ b/crates/admin/src/meta.rs @@ -263,6 +263,8 @@ impl From for AccountModel { auto_download_new_mailboxes: None, download_schedule: None, deleting: false, + archive_rules: None, + extraction_rules: None, } } } diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 170ce91..0344d57 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -64,6 +64,218 @@ pub enum QuotaWindow { Monthly, } +/// Include/exclude filter rule. +/// +/// - `include` non-empty: only values matching these patterns pass. +/// - `exclude` non-empty: values matching these patterns are rejected. +/// - Both empty: all values pass. +/// - Both set: include checked first, then exclude. +/// +/// Extension patterns use case-insensitive exact match; all others use regex. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct FilterRule { + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub exclude: Vec, +} + +impl FilterRule { + pub fn is_empty(&self) -> bool { + self.include.is_empty() && self.exclude.is_empty() + } + + fn matches_exact(&self, value: &str) -> bool { + if !self.include.is_empty() + && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) + { + return false; + } + if !self.exclude.is_empty() + && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) + { + return false; + } + true + } + + fn matches_regex(&self, value: &str) -> bool { + if !self.include.is_empty() && !matches_any_regex(&self.include, value) { + return false; + } + if !self.exclude.is_empty() && matches_any_regex(&self.exclude, value) { + return false; + } + true + } + + fn validate_regex(&self, field: &str) -> Result<(), String> { + validate_patterns(&self.include, &format!("{field}.include"))?; + validate_patterns(&self.exclude, &format!("{field}.exclude"))?; + Ok(()) + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ExtractionRules { + /// Type 0: Master switch. + #[serde(default)] + pub enabled: bool, + /// Type 1: File extensions (exact match, e.g. `{"include": ["pdf","docx"]}`). + #[serde(default)] + pub extensions: FilterRule, + /// Type 2: Folder patterns (regex, e.g. `{"include": ["^INBOX/Invoices"]}`). + #[serde(default)] + pub folders: FilterRule, + /// Type 3: Attachment filename patterns (regex). + #[serde(default)] + pub attachment_names: FilterRule, + /// Type 4: Sender patterns (regex). + #[serde(default)] + pub senders: FilterRule, +} + +impl ExtractionRules { + /// Returns `true` if the attachment should be extracted under these rules. + pub fn should_extract( + &self, + ext: &str, + folder: Option<&str>, + attachment_name: Option<&str>, + sender: Option<&str>, + ) -> bool { + if !self.enabled { + return false; + } + if !self.extensions.matches_exact(ext) { + return false; + } + if !self.folders.is_empty() { + if let Some(folder) = folder { + if !self.folders.matches_regex(folder) { + return false; + } + } + } + if !self.attachment_names.is_empty() { + if let Some(name) = attachment_name { + if !self.attachment_names.matches_regex(name) { + return false; + } + } + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + true + } + + pub fn validate(&self) -> Result<(), String> { + self.folders.validate_regex("folders")?; + self.attachment_names.validate_regex("attachment_names")?; + self.senders.validate_regex("senders")?; + Ok(()) + } +} + +/// Archive filtering rules — skip unwanted emails before storage. +/// +/// Rule types: +/// 0 — Master switch +/// 1 — Sender filter (regex) +/// 2 — Subject filter (regex) +/// 3 — Skip emails larger than this (bytes) +/// 4 — Skip emails with spam headers (X-Spam-Flag, X-Spam) +/// +/// `None` = archive everything (backward compatible). +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] +pub struct ArchiveRules { + /// Type 0: Master switch. `false` = archive everything. + #[serde(default)] + pub enabled: bool, + /// Type 1: Sender filter (regex, include/exclude). + #[serde(default)] + pub senders: FilterRule, + /// Type 2: Subject filter (regex, include/exclude). + #[serde(default)] + pub subjects: FilterRule, + /// Type 3: Skip emails larger than this (bytes). `None` = no size limit. + #[serde(default)] + pub skip_larger_than: Option, + /// Type 4: Spam header names to check (e.g. `["X-Spam-Flag", "X-Spam"]`). + /// When the value is `yes` or `true` (case-insensitive), the email is skipped. + /// Empty = don't check. Common headers: `X-Spam-Flag` (SpamAssassin), + /// `X-Spam` (rspamd), `X-MS-Exchange-Organization-SCL` (Exchange). + #[serde(default)] + pub spam_headers: Vec, +} + +impl ArchiveRules { + /// Returns `true` if the email should be archived under these rules. + pub fn should_archive( + &self, + sender: Option<&str>, + subject: Option<&str>, + size: u32, + is_spam: bool, + ) -> bool { + if !self.enabled { + return true; + } + if !self.senders.is_empty() { + if let Some(sender) = sender { + if !self.senders.matches_regex(sender) { + return false; + } + } + } + if !self.subjects.is_empty() { + if let Some(subject) = subject { + if !self.subjects.matches_regex(subject) { + return false; + } + } + } + if let Some(limit) = self.skip_larger_than { + if size as u64 > limit { + return false; + } + } + if !self.spam_headers.is_empty() && is_spam { + return false; + } + true + } + + /// Validate all regex patterns are well-formed. + pub fn validate(&self) -> Result<(), String> { + self.senders.validate_regex("senders")?; + self.subjects.validate_regex("subjects")?; + Ok(()) + } +} + +fn matches_any_regex(patterns: &[String], value: &str) -> bool { + patterns + .iter() + .any(|p| regex::Regex::new(p).map(|re| re.is_match(value)).unwrap_or(false)) +} + +fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> { + for p in patterns { + regex::Regex::new(p) + .map_err(|e| format!("{} pattern '{}' is invalid regex: {}", field_name, p, e))?; + } + Ok(()) +} + #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] #[cfg_attr(feature = "web-api", derive(poem_openapi::Object))] pub struct Account { @@ -99,6 +311,14 @@ pub struct Account { pub download_schedule: Option, #[serde(default)] pub deleting: bool, + /// Email-level filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + #[serde(default)] + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + #[serde(default)] + pub extraction_rules: Option, } impl MemDbModel for Account { @@ -139,6 +359,8 @@ impl Account { imap_quota_window: request.imap_quota_window, download_schedule: request.download_schedule, deleting: false, + archive_rules: request.archive_rules, + extraction_rules: request.extraction_rules, }) } @@ -477,7 +699,304 @@ impl Account { if request.clear_download_schedule == Some(true) { new.download_schedule = None; } + if request.extraction_rules.is_some() { + new.extraction_rules = request.extraction_rules; + } + if request.archive_rules.is_some() { + new.archive_rules = request.archive_rules; + } new.updated_at = utc_now!(); Ok(new) } } + +#[cfg(test)] +mod tests { + use super::*; + + // ── FilterRule ─────────────────────────────────────────────────── + + #[test] + fn filter_rule_include_only() { + let r = FilterRule { + include: vec![r"@ok\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("spam@bad.com")); + } + + #[test] + fn filter_rule_exclude_only() { + let r = FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@ok.com")); + assert!(!r.matches_regex("bot@spam.com")); + } + + #[test] + fn filter_rule_include_then_exclude() { + let r = FilterRule { + include: vec![r"@company\.com$".into()], + exclude: vec![r"noreply@company\.com$".into()], + ..Default::default() + }; + assert!(r.matches_regex("bob@company.com")); + assert!(!r.matches_regex("noreply@company.com")); + assert!(!r.matches_regex("spam@other.com")); + } + + #[test] + fn filter_rule_exact_match() { + let r = FilterRule { + include: vec!["pdf".into(), "docx".into()], + exclude: vec!["xlsx".into()], + ..Default::default() + }; + assert!(r.matches_exact("pdf")); + assert!(r.matches_exact("docx")); + assert!(r.matches_exact("DOCX")); // case-insensitive + assert!(!r.matches_exact("xlsx")); + assert!(!r.matches_exact("txt")); + } + + // ── ExtractionRules ───────────────────────────────────────────── + + #[test] + fn extraction_rules_master_switch() { + let rules = ExtractionRules { + enabled: false, + ..Default::default() + }; + assert!(!rules.should_extract("pdf", None, None, None)); + } + + #[test] + fn extraction_rules_extension_include() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + include: vec!["pdf".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("docx", None, None, None)); + } + + #[test] + fn extraction_rules_extension_exclude() { + let rules = ExtractionRules { + enabled: true, + extensions: FilterRule { + exclude: vec!["xlsx".into(), "pptx".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, None, None)); + assert!(!rules.should_extract("xlsx", None, None, None)); + } + + #[test] + fn extraction_rules_folder_regex() { + let rules = ExtractionRules { + enabled: true, + folders: FilterRule { + include: vec![r"^INBOX/Invoices".into(), r"Contracts$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", Some("INBOX/Invoices"), None, None)); + assert!(rules.should_extract("pdf", Some("Finance/Contracts"), None, None)); + assert!(!rules.should_extract("pdf", Some("INBOX/Junk"), None, None)); + } + + #[test] + fn extraction_rules_attachment_name_regex() { + let rules = ExtractionRules { + enabled: true, + attachment_names: FilterRule { + include: vec![r"^invoice-.*\.pdf$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_extract("pdf", None, Some("invoice-2024.pdf"), None)); + assert!(!rules.should_extract("pdf", None, Some("newsletter.pdf"), None)); + } + + #[test] + fn extraction_rules_sender_regex() { + let rules = ExtractionRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@noreply\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + // non-excluded sender passes + assert!(rules.should_extract("pdf", None, None, Some("bob@ok.com"))); + // excluded sender blocked + assert!(!rules.should_extract("pdf", None, None, Some("bot@noreply.com"))); + } + + #[test] + fn extraction_rules_empty_filters_pass_everything() { + let rules = ExtractionRules { + enabled: true, + ..Default::default() + }; + assert!(rules.should_extract( + "anything", + Some("any/folder"), + Some("any.pdf"), + Some("any@x.com") + )); + } + + // ── ArchiveRules ──────────────────────────────────────────────── + + #[test] + fn archive_rules_disabled_archives_everything() { + let rules = ArchiveRules { + enabled: false, + ..Default::default() + }; + assert!(rules.should_archive( + Some("spam@x.com"), + Some("BUY NOW"), + 999, + false + )); + } + + #[test] + fn archive_rules_sender_exclude() { + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(Some("bot@spam.com"), None, 100, false)); + assert!(rules.should_archive(Some("friend@ok.com"), None, 100, false)); + } + + #[test] + fn archive_rules_subject_exclude() { + let rules = ArchiveRules { + enabled: true, + subjects: FilterRule { + exclude: vec![r"(?i)unsubscribe|buy now|limited offer".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(!rules.should_archive(None, Some("UNSUBSCRIBE NOW"), 100, false)); + assert!(!rules.should_archive(None, Some("Limited Offer!!"), 100, false)); + assert!(rules.should_archive(None, Some("Meeting tomorrow"), 100, false)); + } + + #[test] + fn archive_rules_sender_include() { + // Only archive emails from specific senders + let rules = ArchiveRules { + enabled: true, + senders: FilterRule { + include: vec![r"@partner\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.should_archive(Some("bob@partner.com"), None, 100, false)); + assert!(!rules.should_archive(Some("spam@random.com"), None, 100, false)); + } + + #[test] + fn archive_rules_skip_larger_than() { + let rules = ArchiveRules { + enabled: true, + skip_larger_than: Some(50_000_000), + ..Default::default() + }; + assert!(rules.should_archive(None, None, 1_000_000, false)); + assert!(!rules.should_archive(None, None, 60_000_000, false)); + } + + #[test] + fn archive_rules_skip_spam_headers() { + let rules = ArchiveRules { + enabled: true, + spam_headers: vec!["X-Spam-Flag".into()], + ..Default::default() + }; + assert!(!rules.should_archive(None, None, 100, true)); + assert!(rules.should_archive(None, None, 100, false)); + } + + // ── Validation ────────────────────────────────────────────────── + + #[test] + fn validate_extraction_rules_valid() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec![r"^INBOX/.*".into()], + ..Default::default() + }, + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_extraction_rules_invalid_regex() { + let rules = ExtractionRules { + folders: FilterRule { + include: vec!["***bad[".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } + + #[test] + fn validate_archive_rules_valid() { + let rules = ArchiveRules { + senders: FilterRule { + exclude: vec![r"@spam\.com$".into()], + ..Default::default() + }, + subjects: FilterRule { + include: vec![r"(?i)invoice".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_ok()); + } + + #[test] + fn validate_archive_rules_invalid_regex() { + let rules = ArchiveRules { + senders: FilterRule { + include: vec!["[unclosed".into()], + ..Default::default() + }, + ..Default::default() + }; + assert!(rules.validate().is_err()); + } +} diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index e796900..798a7aa 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -19,7 +19,7 @@ use std::str::FromStr; use crate::account::entity::ImapConfig; -use crate::account::migration::{AccountModel, AccountType, QuotaWindow}; +use crate::account::migration::{AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow}; use crate::account::since::{DateSince, RelativeDate}; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -56,6 +56,12 @@ pub struct AccountCreateRequest { pub imap_quota_window: Option, pub auto_download_new_mailboxes: Option, pub download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = archive everything (backward compatible). + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = extract everything (backward compatible). + pub extraction_rules: Option, } impl AccountCreateRequest { @@ -106,6 +112,16 @@ impl AccountCreateRequest { } AccountType::NoSync => {} } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(AccountModel::new(user_id, self)?) } @@ -180,6 +196,12 @@ pub struct AccountUpdateRequest { pub auto_download_new_mailboxes: Option, pub download_schedule: Option, pub clear_download_schedule: Option, + /// Email archive filtering rules (Pro feature). + /// `None` = no change. Use `Some(ArchiveRules { .. })` to set. + pub archive_rules: Option, + /// Attachment text extraction rules (Pro feature). + /// `None` = no change. Use `Some(ExtractionRules { .. })` to set. + pub extraction_rules: Option, } impl AccountUpdateRequest { @@ -235,6 +257,16 @@ impl AccountUpdateRequest { validate_cron_expression(schedule)?; } } + if let Some(ref rules) = self.extraction_rules { + rules.validate().map_err(|e| { + raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } + if let Some(ref rules) = self.archive_rules { + rules.validate().map_err(|e| { + raise_error!(format!("archive_rules: {}", e), ErrorCode::InvalidParameter) + })?; + } Ok(()) } } diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index db21626..c412c7d 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -16,6 +16,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use crate::account::migration::AccountModel; use crate::cache::imap::mailbox::MailBox; use crate::common::AddrVec; use crate::envelope::meta::parse_bichon_metadata; @@ -112,6 +113,34 @@ async fn extract_envelope_core( ) })?; + if let Ok(account) = AccountModel::get(account_id) { + if let Some(ref rules) = account.archive_rules { + let sender = message.from().and_then(|addr| { + AddrVec::from(addr).0.into_iter().next().and_then(|a| a.address) + }); + let subject = message.subject().map(|s| s.to_string()); + + let is_spam = !rules.spam_headers.is_empty() + && rules.spam_headers.iter().any(|h| { + message + .header_raw(h.clone()) + .map(|v| matches!(v.trim().to_lowercase().as_str(), "yes" | "true")) + .unwrap_or(false) + }); + + if !rules.should_archive(sender.as_deref(), subject.as_deref(), size, is_spam) { + tracing::debug!( + account_id, + uid, + sender = sender.as_deref().unwrap_or("?"), + subject = subject.as_deref().unwrap_or("?"), + "Email filtered out by archive rules" + ); + return Ok(()); + } + } + } + let preview_limit = 100; let text = if let Some(text) = message.body_text(0).map(|cow| cow.into_owned()) { text @@ -173,7 +202,7 @@ async fn extract_envelope_core( .and_then(|add| add.address) .unwrap_or_else(|| "unknown".to_string()); let attachment_count = message.attachment_count(); - let attachments = detach_and_store_attachments(body, &message, &email_content_hash).await; + let attachments = detach_and_store_attachments(body, &message, &email_content_hash, account_id, mailbox_id).await; let envelope_id = Uuid::new_v4().to_string(); let now = utc_now!(); @@ -399,7 +428,27 @@ pub async fn detach_and_store_attachments( original_body: &[u8], message: &Message<'_>, eml_content_hash: &str, + account_id: u64, + mailbox_id: u64, ) -> Vec { + let rules = if account_id > 0 { + AccountModel::get(account_id) + .ok() + .and_then(|a| a.extraction_rules) + } else { + None + }; + + let mailbox_name = match rules.as_ref().map(|r| !r.folders.is_empty()) { + Some(true) => MailBox::get(mailbox_id).ok().map(|mb| mb.name), + _ => None, + }; + + let sender = message + .from() + .and_then(|addr| AddrVec::from(addr).0.into_iter().next()) + .and_then(|add| add.address); + let mut stripped_eml = original_body.to_vec(); let mut attachment_infos = Vec::new(); // Step 1: Collect and sort attachment ranges in reverse to maintain offset integrity @@ -468,19 +517,30 @@ pub async fn detach_and_store_attachments( }) .unwrap_or_else(|| "application/octet-stream".to_string()); let has_cid = att.content_id().is_some(); - let ext = att - .attachment_name() + let att_name = att.attachment_name().map(|n| n.to_string()); + let ext = att_name + .as_deref() .and_then(|n| { - std::path::Path::new(&n) + std::path::Path::new(n) .extension() .and_then(|e| e.to_str()) .map(|s| s.to_ascii_lowercase()) }) .unwrap_or_default(); + let should_extract = rules.as_ref().map_or(true, |r| { + r.should_extract( + &ext, + mailbox_name.as_deref(), + att_name.as_deref(), + sender.as_deref(), + ) + }); + if !inline || !has_cid { let decoded_len = att.contents().len(); - if decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES + if should_extract + && decoded_len <= crate::ext::text_extractor::MAX_EXTRACT_BYTES && crate::ext::text_extractor::should_try_extract(&file_type, &ext) { text_candidates.push(TextCandidate { @@ -731,7 +791,7 @@ async fn recover_message_blob(envelope: &Envelope) -> BichonResult { ErrorCode::InternalError ) })?; - detach_and_store_attachments(&raw_body, &message, &fetched_hash).await; + detach_and_store_attachments(&raw_body, &message, &fetched_hash, envelope.account_id, envelope.mailbox_id).await; Ok(Bytes::from(raw_body)) } @@ -828,6 +888,8 @@ mod test { truncated, &message, "test_content_hash", + 0, + 0, ) .await; From af089958e9f4c26e62f1009b9e6d1de51af4011a Mon Sep 17 00:00:00 2001 From: rustmailer Date: Wed, 24 Jun 2026 23:11:52 +0800 Subject: [PATCH 49/66] Update README.md --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index a4b725a..68ee074 100644 --- a/README.md +++ b/README.md @@ -715,6 +715,31 @@ cargo test Feel free to open an [Issue](https://github.com/rustmailer/bichon/issues) or join the [Discord](https://discord.gg/Bq4M2cDmF4) to discuss ideas. +#### Guidelines + +1. **AI-assisted, not AI-authored.** Use AI to help analyze, debug, or draft code when unsure — but understand and review every change yourself before submitting. Don't submit unreviewed AI-generated content. +2. **Keep PRs scoped to one issue.** Don't bundle unrelated changes (CI config, dependency bumps, fixes to other modules) into the same PR. Split them into separate PRs. +3. **Frontend/backend changes go together.** If a change affects an API, data structure, or behavior with a frontend consumer, update the frontend in the same PR (or a clearly linked companion PR). +4. **Unit tests are required.** New or fixed logic must include tests that reproduce the original issue and verify the fix. PRs without tests won't be merged. +5. **Maintain backward compatibility.** Changes to data formats, protocols, configs, or APIs must state whether they're backward compatible. If not, include a migration plan. +6. **State the blast radius.** PR descriptions must specify which modules/APIs/data are affected and any downstream impact. + +### Commit Messages + +Format: `(): ` + +- **type**: `fix`, `feat`, `refactor`, `ci`, `test`, `docs`, `chore` +- **scope**: affected module/component (e.g. `rustmailer#286`, `dedup_cache`) +- **subject**: imperative, present tense, no period + +Rules: +- One logical change per commit — don't mix a fix with CI tweaks or unrelated module changes. +- Reference the issue number when applicable (e.g. `fix(#286): ...`). +- Body explains *why*, not just *what* — include root cause and how it was verified for non-trivial fixes. +- Rebase before submitting — squash WIP/fixup commits into a clean, logical sequence. +- No vague messages like `update`, `fix bug`, `wip`. + + ## Tech Stack | Layer | Technology | From 8542ba0d2881534379989dbfbcd5c7a3d0bd74c7 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 25 Jun 2026 02:00:20 +0800 Subject: [PATCH 50/66] feat: Display Name / Alias for IMAP Accounts #306 --- crates/core/src/account/migration.rs | 24 +- crates/core/src/account/payload.rs | 15 +- crates/core/src/envelope/extractor.rs | 2 + crates/core/src/migrate/store.rs | 1 + crates/core/src/store/envelope.rs | 1 + crates/core/src/store/tantivy/model.rs | 3 +- web/src/api/account/api.ts | 1 + web/src/api/index.ts | 1 + web/src/features/dashboard/index.tsx | 45 +++- web/src/features/search/account-popover.tsx | 25 +- web/src/features/search/mail-list-table.tsx | 4 +- web/src/features/search/mail-list.tsx | 278 -------------------- 12 files changed, 88 insertions(+), 312 deletions(-) delete mode 100644 web/src/features/search/mail-list.tsx diff --git a/crates/core/src/account/migration.rs b/crates/core/src/account/migration.rs index 0344d57..cb3ea52 100644 --- a/crates/core/src/account/migration.rs +++ b/crates/core/src/account/migration.rs @@ -87,14 +87,10 @@ impl FilterRule { } fn matches_exact(&self, value: &str) -> bool { - if !self.include.is_empty() - && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) - { + if !self.include.is_empty() && !self.include.iter().any(|e| e.eq_ignore_ascii_case(value)) { return false; } - if !self.exclude.is_empty() - && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) - { + if !self.exclude.is_empty() && self.exclude.iter().any(|e| e.eq_ignore_ascii_case(value)) { return false; } true @@ -263,9 +259,11 @@ impl ArchiveRules { } fn matches_any_regex(patterns: &[String], value: &str) -> bool { - patterns - .iter() - .any(|p| regex::Regex::new(p).map(|re| re.is_match(value)).unwrap_or(false)) + patterns.iter().any(|p| { + regex::Regex::new(p) + .map(|re| re.is_match(value)) + .unwrap_or(false) + }) } fn validate_patterns(patterns: &[String], field_name: &str) -> Result<(), String> { @@ -584,6 +582,7 @@ impl Account { .map(|account: AccountModel| MinimalAccount { id: account.id, email: account.email, + name: account.account_name, }) .collect::>(); Ok(result) @@ -868,12 +867,7 @@ mod tests { enabled: false, ..Default::default() }; - assert!(rules.should_archive( - Some("spam@x.com"), - Some("BUY NOW"), - 999, - false - )); + assert!(rules.should_archive(Some("spam@x.com"), Some("BUY NOW"), 999, false)); } #[test] diff --git a/crates/core/src/account/payload.rs b/crates/core/src/account/payload.rs index 798a7aa..b2c55f7 100644 --- a/crates/core/src/account/payload.rs +++ b/crates/core/src/account/payload.rs @@ -19,7 +19,9 @@ use std::str::FromStr; use crate::account::entity::ImapConfig; -use crate::account::migration::{AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow}; +use crate::account::migration::{ + AccountModel, AccountType, ArchiveRules, ExtractionRules, QuotaWindow, +}; use crate::account::since::{DateSince, RelativeDate}; use crate::error::code::ErrorCode; use crate::error::BichonResult; @@ -114,7 +116,10 @@ impl AccountCreateRequest { } if let Some(ref rules) = self.extraction_rules { rules.validate().map_err(|e| { - raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + raise_error!( + format!("extraction_rules: {}", e), + ErrorCode::InvalidParameter + ) })?; } if let Some(ref rules) = self.archive_rules { @@ -259,7 +264,10 @@ impl AccountUpdateRequest { } if let Some(ref rules) = self.extraction_rules { rules.validate().map_err(|e| { - raise_error!(format!("extraction_rules: {}", e), ErrorCode::InvalidParameter) + raise_error!( + format!("extraction_rules: {}", e), + ErrorCode::InvalidParameter + ) })?; } if let Some(ref rules) = self.archive_rules { @@ -293,6 +301,7 @@ fn validate_cron_expression(expr: &str) -> BichonResult<()> { pub struct MinimalAccount { pub id: u64, pub email: String, + pub name: Option, } pub fn filter_accessible_accounts<'a>( diff --git a/crates/core/src/envelope/extractor.rs b/crates/core/src/envelope/extractor.rs index c412c7d..88195a7 100644 --- a/crates/core/src/envelope/extractor.rs +++ b/crates/core/src/envelope/extractor.rs @@ -296,6 +296,7 @@ async fn extract_envelope_core( account_email: None, mailbox_name: None, content_hash: email_content_hash.clone(), + account_name: None, }; // 'attachments' contains both regular and inline attachments let ea = EnvelopeWithAttachments { @@ -390,6 +391,7 @@ pub fn extract_envelope_from_nested_message( regular_attachment_count: Default::default(), tags: Default::default(), account_email: Default::default(), + account_name: Default::default(), mailbox_name: Default::default(), content_hash: Default::default(), }; diff --git a/crates/core/src/migrate/store.rs b/crates/core/src/migrate/store.rs index 73d7bdb..8dc203f 100644 --- a/crates/core/src/migrate/store.rs +++ b/crates/core/src/migrate/store.rs @@ -404,6 +404,7 @@ impl NewIndexWriter { regular_attachment_count: attachment_docs.len(), tags: None, account_email: None, + account_name: None, mailbox_name: None, content_hash: email_content_hash, }; diff --git a/crates/core/src/store/envelope.rs b/crates/core/src/store/envelope.rs index 6767f29..ca39460 100644 --- a/crates/core/src/store/envelope.rs +++ b/crates/core/src/store/envelope.rs @@ -27,6 +27,7 @@ pub struct Envelope { pub message_id: String, pub account_id: u64, pub account_email: Option, + pub account_name: Option, pub mailbox_id: u64, pub mailbox_name: Option, pub uid: u32, diff --git a/crates/core/src/store/tantivy/model.rs b/crates/core/src/store/tantivy/model.rs index f05fffa..61364c3 100644 --- a/crates/core/src/store/tantivy/model.rs +++ b/crates/core/src/store/tantivy/model.rs @@ -155,7 +155,8 @@ impl EnvelopeWithAttachments { id: extract_string_field(doc, fields.f_id, F_ID)?, message_id: extract_string_field(doc, fields.f_message_id, F_MESSAGE_ID)?, account_id, - account_email: Some(account.email), + account_email: Some(account.email), //https://github.com/rustmailer/bichon/issues/306 + account_name: account.account_name, mailbox_id, mailbox_name: Some(mailbox.name), uid: extract_u64_field(doc, fields.f_uid, F_UID)? as u32, diff --git a/web/src/api/account/api.ts b/web/src/api/account/api.ts index 5fb2218..735fef4 100644 --- a/web/src/api/account/api.ts +++ b/web/src/api/account/api.ts @@ -23,6 +23,7 @@ import { PaginatedResponse } from ".."; export interface MinimalAccount { id: number; email: string; + name?: string; } export const minimal_account_list = async () => { diff --git a/web/src/api/index.ts b/web/src/api/index.ts index 9d06c44..3484257 100644 --- a/web/src/api/index.ts +++ b/web/src/api/index.ts @@ -32,6 +32,7 @@ export interface EmailEnvelope { account_id: number; mailbox_id: number; account_email: string; + account_name?: string; mailbox_name: string; uid: number; subject: string; diff --git a/web/src/features/dashboard/index.tsx b/web/src/features/dashboard/index.tsx index f303f05..71c8463 100644 --- a/web/src/features/dashboard/index.tsx +++ b/web/src/features/dashboard/index.tsx @@ -25,6 +25,11 @@ import LongText from '@/components/long-text'; import { getToken } from '@/stores/authStore'; import { useNavigate } from '@tanstack/react-router'; import useMinimalAccountList from '@/hooks/use-minimal-account-list'; +import { + Tooltip as TooltipUI, + TooltipContent, + TooltipTrigger, +} from '@/components/ui/tooltip'; interface DailyActivity { date: string; @@ -128,6 +133,12 @@ export default function MailArchiveDashboard() { return account ? account.id : null; }; + const getAccountNameByEmail = (email: string): string | null => { + if (!minimalList) return null; + const account = minimalList.find(a => a.email === email); + return account?.name || null; + }; + const handleQuickSearch = (filter: Record) => { navigate({ to: '/search', @@ -519,16 +530,30 @@ export default function MailArchiveDashboard() {
- + {(() => { + const name = getAccountNameByEmail(acc.key); + const btn = ( + + ); + if (name) { + return ( + + {btn} + {acc.key} + + ); + } + return btn; + })()}
diff --git a/web/src/features/search/account-popover.tsx b/web/src/features/search/account-popover.tsx index 0e7e72b..5b02a4c 100644 --- a/web/src/features/search/account-popover.tsx +++ b/web/src/features/search/account-popover.tsx @@ -31,6 +31,11 @@ import { PopoverContent, PopoverTrigger, } from '@/components/ui/popover' +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from '@/components/ui/tooltip' import useMinimalAccountList from '@/hooks/use-minimal-account-list' import { cn } from '@/lib/utils' @@ -83,6 +88,7 @@ export function AccountPopover() { .filter(a => !q || a.email.toLowerCase().includes(q) || + a.name?.toLowerCase().includes(q) || String(a.id).includes(q) ) .sort((a, b) => { @@ -184,9 +190,22 @@ export function AccountPopover() { className="flex-1 truncate text-xs cursor-pointer" >
- - {account.email} - + {account.name ? ( + + + + {account.name} + + + + {account.email} + + + ) : ( + + {account.email} + + )} #{account.id} diff --git a/web/src/features/search/mail-list-table.tsx b/web/src/features/search/mail-list-table.tsx index bd13c42..196b959 100755 --- a/web/src/features/search/mail-list-table.tsx +++ b/web/src/features/search/mail-list-table.tsx @@ -89,9 +89,9 @@ export function MailListTable({ accessorKey: "source", header: t('search.source'), cell: ({ row }) => { - const { from, account_email, mailbox_name, account_id, mailbox_id } = row.original; + const { from, account_email, account_name, mailbox_name, account_id, mailbox_id } = row.original; const { setFilter } = useSearchMessages(); - const accountPrefix = account_email.split('@')[0]; + const accountPrefix = account_name ?? account_email.split('@')[0];//https://github.com/rustmailer/bichon/issues/306 return (
diff --git a/web/src/features/search/mail-list.tsx b/web/src/features/search/mail-list.tsx deleted file mode 100644 index a3c747f..0000000 --- a/web/src/features/search/mail-list.tsx +++ /dev/null @@ -1,278 +0,0 @@ -// -// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) -// -// This file is part of the Bichon Email Archiving Project -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - - -import { cn, dateFnsLocaleMap, formatBytes } from "@/lib/utils" -import { formatDistanceToNow } from "date-fns" -import { MailIcon, MoreVertical, Paperclip, TagIcon, Trash2 } from "lucide-react" -import { Skeleton } from "@/components/ui/skeleton" -import { Checkbox } from "@/components/ui/checkbox" -import { EmailEnvelope } from "@/api" -import { useSearchContext } from "./context" -import { MailBulkActions } from "./bulk-actions" -import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu" -import { Button } from "@/components/ui/button" -import { Badge } from "@/components/ui/badge" -import { useTranslation } from 'react-i18next' -import { enUS } from "date-fns/locale" - -interface MailListProps { - items: EmailEnvelope[] - isLoading: boolean - onEnvelopeChanged: (envelope: EmailEnvelope) => void -} - -export function MailList({ - items, - isLoading, - onEnvelopeChanged -}: MailListProps) { - const { t, i18n } = useTranslation() - - const locale = dateFnsLocaleMap[i18n.language.toLowerCase()] ?? enUS; - const { setOpen, currentEnvelope, setCurrentEnvelope, selected, setSelected, setToDelete } = useSearchContext() - - const handleToggleAll = () => { - const total = Array.from(selected.values()) - .reduce((sum, set) => sum + set.size, 0); - - if (total === items.length && items.length > 0) { - setSelected(new Map()); - } else { - setSelected(prev => { - const next = new Map(prev); - for (const item of items) { - const set = new Set(next.get(item.account_id) || []); - set.add(item.id); - next.set(item.account_id, set); - } - return next; - }); - } - } - - const toggleToDelete = (accountId: number, mailId: string) => { - setToDelete(prev => { - const next = new Map(prev); - const set = new Set(next.get(accountId) || []); - - if (set.has(mailId)) { - set.delete(mailId); - if (set.size === 0) next.delete(accountId); - else next.set(accountId, set); - } else { - set.add(mailId); - next.set(accountId, set); - } - - return next; - }); - }; - - const toggleSelected = (accountId: number, mailId: string) => { - setSelected(prev => { - const next = new Map(prev); - const set = new Set(next.get(accountId) || []); - - if (set.has(mailId)) { - set.delete(mailId); - if (set.size === 0) next.delete(accountId); - else next.set(accountId, set); - } else { - set.add(mailId); - next.set(accountId, set); - } - - return next; - }); - } - - const totalSelected = Array.from(selected.values()) - .reduce((sum, set) => sum + set.size, 0); - - const hasSelected = (accountId: number, mailId: string) => { - return selected.get(accountId)?.has(mailId) ?? false; - } - - const handleDelete = (envelope: EmailEnvelope) => { - setToDelete(new Map()); - toggleToDelete(envelope.account_id, envelope.id) - setOpen("delete") - } - - if (isLoading) { - return ( -
- {Array.from({ length: 8 }).map((_, i) => ( -
- - - - -
- ))} -
- ) - } - - return ( -
- {items.length > 0 && ( -
- 0 - ? true - : totalSelected > 0 - ? "indeterminate" - : false - } - onCheckedChange={handleToggleAll} - className="h-4 w-4" - /> - - {totalSelected > 0 - ? `${t('search.bulkActions.selected', { count: totalSelected })}` - : t('common.selectAll')} - -
- )} - - {items.map((item, index) => { - const hasAttachments = item.regular_attachment_count > 0 - const isSelectedRow = currentEnvelope?.id === item.id - const isChecked = hasSelected(item.account_id, item.id) - - return ( -
{ - const target = e.target as HTMLElement - if (target.closest('input[type="checkbox"], button')) return - onEnvelopeChanged(item) - }} - > - toggleSelected(item.account_id, item.id)} - onClick={(e) => e.stopPropagation()} - className="h-4 w-4 shrink-0" - /> - - -
- -
-
-

{item.from}

-

- {item.subject} -

-
-
- {item.account_email} - • - {item.mailbox_name} -
-

- {item.subject} -

- -
- {item.tags?.map((tag, i) => ( - {tag} - ))} -
-
-
- - {hasAttachments && ( -
- - {item.regular_attachment_count} -
- )} - - {formatBytes(item.size)} - - - {item.date && formatDistanceToNow(new Date(item.date), { addSuffix: true, locale })} - - - - - - - - - e.stopPropagation()} - onSelect={(e) => { - e.stopPropagation(); - setCurrentEnvelope(item); - setOpen("edit-tags"); - }} - > - - {t('search.editTag')} - - e.stopPropagation()} - onSelect={(e) => { - e.stopPropagation(); - setCurrentEnvelope(item); - setOpen("restore"); - }} - > - - {t('restore_message.restore_to_imap')} - - e.stopPropagation()} - onSelect={(e) => { - e.stopPropagation(); - handleDelete(item); - }} - > - - {t('common.delete')} - - - -
-
-
- ) - })} - {totalSelected > 0 && } -
- ) -} From cad447275f1494ad856bd2691b70d46737cc1149 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 25 Jun 2026 02:36:24 +0800 Subject: [PATCH 51/66] feat: search for an email address simultaneously in to, cc and bcc #304 --- crates/core/src/message/search.rs | 4 +++ crates/core/src/store/tantivy/envelope.rs | 28 +++++++++++++++++++++ web/src/features/search/contact-popover.tsx | 8 +++--- web/src/locales/ar.json | 2 ++ web/src/locales/da.json | 2 ++ web/src/locales/de.json | 2 ++ web/src/locales/en.json | 2 ++ web/src/locales/es.json | 2 ++ web/src/locales/fi.json | 2 ++ web/src/locales/fr.json | 2 ++ web/src/locales/it.json | 2 ++ web/src/locales/jp.json | 2 ++ web/src/locales/ko.json | 2 ++ web/src/locales/nl.json | 2 ++ web/src/locales/no.json | 2 ++ web/src/locales/pl.json | 2 ++ web/src/locales/pt.json | 2 ++ web/src/locales/ru.json | 2 ++ web/src/locales/sv.json | 2 ++ web/src/locales/zh-tw.json | 2 ++ web/src/locales/zh.json | 2 ++ 21 files changed, 72 insertions(+), 4 deletions(-) diff --git a/crates/core/src/message/search.rs b/crates/core/src/message/search.rs index 910fc15..f7cdbcd 100644 --- a/crates/core/src/message/search.rs +++ b/crates/core/src/message/search.rs @@ -43,6 +43,10 @@ pub struct EmailSearchFilter { pub to: Option, pub cc: Option, pub bcc: Option, + /// Matches if the address appears in `to`, `cc`, or `bcc` (OR semantics). + pub any_recipient: Option, + /// Matches if the address appears in `from`, `to`, `cc`, or `bcc` (OR semantics). + pub any_participant: Option, pub since: Option, pub before: Option, /// Lower bound (inclusive) on the IMAP server INTERNALDATE timestamp. diff --git a/crates/core/src/store/tantivy/envelope.rs b/crates/core/src/store/tantivy/envelope.rs index a7ccac7..b47d8f6 100644 --- a/crates/core/src/store/tantivy/envelope.rs +++ b/crates/core/src/store/tantivy/envelope.rs @@ -461,6 +461,34 @@ impl IndexManager { } } + // any_recipient: OR across to, cc, bcc + if let Some(ref v) = filter.any_recipient { + let mut recipient_queries: Vec<(Occur, Box)> = Vec::new(); + for field in [f.f_to_text, f.f_cc_text, f.f_bcc_text] { + let query_parser = QueryParser::for_index(&self.index, vec![field]); + if let Ok(q) = query_parser.parse_query(v) { + recipient_queries.push((Occur::Should, q)); + } + } + if !recipient_queries.is_empty() { + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(recipient_queries)))); + } + } + + // any_participant: OR across from, to, cc, bcc + if let Some(ref v) = filter.any_participant { + let mut participant_queries: Vec<(Occur, Box)> = Vec::new(); + for field in [f.f_from_text, f.f_to_text, f.f_cc_text, f.f_bcc_text] { + let query_parser = QueryParser::for_index(&self.index, vec![field]); + if let Ok(q) = query_parser.parse_query(v) { + participant_queries.push((Occur::Should, q)); + } + } + if !participant_queries.is_empty() { + subqueries.push((Occur::Must, Box::new(BooleanQuery::new(participant_queries)))); + } + } + if let Some(has) = filter.has_attachment { let lower: Bound; let upper: Bound; diff --git a/web/src/features/search/contact-popover.tsx b/web/src/features/search/contact-popover.tsx index 2a9fe21..68b0d40 100644 --- a/web/src/features/search/contact-popover.tsx +++ b/web/src/features/search/contact-popover.tsx @@ -30,13 +30,13 @@ import { CommandGroup, CommandInput, CommandItem, - CommandList, } from "@/components/ui/command" +import { ScrollArea } from "@/components/ui/scroll-area" export function MailFilterPopover() { const { t } = useTranslation() const { filter, setFilter } = useSearchContext() - const fields = ['from', 'to', 'cc', 'bcc'] as const + const fields = ['from', 'to', 'cc', 'bcc', 'any_recipient', 'any_participant'] as const const activeCount = fields.filter(k => !!filter[k]).length @@ -196,7 +196,7 @@ function ContactSelectorField({ value={searchTerm} onValueChange={setSearchTerm} /> - + {isLoading && (
{t('search_contacts.loading')}
)} @@ -227,7 +227,7 @@ function ContactSelectorField({
)} - + diff --git a/web/src/locales/ar.json b/web/src/locales/ar.json index 4fb65b7..b16637f 100644 --- a/web/src/locales/ar.json +++ b/web/src/locales/ar.json @@ -1001,6 +1001,8 @@ "advanced": "متقدم", "advancedFilters": "فلاتر متقدمة", "any": "الكل", + "any_participant": "أي طرف في المراسلة", + "any_recipient": "أي مستلم", "attachmentName": "اسم المرفق", "attachmentsSize": "المرفقات والحجم", "bcc": "نسخة مخفية", diff --git a/web/src/locales/da.json b/web/src/locales/da.json index df99490..baaaf3d 100644 --- a/web/src/locales/da.json +++ b/web/src/locales/da.json @@ -1001,6 +1001,8 @@ "advanced": "Avanceret", "advancedFilters": "Avancerede filtre", "any": "Alle", + "any_participant": "Enhver korrespondent", + "any_recipient": "Enhver modtager", "attachmentName": "Vedhæftningsfilnavn", "attachmentsSize": "Vedhæftninger & Størrelse", "bcc": "Blindkopi (BCC)", diff --git a/web/src/locales/de.json b/web/src/locales/de.json index 179558c..65faf23 100644 --- a/web/src/locales/de.json +++ b/web/src/locales/de.json @@ -1001,6 +1001,8 @@ "advanced": "Erweitert", "advancedFilters": "Erweiterte Filter", "any": "Beliebig", + "any_participant": "Beliebiger Beteiligter", + "any_recipient": "Beliebiger Empfänger", "attachmentName": "Anhangsname", "attachmentsSize": "Anhänge & Größe", "bcc": "BCC", diff --git a/web/src/locales/en.json b/web/src/locales/en.json index 35eadb6..ee5b506 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -1002,6 +1002,8 @@ "advanced": "Advanced", "advancedFilters": "Advanced Filters", "any": "Any", + "any_participant": "Any correspondent", + "any_recipient": "Any recipient", "attachmentName": "Attachment Name", "attachmentsSize": "Attachments & Size", "bcc": "BCC", diff --git a/web/src/locales/es.json b/web/src/locales/es.json index 0c4c3e9..cf8a5eb 100644 --- a/web/src/locales/es.json +++ b/web/src/locales/es.json @@ -1001,6 +1001,8 @@ "advanced": "Avanzado", "advancedFilters": "Filtros avanzados", "any": "Cualquiera", + "any_participant": "Cualquier participante", + "any_recipient": "Cualquier destinatario", "attachmentName": "Nombre del adjunto", "attachmentsSize": "Adjuntos y tamaño", "bcc": "CCO", diff --git a/web/src/locales/fi.json b/web/src/locales/fi.json index 9c78869..4bc1c2d 100644 --- a/web/src/locales/fi.json +++ b/web/src/locales/fi.json @@ -1001,6 +1001,8 @@ "advanced": "Tarkennettu", "advancedFilters": "Tarkennetut suodattimet", "any": "Kaikki", + "any_participant": "Kuka tahansa osallistuja", + "any_recipient": "Kuka tahansa vastaanottaja", "attachmentName": "Liitteen nimi", "attachmentsSize": "Liitteet ja koko", "bcc": "Piilokopio", diff --git a/web/src/locales/fr.json b/web/src/locales/fr.json index 71f8e4d..2f72a60 100644 --- a/web/src/locales/fr.json +++ b/web/src/locales/fr.json @@ -1001,6 +1001,8 @@ "advanced": "Avancé", "advancedFilters": "Filtres Avancés", "any": "Toutes", + "any_participant": "N'importe quel correspondant", + "any_recipient": "N'importe quel destinataire", "attachmentName": "Nom de la Pièce Jointe", "attachmentsSize": "Pièces Jointes et Taille", "bcc": "Cci", diff --git a/web/src/locales/it.json b/web/src/locales/it.json index 60496a9..b50737a 100644 --- a/web/src/locales/it.json +++ b/web/src/locales/it.json @@ -1001,6 +1001,8 @@ "advanced": "Avanzato", "advancedFilters": "Filtri Avanzati", "any": "Qualsiasi", + "any_participant": "Qualsiasi partecipante", + "any_recipient": "Qualsiasi destinatario", "attachmentName": "Nome Allegato", "attachmentsSize": "Allegati e Dimensione", "bcc": "BCC", diff --git a/web/src/locales/jp.json b/web/src/locales/jp.json index 59fe40b..d566370 100644 --- a/web/src/locales/jp.json +++ b/web/src/locales/jp.json @@ -1001,6 +1001,8 @@ "advanced": "高度な設定", "advancedFilters": "高度なフィルター", "any": "指定なし", + "any_participant": "すべての関係者", + "any_recipient": "すべての受信者", "attachmentName": "添付ファイル名", "attachmentsSize": "添付ファイルとサイズ", "bcc": "BCC（ブラインド写し）", diff --git a/web/src/locales/ko.json b/web/src/locales/ko.json index 6da8d17..53fa66a 100644 --- a/web/src/locales/ko.json +++ b/web/src/locales/ko.json @@ -1001,6 +1001,8 @@ "advanced": "고급", "advancedFilters": "고급 필터", "any": "전체", + "any_participant": "모든 관련자", + "any_recipient": "모든 수신자", "attachmentName": "첨부 파일 이름", "attachmentsSize": "첨부 파일 및 크기", "bcc": "숨은 참조", diff --git a/web/src/locales/nl.json b/web/src/locales/nl.json index 85d4066..db2d8b8 100644 --- a/web/src/locales/nl.json +++ b/web/src/locales/nl.json @@ -1001,6 +1001,8 @@ "advanced": "Geavanceerd", "advancedFilters": "Geavanceerde Filters", "any": "Alle", + "any_participant": "Elke correspondent", + "any_recipient": "Elke ontvanger", "attachmentName": "Naam Bijlage", "attachmentsSize": "Bijlagen & Grootte", "bcc": "BCC", diff --git a/web/src/locales/no.json b/web/src/locales/no.json index 0c21ca2..99de43d 100644 --- a/web/src/locales/no.json +++ b/web/src/locales/no.json @@ -1001,6 +1001,8 @@ "advanced": "Avansert", "advancedFilters": "Avanserte filtre", "any": "Alle", + "any_participant": "Enhver korrespondent", + "any_recipient": "Enhver mottaker", "attachmentName": "Vedleggsnavn", "attachmentsSize": "Vedlegg og størrelse", "bcc": "Blindkopi (BCC)", diff --git a/web/src/locales/pl.json b/web/src/locales/pl.json index 4445656..6fb5420 100644 --- a/web/src/locales/pl.json +++ b/web/src/locales/pl.json @@ -1001,6 +1001,8 @@ "advanced": "Zaawansowane", "advancedFilters": "Zaawansowane filtry", "any": "Dowolny", + "any_participant": "Dowolny uczestnik", + "any_recipient": "Dowolny odbiorca", "attachmentName": "Nazwa załącznika", "attachmentsSize": "Załączniki & Rozmiar", "bcc": "UDW", diff --git a/web/src/locales/pt.json b/web/src/locales/pt.json index 51ce366..3744593 100644 --- a/web/src/locales/pt.json +++ b/web/src/locales/pt.json @@ -1001,6 +1001,8 @@ "advanced": "Avançado", "advancedFilters": "Filtros Avançados", "any": "Qualquer", + "any_participant": "Qualquer correspondente", + "any_recipient": "Qualquer destinatario", "attachmentName": "Nome do Anexo", "attachmentsSize": "Anexos e Tamanho", "bcc": "BCC", diff --git a/web/src/locales/ru.json b/web/src/locales/ru.json index f18284a..8715209 100644 --- a/web/src/locales/ru.json +++ b/web/src/locales/ru.json @@ -1001,6 +1001,8 @@ "advanced": "Дополнительно", "advancedFilters": "Расширенные фильтры", "any": "Любой", + "any_participant": "Любой участник переписки", + "any_recipient": "Любой получатель", "attachmentName": "Имя вложения", "attachmentsSize": "Вложения и Размер", "bcc": "Скрытая копия", diff --git a/web/src/locales/sv.json b/web/src/locales/sv.json index a021acf..3ffb856 100644 --- a/web/src/locales/sv.json +++ b/web/src/locales/sv.json @@ -1001,6 +1001,8 @@ "advanced": "Avancerat", "advancedFilters": "Avancerade filter", "any": "Alla", + "any_participant": "Valfri korrespondent", + "any_recipient": "Valfri mottagare", "attachmentName": "Filnamn på bilaga", "attachmentsSize": "Bilagor & storlek", "bcc": "Hemlig kopia (BCC)", diff --git a/web/src/locales/zh-tw.json b/web/src/locales/zh-tw.json index a348b89..0e672db 100644 --- a/web/src/locales/zh-tw.json +++ b/web/src/locales/zh-tw.json @@ -1001,6 +1001,8 @@ "advanced": "進階", "advancedFilters": "進階篩選", "any": "不限", + "any_participant": "任意關聯人", + "any_recipient": "任意收件人", "attachmentName": "附件名稱", "attachmentsSize": "附件與大小", "bcc": "密件副本 (BCC)", diff --git a/web/src/locales/zh.json b/web/src/locales/zh.json index 4d23a90..784db20 100644 --- a/web/src/locales/zh.json +++ b/web/src/locales/zh.json @@ -1001,6 +1001,8 @@ "advanced": "高级", "advancedFilters": "高级筛选", "any": "不限", + "any_participant": "任意关联人", + "any_recipient": "任意收件人", "attachmentName": "附件名称", "attachmentsSize": "附件和大小", "bcc": "密送", From db36272bae1303a6b570247bf9a792a263138131 Mon Sep 17 00:00:00 2001 From: rustmailer Date: Thu, 25 Jun 2026 04:51:05 +0800 Subject: [PATCH 52/66] feat: add in-browser attachment preview for images, PDFs, and text files #303 --- crates/server/src/rest/api/message.rs | 27 ++ web/src/api/mailbox/envelope/api.ts | 12 + .../attachment/attachment-preview.tsx | 250 ++++++++++++++++++ .../features/attachment/mail-message-view.tsx | 43 ++- .../table/data-table-row-actions.tsx | 26 +- web/src/features/search/mail-message-view.tsx | 43 ++- web/src/locales/ar.json | 8 + web/src/locales/da.json | 8 + web/src/locales/de.json | 8 + web/src/locales/en.json | 9 + web/src/locales/es.json | 8 + web/src/locales/fi.json | 8 + web/src/locales/fr.json | 8 + web/src/locales/it.json | 8 + web/src/locales/jp.json | 8 + web/src/locales/ko.json | 8 + web/src/locales/nl.json | 8 + web/src/locales/no.json | 8 + web/src/locales/pl.json | 8 + web/src/locales/pt.json | 8 + web/src/locales/ru.json | 8 + web/src/locales/sv.json | 8 + web/src/locales/zh-tw.json | 8 + web/src/locales/zh.json | 8 + 24 files changed, 533 insertions(+), 13 deletions(-) create mode 100644 web/src/features/attachment/attachment-preview.tsx diff --git a/crates/server/src/rest/api/message.rs b/crates/server/src/rest/api/message.rs index e4f4477..b717d99 100644 --- a/crates/server/src/rest/api/message.rs +++ b/crates/server/src/rest/api/message.rs @@ -280,6 +280,33 @@ impl MessageApi { Ok(attachment) } + /// Returns raw attachment content for in-browser preview with + /// `Content-Disposition: inline` and the correct MIME type. + #[oai( + path = "/preview-attachment/:account_id/:envelope_id", + method = "get", + operation_id = "preview_attachment" + )] + async fn preview_attachment( + &self, + /// The ID of the account. + account_id: Path, + /// The ID of the message containing the attachment. + envelope_id: Path, + /// The content_hash of the attachment to preview. + content_hash: Query, + context: WrappedContext, + ) -> ApiResult> { + let account_id = account_id.0; + let envelope_id = envelope_id.0.trim().to_string(); + AccountModel::check_account_exists(account_id)?; + context.require_permission(Some(account_id), Permission::DATA_READ)?; + let content_hash = content_hash.0.trim(); + let reader = retrieve_attachment_content(account_id, envelope_id, content_hash)?; + let body = Body::from_async_read(reader); + Ok(Attachment::new(body).attachment_type(AttachmentType::Inline)) + } + /// Downloads an attachment from within a nested email (EML file). #[oai( path = "/download-nested-attachment/:account_id/:envelope_id", diff --git a/web/src/api/mailbox/envelope/api.ts b/web/src/api/mailbox/envelope/api.ts index a5a3dab..02cee15 100644 --- a/web/src/api/mailbox/envelope/api.ts +++ b/web/src/api/mailbox/envelope/api.ts @@ -41,6 +41,18 @@ export const download_attachment = async (accountId: number, id: string, content saveAs(blob, fileName); }; +/** Fetch raw attachment content for in-browser preview (Content-Disposition: inline). */ +export const preview_attachment = async (accountId: number, id: string, content_hash: string) => { + const response = await axiosInstance.get( + `api/v1/preview-attachment/${accountId}/${id}`, + { + params: { content_hash }, + responseType: 'blob', + } + ); + return response.data as Blob; +}; + export const download_nested_attachment = async (accountId: number, id: string, content_hash: string, nested_content_hash: string, fileName: string) => { const response = await axiosInstance.get(`api/v1/download-nested-attachment/${accountId}/${id}?content_hash=${content_hash}&nested_content_hash=${nested_content_hash}`, { responseType: 'blob' }); const blob = new Blob([response.data]); diff --git a/web/src/features/attachment/attachment-preview.tsx b/web/src/features/attachment/attachment-preview.tsx new file mode 100644 index 0000000..5d13f0c --- /dev/null +++ b/web/src/features/attachment/attachment-preview.tsx @@ -0,0 +1,250 @@ +// +// Copyright (c) 2025-2026 rustmailer.com (https://rustmailer.com) +// +// This file is part of the Bichon Email Archiving Project +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { useEffect, useMemo, useState } from 'react'; +import { useMutation } from '@tanstack/react-query'; +import { Download, FileIcon, ZoomIn, ZoomOut, RotateCcw } from 'lucide-react'; + +import { Button } from '@/components/ui/button'; +import { Dialog, DialogContent } from '@/components/ui/dialog'; +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'; +import { Separator } from '@/components/ui/separator'; +import { Skeleton } from '@/components/ui/skeleton'; +import { toast } from '@/hooks/use-toast'; +import { useTranslation } from 'react-i18next'; + +import { preview_attachment, download_attachment } from '@/api/mailbox/envelope/api'; +import { getFileConfig } from './mail-message-view'; + +const PREVIEWABLE_IMAGE = /^image\/(png|jpeg|gif|webp|svg\+xml)$/; +const PREVIEWABLE_TEXT = /^(text\/(plain|csv|html|xml|css|javascript|markdown)|application\/(json|xml|javascript|x-httpd-php|x-sh|x-perl|x-python|x-ruby))$/; + +interface AttachmentPreviewProps { + open: boolean; + onOpenChange: (open: boolean) => void; + accountId: number; + envelopeId: string; + contentHash: string; + contentType: string; + fileName: string; +} + +function isImagePreview(contentType: string) { + return PREVIEWABLE_IMAGE.test(contentType); +} + +function isPdfPreview(contentType: string) { + return contentType === 'application/pdf'; +} + +function isTextPreview(contentType: string) { + return PREVIEWABLE_TEXT.test(contentType); +} + +export default function AttachmentPreview({ + open, + onOpenChange, + accountId, + envelopeId, + contentHash, + contentType, + fileName, +}: AttachmentPreviewProps) { + const { t } = useTranslation(); + const [blobUrl, setBlobUrl] = useState(null); + const [textContent, setTextContent] = useState(null); + const [imageZoom, setImageZoom] = useState(1); + + const previewMutation = useMutation({ + mutationFn: () => preview_attachment(accountId, envelopeId, contentHash), + onSuccess: (blob) => { + if (isTextPreview(contentType)) { + blob.text().then(setTextContent); + } else { + // Re-wrap with the actual MIME type so browsers render PDFs/images inline + // instead of triggering a download (the HTTP response uses application/octet-stream). + const typedBlob = new Blob([blob], { type: contentType }); + setBlobUrl(URL.createObjectURL(typedBlob)); + } + }, + onError: (error: any) => { + toast({ + title: t('attachment_preview.failedToLoad'), + description: error.message, + variant: 'destructive', + }); + }, + }); + + useEffect(() => { + if (open) { + setBlobUrl(null); + setTextContent(null); + setImageZoom(1); + previewMutation.mutate(); + } + }, [open]); + + useEffect(() => { + return () => { + if (blobUrl) URL.revokeObjectURL(blobUrl); + }; + }, [blobUrl]); + + const handleDownload = () => { + download_attachment(accountId, envelopeId, contentHash, fileName); + }; + + const { icon, color } = useMemo(() => getFileConfig(contentType), [contentType]); + + const isImage = isImagePreview(contentType); + const isPdf = isPdfPreview(contentType); + const isText = isTextPreview(contentType); + + return ( + + { + // Don't close when interacting with the PDF viewer toolbar + if (isPdf) e.preventDefault(); + }} + > + {/* Toolbar */} +
+
+
{icon}
+ + {fileName} + +
+
+ {isImage && blobUrl && ( + <> + + + + + {t('attachment_preview.zoomIn')} + + + + + + {t('attachment_preview.zoomOut')} + + + + + + {t('attachment_preview.resetZoom')} + + + + )} + + + + + {t('attachment.download')} + +
+
+ + {/* Preview body */} +
+ {previewMutation.isPending ? ( +
+
+ + + +
+
+ ) : isImage && blobUrl ? ( +
+ {fileName} +
+ ) : isPdf && blobUrl ? ( +