From c46c301371f0906daa49604f72513c95a64d49a2 Mon Sep 17 00:00:00 2001 From: charlesgauthereau Date: Mon, 16 Mar 2026 08:55:16 +0100 Subject: [PATCH 01/25] fix: s3 port type mismatch, add string_or_number_to_string deserializer --- Cargo.lock | 105 ++++++++++---------- docker-compose.yml | 6 +- docker/Dockerfile | 2 +- src/services/storage/providers/s3/models.rs | 3 + src/utils/deserializer.rs | 17 ++++ 5 files changed, 76 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e49eb4..dd3b6d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,9 +82,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.125.0" +version = "1.126.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed" +checksum = "7878050a2321d215eec9db8be09f8db59418b53860ae86cc7042b4094d6cb2bb" dependencies = [ "aws-credential-types", "aws-runtime", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "bollard" -version = "0.20.1" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "227aa051deec8d16bd9c34605e7aaf153f240e35483dd42f6f78903847934738" +checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4" dependencies = [ "async-stream", "base64 0.22.1", @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -1199,9 +1199,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -1209,11 +1209,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -1223,9 +1222,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", @@ -1289,9 +1288,9 @@ dependencies = [ [[package]] name = "derive-where" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", @@ -1450,7 +1449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1957,9 +1956,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hybrid-array" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b229d73f5803b562cc26e4da0396c8610a4ee209f4fac8fa4f8d709166dc45" +checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1" dependencies = [ "typenum", ] @@ -2089,7 +2088,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -2680,7 +2679,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2821,9 +2820,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" dependencies = [ "critical-section", "portable-atomic", @@ -2831,9 +2830,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.75" +version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ "bitflags 2.11.0", "cfg-if", @@ -2863,9 +2862,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.111" +version = "0.9.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb" dependencies = [ "cc", "libc", @@ -3256,7 +3255,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.37", - "socket2 0.5.10", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -3265,9 +3264,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", @@ -3294,7 +3293,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -3647,7 +3646,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3718,7 +3717,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3772,9 +3771,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -3961,9 +3960,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" dependencies = [ "base64 0.22.1", "chrono", @@ -3980,9 +3979,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.17.0" +version = "3.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" dependencies = [ "darling", "proc-macro2", @@ -4094,7 +4093,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4248,15 +4247,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4408,9 +4407,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" dependencies = [ "tinyvec_macros", ] @@ -4720,9 +4719,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "chrono", "matchers", @@ -5171,7 +5170,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -5736,18 +5735,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/docker-compose.yml b/docker-compose.yml index d6fbc5c..80cbcc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,14 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git - - cargo-target:/app/target +# - cargo-target:/app/target # - sqlite-data:/sqlite-data/workspace/data # - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data environment: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiOGYwMmExZTAtNDY0NC00MWFmLWIzYjctYjZkYWNjNzQ4OWVhIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZWE5OWE5ZjMtNDhkYy00MjMyLTkzMTAtNTc5YWVkY2IyZjhmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" extra_hosts: @@ -130,7 +130,7 @@ services: volumes: cargo-registry: cargo-git: - cargo-target: +# cargo-target: postgres-data: # mariadb-data: diff --git a/docker/Dockerfile b/docker/Dockerfile index 66e38d9..56437d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # ========================= # Base image (shared) # ========================= -FROM rust:1.92.0 AS base +FROM rust:1.94.0 AS base RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ pkg-config \ diff --git a/src/services/storage/providers/s3/models.rs b/src/services/storage/providers/s3/models.rs index 2410fd6..ab3036c 100644 --- a/src/services/storage/providers/s3/models.rs +++ b/src/services/storage/providers/s3/models.rs @@ -1,4 +1,5 @@ use serde::{Deserialize, Serialize}; +use crate::utils::deserializer::string_or_number_to_string; #[derive(Debug, Deserialize, Serialize)] pub struct S3ProviderConfig { @@ -8,5 +9,7 @@ pub struct S3ProviderConfig { pub end_point_url: String, pub ssl: bool, pub region: Option, + #[serde(default, deserialize_with = "string_or_number_to_string")] pub port: Option, } + diff --git a/src/utils/deserializer.rs b/src/utils/deserializer.rs index 01e77f9..e4d094a 100644 --- a/src/utils/deserializer.rs +++ b/src/utils/deserializer.rs @@ -1,5 +1,7 @@ use serde::{Deserialize, Deserializer}; use toml::Value; +use serde_json::{Value as ValueJson, }; + pub fn deserialize_snake_case<'de, D>(deserializer: D) -> Result where @@ -36,3 +38,18 @@ pub fn camel_to_snake(s: &str) -> String { } out } + + +pub fn string_or_number_to_string<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + + match value { + Some(ValueJson::String(s)) => Ok(Some(s)), + Some(ValueJson::Number(n)) => Ok(Some(n.to_string())), + Some(_) => Err(serde::de::Error::custom("port must be string or number")), + None => Ok(None), + } +} From 475b937891c8812ac2341d8295197234aed0c422 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 09:28:25 +0100 Subject: [PATCH 02/25] fix: adding some tests --- src/core/agent.rs | 23 +++-- src/core/context.rs | 2 +- src/core/mod.rs | 2 +- src/domain/factory.rs | 7 +- src/domain/mod.rs | 7 +- src/domain/mongodb/connection.rs | 8 +- src/domain/mongodb/database.rs | 2 +- src/domain/mongodb/mod.rs | 4 +- src/domain/mysql/backup.rs | 2 +- src/domain/mysql/connection.rs | 16 ++-- src/domain/mysql/database.rs | 22 ++--- src/domain/mysql/mod.rs | 4 +- src/domain/mysql/ping.rs | 1 - src/domain/postgres/backup.rs | 4 +- src/domain/postgres/format.rs | 2 +- src/domain/postgres/mod.rs | 6 +- src/domain/postgres/ping.rs | 6 +- src/domain/redis/ping.rs | 9 +- src/domain/sqlite/backup.rs | 4 +- src/domain/sqlite/mod.rs | 4 +- src/domain/valkey/ping.rs | 9 +- src/main.rs | 5 +- src/services/api/client.rs | 9 +- .../api/endpoints/agent/backup/mod.rs | 1 - .../api/endpoints/agent/backup/upload/init.rs | 2 +- .../api/endpoints/agent/backup/upload/mod.rs | 2 +- src/services/api/endpoints/agent/mod.rs | 4 +- src/services/api/endpoints/agent/status.rs | 5 +- src/services/api/endpoints/mod.rs | 2 +- src/services/api/error.rs | 5 +- src/services/api/mod.rs | 3 +- src/services/api/models/agent/mod.rs | 4 +- src/services/api/models/agent/restore.rs | 2 +- src/services/api/models/agent/status.rs | 2 +- src/services/api/models/mod.rs | 1 - src/services/backup/compressor.rs | 14 +-- src/services/backup/dispatcher.rs | 6 +- src/services/backup/executor.rs | 10 +-- src/services/backup/helpers.rs | 6 +- src/services/backup/mod.rs | 12 +-- src/services/backup/models.rs | 4 +- src/services/backup/runner.rs | 10 +-- src/services/backup/service.rs | 4 +- src/services/backup/uploader.rs | 85 ++++++++++--------- src/services/config.rs | 44 +++++++--- src/services/cron.rs | 2 +- src/services/mod.rs | 6 +- src/services/restore/archive.rs | 7 +- src/services/restore/dispatcher.rs | 13 +-- src/services/restore/downloader.rs | 13 +-- src/services/restore/executor.rs | 12 +-- src/services/restore/mod.rs | 12 +-- src/services/restore/models.rs | 2 +- src/services/restore/result.rs | 13 +-- src/services/restore/runner.rs | 9 +- src/services/restore/service.rs | 4 +- src/services/status.rs | 11 ++- src/services/storage/mod.rs | 6 +- .../storage/providers/google_drive/helpers.rs | 27 ++++-- .../storage/providers/google_drive/mod.rs | 25 ++---- .../storage/providers/google_drive/models.rs | 2 - src/services/storage/providers/local.rs | 28 +++--- src/services/storage/providers/mod.rs | 2 +- src/services/storage/providers/s3/mod.rs | 27 ++++-- src/services/storage/providers/s3/models.rs | 3 +- src/tasks/mod.rs | 2 +- src/tasks/ping.rs | 2 +- src/tests/domain/mod.rs | 2 +- src/tests/domain/redis.rs | 9 +- src/tests/domain/valkey.rs | 17 ++-- src/tests/mod.rs | 7 +- src/tests/utils/common_tests.rs | 11 ++- src/tests/utils/compress_tests.rs | 9 +- src/tests/utils/deserializer.rs | 11 +-- src/tests/utils/edge_key_tests.rs | 11 ++- src/tests/utils/mod.rs | 6 +- src/tests/utils/normalize_cron_tests.rs | 2 +- src/utils/common.rs | 3 +- src/utils/compress.rs | 2 +- src/utils/deserializer.rs | 4 +- src/utils/edge_key.rs | 2 +- src/utils/file.rs | 8 +- src/utils/mod.rs | 12 +-- src/utils/redis_client.rs | 5 +- src/utils/task_manager/cron.rs | 17 ++-- src/utils/task_manager/scheduler.rs | 16 ++-- src/utils/task_manager/tasks.rs | 2 +- src/utils/text.rs | 2 +- 88 files changed, 382 insertions(+), 390 deletions(-) diff --git a/src/core/agent.rs b/src/core/agent.rs index 23e56f1..bd20108 100644 --- a/src/core/agent.rs +++ b/src/core/agent.rs @@ -4,11 +4,11 @@ use crate::core::context::Context; use crate::services::backup::BackupService; use crate::services::config::ConfigService; use crate::services::cron::CronService; +use crate::services::restore::RestoreService; use crate::services::status::StatusService; use crate::utils::common::BackupMethod; use std::sync::Arc; use tracing::info; -use crate::services::restore::RestoreService; pub struct Agent { ctx: Arc, @@ -43,23 +43,30 @@ impl Agent { let ping_result = self.status_service.ping(&config.databases).await?; for db in ping_result.databases.iter() { - let database = config.databases.iter().find(|cfg_db|cfg_db.generated_id == db.generated_id).unwrap(); + let database = config + .databases + .iter() + .find(|cfg_db| cfg_db.generated_id == db.generated_id) + .unwrap(); info!( "Generated Id: {} | backup action: {} | restore action: {} | Database Name: {}", - db.generated_id, db.data.backup.action, db.data.restore.action, database.name, + db.generated_id, db.data.backup.action, db.data.restore.action, database.name, ); let _ = self.cron_service.sync(db).await; if db.data.backup.action { let _ = self .backup_service - .dispatch(&db.generated_id, &config, method.clone(), &db.storages, db.encrypt) + .dispatch( + &db.generated_id, + &config, + method.clone(), + &db.storages, + db.encrypt, + ) .await; } else if db.data.restore.action { - let _ = self - .restore_service - .dispatch(db, &config) - .await; + let _ = self.restore_service.dispatch(db, &config).await; } } diff --git a/src/core/context.rs b/src/core/context.rs index 00f7988..c37823d 100644 --- a/src/core/context.rs +++ b/src/core/context.rs @@ -35,7 +35,7 @@ impl Context { panic!("Cannot initialize AgentContext due to invalid EDGE_KEY"); } }; - + let server_url = format!("{}/api", edge_key.server_url); let api_client = ApiClient::new(server_url); diff --git a/src/core/mod.rs b/src/core/mod.rs index 869472d..6cc8135 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,2 +1,2 @@ -pub mod context; pub mod agent; +pub mod context; diff --git a/src/domain/factory.rs b/src/domain/factory.rs index eadd24c..e464967 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -4,11 +4,11 @@ use crate::domain::postgres::database::PostgresDatabase; use crate::domain::postgres::{detect_format_from_file, detect_format_from_size}; use crate::domain::redis::database::RedisDatabase; use crate::domain::sqlite::database::SqliteDatabase; +use crate::domain::valkey::database::ValkeyDatabase; use crate::services::config::{DatabaseConfig, DbType}; use anyhow::Result; use std::path::{Path, PathBuf}; use std::sync::Arc; -use crate::domain::valkey::database::ValkeyDatabase; #[async_trait::async_trait] pub trait Database: Send + Sync { @@ -32,7 +32,7 @@ impl DatabaseFactory { DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), DbType::Redis => Arc::new(RedisDatabase::new(cfg)), - DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)) + DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)), } } @@ -47,8 +47,7 @@ impl DatabaseFactory { DbType::MongoDB => Arc::new(MongoDatabase::new(cfg)), DbType::Sqlite => Arc::new(SqliteDatabase::new(cfg)), DbType::Redis => Arc::new(RedisDatabase::new(cfg)), - DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)) - + DbType::Valkey => Arc::new(ValkeyDatabase::new(cfg)), } } } diff --git a/src/domain/mod.rs b/src/domain/mod.rs index ea49a82..534c955 100644 --- a/src/domain/mod.rs +++ b/src/domain/mod.rs @@ -1,8 +1,7 @@ pub mod factory; -pub mod postgres; -pub mod mysql; mod mongodb; -mod sqlite; +pub mod mysql; +pub mod postgres; mod redis; +mod sqlite; mod valkey; - diff --git a/src/domain/mongodb/connection.rs b/src/domain/mongodb/connection.rs index 5d3a43c..311895b 100644 --- a/src/domain/mongodb/connection.rs +++ b/src/domain/mongodb/connection.rs @@ -16,9 +16,11 @@ pub fn select_mongo_path() -> std::path::PathBuf { } pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result { - if cfg.username.is_empty() || cfg.password.is_empty() { - Ok(format!("mongodb://{}:{}/{}", cfg.host, cfg.port, cfg.database)) + Ok(format!( + "mongodb://{}:{}/{}", + cfg.host, cfg.port, cfg.database + )) } else { Ok(format!( "mongodb://{}:{}@{}:{}/{}?authSource=admin", @@ -26,5 +28,3 @@ pub fn get_mongo_uri(cfg: DatabaseConfig) -> Result { )) } } - - diff --git a/src/domain/mongodb/database.rs b/src/domain/mongodb/database.rs index f0ec057..7850896 100644 --- a/src/domain/mongodb/database.rs +++ b/src/domain/mongodb/database.rs @@ -39,7 +39,7 @@ impl Database for MongoDatabase { res } - async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { + async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { let test_mode = is_test.unwrap_or(false); if !test_mode { FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; diff --git a/src/domain/mongodb/mod.rs b/src/domain/mongodb/mod.rs index 74afdd4..a4869e1 100644 --- a/src/domain/mongodb/mod.rs +++ b/src/domain/mongodb/mod.rs @@ -1,5 +1,5 @@ mod backup; -mod restore; +mod connection; pub mod database; mod ping; -mod connection; \ No newline at end of file +mod restore; diff --git a/src/domain/mysql/backup.rs b/src/domain/mysql/backup.rs index 1413bd4..2ec7531 100644 --- a/src/domain/mysql/backup.rs +++ b/src/domain/mysql/backup.rs @@ -1,4 +1,4 @@ -use crate::domain::mysql::connection::{server_version}; +use crate::domain::mysql::connection::server_version; use crate::services::config::DatabaseConfig; use anyhow::{Context, Result}; use std::collections::HashMap; diff --git a/src/domain/mysql/connection.rs b/src/domain/mysql/connection.rs index db26bbb..d02d8dc 100644 --- a/src/domain/mysql/connection.rs +++ b/src/domain/mysql/connection.rs @@ -1,14 +1,17 @@ use crate::services::config::DatabaseConfig; -use std::process::Command; use anyhow::Result; +use std::process::Command; pub async fn server_version(cfg: &DatabaseConfig) -> Result { - let output = Command::new("mysql") - .arg("--host").arg(&cfg.host) - .arg("--port").arg(cfg.port.to_string()) - .arg("--user").arg(&cfg.username) - .arg("-e").arg("SELECT VERSION();") + .arg("--host") + .arg(&cfg.host) + .arg("--port") + .arg(cfg.port.to_string()) + .arg("--user") + .arg(&cfg.username) + .arg("-e") + .arg("SELECT VERSION();") .env("MYSQL_PWD", &cfg.password) .output()?; @@ -26,4 +29,3 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { Ok(version) } - diff --git a/src/domain/mysql/database.rs b/src/domain/mysql/database.rs index d616f83..e90b3e1 100644 --- a/src/domain/mysql/database.rs +++ b/src/domain/mysql/database.rs @@ -1,14 +1,11 @@ -use std::collections::HashMap; -use anyhow::Result; -use async_trait::async_trait; -use std::path::{Path, PathBuf}; -use super::{ - backup, - ping, restore, -}; +use super::{backup, ping, restore}; use crate::domain::factory::Database; use crate::services::config::DatabaseConfig; use crate::utils::locks::{DbOpLock, FileLock}; +use anyhow::Result; +use async_trait::async_trait; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; pub struct MySQLDatabase { cfg: DatabaseConfig, @@ -36,13 +33,18 @@ impl Database for MySQLDatabase { ping::run(self.cfg.clone(), self.build_env().clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { let test_mode = is_test.unwrap_or(false); if !test_mode { FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; } - let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.build_env().clone(), self.file_extension()).await; + let res = backup::run( + self.cfg.clone(), + dir.to_path_buf(), + self.build_env().clone(), + self.file_extension(), + ) + .await; if !test_mode { FileLock::release(&self.cfg.generated_id).await?; } diff --git a/src/domain/mysql/mod.rs b/src/domain/mysql/mod.rs index 1bfe59e..33dd5f4 100644 --- a/src/domain/mysql/mod.rs +++ b/src/domain/mysql/mod.rs @@ -1,5 +1,5 @@ pub mod backup; +mod connection; pub mod database; -mod restore; mod ping; -mod connection; \ No newline at end of file +mod restore; diff --git a/src/domain/mysql/ping.rs b/src/domain/mysql/ping.rs index dd2491f..fde0407 100644 --- a/src/domain/mysql/ping.rs +++ b/src/domain/mysql/ping.rs @@ -4,7 +4,6 @@ use tokio::process::Command; use tokio::time::{Duration, timeout}; pub async fn run(cfg: DatabaseConfig, env: HashMap) -> anyhow::Result { - let mut cmd = Command::new("mysqladmin"); cmd.arg("--host") .arg(cfg.host) diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index f6f71be..a7d83b4 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -11,7 +11,7 @@ pub async fn run( cfg: DatabaseConfig, format: PostgresDumpFormat, backup_dir: PathBuf, - is_test: Option + is_test: Option, ) -> Result { tokio::task::spawn_blocking(move || -> Result { debug!("Starting backup for database {}", cfg.name); @@ -28,7 +28,7 @@ pub async fn run( }; let pg_dump = select_pg_path(&version, is_test).join("pg_dump"); - + debug!("Using pg_dump at {:?}", pg_dump); match format { diff --git a/src/domain/postgres/format.rs b/src/domain/postgres/format.rs index e76f18a..5415fd7 100644 --- a/src/domain/postgres/format.rs +++ b/src/domain/postgres/format.rs @@ -2,4 +2,4 @@ pub enum PostgresDumpFormat { Fc, Fd, -} \ No newline at end of file +} diff --git a/src/domain/postgres/mod.rs b/src/domain/postgres/mod.rs index f7b39e9..cd42ace 100644 --- a/src/domain/postgres/mod.rs +++ b/src/domain/postgres/mod.rs @@ -1,8 +1,8 @@ pub mod backup; -pub mod database; -mod restore; mod connection; +pub mod database; mod format; mod ping; +mod restore; -pub use connection::{detect_format_from_size, detect_format_from_file}; +pub use connection::{detect_format_from_file, detect_format_from_size}; diff --git a/src/domain/postgres/ping.rs b/src/domain/postgres/ping.rs index b1e5421..5d859f1 100644 --- a/src/domain/postgres/ping.rs +++ b/src/domain/postgres/ping.rs @@ -1,8 +1,6 @@ use super::connection::connect; use crate::services::config::DatabaseConfig; -pub async fn run( - cfg: DatabaseConfig, -) -> anyhow::Result { +pub async fn run(cfg: DatabaseConfig) -> anyhow::Result { Ok(connect(&cfg).await.is_ok()) -} \ No newline at end of file +} diff --git a/src/domain/redis/ping.rs b/src/domain/redis/ping.rs index d06df71..82b2b9b 100644 --- a/src/domain/redis/ping.rs +++ b/src/domain/redis/ping.rs @@ -1,8 +1,8 @@ -use tracing::{debug, info}; use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; use tokio::process::Command; -use tokio::time::{timeout, Duration}; -use anyhow::{Result, Context}; +use tokio::time::{Duration, timeout}; +use tracing::{debug, info}; pub async fn run(cfg: DatabaseConfig) -> Result { let mut cmd = Command::new("redis-cli"); @@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result { debug!("Command Ping: {:?}", cmd); - let result = timeout(Duration::from_secs(10), cmd.output()).await; match result { @@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result { Ok(false) } } -} \ No newline at end of file +} diff --git a/src/domain/sqlite/backup.rs b/src/domain/sqlite/backup.rs index 578cfd6..7b4b21c 100644 --- a/src/domain/sqlite/backup.rs +++ b/src/domain/sqlite/backup.rs @@ -41,5 +41,5 @@ pub async fn run( info!("SQLite backup completed for {}", cfg.name); Ok(file_path) }) - .await? -} \ No newline at end of file + .await? +} diff --git a/src/domain/sqlite/mod.rs b/src/domain/sqlite/mod.rs index 5efda1c..325a54a 100644 --- a/src/domain/sqlite/mod.rs +++ b/src/domain/sqlite/mod.rs @@ -1,4 +1,4 @@ mod backup; -mod restore; +pub mod database; mod ping; -pub mod database; \ No newline at end of file +mod restore; diff --git a/src/domain/valkey/ping.rs b/src/domain/valkey/ping.rs index eb6f47c..851c6a6 100644 --- a/src/domain/valkey/ping.rs +++ b/src/domain/valkey/ping.rs @@ -1,8 +1,8 @@ -use tracing::{debug, info}; use crate::services::config::DatabaseConfig; +use anyhow::{Context, Result}; use tokio::process::Command; -use tokio::time::{timeout, Duration}; -use anyhow::{Result, Context}; +use tokio::time::{Duration, timeout}; +use tracing::{debug, info}; pub async fn run(cfg: DatabaseConfig) -> Result { let mut cmd = Command::new("valkey-cli"); @@ -23,7 +23,6 @@ pub async fn run(cfg: DatabaseConfig) -> Result { debug!("Command Ping: {:?}", cmd); - let result = timeout(Duration::from_secs(10), cmd.output()).await; match result { @@ -52,4 +51,4 @@ pub async fn run(cfg: DatabaseConfig) -> Result { Ok(false) } } -} \ No newline at end of file +} diff --git a/src/main.rs b/src/main.rs index a391593..5c7a3f5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,19 +3,18 @@ mod domain; mod services; mod settings; mod tasks; -mod utils; #[cfg(test)] mod tests; +mod utils; use crate::tasks::ping::ping_server; use crate::utils::locks::FileLock; +use crate::utils::logging; use utils::redis_client; use utils::task_manager::scheduler; -use crate::utils::logging; #[tokio::main] async fn main() { - logging::init_logger(); // Remove all locks on startup diff --git a/src/services/api/client.rs b/src/services/api/client.rs index f11ba33..f1e396c 100644 --- a/src/services/api/client.rs +++ b/src/services/api/client.rs @@ -1,9 +1,9 @@ #![allow(dead_code)] +use crate::services::api::ApiError; use reqwest::{Client, Method}; use serde::de::DeserializeOwned; use std::time::Duration; -use crate::services::api::ApiError; #[derive(Clone, Debug)] pub struct ApiClient { @@ -57,12 +57,7 @@ impl ApiClient { { let url = format!("{}{}", self.base_url, path); - let res = self - .http - .request(method, &url) - .json(body) - .send() - .await?; + let res = self.http.request(method, &url).json(body).send().await?; let status = res.status(); let body_text = res.text().await.unwrap_or_default(); diff --git a/src/services/api/endpoints/agent/backup/mod.rs b/src/services/api/endpoints/agent/backup/mod.rs index 114d146..5a489e4 100644 --- a/src/services/api/endpoints/agent/backup/mod.rs +++ b/src/services/api/endpoints/agent/backup/mod.rs @@ -23,7 +23,6 @@ pub struct BackupUpdateRequest { pub generated_id: String, } - impl ApiClient { pub async fn backup_create( &self, diff --git a/src/services/api/endpoints/agent/backup/upload/init.rs b/src/services/api/endpoints/agent/backup/upload/init.rs index 8b3d52f..cd39651 100644 --- a/src/services/api/endpoints/agent/backup/upload/init.rs +++ b/src/services/api/endpoints/agent/backup/upload/init.rs @@ -1,8 +1,8 @@ +use crate::services::api::models::agent::backup::BackupUploadResponse; use crate::services::api::{ApiClient, ApiError}; use anyhow::Result; use reqwest::Method; use serde::Serialize; -use crate::services::api::models::agent::backup::BackupUploadResponse; #[derive(Serialize)] pub struct InitUploadRequest { diff --git a/src/services/api/endpoints/agent/backup/upload/mod.rs b/src/services/api/endpoints/agent/backup/upload/mod.rs index 832c46e..3ed8b62 100644 --- a/src/services/api/endpoints/agent/backup/upload/mod.rs +++ b/src/services/api/endpoints/agent/backup/upload/mod.rs @@ -1,2 +1,2 @@ pub mod init; -pub mod status; \ No newline at end of file +pub mod status; diff --git a/src/services/api/endpoints/agent/mod.rs b/src/services/api/endpoints/agent/mod.rs index 161e4e6..49027be 100644 --- a/src/services/api/endpoints/agent/mod.rs +++ b/src/services/api/endpoints/agent/mod.rs @@ -1,3 +1,3 @@ -pub mod status; pub mod backup; -pub mod restore; \ No newline at end of file +pub mod restore; +pub mod status; diff --git a/src/services/api/endpoints/agent/status.rs b/src/services/api/endpoints/agent/status.rs index fe05790..868a4d4 100644 --- a/src/services/api/endpoints/agent/status.rs +++ b/src/services/api/endpoints/agent/status.rs @@ -31,6 +31,7 @@ impl ApiClient { let agent_id = agent_id.into(); let path = format!("/agent/{}/status", agent_id); - self.request_with_body(Method::POST, path.as_str(), &body).await + self.request_with_body(Method::POST, path.as_str(), &body) + .await } -} \ No newline at end of file +} diff --git a/src/services/api/endpoints/mod.rs b/src/services/api/endpoints/mod.rs index e779f0f..f498649 100644 --- a/src/services/api/endpoints/mod.rs +++ b/src/services/api/endpoints/mod.rs @@ -1,3 +1,3 @@ pub mod agent; -pub use agent::status; \ No newline at end of file +pub use agent::status; diff --git a/src/services/api/error.rs b/src/services/api/error.rs index ac82449..4862ee5 100644 --- a/src/services/api/error.rs +++ b/src/services/api/error.rs @@ -11,10 +11,7 @@ pub enum ApiError { Serialization(#[from] serde_json::Error), #[error("api error: status={status}, body={body}")] - HttpResponse { - status: StatusCode, - body: String, - }, + HttpResponse { status: StatusCode, body: String }, #[error("api returned unexpected response")] UnexpectedResponse, diff --git a/src/services/api/mod.rs b/src/services/api/mod.rs index 5c1e7d9..be0520a 100644 --- a/src/services/api/mod.rs +++ b/src/services/api/mod.rs @@ -1,8 +1,7 @@ pub mod client; +pub mod endpoints; pub mod error; pub mod models; -pub mod endpoints; pub use client::ApiClient; pub use error::ApiError; - diff --git a/src/services/api/models/agent/mod.rs b/src/services/api/models/agent/mod.rs index 161e4e6..49027be 100644 --- a/src/services/api/models/agent/mod.rs +++ b/src/services/api/models/agent/mod.rs @@ -1,3 +1,3 @@ -pub mod status; pub mod backup; -pub mod restore; \ No newline at end of file +pub mod restore; +pub mod status; diff --git a/src/services/api/models/agent/restore.rs b/src/services/api/models/agent/restore.rs index 288d4c8..d9abfb5 100644 --- a/src/services/api/models/agent/restore.rs +++ b/src/services/api/models/agent/restore.rs @@ -4,4 +4,4 @@ use serde::{Deserialize, Serialize}; pub struct ResultRestoreResponse { pub message: String, pub status: bool, -} \ No newline at end of file +} diff --git a/src/services/api/models/agent/status.rs b/src/services/api/models/agent/status.rs index 992ae2c..8b1b145 100644 --- a/src/services/api/models/agent/status.rs +++ b/src/services/api/models/agent/status.rs @@ -1,8 +1,8 @@ #![allow(dead_code)] +use crate::utils::deserializer::deserialize_snake_case; use serde::{Deserialize, Serialize}; use toml::Value; -use crate::utils::deserializer::deserialize_snake_case; #[derive(Debug, Deserialize)] pub struct PingResult { diff --git a/src/services/api/models/mod.rs b/src/services/api/models/mod.rs index 4be98f7..f17bc55 100644 --- a/src/services/api/models/mod.rs +++ b/src/services/api/models/mod.rs @@ -1,2 +1 @@ pub mod agent; - diff --git a/src/services/backup/compressor.rs b/src/services/backup/compressor.rs index d7ca3e3..ad36a09 100644 --- a/src/services/backup/compressor.rs +++ b/src/services/backup/compressor.rs @@ -1,20 +1,14 @@ use super::service::BackupService; use crate::utils::compress::compress_to_tar_gz_large; -use std::path::PathBuf; use anyhow::Result; +use std::path::PathBuf; impl BackupService { - - pub async fn compress_backup( - &self, - backup_file: Option, - ) -> Result { - - let file = backup_file - .ok_or_else(|| anyhow::anyhow!("No backup file generated"))?; + pub async fn compress_backup(&self, backup_file: Option) -> Result { + let file = backup_file.ok_or_else(|| anyhow::anyhow!("No backup file generated"))?; let compression = compress_to_tar_gz_large(&file).await?; Ok(compression.compressed_path) } -} \ No newline at end of file +} diff --git a/src/services/backup/dispatcher.rs b/src/services/backup/dispatcher.rs index b6aa2bc..bf9145a 100644 --- a/src/services/backup/dispatcher.rs +++ b/src/services/backup/dispatcher.rs @@ -1,11 +1,10 @@ use super::service::BackupService; -use crate::services::config::DatabasesConfig; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::config::DatabasesConfig; use crate::utils::common::BackupMethod; use tracing::error; impl BackupService { - pub async fn dispatch( &self, generated_id: &String, @@ -14,7 +13,6 @@ impl BackupService { storages: &Vec, encrypt: bool, ) { - let Some(cfg) = config .databases .iter() @@ -41,4 +39,4 @@ impl BackupService { } }); } -} \ No newline at end of file +} diff --git a/src/services/backup/executor.rs b/src/services/backup/executor.rs index 203b21f..4b88c86 100644 --- a/src/services/backup/executor.rs +++ b/src/services/backup/executor.rs @@ -1,14 +1,13 @@ use super::service::BackupService; -use crate::services::config::DatabaseConfig; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::config::DatabaseConfig; use crate::utils::common::BackupMethod; use crate::utils::locks::FileLock; -use tempfile::TempDir; use anyhow::Result; +use tempfile::TempDir; impl BackupService { - pub async fn execute_backup( &self, generated_id: String, @@ -17,11 +16,10 @@ impl BackupService { storages: Vec, encrypt: bool, ) -> Result<()> { - if FileLock::is_locked(&generated_id).await? { anyhow::bail!("backup already running"); } - + let backup = self.create_backup_record(&generated_id, &method).await?; let backup_id = backup.backup.id; @@ -46,4 +44,4 @@ impl BackupService { Ok(()) } -} \ No newline at end of file +} diff --git a/src/services/backup/helpers.rs b/src/services/backup/helpers.rs index a5a0130..3d63a82 100644 --- a/src/services/backup/helpers.rs +++ b/src/services/backup/helpers.rs @@ -1,16 +1,14 @@ use super::service::BackupService; +use crate::services::api::models::agent::backup::BackupResponse; use crate::utils::common::BackupMethod; use anyhow::{Result, anyhow}; -use crate::services::api::models::agent::backup::BackupResponse; impl BackupService { - pub async fn create_backup_record( &self, generated_id: &str, method: &BackupMethod, ) -> Result { - let response = self .ctx .api @@ -23,4 +21,4 @@ impl BackupService { response.ok_or_else(|| anyhow!("backup_create returned empty response")) } -} \ No newline at end of file +} diff --git a/src/services/backup/mod.rs b/src/services/backup/mod.rs index 2e1b5f0..98f1aec 100644 --- a/src/services/backup/mod.rs +++ b/src/services/backup/mod.rs @@ -1,11 +1,11 @@ -pub mod service; +pub mod compressor; pub mod dispatcher; pub mod executor; -pub mod compressor; -pub mod uploader; -pub mod result; -pub mod models; pub mod helpers; +pub mod models; +pub mod result; pub mod runner; +pub mod service; +pub mod uploader; -pub use service::BackupService; \ No newline at end of file +pub use service::BackupService; diff --git a/src/services/backup/models.rs b/src/services/backup/models.rs index f9db440..def8787 100644 --- a/src/services/backup/models.rs +++ b/src/services/backup/models.rs @@ -1,7 +1,7 @@ #![allow(dead_code)] -use std::path::PathBuf; use crate::services::config::DbType; +use std::path::PathBuf; #[derive(Debug, Clone)] pub struct BackupResult { @@ -19,4 +19,4 @@ pub struct UploadResult { pub error: Option, pub remote_file_path: Option, pub total_size: Option, -} \ No newline at end of file +} diff --git a/src/services/backup/runner.rs b/src/services/backup/runner.rs index 3e7708c..e617ed6 100644 --- a/src/services/backup/runner.rs +++ b/src/services/backup/runner.rs @@ -9,12 +9,7 @@ use std::path::Path; use tracing::{error, info}; impl BackupService { - - pub async fn run( - cfg: DatabaseConfig, - tmp_path: &Path, - ) -> Result { - + pub async fn run(cfg: DatabaseConfig, tmp_path: &Path) -> Result { let db = DatabaseFactory::create_for_backup(cfg.clone()).await; let generated_id = cfg.generated_id.clone(); @@ -41,7 +36,6 @@ impl BackupService { } match db.backup(tmp_path, Some(false)).await { - Ok(file) => Ok(BackupResult { generated_id, db_type, @@ -67,4 +61,4 @@ impl BackupService { }), } } -} \ No newline at end of file +} diff --git a/src/services/backup/service.rs b/src/services/backup/service.rs index 043e88d..31f6c51 100644 --- a/src/services/backup/service.rs +++ b/src/services/backup/service.rs @@ -1,5 +1,5 @@ -use std::sync::Arc; use crate::core::context::Context as CoreContext; +use std::sync::Arc; pub struct BackupService { pub ctx: Arc, @@ -9,4 +9,4 @@ impl BackupService { pub fn new(ctx: Arc) -> Self { Self { ctx } } -} \ No newline at end of file +} diff --git a/src/services/backup/uploader.rs b/src/services/backup/uploader.rs index 8136f27..35ecdf5 100644 --- a/src/services/backup/uploader.rs +++ b/src/services/backup/uploader.rs @@ -1,16 +1,15 @@ -use super::service::BackupService; use super::models::{BackupResult, UploadResult}; +use super::service::BackupService; -use crate::services::storage; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::storage; use crate::utils::common::BackupMethod; -use futures::future::join_all; use anyhow::{Result, bail}; -use tracing::{info, error}; +use futures::future::join_all; +use tracing::{error, info}; impl BackupService { - pub async fn upload( &self, result: BackupResult, @@ -19,7 +18,6 @@ impl BackupService { encrypt: bool, backup_id: &String, ) -> Result> { - if result.code.as_deref() == Some("backup_already_in_progress") { info!("Skipping send: backup already in progress"); bail!("backup_already_in_progress"); @@ -28,7 +26,6 @@ impl BackupService { let ctx = self.ctx.clone(); let futures = storages.into_iter().map(|storage| { - let ctx_clone = ctx.clone(); let result_clone = result.clone(); let provider = storage::get_provider(&storage); @@ -37,13 +34,16 @@ impl BackupService { let generated_id = result_clone.generated_id.clone(); async move { - - info!("Uploading storage -> {:?} for {:?}", storage.provider, storage_id); + info!( + "Uploading storage -> {:?} for {:?}", + storage.provider, storage_id + ); /* INIT STEP */ - let init = match ctx_clone.api + let init = match ctx_clone + .api .backup_upload_init( ctx_clone.edge_key.agent_id.clone(), generated_id.clone(), @@ -107,7 +107,11 @@ impl BackupService { ) .await; - let status = if upload_result.success { "success" } else { "failed" }; + let status = if upload_result.success { + "success" + } else { + "failed" + }; if status != "success" { return upload_result; @@ -115,49 +119,48 @@ impl BackupService { info!( "Storage {} uploaded to remote path {:?}", - storage_id, - upload_result.remote_file_path + storage_id, upload_result.remote_file_path ); /* METADATA VALIDATION */ - let (remote_path, total_size) = match ( - &upload_result.remote_file_path, - upload_result.total_size, - ) { - (Some(path), Some(size)) => (path.clone(), size), - _ => { - return UploadResult { - storage_id, - success: false, - error: Some("remote_file_path or total_size missing".into()), - remote_file_path: None, - total_size: None, - }; - } - }; + let (remote_path, total_size) = + match (&upload_result.remote_file_path, upload_result.total_size) { + (Some(path), Some(size)) => (path.clone(), size), + _ => { + return UploadResult { + storage_id, + success: false, + error: Some("remote_file_path or total_size missing".into()), + remote_file_path: None, + total_size: None, + }; + } + }; /* STATUS UPDATE */ - match ctx_clone.api.backup_upload_status( - ctx_clone.edge_key.agent_id.clone(), - generated_id, - backup_storage_id, - status, - remote_path, - total_size, - backup_id, - ).await { - + match ctx_clone + .api + .backup_upload_status( + ctx_clone.edge_key.agent_id.clone(), + generated_id, + backup_storage_id, + status, + remote_path, + total_size, + backup_id, + ) + .await + { Ok(_) => upload_result, Err(err) => { error!( "backup_upload_status failed (storage_id={}): {}", - storage_id, - err + storage_id, err ); UploadResult { @@ -178,4 +181,4 @@ impl BackupService { Ok(results) } -} \ No newline at end of file +} diff --git a/src/services/config.rs b/src/services/config.rs index 69460dd..d68989c 100644 --- a/src/services/config.rs +++ b/src/services/config.rs @@ -20,7 +20,7 @@ pub enum DbType { MongoDB, Sqlite, Redis, - Valkey + Valkey, } impl DbType { @@ -58,7 +58,6 @@ pub struct DatabasesConfig { pub databases: Vec, } - #[allow(dead_code)] #[derive(Debug, Deserialize, Clone)] pub struct InputDatabaseConfig { @@ -80,7 +79,6 @@ pub struct InputDatabasesConfig { pub databases: Vec, } - pub struct ConfigService { ctx: Arc, } @@ -133,17 +131,27 @@ impl ConfigService { _ => return Err("Unsupported config file format. Use .json or .toml".to_string()), }; - fn required(opt: &Option, db_name: &str, field_name: &str) -> Result { + fn required( + opt: &Option, + db_name: &str, + field_name: &str, + ) -> Result { match opt { Some(v) => Ok(v.clone()), None => { - let msg = format!("Missing required field '{}' for database '{}'", field_name, db_name); + let msg = format!( + "Missing required field '{}' for database '{}'", + field_name, db_name + ); Err(msg) } } } - fn optional(opt: &Option) -> T where T: Default { + fn optional(opt: &Option) -> T + where + T: Default, + { opt.clone().unwrap_or_default() } @@ -155,28 +163,42 @@ impl ConfigService { } let username = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.username, &db.name, "username")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => { + required(&db.username, &db.name, "username")? + } _ => optional(&db.username), }; let password = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb => required(&db.password, &db.name, "password")?, + DbType::Postgresql | DbType::Mysql | DbType::Mariadb => { + required(&db.password, &db.name, "password")? + } _ => optional(&db.password), }; let host = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.host, &db.name, "host")?, + DbType::Postgresql + | DbType::Mysql + | DbType::Mariadb + | DbType::MongoDB + | DbType::Redis + | DbType::Valkey => required(&db.host, &db.name, "host")?, DbType::Sqlite => optional(&db.host), }; let port = match db.db_type { - DbType::Postgresql | DbType::Mysql | DbType::Mariadb | DbType::MongoDB | DbType::Redis | DbType::Valkey => required(&db.port, &db.name, "port")?, + DbType::Postgresql + | DbType::Mysql + | DbType::Mariadb + | DbType::MongoDB + | DbType::Redis + | DbType::Valkey => required(&db.port, &db.name, "port")?, DbType::Sqlite => db.port.unwrap_or(0), }; let database_name = match db.db_type { DbType::Sqlite | DbType::Redis | DbType::Valkey => optional(&db.database), - _ => required(&db.database, &db.name, "database")? + _ => required(&db.database, &db.name, "database")?, }; let path_val = match db.db_type { diff --git a/src/services/cron.rs b/src/services/cron.rs index 9895bb2..87aa1a3 100644 --- a/src/services/cron.rs +++ b/src/services/cron.rs @@ -1,13 +1,13 @@ #![allow(dead_code)] use crate::core::context::Context; +use crate::services::api::models::agent::status::DatabaseStatus; use crate::utils::common::vec_to_option_json; use crate::utils::redis_client; use crate::utils::task_manager::cron::check_and_update_cron; use redis::aio::MultiplexedConnection; use serde_json::{Value, json}; use std::sync::Arc; -use crate::services::api::models::agent::status::DatabaseStatus; pub struct CronService { ctx: Arc, diff --git a/src/services/mod.rs b/src/services/mod.rs index 009cfbc..c4f3f04 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -1,7 +1,7 @@ +pub mod api; +pub mod backup; pub mod config; pub mod cron; -pub mod backup; pub mod restore; +pub mod status; mod storage; -pub mod api; -pub mod status; \ No newline at end of file diff --git a/src/services/restore/archive.rs b/src/services/restore/archive.rs index 6cb457c..be87159 100644 --- a/src/services/restore/archive.rs +++ b/src/services/restore/archive.rs @@ -7,13 +7,11 @@ use anyhow::Result; use std::path::{Path, PathBuf}; impl RestoreService { - pub async fn prepare_archive( &self, downloaded_file: PathBuf, tmp_path: &Path, ) -> Result { - let filename = downloaded_file .file_name() .unwrap() @@ -31,7 +29,6 @@ impl RestoreService { let mut archive = downloaded_file.clone(); if encrypted { - let new_name = filename.strip_suffix(".enc").unwrap(); let decrypted = tmp_path.join(new_name); @@ -41,7 +38,7 @@ impl RestoreService { decrypted.clone(), self.ctx.edge_key.master_key_b64.clone(), ) - .await?; + .await?; archive = decrypted; } @@ -58,4 +55,4 @@ impl RestoreService { Ok(archive) } } -} \ No newline at end of file +} diff --git a/src/services/restore/dispatcher.rs b/src/services/restore/dispatcher.rs index 12a0fd5..bf0c9c4 100644 --- a/src/services/restore/dispatcher.rs +++ b/src/services/restore/dispatcher.rs @@ -1,13 +1,11 @@ use super::service::RestoreService; -use crate::services::config::DatabasesConfig; use crate::services::api::models::agent::status::DatabaseStatus; +use crate::services::config::DatabasesConfig; use tracing::error; impl RestoreService { - pub async fn dispatch(&self, db: &DatabaseStatus, config: &DatabasesConfig) { - let Some(cfg) = config .databases .iter() @@ -29,14 +27,9 @@ impl RestoreService { let db_cfg = cfg.clone(); tokio::spawn(async move { - - if let Err(e) = service - .execute_restore(db_cfg, file_to_restore) - .await - { + if let Err(e) = service.execute_restore(db_cfg, file_to_restore).await { error!("Restore failed: {}", e); } - }); } -} \ No newline at end of file +} diff --git a/src/services/restore/downloader.rs b/src/services/restore/downloader.rs index 6646b8c..0b6cf41 100644 --- a/src/services/restore/downloader.rs +++ b/src/services/restore/downloader.rs @@ -1,18 +1,12 @@ use super::service::RestoreService; -use reqwest::{Client, Url}; use anyhow::Result; +use reqwest::{Client, Url}; use std::path::{Path, PathBuf}; use tracing::info; impl RestoreService { - - pub async fn download_backup( - &self, - file_url: &str, - tmp_path: &Path, - ) -> Result { - + pub async fn download_backup(&self, file_url: &str, tmp_path: &Path) -> Result { let client = Client::new(); let response = client.get(file_url).send().await?; @@ -28,7 +22,6 @@ impl RestoreService { .and_then(|s| s.split("filename=").nth(1)) .map(|f| f.trim_matches('"').to_string()); - let filename_from_url = Url::parse(file_url).ok().and_then(|u| { u.path_segments()? .last() @@ -50,4 +43,4 @@ impl RestoreService { Ok(path) } -} \ No newline at end of file +} diff --git a/src/services/restore/executor.rs b/src/services/restore/executor.rs index 4cabb5b..b790623 100644 --- a/src/services/restore/executor.rs +++ b/src/services/restore/executor.rs @@ -1,17 +1,11 @@ use super::service::RestoreService; use crate::services::config::DatabaseConfig; -use tempfile::TempDir; use anyhow::Result; +use tempfile::TempDir; use tracing::info; impl RestoreService { - - pub async fn execute_restore( - &self, - cfg: DatabaseConfig, - file_url: String, - ) -> Result<()> { - + pub async fn execute_restore(&self, cfg: DatabaseConfig, file_url: String) -> Result<()> { let temp_dir = TempDir::new()?; let tmp_path = temp_dir.path(); @@ -27,4 +21,4 @@ impl RestoreService { Ok(()) } -} \ No newline at end of file +} diff --git a/src/services/restore/mod.rs b/src/services/restore/mod.rs index 4ca2ebe..72f54ee 100644 --- a/src/services/restore/mod.rs +++ b/src/services/restore/mod.rs @@ -1,10 +1,10 @@ -pub mod service; +pub mod archive; pub mod dispatcher; -pub mod executor; pub mod downloader; -pub mod archive; -pub mod runner; -pub mod result; +pub mod executor; pub mod models; +pub mod result; +pub mod runner; +pub mod service; -pub use service::RestoreService; \ No newline at end of file +pub use service::RestoreService; diff --git a/src/services/restore/models.rs b/src/services/restore/models.rs index e987903..d459114 100644 --- a/src/services/restore/models.rs +++ b/src/services/restore/models.rs @@ -5,4 +5,4 @@ pub struct RestoreResult { #[serde(rename = "generatedId")] pub generated_id: String, pub status: String, -} \ No newline at end of file +} diff --git a/src/services/restore/result.rs b/src/services/restore/result.rs index d7325bf..d3f61c1 100644 --- a/src/services/restore/result.rs +++ b/src/services/restore/result.rs @@ -1,24 +1,25 @@ -use super::service::RestoreService; use super::models::RestoreResult; +use super::service::RestoreService; -use tracing::{info, error}; +use tracing::{error, info}; impl RestoreService { pub async fn send_result(&self, result: RestoreResult) { - info!( "[RestoreService] DB: {} | Status: {}", result.generated_id, result.status ); - match self.ctx + match self + .ctx .api .restore_result( self.ctx.edge_key.agent_id.clone(), &result.generated_id, &result.status, ) - .await { + .await + { Ok(_) => { info!("Restoration result sent successfully"); } @@ -27,4 +28,4 @@ impl RestoreService { } } } -} \ No newline at end of file +} diff --git a/src/services/restore/runner.rs b/src/services/restore/runner.rs index 36751dd..1ba424b 100644 --- a/src/services/restore/runner.rs +++ b/src/services/restore/runner.rs @@ -1,21 +1,19 @@ -use super::service::RestoreService; use super::models::RestoreResult; +use super::service::RestoreService; use crate::domain::factory::DatabaseFactory; use crate::services::config::DatabaseConfig; use anyhow::Result; use std::path::PathBuf; -use tracing::{info, error}; +use tracing::{error, info}; impl RestoreService { - pub async fn run_restore( &self, cfg: DatabaseConfig, backup_file: PathBuf, ) -> Result { - let generated_id = cfg.generated_id.clone(); let db = DatabaseFactory::create_for_restore(cfg.clone(), &backup_file).await; @@ -32,7 +30,6 @@ impl RestoreService { } match db.restore(&backup_file, Some(false)).await { - Ok(_) => Ok(RestoreResult { generated_id, status: "success".into(), @@ -48,4 +45,4 @@ impl RestoreService { } } } -} \ No newline at end of file +} diff --git a/src/services/restore/service.rs b/src/services/restore/service.rs index 5b6be8d..b2f59bb 100644 --- a/src/services/restore/service.rs +++ b/src/services/restore/service.rs @@ -1,5 +1,5 @@ -use std::sync::Arc; use crate::core::context::Context; +use std::sync::Arc; pub struct RestoreService { pub ctx: Arc, @@ -9,4 +9,4 @@ impl RestoreService { pub fn new(ctx: Arc) -> Self { Self { ctx } } -} \ No newline at end of file +} diff --git a/src/services/status.rs b/src/services/status.rs index fe2dba3..0ddfba1 100644 --- a/src/services/status.rs +++ b/src/services/status.rs @@ -1,13 +1,13 @@ #![allow(dead_code)] use crate::core::context::Context; +use crate::services::api::endpoints::status::DatabasePayload; +use crate::services::api::models::agent::status::PingResult; use crate::services::config::DatabaseConfig; use crate::settings::CONFIG; use reqwest::Client; use std::error::Error; use std::sync::Arc; -use crate::services::api::endpoints::status::DatabasePayload; -use crate::services::api::models::agent::status::PingResult; pub struct StatusService { ctx: Arc, @@ -35,7 +35,12 @@ impl StatusService { .collect(); let version_str = CONFIG.app_version.as_str(); - let result = self.ctx.api.agent_status(&edge_key.agent_id, &version_str, databases_payload).await?.unwrap(); + let result = self + .ctx + .api + .agent_status(&edge_key.agent_id, &version_str, databases_payload) + .await? + .unwrap(); Ok(result) } } diff --git a/src/services/storage/mod.rs b/src/services/storage/mod.rs index e44d142..02d3aaf 100644 --- a/src/services/storage/mod.rs +++ b/src/services/storage/mod.rs @@ -1,15 +1,15 @@ pub mod providers; use crate::core::context::Context; +use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::models::{BackupResult, UploadResult}; use crate::utils::common::BackupMethod; use async_trait::async_trait; +use providers::google_drive; use providers::local; use providers::s3; -use providers::google_drive; use std::sync::Arc; use tracing::{error, info}; -use crate::services::api::models::agent::status::DatabaseStorage; -use crate::services::backup::models::{BackupResult, UploadResult}; #[async_trait] pub trait StorageProvider: Send + Sync { diff --git a/src/services/storage/providers/google_drive/helpers.rs b/src/services/storage/providers/google_drive/helpers.rs index e7744bd..688bfa4 100644 --- a/src/services/storage/providers/google_drive/helpers.rs +++ b/src/services/storage/providers/google_drive/helpers.rs @@ -1,15 +1,15 @@ +use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; use anyhow::{Context, Result, anyhow}; +use bytes::Bytes; +use futures::Stream; use futures::StreamExt; use oauth2::{ AuthUrl, ClientId, ClientSecret, RefreshToken, TokenResponse, TokenUrl, basic::BasicClient, reqwest::Client as OAuth2ReqwestClient, }; use reqwest::{Client as ReqwestClient, StatusCode}; -use serde_json::{Value, json}; -use futures::{Stream}; -use bytes::Bytes; use reqwest::{Client, header}; -use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; +use serde_json::{Value, json}; pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Result { let http_client = OAuth2ReqwestClient::new(); @@ -34,7 +34,10 @@ pub async fn get_google_drive_token(config: &GoogleDriveProviderConfig) -> Resul Ok(token_result.access_token().secret().clone()) } -pub async fn ensure_folder_path(config: &GoogleDriveProviderConfig, path_parts: &[&str]) -> Result { +pub async fn ensure_folder_path( + config: &GoogleDriveProviderConfig, + path_parts: &[&str], +) -> Result { if path_parts.is_empty() { return Ok(config.folder_id.clone()); } @@ -152,7 +155,10 @@ pub async fn upload_stream_to_google_drive( let folder_id = ensure_folder_path(config, folder_path).await?; - if find_file_by_name(config, file_name, &folder_id).await?.is_some() { + if find_file_by_name(config, file_name, &folder_id) + .await? + .is_some() + { return Err(anyhow::anyhow!("File already exists: {}", full_path)); } @@ -172,7 +178,7 @@ pub async fn upload_stream_to_google_drive( .post("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable") .bearer_auth(&token) .header("X-Upload-Content-Type", mime) - .header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot + .header("X-Upload-Content-Length", total_size.to_string()) // Helps a lot .json(&metadata) .send() .await @@ -237,12 +243,15 @@ pub async fn upload_stream_to_google_drive( .put(&upload_url) .header("Content-Range", &content_range) .header("Content-Length", chunk_bytes.len().to_string()) - .body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed + .body(chunk_bytes.clone()) // clone is cheap if small; optimize later if needed .send() .await; match res { - Ok(resp) if resp.status().is_success() || resp.status() == StatusCode::PERMANENT_REDIRECT => { + Ok(resp) + if resp.status().is_success() + || resp.status() == StatusCode::PERMANENT_REDIRECT => + { // 200 or 308 = good uploaded += chunk_bytes.len() as u64; tracing::info!("Uploaded {}/{} bytes", uploaded, total_size); diff --git a/src/services/storage/providers/google_drive/mod.rs b/src/services/storage/providers/google_drive/mod.rs index c5182d5..864cf75 100644 --- a/src/services/storage/providers/google_drive/mod.rs +++ b/src/services/storage/providers/google_drive/mod.rs @@ -1,9 +1,12 @@ mod helpers; -mod models; +mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; +use crate::services::storage::providers::google_drive::helpers::upload_stream_to_google_drive; +use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; use crate::utils::common::BackupMethod; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; @@ -11,9 +14,6 @@ use async_trait::async_trait; use std::sync::Arc; use tokio::fs; use tracing::{error, info}; -use crate::services::backup::models::{BackupResult, UploadResult}; -use crate::services::storage::providers::google_drive::helpers::{upload_stream_to_google_drive}; -use crate::services::storage::providers::google_drive::models::GoogleDriveProviderConfig; pub struct GoogleDriveProvider {} @@ -53,13 +53,7 @@ impl StorageProvider for GoogleDriveProvider { let encrypt = encrypt.unwrap_or(false); - let upload = match build_stream( - &file_path, - encrypt, - &ctx.edge_key.master_key_b64 - ) - .await - { + let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await { Ok(u) => u, Err(e) => { error!("Stream build failed: {}", e); @@ -86,7 +80,6 @@ impl StorageProvider for GoogleDriveProvider { } }; - let file_name = full_file_name(encrypt); info!("Uploading file {}", file_name); @@ -96,12 +89,13 @@ impl StorageProvider for GoogleDriveProvider { match upload_stream_to_google_drive( &config, &remote_file_path, - upload.stream, + upload.stream, total_size, Some("application/octet-stream"), - ).await { + ) + .await + { Ok(_file_id) => { - info!("Google Drive upload successful"); UploadResult { @@ -124,6 +118,5 @@ impl StorageProvider for GoogleDriveProvider { } } } - } } diff --git a/src/services/storage/providers/google_drive/models.rs b/src/services/storage/providers/google_drive/models.rs index 98f725b..401edfd 100644 --- a/src/services/storage/providers/google_drive/models.rs +++ b/src/services/storage/providers/google_drive/models.rs @@ -7,5 +7,3 @@ pub struct GoogleDriveProviderConfig { pub refresh_token: String, pub folder_id: String, } - - diff --git a/src/services/storage/providers/local.rs b/src/services/storage/providers/local.rs index 80eae97..b857ad7 100644 --- a/src/services/storage/providers/local.rs +++ b/src/services/storage/providers/local.rs @@ -1,5 +1,6 @@ use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::utils::common::BackupMethod; use crate::utils::file::{full_file_name, full_file_path}; @@ -10,7 +11,6 @@ use reqwest::header::{HeaderMap, HeaderValue}; use std::sync::Arc; use tokio::fs; use tracing::error; -use crate::services::backup::models::{BackupResult, UploadResult}; pub struct LocalProvider; @@ -53,13 +53,7 @@ impl StorageProvider for LocalProvider { } }; - let upload = match build_stream( - &file_path, - encrypt, - &ctx.edge_key.master_key_b64 - ) - .await - { + let upload = match build_stream(&file_path, encrypt, &ctx.edge_key.master_key_b64).await { Ok(u) => u, Err(e) => { error!("Stream build failed: {}", e); @@ -68,7 +62,7 @@ impl StorageProvider for LocalProvider { success: false, error: Some(e.to_string()), remote_file_path: None, - total_size: None + total_size: None, }; } }; @@ -76,7 +70,10 @@ impl StorageProvider for LocalProvider { let mut extra_headers = HeaderMap::new(); extra_headers.insert("X-File-Name", HeaderValue::from_str(&file_name).unwrap()); - extra_headers.insert("X-File-Size", HeaderValue::from_str(&total_size.to_string()).unwrap()); + extra_headers.insert( + "X-File-Size", + HeaderValue::from_str(&total_size.to_string()).unwrap(), + ); extra_headers.insert( "X-File-Path", HeaderValue::from_str(&remote_file_path).unwrap(), @@ -90,10 +87,17 @@ impl StorageProvider for LocalProvider { "X-Method", HeaderValue::from_str(&method.to_string()).unwrap(), ); - + let tus_endpoint = format!("{}/tus/files", ctx.edge_key.server_url); - match upload_to_tus_stream_with_headers(upload.stream, &tus_endpoint, extra_headers, total_size).await { + match upload_to_tus_stream_with_headers( + upload.stream, + &tus_endpoint, + extra_headers, + total_size, + ) + .await + { Ok(_) => UploadResult { storage_id: storage.id.clone(), success: true, diff --git a/src/services/storage/providers/mod.rs b/src/services/storage/providers/mod.rs index 1d01d6d..9dd23be 100644 --- a/src/services/storage/providers/mod.rs +++ b/src/services/storage/providers/mod.rs @@ -1,3 +1,3 @@ +pub mod google_drive; pub mod local; pub mod s3; -pub mod google_drive; \ No newline at end of file diff --git a/src/services/storage/providers/s3/mod.rs b/src/services/storage/providers/s3/mod.rs index dd6fccf..368d021 100644 --- a/src/services/storage/providers/s3/mod.rs +++ b/src/services/storage/providers/s3/mod.rs @@ -2,26 +2,26 @@ mod models; use crate::core::context::Context; use crate::services::api::models::agent::status::DatabaseStorage; +use crate::services::backup::models::{BackupResult, UploadResult}; use crate::services::storage::StorageProvider; use crate::services::storage::providers::s3::models::S3ProviderConfig; use crate::utils::common::BackupMethod; use crate::utils::file::{full_file_name, full_file_path}; use crate::utils::stream::build_stream; use async_trait::async_trait; +use aws_config::retry::RetryConfig; use aws_sdk_s3 as s3; use aws_sdk_s3::config::BehaviorVersion; use aws_sdk_s3::config::Region; +use aws_sdk_s3::config::retry::ReconnectMode; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; use futures::StreamExt; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; -use aws_config::retry::RetryConfig; -use aws_sdk_s3::config::retry::ReconnectMode; use tokio::fs; use tracing::{error, info}; -use crate::services::backup::models::{BackupResult, UploadResult}; pub struct S3Provider {} @@ -97,15 +97,28 @@ impl StorageProvider for S3Provider { ); let region = Region::new(config.region.clone().unwrap_or("us-east-1".to_string())); - + let endpoint = if let Some(port) = &config.port { if port.trim().is_empty() { - format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) + format!( + "{}://{}", + if config.ssl { "https" } else { "http" }, + config.end_point_url + ) } else { - format!("{}://{}:{}", if config.ssl { "https" } else { "http" }, config.end_point_url, port) + format!( + "{}://{}:{}", + if config.ssl { "https" } else { "http" }, + config.end_point_url, + port + ) } } else { - format!("{}://{}", if config.ssl { "https" } else { "http" }, config.end_point_url) + format!( + "{}://{}", + if config.ssl { "https" } else { "http" }, + config.end_point_url + ) }; info!("S3 endpoint to {}", &endpoint); diff --git a/src/services/storage/providers/s3/models.rs b/src/services/storage/providers/s3/models.rs index ab3036c..63b795d 100644 --- a/src/services/storage/providers/s3/models.rs +++ b/src/services/storage/providers/s3/models.rs @@ -1,5 +1,5 @@ -use serde::{Deserialize, Serialize}; use crate::utils::deserializer::string_or_number_to_string; +use serde::{Deserialize, Serialize}; #[derive(Debug, Deserialize, Serialize)] pub struct S3ProviderConfig { @@ -12,4 +12,3 @@ pub struct S3ProviderConfig { #[serde(default, deserialize_with = "string_or_number_to_string")] pub port: Option, } - diff --git a/src/tasks/mod.rs b/src/tasks/mod.rs index 924dabf..a766209 100644 --- a/src/tasks/mod.rs +++ b/src/tasks/mod.rs @@ -1 +1 @@ -pub mod ping; \ No newline at end of file +pub mod ping; diff --git a/src/tasks/ping.rs b/src/tasks/ping.rs index 70d57ec..04771e0 100644 --- a/src/tasks/ping.rs +++ b/src/tasks/ping.rs @@ -21,4 +21,4 @@ pub async fn ping_server() { } tokio::time::sleep(Duration::from_secs(CONFIG.pooling as u64)).await; } -} \ No newline at end of file +} diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index 7668a49..16bb1a9 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -1,3 +1,3 @@ mod postgres; mod redis; -mod valkey; \ No newline at end of file +mod valkey; diff --git a/src/tests/domain/redis.rs b/src/tests/domain/redis.rs index f704aab..fa4d4a8 100644 --- a/src/tests/domain/redis.rs +++ b/src/tests/domain/redis.rs @@ -1,6 +1,6 @@ use tempfile::TempDir; -use testcontainers::runners::AsyncRunner; use testcontainers::ContainerAsync; +use testcontainers::runners::AsyncRunner; use testcontainers_modules::redis::Redis; use url::Host; @@ -16,10 +16,7 @@ async fn create_config() -> (ContainerAsync, DatabaseConfig) { .await .unwrap_or(Host::parse("127.0.0.1").unwrap()); - let port = container - .get_host_port_ipv4(6379) - .await - .unwrap_or(6379); + let port = container.get_host_port_ipv4(6379).await.unwrap_or(6379); let config = DatabaseConfig { name: "Test Redis".to_string(), @@ -62,4 +59,4 @@ async fn redis_backup_test() { let file_path = db.backup(backup_path, Some(true)).await.unwrap(); assert!(file_path.is_file()); -} \ No newline at end of file +} diff --git a/src/tests/domain/valkey.rs b/src/tests/domain/valkey.rs index 840f35e..6619cfd 100644 --- a/src/tests/domain/valkey.rs +++ b/src/tests/domain/valkey.rs @@ -1,11 +1,11 @@ -use tempfile::TempDir; -use testcontainers::runners::AsyncRunner; -use testcontainers::ContainerAsync; -use testcontainers_modules::valkey::{Valkey}; -use url::Host; use crate::domain::factory::DatabaseFactory; use crate::services::config::{DatabaseConfig, DbType}; use crate::tests::init_tracing_for_test; +use tempfile::TempDir; +use testcontainers::ContainerAsync; +use testcontainers::runners::AsyncRunner; +use testcontainers_modules::valkey::Valkey; +use url::Host; async fn create_config() -> (ContainerAsync, DatabaseConfig) { let container = Valkey::default().start().await.unwrap(); @@ -15,10 +15,7 @@ async fn create_config() -> (ContainerAsync, DatabaseConfig) { .await .unwrap_or(Host::parse("127.0.0.1").unwrap()); - let port = container - .get_host_port_ipv4(6379) - .await - .unwrap_or(6379); + let port = container.get_host_port_ipv4(6379).await.unwrap_or(6379); let config = DatabaseConfig { name: "Test Valkey".to_string(), @@ -61,4 +58,4 @@ async fn valkey_backup_test() { let file_path = db.backup(backup_path, Some(true)).await.unwrap(); assert!(file_path.is_file()); -} \ No newline at end of file +} diff --git a/src/tests/mod.rs b/src/tests/mod.rs index 6ece72c..168c8ef 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,7 +1,6 @@ -mod utils; mod domain; - - +mod services; +mod utils; use once_cell::sync::Lazy; use tracing_subscriber; @@ -15,4 +14,4 @@ static TRACING: Lazy<()> = Lazy::new(|| { fn init_tracing_for_test() -> () { Lazy::force(&TRACING); -} \ No newline at end of file +} diff --git a/src/tests/utils/common_tests.rs b/src/tests/utils/common_tests.rs index ad5ccbd..cff10da 100644 --- a/src/tests/utils/common_tests.rs +++ b/src/tests/utils/common_tests.rs @@ -1,5 +1,5 @@ +use crate::utils::common::{BackupMethod, vec_to_option_json}; use serde_json::json; -use crate::utils::common::{vec_to_option_json, BackupMethod}; #[test] fn backup_method_to_string_automatic() { @@ -39,8 +39,11 @@ fn vec_to_option_json_serializes_struct_vector() { let v = vec![Item { id: 1 }, Item { id: 2 }]; let result = vec_to_option_json(v); - assert_eq!(result, Some(json!([ + assert_eq!( + result, + Some(json!([ { "id": 1 }, { "id": 2 } - ]))); -} \ No newline at end of file + ])) + ); +} diff --git a/src/tests/utils/compress_tests.rs b/src/tests/utils/compress_tests.rs index f4d3b42..b735d95 100644 --- a/src/tests/utils/compress_tests.rs +++ b/src/tests/utils/compress_tests.rs @@ -1,7 +1,7 @@ -use tempfile::tempdir; -use tokio::fs::{write, read}; -use anyhow::Result; use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz}; +use anyhow::Result; +use tempfile::tempdir; +use tokio::fs::{read, write}; #[tokio::test] async fn compress_creates_tar_gz() -> Result<()> { @@ -39,7 +39,8 @@ async fn decompress_restores_file() -> Result<()> { let output_dir = tmp.path().join("out"); tokio::fs::create_dir_all(&output_dir).await?; - let extracted_files = decompress_large_tar_gz(&compress_result.compressed_path, &output_dir).await?; + let extracted_files = + decompress_large_tar_gz(&compress_result.compressed_path, &output_dir).await?; assert_eq!(extracted_files.len(), 1); let extracted_content = read(&extracted_files[0]).await?; diff --git a/src/tests/utils/deserializer.rs b/src/tests/utils/deserializer.rs index df1fd4e..2a0c8aa 100644 --- a/src/tests/utils/deserializer.rs +++ b/src/tests/utils/deserializer.rs @@ -1,9 +1,9 @@ #[cfg(test)] mod tests { + use crate::utils::deserializer::{camel_to_snake, deserialize_snake_case, to_snake_case}; use serde::Deserialize; - use toml::map::Map; use toml::Value; - use crate::utils::deserializer::{camel_to_snake, deserialize_snake_case, to_snake_case}; + use toml::map::Map; #[test] fn camel_to_snake_simple() { @@ -45,10 +45,7 @@ mod tests { let mut table2 = Map::new(); table2.insert("AnotherKey".into(), Value::Integer(2)); - let value = Value::Array(vec![ - Value::Table(table1), - Value::Table(table2), - ]); + let value = Value::Array(vec![Value::Table(table1), Value::Table(table2)]); let mut expected_table1 = Map::new(); expected_table1.insert("camel_key".into(), Value::Integer(1)); @@ -101,4 +98,4 @@ mod tests { let result = to_snake_case(value.clone()); assert_eq!(result, value); } -} \ No newline at end of file +} diff --git a/src/tests/utils/edge_key_tests.rs b/src/tests/utils/edge_key_tests.rs index 948c61f..e38d1f8 100644 --- a/src/tests/utils/edge_key_tests.rs +++ b/src/tests/utils/edge_key_tests.rs @@ -1,6 +1,6 @@ -use base64::{engine::general_purpose, Engine as _}; +use crate::utils::edge_key::{EdgeKeyError, decode_edge_key}; +use base64::{Engine as _, engine::general_purpose}; use serde_json::json; -use crate::utils::edge_key::{decode_edge_key, EdgeKeyError}; #[test] fn decode_valid_edge_key() { @@ -9,7 +9,10 @@ fn decode_valid_edge_key() { assert_eq!(decoded.server_url, "http://localhost:8887"); assert_eq!(decoded.agent_id, "625043cf-7c00-43c8-bcc9-d35199896dcd"); - assert_eq!(decoded.master_key_b64, "BXV3XolC656SV7dNgcWPGQlk++rpLI6lGDi7CPB5ieo="); + assert_eq!( + decoded.master_key_b64, + "BXV3XolC656SV7dNgcWPGQlk++rpLI6lGDi7CPB5ieo=" + ); } #[test] @@ -19,7 +22,7 @@ fn decode_edge_key_missing_field() { "agentId": "123" // masterKeyB64 is missing }) - .to_string(); + .to_string(); let b64 = general_purpose::URL_SAFE.encode(incomplete_json); let result = decode_edge_key(&b64); diff --git a/src/tests/utils/mod.rs b/src/tests/utils/mod.rs index c942141..858d6c1 100644 --- a/src/tests/utils/mod.rs +++ b/src/tests/utils/mod.rs @@ -1,5 +1,7 @@ -mod normalize_cron_tests; mod common_tests; mod compress_tests; mod deserializer; -mod edge_key_tests; \ No newline at end of file +mod edge_key_tests; +mod file_tests; +mod normalize_cron_tests; +mod stream_tests; diff --git a/src/tests/utils/normalize_cron_tests.rs b/src/tests/utils/normalize_cron_tests.rs index 2c9a1ab..2144e05 100644 --- a/src/tests/utils/normalize_cron_tests.rs +++ b/src/tests/utils/normalize_cron_tests.rs @@ -1,7 +1,7 @@ +use crate::utils::task_manager::cron::next_run_timestamp; use crate::utils::text::normalize_cron; use cron::Schedule; use std::str::FromStr; -use crate::utils::task_manager::cron::next_run_timestamp; #[test] fn normalize_adds_seconds_to_five_field_cron() { diff --git a/src/utils/common.rs b/src/utils/common.rs index afb5017..deb8ec4 100644 --- a/src/utils/common.rs +++ b/src/utils/common.rs @@ -16,11 +16,10 @@ impl ToString for BackupMethod { } } - pub fn vec_to_option_json(v: Vec) -> Option { if v.is_empty() { None } else { Some(serde_json::to_value(v).expect("serialization failed")) } -} \ No newline at end of file +} diff --git a/src/utils/compress.rs b/src/utils/compress.rs index f354b36..64bb945 100644 --- a/src/utils/compress.rs +++ b/src/utils/compress.rs @@ -4,7 +4,7 @@ use async_compression::tokio::write::GzipEncoder as AsyncGzipEncoder; use futures::StreamExt; use std::path::{Path, PathBuf}; use tokio::fs::File; -use tokio::fs::{create_dir_all}; +use tokio::fs::create_dir_all; use tokio::io::AsyncWriteExt; use tokio::io::BufReader; use tokio_tar::Archive; diff --git a/src/utils/deserializer.rs b/src/utils/deserializer.rs index e4d094a..33955ac 100644 --- a/src/utils/deserializer.rs +++ b/src/utils/deserializer.rs @@ -1,7 +1,6 @@ use serde::{Deserialize, Deserializer}; +use serde_json::Value as ValueJson; use toml::Value; -use serde_json::{Value as ValueJson, }; - pub fn deserialize_snake_case<'de, D>(deserializer: D) -> Result where @@ -39,7 +38,6 @@ pub fn camel_to_snake(s: &str) -> String { out } - pub fn string_or_number_to_string<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, diff --git a/src/utils/edge_key.rs b/src/utils/edge_key.rs index 43566c0..d18b87d 100644 --- a/src/utils/edge_key.rs +++ b/src/utils/edge_key.rs @@ -34,7 +34,7 @@ pub fn decode_edge_key(edge_key: &str) -> Result { let decoded_str = String::from_utf8_lossy(&decoded_bytes); let parsed: Value = serde_json::from_str(&decoded_str)?; - + if parsed.get("serverUrl").is_some() && parsed.get("agentId").is_some() && parsed.get("masterKeyB64").is_some() diff --git a/src/utils/file.rs b/src/utils/file.rs index f4a1b9b..2cfc31d 100644 --- a/src/utils/file.rs +++ b/src/utils/file.rs @@ -14,8 +14,8 @@ use base64::Engine; use base64::engine::general_purpose; use bytes::Bytes; use futures::Stream; -use rand::rngs::OsRng; use rand::TryRngCore; +use rand::rngs::OsRng; use tokio::io::{AsyncWriteExt, BufWriter}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; @@ -77,7 +77,8 @@ pub async fn encrypt_file_stream_gcm( rng.try_fill_bytes(&mut base_nonce).unwrap(); let key = Key::::try_from(master_key_bytes.as_slice()) - .map_err(|_| anyhow::anyhow!("Invalid AES-256 key length")).unwrap(); + .map_err(|_| anyhow::anyhow!("Invalid AES-256 key length")) + .unwrap(); let cipher = Aes256Gcm::new(&key); @@ -105,7 +106,8 @@ pub async fn encrypt_file_stream_gcm( nonce_bytes[..8].copy_from_slice(&base_nonce); nonce_bytes[8..].copy_from_slice(&chunk_index.to_be_bytes()); let nonce = Nonce::try_from(&nonce_bytes[..]) - .map_err(|_| anyhow::anyhow!("Invalid nonce length")).unwrap(); + .map_err(|_| anyhow::anyhow!("Invalid nonce length")) + .unwrap(); let ciphertext = cipher.encrypt(&nonce, &buffer[..n]).unwrap(); let mut out = Vec::with_capacity(4 + ciphertext.len()); diff --git a/src/utils/mod.rs b/src/utils/mod.rs index c6cfcf3..ebe8b00 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,12 +1,12 @@ pub mod common; +pub mod compress; +pub mod deserializer; pub mod edge_key; -pub mod redis_client; -pub mod task_manager; -pub mod text; pub mod file; pub mod locks; pub mod logging; +pub mod redis_client; +pub mod stream; +pub mod task_manager; +pub mod text; pub mod tus; -pub mod deserializer; -pub mod compress; -pub mod stream; \ No newline at end of file diff --git a/src/utils/redis_client.rs b/src/utils/redis_client.rs index e7664fb..24159c3 100644 --- a/src/utils/redis_client.rs +++ b/src/utils/redis_client.rs @@ -1,9 +1,8 @@ -use redis::{aio::MultiplexedConnection, Client}; use crate::settings::CONFIG; +use redis::{Client, aio::MultiplexedConnection}; pub async fn redis_connection() -> MultiplexedConnection { - let client = Client::open(CONFIG.redis_url.clone()) - .expect("Invalid Redis URL"); + let client = Client::open(CONFIG.redis_url.clone()).expect("Invalid Redis URL"); client .get_multiplexed_async_connection() diff --git a/src/utils/task_manager/cron.rs b/src/utils/task_manager/cron.rs index a4e5a7a..381ccd4 100644 --- a/src/utils/task_manager/cron.rs +++ b/src/utils/task_manager/cron.rs @@ -50,18 +50,11 @@ pub async fn check_and_update_cron( let metadata_changed = stored.metadata != metadata; if cron_changed || args_changed || metadata_changed { - upsert_task( - conn, - &task_name, - task, - &cron, - args.clone(), - metadata, - ) - .await - .unwrap_or_else(|e| { - tracing::error!("Failed to update task {}: {:?}", task_name, e); - }); + upsert_task(conn, &task_name, task, &cron, args.clone(), metadata) + .await + .unwrap_or_else(|e| { + tracing::error!("Failed to update task {}: {:?}", task_name, e); + }); info!( "Task {} updated (cron: {}, args: {}, metadata: {})", diff --git a/src/utils/task_manager/scheduler.rs b/src/utils/task_manager/scheduler.rs index c5b5fea..1018c79 100644 --- a/src/utils/task_manager/scheduler.rs +++ b/src/utils/task_manager/scheduler.rs @@ -1,4 +1,5 @@ use crate::core::context::Context; +use crate::services::api::models::agent::status::DatabaseStorage; use crate::services::backup::BackupService; use crate::services::config::ConfigService; use crate::utils::common::BackupMethod; @@ -11,7 +12,6 @@ use serde_json::Value; use std::sync::Arc; use tracing::error; use tracing::info; -use crate::services::api::models::agent::status::DatabaseStorage; pub async fn scheduler_loop(mut conn: MultiplexedConnection) { loop { @@ -80,16 +80,22 @@ pub async fn execute_task( let storages_value: &Value = metadata_obj .get("storages") .ok_or_else(|| anyhow::anyhow!("storages key missing"))?; - + let encrypt_value: &Value = metadata_obj .get("encrypt") .ok_or_else(|| anyhow::anyhow!("encrypt key missing"))?; - + let storages: Vec = serde_json::from_value(storages_value.clone())?; - let encrypt : bool = serde_json::from_value(encrypt_value.clone())?; + let encrypt: bool = serde_json::from_value(encrypt_value.clone())?; backup_service - .dispatch(generated_id, &config, BackupMethod::Automatic, &storages, encrypt) + .dispatch( + generated_id, + &config, + BackupMethod::Automatic, + &storages, + encrypt, + ) .await; Ok(()) diff --git a/src/utils/task_manager/tasks.rs b/src/utils/task_manager/tasks.rs index 1ec0acc..28bb115 100644 --- a/src/utils/task_manager/tasks.rs +++ b/src/utils/task_manager/tasks.rs @@ -23,7 +23,7 @@ pub async fn upsert_task( cron: cron.to_string(), args, enabled: true, - metadata + metadata, }; let payload = serde_json::to_string(&entry).unwrap(); diff --git a/src/utils/text.rs b/src/utils/text.rs index 5772fa5..c72b743 100644 --- a/src/utils/text.rs +++ b/src/utils/text.rs @@ -4,4 +4,4 @@ pub fn normalize_cron(expr: &str) -> String { } else { expr.to_string() } -} \ No newline at end of file +} From a613805dd2a1718212302fb250b3a026f8f7250b Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 09:28:29 +0100 Subject: [PATCH 03/25] fix: adding some tests --- src/tests/services/api_models_tests.rs | 112 +++++++++++++++++++++++++ src/tests/services/mod.rs | 1 + src/tests/utils/file_tests.rs | 88 +++++++++++++++++++ src/tests/utils/stream_tests.rs | 54 ++++++++++++ 4 files changed, 255 insertions(+) create mode 100644 src/tests/services/api_models_tests.rs create mode 100644 src/tests/services/mod.rs create mode 100644 src/tests/utils/file_tests.rs create mode 100644 src/tests/utils/stream_tests.rs diff --git a/src/tests/services/api_models_tests.rs b/src/tests/services/api_models_tests.rs new file mode 100644 index 0000000..0cf7bd6 --- /dev/null +++ b/src/tests/services/api_models_tests.rs @@ -0,0 +1,112 @@ +use serde_json::json; + +use crate::services::api::models::agent::backup::{BackupResponse, BackupUploadResponse}; +use crate::services::api::models::agent::restore::ResultRestoreResponse; +use crate::services::api::models::agent::status::PingResult; + +#[test] +fn backup_response_deserializes_nested_backup_id() { + let response: BackupResponse = serde_json::from_value(json!({ + "message": "created", + "backup": { + "id": "backup-123" + } + })) + .unwrap(); + + assert_eq!(response.message, "created"); + assert_eq!(response.backup.id, "backup-123"); +} + +#[test] +fn backup_upload_response_deserializes_storage_payload() { + let response: BackupUploadResponse = serde_json::from_value(json!({ + "message": "uploaded", + "backupStorage": { + "id": "storage-456" + } + })) + .unwrap(); + + assert_eq!(response.message, "uploaded"); + assert_eq!(response.backup_storage.id, "storage-456"); +} + +#[test] +fn restore_response_deserializes_status() { + let response: ResultRestoreResponse = serde_json::from_value(json!({ + "message": "ok", + "status": true + })) + .unwrap(); + + assert_eq!(response.message, "ok"); + assert!(response.status); +} + +#[test] +fn ping_result_deserializes_and_normalizes_storage_config_keys() { + let payload = json!({ + "agent": { + "id": "agent-1", + "lastContact": "2026-03-22T10:00:00Z" + }, + "databases": [{ + "dbms": "postgres", + "generatedId": "db-1", + "storages": [{ + "id": "storage-1", + "provider": "s3", + "config": { + "bucketName": "agent-backups", + "nestedConfig": { + "regionName": "eu-west-3" + }, + "allowedRegions": [ + { "regionCode": "eu-west-3" } + ] + } + }], + "encrypt": true, + "data": { + "backup": { + "action": true, + "cron": "*/5 * * * *" + }, + "restore": { + "action": false, + "file": null, + "metaFile": null + } + } + }] + }); + + let result: PingResult = serde_json::from_value(payload).unwrap(); + let storage = &result.databases[0].storages[0]; + + assert_eq!(result.agent.id, "agent-1"); + assert_eq!(result.agent.last_contact, "2026-03-22T10:00:00Z"); + assert_eq!(result.databases[0].generated_id, "db-1"); + assert_eq!(storage.provider, "s3"); + assert_eq!( + storage.config["bucket_name"].as_str(), + Some("agent-backups") + ); + assert_eq!( + storage.config["nested_config"]["region_name"].as_str(), + Some("eu-west-3") + ); + assert_eq!( + storage.config["allowed_regions"][0]["region_code"].as_str(), + Some("eu-west-3") + ); + assert_eq!( + result.databases[0].data.backup.cron.as_deref(), + Some("*/5 * * * *") + ); + assert!(result.databases[0].data.backup.action); + assert!(!result.databases[0].data.restore.action); + assert!(result.databases[0].data.restore.file.is_none()); + assert!(result.databases[0].data.restore.meta_file.is_none()); +} diff --git a/src/tests/services/mod.rs b/src/tests/services/mod.rs new file mode 100644 index 0000000..4a12318 --- /dev/null +++ b/src/tests/services/mod.rs @@ -0,0 +1 @@ +mod api_models_tests; diff --git a/src/tests/utils/file_tests.rs b/src/tests/utils/file_tests.rs new file mode 100644 index 0000000..26581b9 --- /dev/null +++ b/src/tests/utils/file_tests.rs @@ -0,0 +1,88 @@ +use anyhow::Result; +use base64::{Engine as _, engine::general_purpose}; +use futures::StreamExt; +use serde_json::Value; +use tempfile::tempdir; +use tokio::fs; +use tokio::io::AsyncWriteExt; + +use crate::utils::file::{ + decrypt_file_stream_gcm, encrypt_file_stream_gcm, full_extension, full_file_name, + full_file_path, +}; + +#[test] +fn full_extension_returns_everything_after_first_dot() { + assert_eq!( + full_extension(std::path::Path::new("archive.tar.gz")), + ".tar.gz" + ); + assert_eq!(full_extension(std::path::Path::new("README")), ""); +} + +#[test] +fn full_file_name_matches_expected_suffix() { + let unencrypted = full_file_name(false); + let encrypted = full_file_name(true); + + assert!(unencrypted.ends_with(".tar.gz")); + assert!(encrypted.ends_with(".tar.gz.enc")); +} + +#[test] +fn full_file_path_prefixes_backups_directory_and_date() { + let file_name = "backup.tar.gz".to_string(); + let full_path = full_file_path(&file_name); + + assert!(full_path.starts_with("backups/")); + assert!(full_path.ends_with("/backup.tar.gz")); +} + +#[tokio::test] +async fn encrypt_and_decrypt_round_trip_restores_original_bytes() -> Result<()> { + let tmp = tempdir()?; + let input_path = tmp.path().join("plain.txt"); + let encrypted_path = tmp.path().join("cipher.bin"); + let decrypted_path = tmp.path().join("plain.out.txt"); + let original = b"encryption round-trip payload"; + + fs::write(&input_path, original).await?; + + let key = general_purpose::STANDARD.encode([7u8; 32]); + let mut encrypted_stream = encrypt_file_stream_gcm(input_path.clone(), key.clone()).await?; + let mut encrypted_file = fs::File::create(&encrypted_path).await?; + + while let Some(chunk) = encrypted_stream.next().await { + encrypted_file.write_all(&chunk?).await?; + } + + decrypt_file_stream_gcm(encrypted_path, decrypted_path.clone(), key).await?; + + let decrypted = fs::read(decrypted_path).await?; + assert_eq!(decrypted, original); + + Ok(()) +} + +#[tokio::test] +async fn encrypt_stream_starts_with_json_header_line() -> Result<()> { + let tmp = tempdir()?; + let input_path = tmp.path().join("plain.txt"); + fs::write(&input_path, b"header test").await?; + + let key = general_purpose::STANDARD.encode([9u8; 32]); + let mut encrypted_stream = encrypt_file_stream_gcm(input_path, key).await?; + let first_chunk = encrypted_stream.next().await.unwrap()?; + + let header_end = first_chunk + .iter() + .position(|byte| *byte == b'\n') + .expect("missing header newline"); + let header: Value = serde_json::from_slice(&first_chunk[..header_end])?; + + assert_eq!(header["version"], 1); + assert_eq!(header["cipher"], "AES-256-GCM"); + assert_eq!(header["chunk_size"], 16 * 1024 * 1024); + + Ok(()) +} diff --git a/src/tests/utils/stream_tests.rs b/src/tests/utils/stream_tests.rs new file mode 100644 index 0000000..b31cc24 --- /dev/null +++ b/src/tests/utils/stream_tests.rs @@ -0,0 +1,54 @@ +use anyhow::Result; +use base64::{Engine as _, engine::general_purpose}; +use futures::StreamExt; +use tempfile::tempdir; +use tokio::fs; + +use crate::utils::file::decrypt_file_stream_gcm; +use crate::utils::stream::build_stream; + +#[tokio::test] +async fn build_stream_without_encryption_yields_original_bytes() -> Result<()> { + let tmp = tempdir()?; + let file_path = tmp.path().join("plain.txt"); + let content = b"plain upload stream"; + fs::write(&file_path, content).await?; + + let mut upload_stream = build_stream(&file_path, false, &String::new()) + .await? + .stream; + let mut collected = Vec::new(); + + while let Some(chunk) = upload_stream.next().await { + collected.extend_from_slice(&chunk?); + } + + assert_eq!(collected, content); + + Ok(()) +} + +#[tokio::test] +async fn build_stream_with_encryption_produces_decryptable_content() -> Result<()> { + let tmp = tempdir()?; + let input_path = tmp.path().join("plain.txt"); + let encrypted_path = tmp.path().join("encrypted.bin"); + let decrypted_path = tmp.path().join("decrypted.txt"); + let content = b"encrypted upload stream"; + fs::write(&input_path, content).await?; + + let key = general_purpose::STANDARD.encode([5u8; 32]); + let mut upload_stream = build_stream(&input_path, true, &key).await?.stream; + let mut encrypted_bytes = Vec::new(); + + while let Some(chunk) = upload_stream.next().await { + encrypted_bytes.extend_from_slice(&chunk?); + } + + fs::write(&encrypted_path, encrypted_bytes).await?; + decrypt_file_stream_gcm(encrypted_path, decrypted_path.clone(), key).await?; + + assert_eq!(fs::read(decrypted_path).await?, content); + + Ok(()) +} From 808e42a4727ec59497a0b29eec9ea0d5267b5fe2 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 11:25:36 +0100 Subject: [PATCH 04/25] fix: working on pipeline --- .github/workflows/codecov.yml | 84 ++++++++++++++--- Cargo.toml | 4 +- assets/tools/amd64/mongodb/bin/bsondump | Bin assets/tools/amd64/mongodb/bin/mongodump | Bin assets/tools/amd64/mongodb/bin/mongoexport | Bin assets/tools/amd64/mongodb/bin/mongofiles | Bin assets/tools/amd64/mongodb/bin/mongoimport | Bin assets/tools/amd64/mongodb/bin/mongorestore | Bin assets/tools/amd64/mongodb/bin/mongostat | Bin assets/tools/amd64/mongodb/bin/mongotop | Bin assets/tools/arm64/mongodb/bin/bsondump | Bin assets/tools/arm64/mongodb/bin/mongodump | Bin assets/tools/arm64/mongodb/bin/mongoexport | Bin assets/tools/arm64/mongodb/bin/mongofiles | Bin assets/tools/arm64/mongodb/bin/mongoimport | Bin assets/tools/arm64/mongodb/bin/mongorestore | Bin assets/tools/arm64/mongodb/bin/mongostat | Bin assets/tools/arm64/mongodb/bin/mongotop | Bin docker-compose.test.yml | 17 ++++ docker-compose.yml | 4 +- scripts/tests/requirements.sh | 3 + src/domain/mongodb/backup.rs | 1 + src/domain/postgres/connection.rs | 3 +- src/tests/domain/mariadb.rs | 94 +++++++++++++++++++ src/tests/domain/mod.rs | 3 + src/tests/domain/mongodb.rs | 98 ++++++++++++++++++++ src/tests/domain/mysql.rs | 94 +++++++++++++++++++ 27 files changed, 387 insertions(+), 18 deletions(-) mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/bsondump mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongodump mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongoexport mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongofiles mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongoimport mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongorestore mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongostat mode change 100644 => 100755 assets/tools/amd64/mongodb/bin/mongotop mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/bsondump mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongodump mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongoexport mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongofiles mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongoimport mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongorestore mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongostat mode change 100644 => 100755 assets/tools/arm64/mongodb/bin/mongotop create mode 100644 docker-compose.test.yml create mode 100644 src/tests/domain/mariadb.rs create mode 100644 src/tests/domain/mongodb.rs create mode 100644 src/tests/domain/mysql.rs diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 9a430b3..fd3b147 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,3 +1,62 @@ +#name: Codecov Rust +# +#on: +# push: +# branches: ["main"] +# pull_request: +# branches: ["main"] +# +#env: +# CARGO_TERM_COLOR: always +# +#jobs: +# build: +# runs-on: ubuntu-latest +# +# steps: +# - uses: actions/checkout@v3 +# +# - uses: actions-rs/toolchain@v1 +# with: +# toolchain: stable +# override: true +# components: llvm-tools-preview +# +# - name: Install grcov +# run: cargo install grcov +# +# - name: Install test requirements +# run: bash scripts/tests/requirements.sh +# +# - name: Build +# run: cargo build --verbose +# +# - name: Run tests +# env: +# CARGO_INCREMENTAL: 0 +# RUSTFLAGS: "-C instrument-coverage" +# LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw" +# run: cargo test --verbose +# +# - name: Generate coverage +# run: | +# grcov . \ +# --binary-path ./target/debug/ \ +# -s . \ +# -t lcov \ +# --branch \ +# --ignore-not-existing \ +# -o lcov.info +# +# - name: Upload to Codecov +# uses: codecov/codecov-action@v5 +# with: +# files: lcov.info +# verbose: true +# fail_ci_if_error: true +# env: +# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + name: Codecov Rust on: @@ -16,27 +75,24 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: llvm-tools-preview - - name: Install grcov run: cargo install grcov - - name: Install test requirements - run: bash scripts/tests/requirements.sh - - - name: Build - run: cargo build --verbose + - name: Build test image + run: docker compose -f docker-compose.test.yml build agent-test - name: Run tests env: CARGO_INCREMENTAL: 0 RUSTFLAGS: "-C instrument-coverage" - LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw" - run: cargo test --verbose + LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" + run: | + mkdir -p coverage + docker compose -f docker-compose.test.yml run --rm \ + -e CARGO_INCREMENTAL \ + -e RUSTFLAGS \ + -e LLVM_PROFILE_FILE \ + agent-test cargo test --verbose - name: Generate coverage run: | @@ -55,4 +111,4 @@ jobs: verbose: true fail_ci_if_error: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/Cargo.toml b/Cargo.toml index 3c32d59..26c3908 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ tokio-stream = "0.1.18" aes = "0.9.0-rc.4" typenum = "1.19.0" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "valkey", "mysql", "mariadb", "mongo"] } postgres = "0.19.12" url = "2.5.8" @@ -57,7 +57,7 @@ url = "2.5.8" tokio = { version = "1", features = ["full"] } mockall = "0.13" testcontainers = "0.27.1" -testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis"] } +testcontainers-modules = { version = "0.15.0", features = ["postgres", "redis", "mysql", "mariadb", "mongo"] } wiremock = "0.6" [profile.release] diff --git a/assets/tools/amd64/mongodb/bin/bsondump b/assets/tools/amd64/mongodb/bin/bsondump old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongodump b/assets/tools/amd64/mongodb/bin/mongodump old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongoexport b/assets/tools/amd64/mongodb/bin/mongoexport old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongofiles b/assets/tools/amd64/mongodb/bin/mongofiles old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongoimport b/assets/tools/amd64/mongodb/bin/mongoimport old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongorestore b/assets/tools/amd64/mongodb/bin/mongorestore old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongostat b/assets/tools/amd64/mongodb/bin/mongostat old mode 100644 new mode 100755 diff --git a/assets/tools/amd64/mongodb/bin/mongotop b/assets/tools/amd64/mongodb/bin/mongotop old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/bsondump b/assets/tools/arm64/mongodb/bin/bsondump old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongodump b/assets/tools/arm64/mongodb/bin/mongodump old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongoexport b/assets/tools/arm64/mongodb/bin/mongoexport old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongofiles b/assets/tools/arm64/mongodb/bin/mongofiles old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongoimport b/assets/tools/arm64/mongodb/bin/mongoimport old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongorestore b/assets/tools/arm64/mongodb/bin/mongorestore old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongostat b/assets/tools/arm64/mongodb/bin/mongostat old mode 100644 new mode 100755 diff --git a/assets/tools/arm64/mongodb/bin/mongotop b/assets/tools/arm64/mongodb/bin/mongotop old mode 100644 new mode 100755 diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 0000000..21c9a18 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,17 @@ +services: + agent-test: + build: + context: . + dockerfile: docker/Dockerfile + target: dev + working_dir: /app + volumes: + - .:/app + - /var/run/docker.sock:/var/run/docker.sock + environment: + APP_ENV: test + LOG: debug + TZ: Europe/Paris + EDGE_KEY: "" + + diff --git a/docker-compose.yml b/docker-compose.yml index 51e7d19..0e3b0a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,9 @@ services: # - ./databases.toml:/config/config.toml - cargo-registry:/usr/local/cargo/registry - cargo-git:/usr/local/cargo/git -# - cargo-target:/app/target + - /var/run/docker.sock:/var/run/docker.sock + + # - cargo-target:/app/target # - sqlite-data:/sqlite-data/workspace/data # - ./scripts/sqlite/test-db:/sqlite-data-2/workspace/data environment: diff --git a/scripts/tests/requirements.sh b/scripts/tests/requirements.sh index 324244e..3a9f2ab 100755 --- a/scripts/tests/requirements.sh +++ b/scripts/tests/requirements.sh @@ -70,6 +70,9 @@ elif [[ "$OS_TYPE" == "Darwin" ]]; then echo "macOS detected. Installing prerequisites..." brew install redis brew install valkey + brew install mysql-client + brew tap mongodb/brew + brew install mongodb-database-tools sudo mkdir -p "$POSTGRES_BASE" sudo chown -R "$(whoami)" "$POSTGRES_BASE" diff --git a/src/domain/mongodb/backup.rs b/src/domain/mongodb/backup.rs index d4724b3..7abb6fd 100644 --- a/src/domain/mongodb/backup.rs +++ b/src/domain/mongodb/backup.rs @@ -15,6 +15,7 @@ pub async fn run( let file_path = backup_dir.join(format!("{}{}", cfg.generated_id, file_extension)); let mongodump = select_mongo_path().join("mongodump"); + info!("{:?}", mongodump); let uri = get_mongo_uri(cfg.clone())?; let output = Command::new(mongodump) diff --git a/src/domain/postgres/connection.rs b/src/domain/postgres/connection.rs index 3145f9a..585d369 100644 --- a/src/domain/postgres/connection.rs +++ b/src/domain/postgres/connection.rs @@ -32,7 +32,8 @@ pub fn select_pg_path(version: &str, is_test: Option) -> std::path::PathBu let major = version.split('.').next().unwrap_or("17"); if is_test.unwrap_or(false) { - format!("/usr/local/postgresql/{}/bin", major).into() + // format!("/usr/local/postgresql/{}/bin", major).into() + format!("/usr/lib/postgresql/{}/bin", major).into() } else { format!("/usr/lib/postgresql/{}/bin", major).into() } diff --git a/src/tests/domain/mariadb.rs b/src/tests/domain/mariadb.rs new file mode 100644 index 0000000..9837745 --- /dev/null +++ b/src/tests/domain/mariadb.rs @@ -0,0 +1,94 @@ +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; +use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz}; +use oauth2::url; +use std::path::PathBuf; +use tempfile::TempDir; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; +use testcontainers_modules::mariadb::Mariadb; +use tracing::{error, info}; +use url::Host; + +async fn create_config() -> (ContainerAsync, DatabaseConfig) { + let container = Mariadb::default().with_tag("11.3").start().await.unwrap(); + + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container.get_host_port_ipv4(3306).await.unwrap_or(3306); + + let config = DatabaseConfig { + name: "Test MariaDB".to_string(), + database: "test".to_string(), + db_type: DbType::Mariadb, + username: "root".to_string(), + password: "".to_string(), + port, + host: host.to_string(), + generated_id: "3c4b4eb4-c2c6-4bde-a423-ee1385dcf6d2".to_string(), + path: "".to_string(), + }; + + (container, config) +} + +#[tokio::test] +async fn mariadb_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let reachable = db.ping().await.unwrap_or(false); + + assert!(reachable); +} + +#[tokio::test] +async fn mariadb_backup_restore_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + + assert!(file_path.is_file()); + + let compression = compress_to_tar_gz_large(&file_path).await.unwrap(); + assert!(compression.compressed_path.is_file()); + + let files = decompress_large_tar_gz(compression.compressed_path.as_path(), temp_dir.path()) + .await + .unwrap(); + + let backup_file: PathBuf = if files.len() == 1 { + files[0].clone() + } else { + "".into() + }; + + let db = DatabaseFactory::create_for_restore(config.clone(), &backup_file).await; + let reachable = db.ping().await.unwrap_or(false); + + info!("Reachable: {}", reachable); + assert!(reachable); + + match db.restore(&backup_file, Some(true)).await { + Ok(_) => { + info!("Restore succeeded for {}", config.generated_id); + assert!(true) + } + Err(e) => { + error!("Restore failed for {}: {:?}", config.generated_id, e); + assert!(false) + } + } +} diff --git a/src/tests/domain/mod.rs b/src/tests/domain/mod.rs index 16bb1a9..bfcb388 100644 --- a/src/tests/domain/mod.rs +++ b/src/tests/domain/mod.rs @@ -1,3 +1,6 @@ +mod mariadb; +mod mongodb; +mod mysql; mod postgres; mod redis; mod valkey; diff --git a/src/tests/domain/mongodb.rs b/src/tests/domain/mongodb.rs new file mode 100644 index 0000000..d6e6e99 --- /dev/null +++ b/src/tests/domain/mongodb.rs @@ -0,0 +1,98 @@ +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; +use mongodb::{Client, bson::doc}; +use tempfile::TempDir; +use testcontainers::ContainerAsync; +use testcontainers::runners::AsyncRunner; +use testcontainers_modules::mongo::Mongo; +use tracing::{error, info}; +use url::Host; + +async fn create_config() -> (ContainerAsync, DatabaseConfig) { + let container = Mongo::default().start().await.unwrap(); + + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container.get_host_port_ipv4(27017).await.unwrap_or(27017); + + let config = DatabaseConfig { + name: "Test MongoDB".to_string(), + database: "testdb".to_string(), + db_type: DbType::MongoDB, + username: "".to_string(), + password: "".to_string(), + port, + host: host.to_string(), + generated_id: "96d30a9f-ff4b-47c9-aaab-f3147bb34f16".to_string(), + path: "".to_string(), + }; + + (container, config) +} + +async fn seed_database(config: &DatabaseConfig) { + let client = Client::with_uri_str(format!( + "mongodb://{}:{}/{}", + config.host, config.port, config.database + )) + .await + .unwrap(); + + let collection = client + .database(&config.database) + .collection::("sample"); + + collection + .insert_one(doc! { "name": "hello mongo" }) + .await + .unwrap(); +} + +#[tokio::test] +async fn mongodb_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let reachable = db.ping().await.unwrap_or(false); + + assert!(reachable); +} + +#[tokio::test] +async fn mongodb_backup_restore_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + seed_database(&config).await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + info!("Backup path: {:?}", file_path); + assert!(file_path.is_file()); + + let db = DatabaseFactory::create_for_restore(config.clone(), &file_path).await; + let reachable = db.ping().await.unwrap_or(false); + + info!("Reachable: {}", reachable); + assert!(reachable); + + match db.restore(&file_path, Some(true)).await { + Ok(_) => { + info!("Restore succeeded for {}", config.generated_id); + assert!(true) + } + Err(e) => { + error!("Restore failed for {}: {:?}", config.generated_id, e); + assert!(false) + } + } +} diff --git a/src/tests/domain/mysql.rs b/src/tests/domain/mysql.rs new file mode 100644 index 0000000..5fd0aed --- /dev/null +++ b/src/tests/domain/mysql.rs @@ -0,0 +1,94 @@ +use crate::domain::factory::DatabaseFactory; +use crate::services::config::{DatabaseConfig, DbType}; +use crate::tests::init_tracing_for_test; +use crate::utils::compress::{compress_to_tar_gz_large, decompress_large_tar_gz}; +use oauth2::url; +use std::path::PathBuf; +use tempfile::TempDir; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, ImageExt}; +use testcontainers_modules::mysql::Mysql; +use tracing::{error, info}; +use url::Host; + +async fn create_config() -> (ContainerAsync, DatabaseConfig) { + let container = Mysql::default().with_tag("8.1").start().await.unwrap(); + + let host = container + .get_host() + .await + .unwrap_or(Host::parse("127.0.0.1").unwrap()); + + let port = container.get_host_port_ipv4(3306).await.unwrap_or(3306); + + let config = DatabaseConfig { + name: "Test MySQL".to_string(), + database: "test".to_string(), + db_type: DbType::Mysql, + username: "root".to_string(), + password: "".to_string(), + port, + host: host.to_string(), + generated_id: "0f1bb8f2-35a0-4c91-8098-e36873d3ce31".to_string(), + path: "".to_string(), + }; + + (container, config) +} + +#[tokio::test] +async fn mysql_ping_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let reachable = db.ping().await.unwrap_or(false); + + assert!(reachable); +} + +#[tokio::test] +async fn mysql_backup_restore_test() { + init_tracing_for_test(); + + let (_container, config) = create_config().await; + + let temp_dir = TempDir::new().unwrap(); + let backup_path = temp_dir.path(); + + let db = DatabaseFactory::create_for_backup(config.clone()).await; + let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + + assert!(file_path.is_file()); + + let compression = compress_to_tar_gz_large(&file_path).await.unwrap(); + assert!(compression.compressed_path.is_file()); + + let files = decompress_large_tar_gz(compression.compressed_path.as_path(), temp_dir.path()) + .await + .unwrap(); + + let backup_file: PathBuf = if files.len() == 1 { + files[0].clone() + } else { + "".into() + }; + + let db = DatabaseFactory::create_for_restore(config.clone(), &backup_file).await; + let reachable = db.ping().await.unwrap_or(false); + + info!("Reachable: {}", reachable); + assert!(reachable); + + match db.restore(&backup_file, Some(true)).await { + Ok(_) => { + info!("Restore succeeded for {}", config.generated_id); + assert!(true) + } + Err(e) => { + error!("Restore failed for {}: {:?}", config.generated_id, e); + assert!(false) + } + } +} From 3528f775da79dad580f78477e127d329968239c5 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 11:38:15 +0100 Subject: [PATCH 05/25] fix: working on pipeline --- .github/workflows/codecov.yml | 24 +++++++++++++++++++----- docker-compose.test.yml | 7 ++++++- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index fd3b147..b1e105c 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -56,7 +56,6 @@ # fail_ci_if_error: true # env: # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Codecov Rust on: @@ -69,11 +68,16 @@ env: CARGO_TERM_COLOR: always jobs: - build: + coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview - name: Install grcov run: cargo install grcov @@ -81,23 +85,33 @@ jobs: - name: Build test image run: docker compose -f docker-compose.test.yml build agent-test - - name: Run tests + - name: Run tests in container env: + CARGO_TARGET_DIR: /app/target CARGO_INCREMENTAL: 0 RUSTFLAGS: "-C instrument-coverage" LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | mkdir -p coverage docker compose -f docker-compose.test.yml run --rm \ + -e CARGO_TARGET_DIR \ -e CARGO_INCREMENTAL \ -e RUSTFLAGS \ -e LLVM_PROFILE_FILE \ agent-test cargo test --verbose - - name: Generate coverage + - name: Verify generated coverage files + run: | + ls -la coverage + find coverage -name '*.profraw' | head -50 + ls -la target/debug + + - name: Generate lcov report run: | + LLVM_PATH="$(rustc --print target-libdir | sed 's#/rustlib/.*#/bin#')" grcov . \ --binary-path ./target/debug/ \ + --llvm-path "$LLVM_PATH" \ -s . \ -t lcov \ --branch \ diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 21c9a18..4022ede 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -5,8 +5,11 @@ services: dockerfile: docker/Dockerfile target: dev working_dir: /app + container_name: agent-test volumes: - .:/app + - cargo-registry:/usr/local/cargo/registry + - cargo-git:/usr/local/cargo/git - /var/run/docker.sock:/var/run/docker.sock environment: APP_ENV: test @@ -14,4 +17,6 @@ services: TZ: Europe/Paris EDGE_KEY: "" - +volumes: + cargo-registry: + cargo-git: From 3137c5080c87f8a690f88e9578ab2e0137faf746 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 11:50:13 +0100 Subject: [PATCH 06/25] fix: working on pipeline --- .github/workflows/codecov.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index b1e105c..6526e93 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -108,7 +108,17 @@ jobs: - name: Generate lcov report run: | - LLVM_PATH="$(rustc --print target-libdir | sed 's#/rustlib/.*#/bin#')" + HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" + SYSROOT="$(rustc --print sysroot)" + LLVM_PATH="$SYSROOT/lib/rustlib/$HOST_TRIPLE/bin" + + echo "HOST_TRIPLE=$HOST_TRIPLE" + echo "SYSROOT=$SYSROOT" + echo "LLVM_PATH=$LLVM_PATH" + + test -x "$LLVM_PATH/llvm-profdata" + "$LLVM_PATH/llvm-profdata" --version + grcov . \ --binary-path ./target/debug/ \ --llvm-path "$LLVM_PATH" \ From a3cd16e654970bd5d8fff6a5adbc629ecfdcd441 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 13:53:18 +0100 Subject: [PATCH 07/25] fix: working on pipeline --- .github/workflows/codecov.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 6526e93..d016459 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -122,12 +122,24 @@ jobs: grcov . \ --binary-path ./target/debug/ \ --llvm-path "$LLVM_PATH" \ - -s . \ + -s /app \ -t lcov \ --branch \ --ignore-not-existing \ -o lcov.info + - name: Normalize lcov paths + run: | + sed -i "s#SF:/app/#SF:${GITHUB_WORKSPACE}/#g" lcov.info + echo "First source files in lcov:" + grep '^SF:' lcov.info | head -20 + + - name: Verify lcov paths exist + run: | + grep '^SF:' lcov.info | sed 's/^SF://' | head -20 | while read -r f; do + test -f "$f" && echo "OK $f" || echo "MISSING $f" + done + - name: Upload to Codecov uses: codecov/codecov-action@v5 with: From d8bfbe79a5fa46144fa1be2a88740f27f89eb18d Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 14:06:05 +0100 Subject: [PATCH 08/25] fix: working on pipeline --- .github/workflows/codecov.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index d016459..e890d36 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -103,7 +103,7 @@ jobs: - name: Verify generated coverage files run: | ls -la coverage - find coverage -name '*.profraw' | head -50 + find coverage -name '*.profraw' | head -50 || true ls -la target/debug - name: Generate lcov report @@ -122,7 +122,7 @@ jobs: grcov . \ --binary-path ./target/debug/ \ --llvm-path "$LLVM_PATH" \ - -s /app \ + -s . \ -t lcov \ --branch \ --ignore-not-existing \ @@ -130,7 +130,7 @@ jobs: - name: Normalize lcov paths run: | - sed -i "s#SF:/app/#SF:${GITHUB_WORKSPACE}/#g" lcov.info + sed -i 's#SF:/app/#SF:#g' lcov.info echo "First source files in lcov:" grep '^SF:' lcov.info | head -20 From c3ec58c2ff506f2e604ce2f5e1e159db27513dce Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 14:22:23 +0100 Subject: [PATCH 09/25] fix: working on pipeline --- .github/workflows/codecov.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index e890d36..199e703 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -92,7 +92,9 @@ jobs: RUSTFLAGS: "-C instrument-coverage" LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | + rm -rf coverage mkdir -p coverage + docker compose -f docker-compose.test.yml run --rm \ -e CARGO_TARGET_DIR \ -e CARGO_INCREMENTAL \ @@ -119,15 +121,19 @@ jobs: test -x "$LLVM_PATH/llvm-profdata" "$LLVM_PATH/llvm-profdata" --version - grcov . \ - --binary-path ./target/debug/ \ + grcov coverage \ + --binary-path ./target/debug \ --llvm-path "$LLVM_PATH" \ -s . \ -t lcov \ --branch \ --ignore-not-existing \ + --ignore "/*" \ + --ignore "target/*" \ -o lcov.info + test -s lcov.info || (echo "lcov empty" && exit 1) + - name: Normalize lcov paths run: | sed -i 's#SF:/app/#SF:#g' lcov.info @@ -147,4 +153,4 @@ jobs: verbose: true fail_ci_if_error: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 8a1de258bb676de100140fa548df0e842655e837 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 14:33:50 +0100 Subject: [PATCH 10/25] fix: working on pipeline --- .github/workflows/codecov.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 199e703..07f65e5 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -114,24 +114,28 @@ jobs: SYSROOT="$(rustc --print sysroot)" LLVM_PATH="$SYSROOT/lib/rustlib/$HOST_TRIPLE/bin" - echo "HOST_TRIPLE=$HOST_TRIPLE" - echo "SYSROOT=$SYSROOT" echo "LLVM_PATH=$LLVM_PATH" test -x "$LLVM_PATH/llvm-profdata" - "$LLVM_PATH/llvm-profdata" --version + test -x "$LLVM_PATH/llvm-cov" - grcov coverage \ + # Merge des profils LLVM (obligatoire) + "$LLVM_PATH/llvm-profdata" merge -sparse coverage/*.profraw -o coverage/merged.profdata + + # Génération du rapport lcov + grcov . \ --binary-path ./target/debug \ --llvm-path "$LLVM_PATH" \ - -s . \ - -t lcov \ --branch \ --ignore-not-existing \ --ignore "/*" \ --ignore "target/*" \ + --llvm \ + --instr-profile coverage/merged.profdata \ + -t lcov \ -o lcov.info + # Fail si vide test -s lcov.info || (echo "lcov empty" && exit 1) - name: Normalize lcov paths From 5519e1e5b31417b84c7d26a659aa71243df3b9f8 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 14:44:58 +0100 Subject: [PATCH 11/25] fix: working on pipeline --- .github/workflows/codecov.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 07f65e5..63fef16 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -119,23 +119,22 @@ jobs: test -x "$LLVM_PATH/llvm-profdata" test -x "$LLVM_PATH/llvm-cov" - # Merge des profils LLVM (obligatoire) + # Merge des profils LLVM "$LLVM_PATH/llvm-profdata" merge -sparse coverage/*.profraw -o coverage/merged.profdata - # Génération du rapport lcov - grcov . \ + # Génération du rapport lcov (compatible grcov actuel) + grcov coverage \ --binary-path ./target/debug \ --llvm-path "$LLVM_PATH" \ --branch \ --ignore-not-existing \ --ignore "/*" \ --ignore "target/*" \ - --llvm \ - --instr-profile coverage/merged.profdata \ + --llvm coverage/merged.profdata \ -t lcov \ -o lcov.info - # Fail si vide + # Fail si coverage vide test -s lcov.info || (echo "lcov empty" && exit 1) - name: Normalize lcov paths From a9afc387c1a7dc9faf6d156e5368fa037b575208 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 15:05:33 +0100 Subject: [PATCH 12/25] fix: working on pipeline --- .github/workflows/codecov.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 63fef16..0ccb73b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -119,22 +119,16 @@ jobs: test -x "$LLVM_PATH/llvm-profdata" test -x "$LLVM_PATH/llvm-cov" - # Merge des profils LLVM - "$LLVM_PATH/llvm-profdata" merge -sparse coverage/*.profraw -o coverage/merged.profdata - - # Génération du rapport lcov (compatible grcov actuel) - grcov coverage \ + grcov . \ --binary-path ./target/debug \ --llvm-path "$LLVM_PATH" \ + -s . \ --branch \ --ignore-not-existing \ - --ignore "/*" \ --ignore "target/*" \ - --llvm coverage/merged.profdata \ -t lcov \ -o lcov.info - # Fail si coverage vide test -s lcov.info || (echo "lcov empty" && exit 1) - name: Normalize lcov paths @@ -156,4 +150,4 @@ jobs: verbose: true fail_ci_if_error: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From 0f1ef18334110b0609e669cb041c79f3b10e725e Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 15:16:32 +0100 Subject: [PATCH 13/25] fix: working on pipeline --- .github/workflows/codecov.yml | 41 +++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 0ccb73b..b0c7a65 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -132,10 +132,38 @@ jobs: test -s lcov.info || (echo "lcov empty" && exit 1) - name: Normalize lcov paths + env: + WORKSPACE: ${{ github.workspace }} run: | - sed -i 's#SF:/app/#SF:#g' lcov.info + python3 <<'PY' + from pathlib import Path + import os + + workspace = Path(os.environ["WORKSPACE"]).resolve() + lines = Path("lcov.info").read_text().splitlines() + out = [] + + for line in lines: + if line.startswith("SF:"): + raw = line[3:] + + if raw.startswith("/app/"): + raw = raw[len("/app/"):] + + p = Path(raw) + + if not p.is_absolute(): + p = workspace / p + + out.append(f"SF:{p}") + else: + out.append(line) + + Path("lcov.info").write_text("\n".join(out) + "\n") + PY + echo "First source files in lcov:" - grep '^SF:' lcov.info | head -20 + grep '^SF:' lcov.info | head -20 || true - name: Verify lcov paths exist run: | @@ -143,6 +171,15 @@ jobs: test -f "$f" && echo "OK $f" || echo "MISSING $f" done + - name: Debug lcov stats + run: | + echo "SF count:" + grep -c '^SF:' lcov.info || true + echo "DA count:" + grep -c '^DA:' lcov.info || true + echo "end_of_record count:" + grep -c '^end_of_record' lcov.info || true + - name: Upload to Codecov uses: codecov/codecov-action@v5 with: From 6f1b33b8100fda804b375ffab7e8ac2c12509fc7 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 17:53:10 +0100 Subject: [PATCH 14/25] fix: working on pipeline --- .github/workflows/codecov.yml | 79 +++-------------------------------- 1 file changed, 5 insertions(+), 74 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index b0c7a65..b23e5b7 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -90,10 +90,8 @@ jobs: CARGO_TARGET_DIR: /app/target CARGO_INCREMENTAL: 0 RUSTFLAGS: "-C instrument-coverage" - LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" + LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw" run: | - rm -rf coverage - mkdir -p coverage docker compose -f docker-compose.test.yml run --rm \ -e CARGO_TARGET_DIR \ @@ -102,84 +100,16 @@ jobs: -e LLVM_PROFILE_FILE \ agent-test cargo test --verbose - - name: Verify generated coverage files + - name: Generate coverage run: | - ls -la coverage - find coverage -name '*.profraw' | head -50 || true - ls -la target/debug - - - name: Generate lcov report - run: | - HOST_TRIPLE="$(rustc -vV | sed -n 's/^host: //p')" - SYSROOT="$(rustc --print sysroot)" - LLVM_PATH="$SYSROOT/lib/rustlib/$HOST_TRIPLE/bin" - - echo "LLVM_PATH=$LLVM_PATH" - - test -x "$LLVM_PATH/llvm-profdata" - test -x "$LLVM_PATH/llvm-cov" - grcov . \ - --binary-path ./target/debug \ - --llvm-path "$LLVM_PATH" \ + --binary-path ./target/debug/ \ -s . \ + -t lcov \ --branch \ --ignore-not-existing \ - --ignore "target/*" \ - -t lcov \ -o lcov.info - test -s lcov.info || (echo "lcov empty" && exit 1) - - - name: Normalize lcov paths - env: - WORKSPACE: ${{ github.workspace }} - run: | - python3 <<'PY' - from pathlib import Path - import os - - workspace = Path(os.environ["WORKSPACE"]).resolve() - lines = Path("lcov.info").read_text().splitlines() - out = [] - - for line in lines: - if line.startswith("SF:"): - raw = line[3:] - - if raw.startswith("/app/"): - raw = raw[len("/app/"):] - - p = Path(raw) - - if not p.is_absolute(): - p = workspace / p - - out.append(f"SF:{p}") - else: - out.append(line) - - Path("lcov.info").write_text("\n".join(out) + "\n") - PY - - echo "First source files in lcov:" - grep '^SF:' lcov.info | head -20 || true - - - name: Verify lcov paths exist - run: | - grep '^SF:' lcov.info | sed 's/^SF://' | head -20 | while read -r f; do - test -f "$f" && echo "OK $f" || echo "MISSING $f" - done - - - name: Debug lcov stats - run: | - echo "SF count:" - grep -c '^SF:' lcov.info || true - echo "DA count:" - grep -c '^DA:' lcov.info || true - echo "end_of_record count:" - grep -c '^end_of_record' lcov.info || true - - name: Upload to Codecov uses: codecov/codecov-action@v5 with: @@ -188,3 +118,4 @@ jobs: fail_ci_if_error: true env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + From dddb290d805b9f9937bf4bac293dccf6579481ac Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 18:21:32 +0100 Subject: [PATCH 15/25] fix: working on pipeline --- .github/workflows/codecov.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index b23e5b7..bc9348b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -82,6 +82,9 @@ jobs: - name: Install grcov run: cargo install grcov + - name: Prepare coverage directory + run: mkdir -p coverage + - name: Build test image run: docker compose -f docker-compose.test.yml build agent-test @@ -90,9 +93,8 @@ jobs: CARGO_TARGET_DIR: /app/target CARGO_INCREMENTAL: 0 RUSTFLAGS: "-C instrument-coverage" - LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw" + LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | - docker compose -f docker-compose.test.yml run --rm \ -e CARGO_TARGET_DIR \ -e CARGO_INCREMENTAL \ @@ -100,9 +102,14 @@ jobs: -e LLVM_PROFILE_FILE \ agent-test cargo test --verbose + - name: Debug coverage files + run: | + echo "Listing coverage files:" + ls -R coverage || true + - name: Generate coverage run: | - grcov . \ + grcov ./coverage \ --binary-path ./target/debug/ \ -s . \ -t lcov \ @@ -117,5 +124,4 @@ jobs: verbose: true fail_ci_if_error: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file From 1f1c4f887b2e20e32424386bf9c052e6ca886b54 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 18:42:49 +0100 Subject: [PATCH 16/25] fix: working on pipeline --- .github/workflows/codecov.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index bc9348b..412e662 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -100,27 +100,39 @@ jobs: -e CARGO_INCREMENTAL \ -e RUSTFLAGS \ -e LLVM_PROFILE_FILE \ - agent-test cargo test --verbose + agent-test bash -c "cargo test --verbose && sync" - - name: Debug coverage files + - name: Verify profraw files run: | - echo "Listing coverage files:" - ls -R coverage || true + echo "==== PROFRAW FILES ====" + find coverage -type f || true - name: Generate coverage run: | grcov ./coverage \ --binary-path ./target/debug/ \ - -s . \ + -s /app \ + --prefix-dir /app \ -t lcov \ --branch \ --ignore-not-existing \ + --ignore "/*" \ + --ignore "../*" \ -o lcov.info + - name: Check lcov content + run: | + echo "==== LCOV SIZE ====" + wc -l lcov.info || true + echo "==== LCOV HEAD ====" + head -20 lcov.info || true + - name: Upload to Codecov uses: codecov/codecov-action@v5 with: files: lcov.info + flags: unittests + name: rust-docker-coverage verbose: true fail_ci_if_error: true env: From c61dd671153ecedf2df83bbf8d10d2bfd1cb959f Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 18:57:39 +0100 Subject: [PATCH 17/25] fix: working on pipeline --- .github/workflows/codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 412e662..9aaa6e1 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -111,8 +111,7 @@ jobs: run: | grcov ./coverage \ --binary-path ./target/debug/ \ - -s /app \ - --prefix-dir /app \ + -s . \ -t lcov \ --branch \ --ignore-not-existing \ From 06dd5e188c13b7bcab3948dc2672075280affa3f Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 19:00:02 +0100 Subject: [PATCH 18/25] fix: working on pipeline --- .github/workflows/codecov.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 9aaa6e1..6f31a9e 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -112,11 +112,10 @@ jobs: grcov ./coverage \ --binary-path ./target/debug/ \ -s . \ + --path-mapping /app=$(pwd) \ -t lcov \ --branch \ --ignore-not-existing \ - --ignore "/*" \ - --ignore "../*" \ -o lcov.info - name: Check lcov content From db084b779177b9ca22837a689c822330bbeb26bc Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 19:22:26 +0100 Subject: [PATCH 19/25] fix: working on pipeline --- .github/workflows/codecov.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 6f31a9e..a662573 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -100,19 +100,22 @@ jobs: -e CARGO_INCREMENTAL \ -e RUSTFLAGS \ -e LLVM_PROFILE_FILE \ - agent-test bash -c "cargo test --verbose && sync" + agent-test bash -c "cargo clean && cargo test --verbose && sync" - - name: Verify profraw files + - name: Verify artifacts run: | echo "==== PROFRAW FILES ====" find coverage -type f || true + echo "==== TARGET FILES ====" + ls -R target || true - name: Generate coverage run: | - grcov ./coverage \ - --binary-path ./target/debug/ \ + grcov coverage \ + --binary-path ./target/debug \ -s . \ --path-mapping /app=$(pwd) \ + --llvm \ -t lcov \ --branch \ --ignore-not-existing \ @@ -120,9 +123,7 @@ jobs: - name: Check lcov content run: | - echo "==== LCOV SIZE ====" wc -l lcov.info || true - echo "==== LCOV HEAD ====" head -20 lcov.info || true - name: Upload to Codecov From 65b84a4a7d00058d72840f6ea09abb9a3f71157b Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 19:48:49 +0100 Subject: [PATCH 20/25] fix: working on pipeline --- .github/workflows/codecov.yml | 59 +++++++++++++++++------------------ docker/Dockerfile | 1 + 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index a662573..fb5d340 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -70,7 +70,6 @@ env: jobs: coverage: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 @@ -79,20 +78,14 @@ jobs: with: components: llvm-tools-preview - - name: Install grcov - run: cargo install grcov - - - name: Prepare coverage directory - run: mkdir -p coverage - - - name: Build test image + - name: Build test image (with grcov included) run: docker compose -f docker-compose.test.yml build agent-test - name: Run tests in container env: CARGO_TARGET_DIR: /app/target CARGO_INCREMENTAL: 0 - RUSTFLAGS: "-C instrument-coverage" + RUSTFLAGS: "-C instrument-coverage -C link-dead-code" LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | docker compose -f docker-compose.test.yml run --rm \ @@ -102,36 +95,42 @@ jobs: -e LLVM_PROFILE_FILE \ agent-test bash -c "cargo clean && cargo test --verbose && sync" - - name: Verify artifacts + - name: Verify profraw files exist + run: | + docker compose -f docker-compose.test.yml run --rm agent-test \ + find /app/coverage -type f -name "*.profraw" | wc -l || true + + - name: Generate coverage report inside container run: | - echo "==== PROFRAW FILES ====" - find coverage -type f || true - echo "==== TARGET FILES ====" - ls -R target || true + docker compose -f docker-compose.test.yml run --rm agent-test bash -c " + rustup component add llvm-tools && + grcov /app/coverage \ + --binary-path /app/target/debug \ + -s /app \ + --llvm \ + -t lcov \ + --branch \ + --ignore-not-existing \ + --ignore '/app/target/*' \ + --ignore '/*' \ + -o /app/lcov.info + " - - name: Generate coverage + - name: Copy lcov.info from container to host run: | - grcov coverage \ - --binary-path ./target/debug \ - -s . \ - --path-mapping /app=$(pwd) \ - --llvm \ - -t lcov \ - --branch \ - --ignore-not-existing \ - -o lcov.info + docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info - - name: Check lcov content + - name: Show basic coverage report info (debug) run: | - wc -l lcov.info || true - head -20 lcov.info || true + echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}') + head -n 30 ./lcov.info || true - - name: Upload to Codecov + - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: - files: lcov.info + files: ./lcov.info flags: unittests - name: rust-docker-coverage + name: rust-unit-coverage verbose: true fail_ci_if_error: true env: diff --git a/docker/Dockerfile b/docker/Dockerfile index 5a714fa..0006a89 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,6 +60,7 @@ WORKDIR /app FROM base AS dev RUN cargo install cargo-watch +RUN cargo install grcov --locked COPY Cargo.toml Cargo.lock ./ RUN mkdir src && echo "fn main() {}" > src/main.rs From 2e89d36ff6821745b894b8af3e9c698c1fa7b296 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 20:01:47 +0100 Subject: [PATCH 21/25] fix: working on pipeline --- .github/workflows/codecov.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index fb5d340..207ad8b 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -88,7 +88,7 @@ jobs: RUSTFLAGS: "-C instrument-coverage -C link-dead-code" LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | - docker compose -f docker-compose.test.yml run --rm \ + docker compose -f docker-compose.test.yml run \ -e CARGO_TARGET_DIR \ -e CARGO_INCREMENTAL \ -e RUSTFLAGS \ @@ -120,6 +120,10 @@ jobs: run: | docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info + - name: Remove container + run: | + docker rm agent-test-run + - name: Show basic coverage report info (debug) run: | echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}') From ea0705a8cca239ea341bcddf42127b13b78c3785 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 20:14:24 +0100 Subject: [PATCH 22/25] fix: working on pipeline --- .github/workflows/codecov.yml | 62 +-------------------- scripts/README.md | 4 ++ scripts/tests/requirements.sh | 91 ------------------------------- src/domain/factory.rs | 4 +- src/domain/mongodb/database.rs | 22 ++------ src/domain/mysql/connection.rs | 2 +- src/domain/mysql/database.rs | 22 ++------ src/domain/postgres/backup.rs | 3 +- src/domain/postgres/connection.rs | 10 +--- src/domain/postgres/database.rs | 26 +++------ src/domain/postgres/restore.rs | 3 +- src/domain/redis/database.rs | 13 ++--- src/domain/sqlite/database.rs | 22 ++------ src/domain/valkey/database.rs | 13 ++--- src/services/backup/runner.rs | 2 +- src/services/restore/runner.rs | 2 +- src/tests/domain/mariadb.rs | 4 +- src/tests/domain/mongodb.rs | 4 +- src/tests/domain/mysql.rs | 4 +- src/tests/domain/postgres.rs | 4 +- src/tests/domain/redis.rs | 2 +- src/tests/domain/valkey.rs | 2 +- 22 files changed, 59 insertions(+), 262 deletions(-) delete mode 100755 scripts/tests/requirements.sh diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 207ad8b..1789918 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,61 +1,3 @@ -#name: Codecov Rust -# -#on: -# push: -# branches: ["main"] -# pull_request: -# branches: ["main"] -# -#env: -# CARGO_TERM_COLOR: always -# -#jobs: -# build: -# runs-on: ubuntu-latest -# -# steps: -# - uses: actions/checkout@v3 -# -# - uses: actions-rs/toolchain@v1 -# with: -# toolchain: stable -# override: true -# components: llvm-tools-preview -# -# - name: Install grcov -# run: cargo install grcov -# -# - name: Install test requirements -# run: bash scripts/tests/requirements.sh -# -# - name: Build -# run: cargo build --verbose -# -# - name: Run tests -# env: -# CARGO_INCREMENTAL: 0 -# RUSTFLAGS: "-C instrument-coverage" -# LLVM_PROFILE_FILE: "cargo-test-%p-%m.profraw" -# run: cargo test --verbose -# -# - name: Generate coverage -# run: | -# grcov . \ -# --binary-path ./target/debug/ \ -# -s . \ -# -t lcov \ -# --branch \ -# --ignore-not-existing \ -# -o lcov.info -# -# - name: Upload to Codecov -# uses: codecov/codecov-action@v5 -# with: -# files: lcov.info -# verbose: true -# fail_ci_if_error: true -# env: -# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} name: Codecov Rust on: @@ -97,12 +39,12 @@ jobs: - name: Verify profraw files exist run: | - docker compose -f docker-compose.test.yml run --rm agent-test \ + docker compose -f docker-compose.test.yml run agent-test \ find /app/coverage -type f -name "*.profraw" | wc -l || true - name: Generate coverage report inside container run: | - docker compose -f docker-compose.test.yml run --rm agent-test bash -c " + docker compose -f docker-compose.test.yml run agent-test bash -c " rustup component add llvm-tools && grcov /app/coverage \ --binary-path /app/target/debug \ diff --git a/scripts/README.md b/scripts/README.md index 9045b17..f94a41e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -24,3 +24,7 @@ docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT * FROM users LI ```bash docker exec -it db-sqlite sqlite3 /workspace/data/app.db "SELECT name FROM sqlite_master WHERE type='table';" ``` + +```bash +docker compose -f docker-compose.test.yml run agent-test bash -c "cargo clean && cargo test" +``` \ No newline at end of file diff --git a/scripts/tests/requirements.sh b/scripts/tests/requirements.sh deleted file mode 100755 index 3a9f2ab..0000000 --- a/scripts/tests/requirements.sh +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env bash -set -e - -POSTGRES_BASE="/usr/local/postgresql" -echo "Detecting OS and architecture..." -OS_TYPE="$(uname -s)" -ARCH="$(uname -m)" - -install_pg_binaries() { - echo "Installing PostgreSQL binaries for versions 12-18..." - - for v in 12 13 14 15 16 17 18; do - TARGET_DIR="$POSTGRES_BASE/$v/bin" - sudo mkdir -p "$TARGET_DIR" - - if [[ "$OS_TYPE" == "Linux" ]]; then - if [[ "$ARCH" == "x86_64" ]]; then - SRC_DIR="./assets/tools/amd64/postgresql/postgresql-$v/bin" - elif [[ "$ARCH" == "aarch64" ]]; then - SRC_DIR="./assets/tools/arm64/postgresql/postgresql-$v/bin" - else - echo "Unsupported architecture: $ARCH" - continue - fi - - if [[ -d "$SRC_DIR" ]]; then - echo "Copying PostgreSQL $v binaries from $SRC_DIR to $TARGET_DIR" - sudo cp -r "$SRC_DIR"/* "$TARGET_DIR/" - else - echo "Binaries for PostgreSQL $v not found for Linux, skipping..." - continue - fi - - elif [[ "$OS_TYPE" == "Darwin" ]]; then - PG_SRC="$(brew --prefix postgresql@$v)/bin" 2>/dev/null || true - - if [[ ! -d "$PG_SRC" ]]; then - echo "PostgreSQL $v not installed via Homebrew. Trying to install..." - if ! brew install postgresql@$v; then - echo "PostgreSQL $v not available, skipping..." - continue - fi - PG_SRC="$(brew --prefix postgresql@$v)/bin" - fi - - echo "Copying PostgreSQL $v binaries from $PG_SRC to $TARGET_DIR" - sudo cp -r "$PG_SRC"/* "$TARGET_DIR/" - fi - - sudo chown -R "$(whoami)" "$TARGET_DIR" - chmod +x "$TARGET_DIR"/* - done - - echo "PostgreSQL binaries installed under $POSTGRES_BASE" -} - -if [[ "$OS_TYPE" == "Linux" ]]; then - if command -v apt >/dev/null 2>&1; then - echo "Linux detected with apt. Installing prerequisites..." - sudo apt update - sudo apt install -y wget gnupg lsb-release redis-tools valkey - install_pg_binaries - else - echo "Unsupported Linux distribution. Only apt-based distros are supported." - exit 1 - fi - -elif [[ "$OS_TYPE" == "Darwin" ]]; then - if command -v brew >/dev/null 2>&1; then - echo "macOS detected. Installing prerequisites..." - brew install redis - brew install valkey - brew install mysql-client - brew tap mongodb/brew - brew install mongodb-database-tools - - sudo mkdir -p "$POSTGRES_BASE" - sudo chown -R "$(whoami)" "$POSTGRES_BASE" - - install_pg_binaries - else - echo "Homebrew not found. Please install Homebrew first: https://brew.sh/" - exit 1 - fi - -else - echo "Unsupported OS: $OS_TYPE" - exit 1 -fi - -echo "Tools installation completed successfully." \ No newline at end of file diff --git a/src/domain/factory.rs b/src/domain/factory.rs index e464967..d3b532a 100644 --- a/src/domain/factory.rs +++ b/src/domain/factory.rs @@ -14,8 +14,8 @@ use std::sync::Arc; pub trait Database: Send + Sync { fn file_extension(&self) -> &'static str; async fn ping(&self) -> Result; - async fn backup(&self, backup_dir: &Path, is_test: Option) -> Result; - async fn restore(&self, restore_file: &Path, is_test: Option) -> Result<()>; + async fn backup(&self, backup_dir: &Path) -> Result; + async fn restore(&self, restore_file: &Path) -> Result<()>; } pub struct DatabaseFactory; diff --git a/src/domain/mongodb/database.rs b/src/domain/mongodb/database.rs index 7850896..e4c40b8 100644 --- a/src/domain/mongodb/database.rs +++ b/src/domain/mongodb/database.rs @@ -27,27 +27,17 @@ impl Database for MongoDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; - } + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } } diff --git a/src/domain/mysql/connection.rs b/src/domain/mysql/connection.rs index d02d8dc..f9e14fb 100644 --- a/src/domain/mysql/connection.rs +++ b/src/domain/mysql/connection.rs @@ -22,7 +22,7 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { let version = String::from_utf8_lossy(&output.stdout) .lines() - .nth(1) // skip column header + .nth(1) .unwrap_or_default() .trim() .to_string(); diff --git a/src/domain/mysql/database.rs b/src/domain/mysql/database.rs index e90b3e1..9c39bd6 100644 --- a/src/domain/mysql/database.rs +++ b/src/domain/mysql/database.rs @@ -33,11 +33,8 @@ impl Database for MySQLDatabase { ping::run(self.cfg.clone(), self.build_env().clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run( self.cfg.clone(), dir.to_path_buf(), @@ -45,21 +42,14 @@ impl Database for MySQLDatabase { self.file_extension(), ) .await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; - } + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } } diff --git a/src/domain/postgres/backup.rs b/src/domain/postgres/backup.rs index a7d83b4..9ec9eb4 100644 --- a/src/domain/postgres/backup.rs +++ b/src/domain/postgres/backup.rs @@ -11,7 +11,6 @@ pub async fn run( cfg: DatabaseConfig, format: PostgresDumpFormat, backup_dir: PathBuf, - is_test: Option, ) -> Result { tokio::task::spawn_blocking(move || -> Result { debug!("Starting backup for database {}", cfg.name); @@ -27,7 +26,7 @@ pub async fn run( } }; - let pg_dump = select_pg_path(&version, is_test).join("pg_dump"); + let pg_dump = select_pg_path(&version).join("pg_dump"); debug!("Using pg_dump at {:?}", pg_dump); diff --git a/src/domain/postgres/connection.rs b/src/domain/postgres/connection.rs index 585d369..92648f3 100644 --- a/src/domain/postgres/connection.rs +++ b/src/domain/postgres/connection.rs @@ -28,15 +28,9 @@ pub async fn server_version(cfg: &DatabaseConfig) -> Result { Ok(version) } -pub fn select_pg_path(version: &str, is_test: Option) -> std::path::PathBuf { +pub fn select_pg_path(version: &str) -> std::path::PathBuf { let major = version.split('.').next().unwrap_or("17"); - - if is_test.unwrap_or(false) { - // format!("/usr/local/postgresql/{}/bin", major).into() - format!("/usr/lib/postgresql/{}/bin", major).into() - } else { - format!("/usr/lib/postgresql/{}/bin", major).into() - } + format!("/usr/lib/postgresql/{}/bin", major).into() } pub async fn terminate_connections(cfg: &DatabaseConfig) -> Result<()> { diff --git a/src/domain/postgres/database.rs b/src/domain/postgres/database.rs index 7af5376..12aa7ce 100644 --- a/src/domain/postgres/database.rs +++ b/src/domain/postgres/database.rs @@ -31,27 +31,17 @@ impl Database for PostgresDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } - let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf(), is_test).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; + let res = backup::run(self.cfg.clone(), self.format, dir.to_path_buf()).await; + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; - } - let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf(), is_test).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; + let res = restore::run(self.cfg.clone(), self.format, file.to_path_buf()).await; + FileLock::release(&self.cfg.generated_id).await?; res } } diff --git a/src/domain/postgres/restore.rs b/src/domain/postgres/restore.rs index a5181f7..98a7732 100644 --- a/src/domain/postgres/restore.rs +++ b/src/domain/postgres/restore.rs @@ -11,7 +11,6 @@ pub async fn run( cfg: DatabaseConfig, format: PostgresDumpFormat, restore_file: PathBuf, - is_test: Option, ) -> Result<()> { tokio::task::spawn_blocking(move || -> Result<()> { debug!("Starting restore for database {}", cfg.name); @@ -27,7 +26,7 @@ pub async fn run( } }; - let pg_restore = select_pg_path(&version, is_test).join("pg_restore"); + let pg_restore = select_pg_path(&version).join("pg_restore"); debug!("Using pg_restore at {:?}", pg_restore); diff --git a/src/domain/redis/database.rs b/src/domain/redis/database.rs index e536b18..33e67e2 100644 --- a/src/domain/redis/database.rs +++ b/src/domain/redis/database.rs @@ -27,19 +27,14 @@ impl Database for RedisDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, _file: &Path, _is_test: Option) -> Result<()> { + async fn restore(&self, _file: &Path) -> Result<()> { bail!("Restore not supported for Redis databases") } } diff --git a/src/domain/sqlite/database.rs b/src/domain/sqlite/database.rs index 5bcebab..2579061 100644 --- a/src/domain/sqlite/database.rs +++ b/src/domain/sqlite/database.rs @@ -27,26 +27,16 @@ impl Database for SqliteDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, file: &Path, is_test: Option) -> Result<()> { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; - } + async fn restore(&self, file: &Path) -> Result<()> { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Restore.as_str()).await?; let res = restore::run(self.cfg.clone(), file.to_path_buf()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } } diff --git a/src/domain/valkey/database.rs b/src/domain/valkey/database.rs index a0677ac..8696179 100644 --- a/src/domain/valkey/database.rs +++ b/src/domain/valkey/database.rs @@ -27,19 +27,14 @@ impl Database for ValkeyDatabase { ping::run(self.cfg.clone()).await } - async fn backup(&self, dir: &Path, is_test: Option) -> Result { - let test_mode = is_test.unwrap_or(false); - if !test_mode { - FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; - } + async fn backup(&self, dir: &Path) -> Result { + FileLock::acquire(&self.cfg.generated_id, DbOpLock::Backup.as_str()).await?; let res = backup::run(self.cfg.clone(), dir.to_path_buf(), self.file_extension()).await; - if !test_mode { - FileLock::release(&self.cfg.generated_id).await?; - } + FileLock::release(&self.cfg.generated_id).await?; res } - async fn restore(&self, _file: &Path, _is_test: Option) -> Result<()> { + async fn restore(&self, _file: &Path) -> Result<()> { bail!("Restore not supported for Valkey databases") } } diff --git a/src/services/backup/runner.rs b/src/services/backup/runner.rs index e617ed6..e2bacd9 100644 --- a/src/services/backup/runner.rs +++ b/src/services/backup/runner.rs @@ -35,7 +35,7 @@ impl BackupService { }); } - match db.backup(tmp_path, Some(false)).await { + match db.backup(tmp_path).await { Ok(file) => Ok(BackupResult { generated_id, db_type, diff --git a/src/services/restore/runner.rs b/src/services/restore/runner.rs index 1ba424b..3806865 100644 --- a/src/services/restore/runner.rs +++ b/src/services/restore/runner.rs @@ -29,7 +29,7 @@ impl RestoreService { }); } - match db.restore(&backup_file, Some(false)).await { + match db.restore(&backup_file).await { Ok(_) => Ok(RestoreResult { generated_id, status: "success".into(), diff --git a/src/tests/domain/mariadb.rs b/src/tests/domain/mariadb.rs index 9837745..6538151 100644 --- a/src/tests/domain/mariadb.rs +++ b/src/tests/domain/mariadb.rs @@ -58,7 +58,7 @@ async fn mariadb_backup_restore_test() { let backup_path = temp_dir.path(); let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); assert!(file_path.is_file()); @@ -81,7 +81,7 @@ async fn mariadb_backup_restore_test() { info!("Reachable: {}", reachable); assert!(reachable); - match db.restore(&backup_file, Some(true)).await { + match db.restore(&backup_file).await { Ok(_) => { info!("Restore succeeded for {}", config.generated_id); assert!(true) diff --git a/src/tests/domain/mongodb.rs b/src/tests/domain/mongodb.rs index d6e6e99..1b86ae2 100644 --- a/src/tests/domain/mongodb.rs +++ b/src/tests/domain/mongodb.rs @@ -75,7 +75,7 @@ async fn mongodb_backup_restore_test() { let backup_path = temp_dir.path(); let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); info!("Backup path: {:?}", file_path); assert!(file_path.is_file()); @@ -85,7 +85,7 @@ async fn mongodb_backup_restore_test() { info!("Reachable: {}", reachable); assert!(reachable); - match db.restore(&file_path, Some(true)).await { + match db.restore(&file_path).await { Ok(_) => { info!("Restore succeeded for {}", config.generated_id); assert!(true) diff --git a/src/tests/domain/mysql.rs b/src/tests/domain/mysql.rs index 5fd0aed..20b8861 100644 --- a/src/tests/domain/mysql.rs +++ b/src/tests/domain/mysql.rs @@ -58,7 +58,7 @@ async fn mysql_backup_restore_test() { let backup_path = temp_dir.path(); let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); assert!(file_path.is_file()); @@ -81,7 +81,7 @@ async fn mysql_backup_restore_test() { info!("Reachable: {}", reachable); assert!(reachable); - match db.restore(&backup_file, Some(true)).await { + match db.restore(&backup_file).await { Ok(_) => { info!("Restore succeeded for {}", config.generated_id); assert!(true) diff --git a/src/tests/domain/postgres.rs b/src/tests/domain/postgres.rs index ca0543f..bac4f6d 100644 --- a/src/tests/domain/postgres.rs +++ b/src/tests/domain/postgres.rs @@ -66,7 +66,7 @@ async fn postgres_backup_restore_test() { let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); assert!(file_path.is_file()); @@ -96,7 +96,7 @@ async fn postgres_backup_restore_test() { info!("Running pg_restore: {:?}", backup_file); - match db.restore(&backup_file, Some(true)).await { + match db.restore(&backup_file).await { Ok(_) => { info!("Restore succeeded for {}", config.generated_id); assert!(true) diff --git a/src/tests/domain/redis.rs b/src/tests/domain/redis.rs index fa4d4a8..32e3ebd 100644 --- a/src/tests/domain/redis.rs +++ b/src/tests/domain/redis.rs @@ -56,7 +56,7 @@ async fn redis_backup_test() { let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); assert!(file_path.is_file()); } diff --git a/src/tests/domain/valkey.rs b/src/tests/domain/valkey.rs index 6619cfd..f018541 100644 --- a/src/tests/domain/valkey.rs +++ b/src/tests/domain/valkey.rs @@ -55,7 +55,7 @@ async fn valkey_backup_test() { let db = DatabaseFactory::create_for_backup(config.clone()).await; - let file_path = db.backup(backup_path, Some(true)).await.unwrap(); + let file_path = db.backup(backup_path).await.unwrap(); assert!(file_path.is_file()); } From 4aba2f2c86442b1f587e87cacaf20fcd807f77a7 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 20:29:07 +0100 Subject: [PATCH 23/25] fix: working on pipeline --- .github/workflows/codecov.yml | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 1789918..21c7b5c 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -23,28 +23,30 @@ jobs: - name: Build test image (with grcov included) run: docker compose -f docker-compose.test.yml build agent-test - - name: Run tests in container + - name: Start agent-test container + run: docker compose -f docker-compose.test.yml up -d agent-test + + - name: Run tests inside container env: CARGO_TARGET_DIR: /app/target CARGO_INCREMENTAL: 0 RUSTFLAGS: "-C instrument-coverage -C link-dead-code" LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | - docker compose -f docker-compose.test.yml run \ - -e CARGO_TARGET_DIR \ - -e CARGO_INCREMENTAL \ - -e RUSTFLAGS \ - -e LLVM_PROFILE_FILE \ - agent-test bash -c "cargo clean && cargo test --verbose && sync" + docker compose -f docker-compose.test.yml exec agent-test bash -c " + cargo clean && + cargo test --verbose && + sync + " - name: Verify profraw files exist run: | - docker compose -f docker-compose.test.yml run agent-test \ + docker compose -f docker-compose.test.yml exec agent-test \ find /app/coverage -type f -name "*.profraw" | wc -l || true - - name: Generate coverage report inside container + - name: Generate coverage report run: | - docker compose -f docker-compose.test.yml run agent-test bash -c " + docker compose -f docker-compose.test.yml exec agent-test bash -c " rustup component add llvm-tools && grcov /app/coverage \ --binary-path /app/target/debug \ @@ -58,13 +60,8 @@ jobs: -o /app/lcov.info " - - name: Copy lcov.info from container to host - run: | - docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info - - - name: Remove container - run: | - docker rm agent-test-run + - name: Copy lcov.info to host + run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info - name: Show basic coverage report info (debug) run: | @@ -80,4 +77,7 @@ jobs: verbose: true fail_ci_if_error: true env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Stop and remove container + run: docker compose -f docker-compose.test.yml down \ No newline at end of file From 335c5b75308f73bf93ffd5f3f6061292ad57ee2b Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Sun, 22 Mar 2026 20:31:20 +0100 Subject: [PATCH 24/25] fix: working on pipeline --- .github/workflows/codecov.yml | 143 +++++++++++++++++++++++++++------- 1 file changed, 116 insertions(+), 27 deletions(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 21c7b5c..07efe04 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -1,3 +1,86 @@ +#name: Codecov Rust +# +#on: +# push: +# branches: ["main"] +# pull_request: +# branches: ["main"] +# +#env: +# CARGO_TERM_COLOR: always +# +#jobs: +# coverage: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# +# - name: Install Rust toolchain +# uses: dtolnay/rust-toolchain@stable +# with: +# components: llvm-tools-preview +# +# - name: Build test image (with grcov included) +# run: docker compose -f docker-compose.test.yml build agent-test +# +# - name: Run tests in container +# env: +# CARGO_TARGET_DIR: /app/target +# CARGO_INCREMENTAL: 0 +# RUSTFLAGS: "-C instrument-coverage -C link-dead-code" +# LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" +# run: | +# docker compose -f docker-compose.test.yml run \ +# -e CARGO_TARGET_DIR \ +# -e CARGO_INCREMENTAL \ +# -e RUSTFLAGS \ +# -e LLVM_PROFILE_FILE \ +# agent-test bash -c "cargo clean && cargo test --verbose && sync" +# +# - name: Verify profraw files exist +# run: | +# docker compose -f docker-compose.test.yml run agent-test \ +# find /app/coverage -type f -name "*.profraw" | wc -l || true +# +# - name: Generate coverage report inside container +# run: | +# docker compose -f docker-compose.test.yml run agent-test bash -c " +# rustup component add llvm-tools && +# grcov /app/coverage \ +# --binary-path /app/target/debug \ +# -s /app \ +# --llvm \ +# -t lcov \ +# --branch \ +# --ignore-not-existing \ +# --ignore '/app/target/*' \ +# --ignore '/*' \ +# -o /app/lcov.info +# " +# +# - name: Copy lcov.info from container to host +# run: | +# docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info +# +# - name: Remove container +# run: | +# docker rm agent-test-run +# +# - name: Show basic coverage report info (debug) +# run: | +# echo "lcov.info size:" $(wc -c ./lcov.info | awk '{print $1}') +# head -n 30 ./lcov.info || true +# +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v5 +# with: +# files: ./lcov.info +# flags: unittests +# name: rust-unit-coverage +# verbose: true +# fail_ci_if_error: true +# env: +# CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} name: Codecov Rust on: @@ -8,6 +91,10 @@ on: env: CARGO_TERM_COLOR: always + CARGO_TARGET_DIR: /app/target + CARGO_INCREMENTAL: 0 + RUSTFLAGS: "-C instrument-coverage -C link-dead-code" + LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" jobs: coverage: @@ -27,40 +114,42 @@ jobs: run: docker compose -f docker-compose.test.yml up -d agent-test - name: Run tests inside container - env: - CARGO_TARGET_DIR: /app/target - CARGO_INCREMENTAL: 0 - RUSTFLAGS: "-C instrument-coverage -C link-dead-code" - LLVM_PROFILE_FILE: "/app/coverage/cargo-test-%p-%m.profraw" run: | - docker compose -f docker-compose.test.yml exec agent-test bash -c " - cargo clean && - cargo test --verbose && - sync - " + docker compose -f docker-compose.test.yml exec \ + -e CARGO_TARGET_DIR \ + -e CARGO_INCREMENTAL \ + -e RUSTFLAGS \ + -e LLVM_PROFILE_FILE \ + agent-test bash -c " + cargo clean && + cargo test --verbose && + sync + " - name: Verify profraw files exist run: | - docker compose -f docker-compose.test.yml exec agent-test \ - find /app/coverage -type f -name "*.profraw" | wc -l || true + docker compose -f docker-compose.test.yml exec \ + -e LLVM_PROFILE_FILE \ + agent-test find /app/coverage -type f -name "*.profraw" | wc -l || true - - name: Generate coverage report + - name: Generate coverage report inside container run: | - docker compose -f docker-compose.test.yml exec agent-test bash -c " - rustup component add llvm-tools && - grcov /app/coverage \ - --binary-path /app/target/debug \ - -s /app \ - --llvm \ - -t lcov \ - --branch \ - --ignore-not-existing \ - --ignore '/app/target/*' \ - --ignore '/*' \ - -o /app/lcov.info - " + docker compose -f docker-compose.test.yml exec \ + agent-test bash -c " + rustup component add llvm-tools && + grcov /app/coverage \ + --binary-path /app/target/debug \ + -s /app \ + --llvm \ + -t lcov \ + --branch \ + --ignore-not-existing \ + --ignore '/app/target/*' \ + --ignore '/*' \ + -o /app/lcov.info + " - - name: Copy lcov.info to host + - name: Copy lcov.info from container to host run: docker compose -f docker-compose.test.yml cp agent-test:/app/lcov.info ./lcov.info - name: Show basic coverage report info (debug) From 6331d72d36a06af1fbbb4f9bc5255a9c7c2b0623 Mon Sep 17 00:00:00 2001 From: Charles GTE Date: Fri, 27 Mar 2026 22:13:43 +0100 Subject: [PATCH 25/25] fix: working on pipeline --- docker-compose.yml | 2 +- src/domain/mongodb/ping.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0e3b0a5..7b19a5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,7 @@ services: APP_ENV: development LOG: debug TZ: "Europe/Paris" - EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNWU1OGU2MGEtODhiMy00YTBjLWI0NDktNTQ3OWZhOTQzZDBkIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" + EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiNGI1OTM2MGItNTNkMi00ZTZmLWE1ODctODcyMmQ1NDc1MTNmIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ==" #POOLING: 1 #DATABASES_CONFIG_FILE: "config.toml" extra_hosts: diff --git a/src/domain/mongodb/ping.rs b/src/domain/mongodb/ping.rs index 6a3478b..f70e9ef 100644 --- a/src/domain/mongodb/ping.rs +++ b/src/domain/mongodb/ping.rs @@ -24,7 +24,7 @@ pub async fn run(cfg: DatabaseConfig) -> Result { error!("Full Error: {}", e); error!("Check you database network connectivity"); error!("----------------------------------------"); - Err(anyhow::anyhow!("Ping failed for {}: {}", cfg.name, e)) + Ok(false) } } }