diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed0aae6..d7d3854 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -274,13 +274,26 @@ jobs: run: cargo install cargo-audit - name: Run security audit - # Ignored advisories — all pre-existing on main, not caused by this PR: + # Ignored advisories — all transitive deps, not caused by our code: # - RUSTSEC-2026-0097 (rand 0.9.x unsoundness): only triggers with a # custom logger calling rand::rng(). NQRust-MicroVM does not. Via # quinn-proto (reqwest HTTP/3) and mockito (test-only). # - RUSTSEC-2026-0002 (lru 0.12.x IterMut Stacked Borrows): transitive # via ratatui in the installer TUI; ratatui has not yet bumped lru. - run: cargo audit --ignore RUSTSEC-2026-0097 --ignore RUSTSEC-2026-0002 + # - RUSTSEC-2026-0098/0099/0104 (rustls-webpki 0.101.x): transitive via + # aws-sdk-s3 → aws-smithy-http-client → rustls 0.21. We already use + # rustls-webpki 0.103.13 in the rest of the stack; the AWS SDK pins + # the old version and will upgrade in a future release. + # - RUSTSEC-2025-0141 (bincode unmaintained): bincode is functionally + # stable; migration to postcard/ciborium is a follow-up. + run: | + cargo audit \ + --ignore RUSTSEC-2026-0097 \ + --ignore RUSTSEC-2026-0002 \ + --ignore RUSTSEC-2026-0098 \ + --ignore RUSTSEC-2026-0099 \ + --ignore RUSTSEC-2026-0104 \ + --ignore RUSTSEC-2025-0141 check-installer-syntax: name: Check Installer Syntax diff --git a/Cargo.lock b/Cargo.lock index c5f4c69..6015688 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,7 +35,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -58,15 +58,22 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-types", "axum", + "blake3", "bytes", "chrono", "futures", + "hex", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-util", "hyperlocal", "libc", + "nexus-backup", "nexus-storage", "nexus-types", "num_cpus", @@ -191,10 +198,22 @@ checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" dependencies = [ "base64ct", "blake2", - "cpufeatures", + "cpufeatures 0.2.17", "password-hash", ] +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -237,6 +256,390 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "aws-config" +version = "1.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.3.1", + "time", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e26bbf46abc608f2dc61fd6cb3b7b0665497cc259a21520151ed98f8b37d2c79" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "aws-runtime" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f92058d22a46adf53ec57a6a96f34447daf02bff52e8fb956c66bcd5c6ac12" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.123.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c018f22146966fdd493a664f62ee2483dff256b42a08c125ab6a084bde7b77fe" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "http-body 1.0.1", + "lru 0.16.4", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.98.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89c4f19655ab0856375e169865c91264de965bd74c407c7f1e403184b1049409" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f6ae9b71597dc5fd115d52849d7a5556ad9265885ad3492ea8d73b93bbc46e" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.3.1", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cba48474f1d6807384d06fec085b909f5807e16653c5af5c45dfe89539f0b70" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.64.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a764fa7222922f6c0af8eea478b0ef1ba5ce1222af97e01f33ca5e957bd7f3b9" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c0b3e587fbaa5d7f7e870544508af8ce82ea47cd30376e69e1e37c4ac746f79" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.63.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af4a8a5fe3e4ac7ee871237c340bbce13e982d37543b65700f4419e039f5d78e" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0709f0083aa19b704132684bc26d3c868e06bd428ccc4373b0b55c3e8748a58b" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.12", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.7.0", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.32", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.3", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.62.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b3a779093e18cad88bbae08dc4261e1d95018c4c5b9356a52bcae7c0b6e9bb" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3f39d5bb871aaf461d59144557f16d5927a5248a983a40654d9cf3b9ba183b" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f76a580e3d8f8961e5d48763214025a2af65c2fa4cd1fb7f270a0e107a71b0" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd3dfc18c1ce097cf81fced7192731e63809829c6cbf933c1ec47452d08e1aa" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c55e0837e9b8526f49e0b9bfa9ee18ddee70e853f5bc09c5d11ebceddcb0fec" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.3.1", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576b0d6991c9c32bc14fc340582ef148311f924d41815f641a308b5d11e8e7cd" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.3.1", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c50f3cdf47caa8d01f2be4a6663ea02418e892f9bbfd82c7b9a3a37eaccdd3a" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.9" @@ -249,10 +652,10 @@ dependencies = [ "base64 0.22.1", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-util", "itoa", "matchit", @@ -285,8 +688,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -341,19 +744,38 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -370,6 +792,21 @@ dependencies = [ "which", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit_field" version = "0.10.3" @@ -384,11 +821,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.4" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -400,6 +837,20 @@ dependencies = [ "digest", ] +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -421,9 +872,9 @@ dependencies = [ "futures-core", "futures-util", "hex", - "http", + "http 1.3.1", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -477,6 +928,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cassowary" version = "0.3.0" @@ -499,6 +960,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -523,6 +986,30 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "chacha20poly1305" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", + "zeroize", +] + [[package]] name = "chrono" version = "0.4.42" @@ -545,6 +1032,7 @@ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common", "inout", + "zeroize", ] [[package]] @@ -598,6 +1086,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -648,6 +1145,22 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -663,6 +1176,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.3.0" @@ -678,6 +1200,18 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc-fast" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" +dependencies = [ + "crc", + "digest", + "rustversion", + "spin 0.10.0", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -687,6 +1221,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8c3e73077b4b4a6ab1ea5047c37c57aee77657bc8ecd6f29b0af082d0b0c07" +dependencies = [ + "chrono", + "nom", + "once_cell", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -727,7 +1272,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "crossterm_winapi", "mio", "parking_lot", @@ -791,7 +1336,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -997,6 +1542,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1133,6 +1684,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fastcdc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf51ceb43e96afbfe4dd5c6f6082af5dfd60e220820b8123792d61963f2ce6bc" + [[package]] name = "fastrand" version = "2.3.0" @@ -1188,7 +1745,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.8", ] [[package]] @@ -1199,9 +1756,15 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "foreign-types" @@ -1227,6 +1790,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1421,6 +1990,25 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.11.4", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.12" @@ -1432,7 +2020,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap 2.11.4", "slab", "tokio", @@ -1465,7 +2053,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1473,6 +2061,11 @@ name = "hashbrown" version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -1539,6 +2132,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -1550,6 +2154,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1557,7 +2172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -1568,8 +2183,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -1585,6 +2200,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.7.0" @@ -1595,9 +2234,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1615,7 +2254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -1623,19 +2262,35 @@ dependencies = [ "winapi", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.3.1", + "hyper 1.7.0", "hyper-util", - "rustls", + "rustls 0.23.32", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.3", "tower-service", "webpki-roots", ] @@ -1651,14 +2306,14 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", - "hyper", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.7.0", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -1672,7 +2327,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-util", "pin-project-lite", "tokio", @@ -1894,7 +2549,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "cfg-if", "libc", ] @@ -1954,6 +2609,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", +] + [[package]] name = "jpeg-decoder" version = "0.3.2" @@ -1979,7 +2644,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" dependencies = [ - "spin", + "spin 0.9.8", ] [[package]] @@ -2022,7 +2687,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "libc", "redox_syscall", ] @@ -2080,6 +2745,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.0", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -2094,10 +2768,16 @@ dependencies = [ "anyhow", "argon2", "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-types", + "aws-types", "axum", "base64 0.22.1", "bollard", "chrono", + "cron", "dotenvy", "ed25519-dalek", "futures", @@ -2106,6 +2786,7 @@ dependencies = [ "image", "metrics", "mockito", + "nexus-backup", "nexus-storage", "nexus-types", "openidconnect", @@ -2245,10 +2926,10 @@ dependencies = [ "bytes", "colored", "futures-core", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-util", "log", "pin-project-lite", @@ -2269,14 +2950,33 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 1.3.1", "httparse", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] +[[package]] +name = "nexus-backup" +version = "0.1.0" +dependencies = [ + "async-trait", + "bincode", + "blake3", + "chacha20poly1305", + "fastcdc", + "hex", + "proptest", + "rand 0.8.5", + "serde", + "thiserror 1.0.69", + "tokio", + "uuid", + "zstd", +] + [[package]] name = "nexus-storage" version = "0.1.0" @@ -2307,7 +3007,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "cfg-if", "cfg_aliases", "libc", @@ -2460,7 +3160,7 @@ dependencies = [ "base64 0.22.1", "chrono", "getrandom 0.2.16", - "http", + "http 1.3.1", "rand 0.8.5", "reqwest", "serde", @@ -2509,7 +3209,7 @@ dependencies = [ "dyn-clone", "ed25519-dalek", "hmac", - "http", + "http 1.3.1", "itertools 0.10.5", "log", "oauth2", @@ -2535,7 +3235,7 @@ version = "0.10.76" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "cfg-if", "foreign-types", "libc", @@ -2561,6 +3261,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-src" version = "300.6.0+3.6.2" @@ -2592,6 +3298,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -2729,6 +3441,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "poly1305" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" +dependencies = [ + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "polyval" version = "0.6.2" @@ -2736,7 +3459,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "opaque-debug", "universal-hash", ] @@ -2832,6 +3555,25 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.11.1", + "num-traits", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "qoi" version = "0.4.1" @@ -2841,6 +3583,12 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.33.0" @@ -2863,8 +3611,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", - "socket2", + "rustls 0.23.32", + "socket2 0.6.0", "thiserror 2.0.16", "tokio", "tracing", @@ -2883,7 +3631,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.32", "rustls-pki-types", "slab", "thiserror 2.0.16", @@ -2901,7 +3649,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.0", "tracing", "windows-sys 0.59.0", ] @@ -2980,20 +3728,29 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "ratatui" version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabd94c2f37801c20583fc49dd5cd6b0ba68c716787c2dd6ed18571e1e63117b" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "cassowary", "compact_str", "crossterm", "indoc", "instability", "itertools 0.13.0", - "lru", + "lru 0.12.5", "paste", "strum", "unicode-segmentation", @@ -3027,7 +3784,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", ] [[package]] @@ -3073,6 +3830,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.6" @@ -3090,25 +3853,25 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.7.0", + "hyper-rustls 0.27.7", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.32", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.3", "tower", "tower-http 0.6.6", "tower-service", @@ -3230,7 +3993,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -3243,27 +4006,52 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.11.0", "windows-sys 0.61.1", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.12.0" @@ -3274,12 +4062,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3291,6 +4090,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.20" @@ -3310,7 +4121,7 @@ dependencies = [ "derive_builder", "flate2", "openssl", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "pkg-config", "quick-xml", @@ -3339,6 +4150,15 @@ dependencies = [ "sdd", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.1", +] + [[package]] name = "schemars" version = "0.9.0" @@ -3369,6 +4189,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sdd" version = "3.0.10" @@ -3389,6 +4219,29 @@ dependencies = [ "zeroize", ] +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.1", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -3397,9 +4250,9 @@ checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "serde" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dca6411025b24b60bfa7ec1fe1f8e710ac09782dca409ee8237ba74b51295fd" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -3417,18 +4270,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba2ba63999edb9dac981fb34b3e5c0d111a69b0924e253ed29d83f7c99e966a4" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.226" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db53ae22f34573731bafa1db20f04027b2d25e02d8205921b569171699cdb33" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -3578,7 +4431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -3589,7 +4442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -3675,6 +4528,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.0" @@ -3694,6 +4557,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "spki" version = "0.7.3" @@ -3799,7 +4668,7 @@ checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.4", + "bitflags 2.11.1", "byteorder", "bytes", "chrono", @@ -3843,7 +4712,7 @@ checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", "base64 0.22.1", - "bitflags 2.9.4", + "bitflags 2.11.1", "byteorder", "chrono", "crc", @@ -4156,7 +5025,7 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", "windows-sys 0.59.0", ] @@ -4172,13 +5041,23 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ - "rustls", + "rustls 0.23.32", "tokio", ] @@ -4323,10 +5202,10 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "bytes", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "tower-layer", @@ -4339,11 +5218,11 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ - "bitflags 2.9.4", + "bitflags 2.11.1", "bytes", "futures-util", - "http", - "http-body", + "http 1.3.1", + "http-body 1.0.1", "iri-string", "pin-project-lite", "tower", @@ -4440,7 +5319,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.3.1", "httparse", "log", "rand 0.8.5", @@ -4459,7 +5338,7 @@ dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.3.1", "httparse", "log", "rand 0.8.5", @@ -4474,6 +5353,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.8.1" @@ -4703,6 +5588,21 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5200,9 +6100,9 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http", + "http 1.3.1", "http-body-util", - "hyper", + "hyper 1.7.0", "hyper-util", "log", "once_cell", @@ -5225,6 +6125,12 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yoke" version = "0.8.0" @@ -5345,6 +6251,34 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "zune-inflate" version = "0.2.54" diff --git a/Cargo.toml b/Cargo.toml index 7c8568c..51622fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "apps/agent", "apps/guest-agent", "apps/manager", "apps/installer", +"crates/nexus-backup", "crates/nexus-storage", "crates/nexus-types", ] diff --git a/apps/agent/Cargo.toml b/apps/agent/Cargo.toml index 1647a34..e2f33bd 100644 --- a/apps/agent/Cargo.toml +++ b/apps/agent/Cargo.toml @@ -28,6 +28,13 @@ reqwest = { workspace = true } uuid = { workspace = true } futures = { workspace = true } libc = "0.2" +nexus-backup = { path = "../../crates/nexus-backup" } +aws-sdk-s3 = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-credential-types = "1" +aws-config = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-types = "1" +hex = "0.4" +blake3 = "1" [dev-dependencies] tempfile = "3" diff --git a/apps/agent/src/features/storage/backup.rs b/apps/agent/src/features/storage/backup.rs new file mode 100644 index 0000000..2d36e4c --- /dev/null +++ b/apps/agent/src/features/storage/backup.rs @@ -0,0 +1,253 @@ +//! The chunker pipeline: read snapshot bytes → FastCDC → encrypt → HEAD-or-PUT. + +use crate::features::storage::registry::HostBackendRegistry; +use crate::features::storage::s3::{self, BackupTargetConfig}; +use anyhow::{Context, Result}; +use chrono::Utc; +use nexus_backup::{ + chunk_object_key, decrypt_chunk, decrypt_manifest, encrypt_chunk, encrypt_manifest, + manifest_object_key, ChunkKey, ChunkRef, Chunker, ChunkerParams, Manifest, MANIFEST_VERSION, +}; +use nexus_storage::{AttachedPath, VolumeHandle, VolumeSnapshotHandle}; +use std::sync::Arc; +use std::time::Instant; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; +use uuid::Uuid; + +pub struct BackupParams { + pub backup_id: Uuid, + pub snapshot: VolumeSnapshotHandle, + pub target: BackupTargetConfig, + pub encryption_key: [u8; 32], + pub chunker_params: ChunkerParams, +} + +pub struct BackupOutcome { + pub manifest_object_key: String, + pub chunk_count: u64, + pub bytes_written: u64, + pub bytes_unique: u64, + pub duration_ms: u64, +} + +pub async fn run_backup( + registry: Arc, + params: BackupParams, +) -> Result { + let start = Instant::now(); + let backend = registry + .get(params.snapshot.backend_kind) + .ok_or_else(|| { + anyhow::anyhow!( + "no host backend for kind {:?}", + params.snapshot.backend_kind + ) + })? + .clone(); + + let mut reader = backend + .read_snapshot(¶ms.snapshot) + .await + .context("read_snapshot")?; + let mut chunker = Chunker::new(&mut reader, params.chunker_params); + let s3 = s3::make_client(¶ms.target); + let key = ChunkKey::from_bytes(params.encryption_key); + + let mut chunks: Vec = Vec::new(); + let mut bytes_written: u64 = 0; + let mut bytes_unique: u64 = 0; + let mut total_plaintext: u64 = 0; + + while let Some(chunk) = chunker.next_chunk().await? { + let plaintext_hash: [u8; 32] = *blake3::hash(&chunk.plaintext_bytes).as_bytes(); + let ciphertext = encrypt_chunk(&key, &chunk.plaintext_bytes).context("encrypt_chunk")?; + let chunk_id: [u8; 32] = *blake3::hash(&ciphertext).as_bytes(); + let object_key = chunk_object_key(¶ms.target.prefix, &chunk_id); + + let exists = { + let mut attempt = 0u32; + loop { + match s3::head_object(&s3, ¶ms.target.bucket, &object_key).await { + Ok(v) => break v, + Err(e) if attempt < 4 => { + let backoff_ms = 200u64 * (1u64 << attempt); + tracing::warn!( + "HEAD chunk attempt {} failed: {e}; retrying in {backoff_ms}ms", + attempt + 1 + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + attempt += 1; + } + Err(e) => { + return Err(anyhow::anyhow!("HEAD chunk failed after 5 attempts: {e}")) + } + } + } + }; + bytes_written += ciphertext.len() as u64; + if !exists { + let cipher_len = ciphertext.len() as u64; + let mut attempt = 0u32; + loop { + match s3::put_object(&s3, ¶ms.target.bucket, &object_key, ciphertext.clone()) + .await + { + Ok(()) => break, + Err(e) if attempt < 4 => { + let backoff_ms = 200u64 * (1u64 << attempt); + tracing::warn!( + "PUT chunk attempt {} failed: {e}; retrying in {backoff_ms}ms", + attempt + 1 + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + attempt += 1; + } + Err(e) => { + return Err(anyhow::anyhow!("PUT chunk failed after 5 attempts: {e}")) + } + } + } + bytes_unique += cipher_len; + } + + chunks.push(ChunkRef { + plaintext_offset: chunk.plaintext_offset, + plaintext_length: chunk.plaintext_length, + plaintext_hash, + chunk_id, + ciphertext_length: ciphertext.len() as u32, + }); + total_plaintext += chunk.plaintext_length as u64; + } + + let manifest = Manifest { + version: MANIFEST_VERSION, + backup_id: params.backup_id, + source_volume_id: params.snapshot.source_volume_id, + source_snapshot_id: Some(params.snapshot.snapshot_id), + total_plaintext_size: total_plaintext, + created_at_unix_seconds: Utc::now().timestamp(), + chunks: chunks.clone(), + }; + let manifest_compressed = manifest + .serialize_compressed() + .context("manifest serialize")?; + let manifest_blob = encrypt_manifest(&key, &manifest_compressed).context("encrypt manifest")?; + let mkey = manifest_object_key(¶ms.target.prefix, ¶ms.backup_id); + { + let mut attempt = 0u32; + loop { + match s3::put_object(&s3, ¶ms.target.bucket, &mkey, manifest_blob.clone()).await { + Ok(()) => break, + Err(e) if attempt < 4 => { + let backoff_ms = 200u64 * (1u64 << attempt); + tracing::warn!( + "PUT manifest attempt {} failed: {e}; retrying in {backoff_ms}ms", + attempt + 1 + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + attempt += 1; + } + Err(e) => return Err(anyhow::anyhow!("PUT manifest failed after 5 attempts: {e}")), + } + } + } + + Ok(BackupOutcome { + manifest_object_key: mkey, + chunk_count: chunks.len() as u64, + bytes_written, + bytes_unique, + duration_ms: start.elapsed().as_millis() as u64, + }) +} + +pub struct RestoreParams { + pub target_volume: VolumeHandle, + pub target_attached: AttachedPath, + pub manifest_object_key: String, + pub target: BackupTargetConfig, + pub encryption_key: [u8; 32], +} + +pub struct RestoreOutcome { + pub bytes_written: u64, + pub duration_ms: u64, +} + +pub async fn run_restore(params: RestoreParams) -> Result { + let start = Instant::now(); + let s3 = s3::make_client(¶ms.target); + let key = ChunkKey::from_bytes(params.encryption_key); + + let blob = { + let mut attempt = 0u32; + loop { + match s3::get_object(&s3, ¶ms.target.bucket, ¶ms.manifest_object_key).await { + Ok(v) => break v, + Err(e) if attempt < 4 => { + let backoff_ms = 200u64 * (1u64 << attempt); + tracing::warn!( + "GET manifest attempt {} failed: {e}; retrying in {backoff_ms}ms", + attempt + 1 + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + attempt += 1; + } + Err(e) => return Err(anyhow::anyhow!("GET manifest failed after 5 attempts: {e}")), + } + } + }; + let compressed = decrypt_manifest(&key, &blob).context("decrypt manifest")?; + let manifest = Manifest::deserialize_compressed(&compressed).context("deserialize manifest")?; + + let mut dst = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(false) + .open(params.target_attached.path()) + .await?; + + let mut bytes_written: u64 = 0; + for chunk_ref in &manifest.chunks { + let object_key = chunk_object_key(¶ms.target.prefix, &chunk_ref.chunk_id); + let ciphertext = { + let mut attempt = 0u32; + loop { + match s3::get_object(&s3, ¶ms.target.bucket, &object_key).await { + Ok(v) => break v, + Err(e) if attempt < 4 => { + let backoff_ms = 200u64 * (1u64 << attempt); + tracing::warn!( + "GET chunk {} attempt {} failed: {e}; retrying in {backoff_ms}ms", + hex::encode(chunk_ref.chunk_id), + attempt + 1 + ); + tokio::time::sleep(std::time::Duration::from_millis(backoff_ms)).await; + attempt += 1; + } + Err(e) => { + return Err(anyhow::anyhow!( + "GET chunk {} failed after 5 attempts: {e}", + hex::encode(chunk_ref.chunk_id) + )) + } + } + } + }; + let plaintext = + decrypt_chunk(&key, &ciphertext, &chunk_ref.plaintext_hash).context("decrypt chunk")?; + dst.seek(std::io::SeekFrom::Start(chunk_ref.plaintext_offset)) + .await?; + dst.write_all(&plaintext).await?; + bytes_written += plaintext.len() as u64; + } + dst.flush().await?; + + let _ = params.target_volume; // suppress unused warning; kept for future logging + + Ok(RestoreOutcome { + bytes_written, + duration_ms: start.elapsed().as_millis() as u64, + }) +} diff --git a/apps/agent/src/features/storage/iscsi.rs b/apps/agent/src/features/storage/iscsi.rs index 18e54cf..8f309eb 100644 --- a/apps/agent/src/features/storage/iscsi.rs +++ b/apps/agent/src/features/storage/iscsi.rs @@ -6,7 +6,9 @@ //! Locator format (JSON): {"iqn":"...","lun":N,"dataset":"...","portal":"..."} //! `dataset` is ignored on the host side; it's a control-plane concern. -use nexus_storage::{AttachedPath, BackendKind, HostBackend, StorageError, VolumeHandle}; +use nexus_storage::{ + AttachedPath, BackendKind, HostBackend, StorageError, VolumeHandle, VolumeSnapshotHandle, +}; use serde::Deserialize; use std::path::{Path, PathBuf}; @@ -18,6 +20,39 @@ struct LocatorJson { portal: Option, } +/// An `AsyncRead` wrapper around a `tokio::fs::File` opened on an iSCSI block +/// device. On drop it spawns a best-effort logout so the session is cleaned up +/// even if the caller forgets to call `detach`. +struct IscsiSnapshotReader { + inner: tokio::fs::File, + iqn: String, + portal: String, +} + +impl tokio::io::AsyncRead for IscsiSnapshotReader { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_read(cx, buf) + } +} + +impl Drop for IscsiSnapshotReader { + fn drop(&mut self) { + // Best-effort logout. We cannot await in Drop so spawn a detached task. + let iqn = std::mem::take(&mut self.iqn); + let portal = std::mem::take(&mut self.portal); + tokio::spawn(async move { + let _ = tokio::process::Command::new("iscsiadm") + .args(["-m", "node", "-T", &iqn, "-p", &portal, "--logout"]) + .output() + .await; + }); + } +} + pub struct IscsiHostBackend; impl IscsiHostBackend { @@ -130,6 +165,42 @@ impl HostBackend for IscsiHostBackend { let _ = target_size_bytes; Ok(()) } + + async fn read_snapshot( + &self, + snap: &VolumeSnapshotHandle, + ) -> Result, StorageError> { + // Snapshot's locator has the same JSON shape as a volume's locator — + // {iqn, lun, portal, dataset?} — but the LUN refers to the read-only + // snapshot extent. + let loc = Self::parse_locator(&snap.locator)?; + Self::iscsiadm_login(&loc).await?; + let dev = Self::block_device_path(&loc); + let portal = loc + .portal + .clone() + .unwrap_or_else(|| "127.0.0.1".to_string()); + for _ in 0..30 { + if dev.exists() { + let f = tokio::fs::File::open(&dev).await?; + return Ok(Box::new(IscsiSnapshotReader { + inner: f, + iqn: loc.iqn.clone(), + portal, + })); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + // No device appeared — log out before erroring so we don't leak the session. + Self::iscsiadm_logout(&loc).await.ok(); + Err(StorageError::Backend( + format!( + "snapshot device {} did not appear after iscsi login", + dev.display() + ) + .into(), + )) + } } #[cfg(test)] diff --git a/apps/agent/src/features/storage/local_file.rs b/apps/agent/src/features/storage/local_file.rs index f1ae9c0..1dfac38 100644 --- a/apps/agent/src/features/storage/local_file.rs +++ b/apps/agent/src/features/storage/local_file.rs @@ -1,4 +1,6 @@ -use nexus_storage::{AttachedPath, BackendKind, HostBackend, StorageError, VolumeHandle}; +use nexus_storage::{ + AttachedPath, BackendKind, HostBackend, StorageError, VolumeHandle, VolumeSnapshotHandle, +}; use std::path::{Path, PathBuf}; /// Agent-side LocalFile backend. Trivial: the locator IS the file path. @@ -45,6 +47,15 @@ impl HostBackend for LocalFileHostBackend { dst.flush().await?; Ok(()) } + + async fn read_snapshot( + &self, + snap: &VolumeSnapshotHandle, + ) -> Result, StorageError> { + let path = std::path::PathBuf::from(&snap.locator); + let f = tokio::fs::File::open(&path).await?; + Ok(Box::new(f)) + } } #[cfg(test)] @@ -83,4 +94,29 @@ mod tests { // File extended to target size (sparse tail OK). assert_eq!(std::fs::metadata(&dst).unwrap().len(), 16 * 1024); } + + #[tokio::test] + async fn read_snapshot_returns_file_contents() { + use nexus_storage::{BackendInstanceId, VolumeSnapshotHandle}; + use tokio::io::AsyncReadExt; + use uuid::Uuid; + + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("snap.img"); + std::fs::write(&p, b"snapshot-bytes").unwrap(); + + let snap = VolumeSnapshotHandle { + snapshot_id: Uuid::new_v4(), + source_volume_id: Uuid::new_v4(), + backend_id: BackendInstanceId(Uuid::new_v4()), + backend_kind: BackendKind::LocalFile, + locator: p.display().to_string(), + }; + + let backend = LocalFileHostBackend; + let mut reader = backend.read_snapshot(&snap).await.unwrap(); + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await.unwrap(); + assert_eq!(buf, b"snapshot-bytes"); + } } diff --git a/apps/agent/src/features/storage/mod.rs b/apps/agent/src/features/storage/mod.rs index 6b57753..8d12ecc 100644 --- a/apps/agent/src/features/storage/mod.rs +++ b/apps/agent/src/features/storage/mod.rs @@ -1,4 +1,6 @@ +pub mod backup; pub mod iscsi; pub mod local_file; pub mod registry; pub mod routes; +pub mod s3; diff --git a/apps/agent/src/features/storage/routes.rs b/apps/agent/src/features/storage/routes.rs index 778d246..d99a26f 100644 --- a/apps/agent/src/features/storage/routes.rs +++ b/apps/agent/src/features/storage/routes.rs @@ -1,4 +1,6 @@ +use crate::features::storage::backup::{run_backup, run_restore, BackupParams, RestoreParams}; use crate::features::storage::registry::HostBackendRegistry; +use crate::features::storage::s3::BackupTargetConfig as S3Config; use axum::{ extract::State, http::StatusCode, @@ -6,6 +8,7 @@ use axum::{ routing::{get, post}, Json, Router, }; +use nexus_backup::ChunkerParams as NexusChunkerParams; use nexus_storage::{AttachedPath, BackendKind, VolumeHandle}; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -162,6 +165,145 @@ pub async fn supported_kinds(State(s): State>) -> impl IntoRes (StatusCode::OK, Json(serde_json::json!({"kinds": kinds}))).into_response() } +#[derive(Deserialize)] +pub struct BackupReq { + pub backup_id: uuid::Uuid, + pub snapshot: nexus_storage::VolumeSnapshotHandle, + #[allow(dead_code)] // wire field; backend_kind is read from snapshot.backend_kind + pub backend_kind: nexus_storage::BackendKind, + pub target: BackupTargetWire, + pub encryption_key: [u8; 32], + pub chunker_params: ChunkerParamsWire, +} + +#[derive(Deserialize)] +pub struct BackupTargetWire { + pub endpoint: String, + #[serde(default)] + pub region: Option, + pub bucket: String, + #[serde(default)] + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +#[derive(Deserialize)] +pub struct ChunkerParamsWire { + pub min_size: u32, + pub avg_size: u32, + pub max_size: u32, +} + +#[derive(Serialize)] +pub struct BackupRespWire { + pub manifest_object_key: String, + pub chunk_count: u64, + pub bytes_written: u64, + pub bytes_unique: u64, + pub duration_ms: u64, +} + +pub async fn backup( + State(s): State>, + Json(req): Json, +) -> impl IntoResponse { + let target = S3Config { + endpoint: req.target.endpoint, + region: req.target.region, + bucket: req.target.bucket, + prefix: req.target.prefix, + access_key_id: req.target.access_key_id, + secret_access_key: req.target.secret_access_key, + }; + let params = BackupParams { + backup_id: req.backup_id, + snapshot: req.snapshot, + target, + encryption_key: req.encryption_key, + chunker_params: NexusChunkerParams { + min_size: req.chunker_params.min_size, + avg_size: req.chunker_params.avg_size, + max_size: req.chunker_params.max_size, + }, + }; + match run_backup(Arc::new(s.registry.clone()), params).await { + Ok(o) => ( + StatusCode::OK, + Json(BackupRespWire { + manifest_object_key: o.manifest_object_key, + chunk_count: o.chunk_count, + bytes_written: o.bytes_written, + bytes_unique: o.bytes_unique, + duration_ms: o.duration_ms, + }), + ) + .into_response(), + Err(e) => { + tracing::error!("agent backup failed: {e:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": e.to_string()})), + ) + .into_response() + } + } +} + +#[derive(Deserialize)] +pub struct RestoreReq { + pub target_volume: nexus_storage::VolumeHandle, + pub target_attached: nexus_storage::AttachedPath, + pub manifest_object_key: String, + pub target: BackupTargetWire, + pub encryption_key: [u8; 32], +} + +#[derive(Serialize)] +pub struct RestoreRespWire { + pub bytes_written: u64, + pub duration_ms: u64, +} + +pub async fn restore( + State(_s): State>, + Json(req): Json, +) -> impl IntoResponse { + let target = S3Config { + endpoint: req.target.endpoint, + region: req.target.region, + bucket: req.target.bucket, + prefix: req.target.prefix, + access_key_id: req.target.access_key_id, + secret_access_key: req.target.secret_access_key, + }; + let params = RestoreParams { + target_volume: req.target_volume, + target_attached: req.target_attached, + manifest_object_key: req.manifest_object_key, + target, + encryption_key: req.encryption_key, + }; + match run_restore(params).await { + Ok(o) => ( + StatusCode::OK, + Json(RestoreRespWire { + bytes_written: o.bytes_written, + duration_ms: o.duration_ms, + }), + ) + .into_response(), + Err(e) => { + tracing::error!("agent restore failed: {e:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": e.to_string()})), + ) + .into_response() + } + } +} + pub fn router(state: Arc) -> Router { Router::new() .route("/attach", post(attach)) @@ -169,5 +311,7 @@ pub fn router(state: Arc) -> Router { .route("/populate", post(populate)) .route("/resize2fs", post(resize2fs)) .route("/supported_kinds", get(supported_kinds)) + .route("/backup", post(backup)) + .route("/restore", post(restore)) .with_state(state) } diff --git a/apps/agent/src/features/storage/s3.rs b/apps/agent/src/features/storage/s3.rs new file mode 100644 index 0000000..68e2f37 --- /dev/null +++ b/apps/agent/src/features/storage/s3.rs @@ -0,0 +1,93 @@ +//! Thin async wrapper over aws-sdk-s3 for the backup pipeline. + +use aws_credential_types::Credentials; +use aws_sdk_s3::{ + config::{Builder, Region}, + error::SdkError, + operation::head_object::HeadObjectError, + Client, +}; +use std::time::Duration; + +#[derive(Clone)] +pub struct BackupTargetConfig { + pub endpoint: String, + pub region: Option, + pub bucket: String, + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +pub fn make_client(cfg: &BackupTargetConfig) -> Client { + let creds = Credentials::new( + cfg.access_key_id.clone(), + cfg.secret_access_key.clone(), + None, + None, + "nqrust-backup", + ); + let region = Region::new(cfg.region.clone().unwrap_or_else(|| "us-east-1".into())); + let cfg_built = Builder::new() + .behavior_version_latest() + .endpoint_url(&cfg.endpoint) + .credentials_provider(creds) + .region(region) + .force_path_style(true) + .timeout_config( + aws_sdk_s3::config::timeout::TimeoutConfig::builder() + .operation_timeout(Duration::from_secs(120)) + .build(), + ) + .build(); + Client::from_conf(cfg_built) +} + +#[derive(Debug, thiserror::Error)] +pub enum S3Error { + #[error("s3: {0}")] + Other(String), +} + +pub async fn head_object(client: &Client, bucket: &str, key: &str) -> Result { + match client.head_object().bucket(bucket).key(key).send().await { + Ok(_) => Ok(true), + Err(SdkError::ServiceError(svc)) if matches!(svc.err(), HeadObjectError::NotFound(_)) => { + Ok(false) + } + Err(e) => Err(S3Error::Other(format!("head: {e}"))), + } +} + +pub async fn put_object( + client: &Client, + bucket: &str, + key: &str, + body: Vec, +) -> Result<(), S3Error> { + client + .put_object() + .bucket(bucket) + .key(key) + .body(body.into()) + .send() + .await + .map_err(|e| S3Error::Other(format!("put: {e}")))?; + Ok(()) +} + +pub async fn get_object(client: &Client, bucket: &str, key: &str) -> Result, S3Error> { + let resp = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .map_err(|e| S3Error::Other(format!("get: {e}")))?; + let body = resp + .body + .collect() + .await + .map_err(|e| S3Error::Other(format!("get body: {e}")))?; + Ok(body.into_bytes().to_vec()) +} diff --git a/apps/manager/Cargo.toml b/apps/manager/Cargo.toml index 433c922..a1ca935 100644 --- a/apps/manager/Cargo.toml +++ b/apps/manager/Cargo.toml @@ -50,6 +50,13 @@ samael = "0.0.17" openssl = { version = "0.10", features = ["vendored"] } aes-gcm = "0.10" url = "2" +nexus-backup = { path = "../../crates/nexus-backup" } +aws-sdk-s3 = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-credential-types = "1" +aws-config = { version = "1", default-features = false, features = ["rustls", "rt-tokio"] } +aws-types = "1" +aws-smithy-types = "1" +cron = "0.12" [dev-dependencies] wiremock = "0.6" diff --git a/apps/manager/migrations/0036_backup.sql b/apps/manager/migrations/0036_backup.sql new file mode 100644 index 0000000..22fb0b1 --- /dev/null +++ b/apps/manager/migrations/0036_backup.sql @@ -0,0 +1,62 @@ +-- 0036_backup.sql — Chunked encrypted backup pipeline. + +CREATE TABLE IF NOT EXISTS backup_target ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + endpoint TEXT NOT NULL, + region TEXT, + bucket TEXT NOT NULL, + prefix TEXT NOT NULL DEFAULT '', + access_key_id TEXT NOT NULL, + encrypted_secret_access_key BYTEA NOT NULL, + encrypted_target_key BYTEA NOT NULL, + gc_hour SMALLINT NOT NULL DEFAULT 3 CHECK (gc_hour BETWEEN 0 AND 23), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + deleted_at TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS backup ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_volume_id UUID REFERENCES volume(id) ON DELETE SET NULL, + source_snapshot_id UUID, + target_id UUID NOT NULL REFERENCES backup_target(id), + manifest_object_key TEXT, + size_bytes BIGINT NOT NULL DEFAULT 0, + unique_bytes BIGINT NOT NULL DEFAULT 0, + chunk_count BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'running' + CHECK (status IN ('running', 'completed', 'failed', 'pruning')), + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_backup_volume ON backup(source_volume_id); +CREATE INDEX IF NOT EXISTS idx_backup_target ON backup(target_id); +CREATE INDEX IF NOT EXISTS idx_backup_status_updated ON backup(status, updated_at) + WHERE status = 'running'; + +ALTER TABLE volume ADD COLUMN IF NOT EXISTS backup_cron TEXT; +ALTER TABLE volume ADD COLUMN IF NOT EXISTS backup_retain_count INT; +ALTER TABLE volume ADD COLUMN IF NOT EXISTS backup_target_id UUID + REFERENCES backup_target(id); + +CREATE TABLE IF NOT EXISTS backup_gc_run ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + target_id UUID NOT NULL REFERENCES backup_target(id) ON DELETE CASCADE, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + bytes_freed BIGINT NOT NULL DEFAULT 0, + chunks_deleted BIGINT NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'running' + CHECK (status IN ('running', 'completed', 'failed')), + error_message TEXT +); +CREATE INDEX IF NOT EXISTS idx_backup_gc_run_target ON backup_gc_run(target_id, started_at DESC); + +COMMENT ON COLUMN backup_target.encrypted_secret_access_key IS + 'AES-GCM(envelope_key) over the S3 secret access key.'; +COMMENT ON COLUMN backup_target.encrypted_target_key IS + 'AES-GCM(envelope_key) over the per-target XChaCha20-Poly1305 key used for chunk + manifest encryption.'; +COMMENT ON COLUMN backup.unique_bytes IS + 'Post-dedup ciphertext bytes that this backup actually wrote (chunks not skipped by HEAD).'; diff --git a/apps/manager/src/features/backup_targets/envelope.rs b/apps/manager/src/features/backup_targets/envelope.rs new file mode 100644 index 0000000..9c91ecb --- /dev/null +++ b/apps/manager/src/features/backup_targets/envelope.rs @@ -0,0 +1,104 @@ +//! AES-GCM(envelope_key) wrap/unwrap for backup target secrets. +//! Reuses the same MANAGER_ENVELOPE_KEY env var the SSO module uses. + +use aes_gcm::{aead::Aead, Aes256Gcm, Key, KeyInit, Nonce}; +use anyhow::{anyhow, Context, Result}; + +const NONCE_LEN: usize = 12; + +fn cipher() -> Result { + let raw = std::env::var("MANAGER_ENVELOPE_KEY").context("MANAGER_ENVELOPE_KEY not set")?; + let bytes = hex::decode(raw).context("MANAGER_ENVELOPE_KEY must be hex-encoded")?; + if bytes.len() != 32 { + return Err(anyhow!( + "MANAGER_ENVELOPE_KEY must be 32 bytes (64 hex chars), got {}", + bytes.len() + )); + } + let key = Key::::from_slice(&bytes); + Ok(Aes256Gcm::new(key)) +} + +pub fn wrap(plaintext: &[u8]) -> Result> { + use rand::RngCore; + let c = cipher()?; + let mut nonce_bytes = [0u8; NONCE_LEN]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = Nonce::from_slice(&nonce_bytes); + let ct = c + .encrypt(nonce, plaintext) + .map_err(|e| anyhow!("aes-gcm encrypt: {e}"))?; + let mut out = Vec::with_capacity(NONCE_LEN + ct.len()); + out.extend_from_slice(&nonce_bytes); + out.extend_from_slice(&ct); + Ok(out) +} + +#[allow(dead_code)] +pub fn unwrap_to_string(blob: &[u8]) -> Result { + let bytes = unwrap(blob)?; + String::from_utf8(bytes).context("decrypted secret is not utf-8") +} + +#[allow(dead_code)] +pub fn unwrap_to_array(blob: &[u8]) -> Result<[u8; N]> { + let bytes = unwrap(blob)?; + if bytes.len() != N { + return Err(anyhow!( + "decrypted blob is {} bytes, expected {}", + bytes.len(), + N + )); + } + let mut out = [0u8; N]; + out.copy_from_slice(&bytes); + Ok(out) +} + +#[allow(dead_code)] +fn unwrap(blob: &[u8]) -> Result> { + if blob.len() < NONCE_LEN { + return Err(anyhow!("envelope blob too short")); + } + let c = cipher()?; + let nonce = Nonce::from_slice(&blob[..NONCE_LEN]); + c.decrypt(nonce, &blob[NONCE_LEN..]) + .map_err(|_| anyhow!("envelope decrypt: auth failed")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with_key(f: F) { + std::env::set_var("MANAGER_ENVELOPE_KEY", "00".repeat(32)); + f(); + } + + #[test] + fn wrap_unwrap_string() { + with_key(|| { + let blob = wrap(b"secret-access-key").unwrap(); + let s = unwrap_to_string(&blob).unwrap(); + assert_eq!(s, "secret-access-key"); + }); + } + + #[test] + fn wrap_unwrap_array() { + with_key(|| { + let blob = wrap(&[0xAAu8; 32]).unwrap(); + let a: [u8; 32] = unwrap_to_array(&blob).unwrap(); + assert_eq!(a, [0xAAu8; 32]); + }); + } + + #[test] + fn tampered_blob_rejected() { + with_key(|| { + let mut blob = wrap(b"hello").unwrap(); + blob[20] ^= 1; + assert!(unwrap_to_string(&blob).is_err()); + }); + } +} diff --git a/apps/manager/src/features/backup_targets/mod.rs b/apps/manager/src/features/backup_targets/mod.rs new file mode 100644 index 0000000..0031914 --- /dev/null +++ b/apps/manager/src/features/backup_targets/mod.rs @@ -0,0 +1,20 @@ +pub mod envelope; +pub mod repo; +pub mod routes; + +use axum::{ + routing::{get, post}, + Router, +}; + +pub fn router() -> Router { + Router::new() + .route("/", post(routes::create).get(routes::list)) + .route( + "/:id", + get(routes::get_one) + .patch(routes::update) + .delete(routes::soft_delete), + ) + .route("/:id/gc", post(routes::trigger_gc)) +} diff --git a/apps/manager/src/features/backup_targets/repo.rs b/apps/manager/src/features/backup_targets/repo.rs new file mode 100644 index 0000000..e0f7967 --- /dev/null +++ b/apps/manager/src/features/backup_targets/repo.rs @@ -0,0 +1,102 @@ +use chrono::{DateTime, Utc}; +use sqlx::PgPool; +use uuid::Uuid; + +#[derive(Clone)] +pub struct BackupTargetRepository { + pool: PgPool, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +#[allow(dead_code)] +pub struct BackupTargetRow { + pub id: Uuid, + pub name: String, + pub endpoint: String, + pub region: Option, + pub bucket: String, + pub prefix: String, + pub access_key_id: String, + pub encrypted_secret_access_key: Vec, + pub encrypted_target_key: Vec, + pub gc_hour: i16, + pub created_at: DateTime, + pub deleted_at: Option>, +} + +pub struct CreateParams<'a> { + pub name: &'a str, + pub endpoint: &'a str, + pub region: Option<&'a str>, + pub bucket: &'a str, + pub prefix: &'a str, + pub access_key_id: &'a str, + pub encrypted_secret_access_key: &'a [u8], + pub encrypted_target_key: &'a [u8], + pub gc_hour: i16, +} + +impl BackupTargetRepository { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } + + pub async fn list_active(&self) -> sqlx::Result> { + sqlx::query_as::<_, BackupTargetRow>( + r#"SELECT * FROM backup_target WHERE deleted_at IS NULL ORDER BY name"#, + ) + .fetch_all(&self.pool) + .await + } + + pub async fn get(&self, id: Uuid) -> sqlx::Result> { + sqlx::query_as::<_, BackupTargetRow>( + r#"SELECT * FROM backup_target WHERE id = $1 AND deleted_at IS NULL"#, + ) + .bind(id) + .fetch_optional(&self.pool) + .await + } + + pub async fn create(&self, p: CreateParams<'_>) -> sqlx::Result { + sqlx::query_as::<_, BackupTargetRow>( + r#" + INSERT INTO backup_target + (name, endpoint, region, bucket, prefix, access_key_id, + encrypted_secret_access_key, encrypted_target_key, gc_hour) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + RETURNING * + "#, + ) + .bind(p.name) + .bind(p.endpoint) + .bind(p.region) + .bind(p.bucket) + .bind(p.prefix) + .bind(p.access_key_id) + .bind(p.encrypted_secret_access_key) + .bind(p.encrypted_target_key) + .bind(p.gc_hour) + .fetch_one(&self.pool) + .await + } + + pub async fn soft_delete(&self, id: Uuid) -> sqlx::Result<()> { + sqlx::query( + r#"UPDATE backup_target SET deleted_at = now() WHERE id = $1 AND deleted_at IS NULL"#, + ) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn count_backups_for_target(&self, id: Uuid) -> sqlx::Result { + sqlx::query_scalar::<_, i64>( + r#"SELECT COUNT(*) FROM backup WHERE target_id = $1 AND status IN ('running','completed')"#, + ) + .bind(id) + .fetch_one(&self.pool) + .await + } +} diff --git a/apps/manager/src/features/backup_targets/routes.rs b/apps/manager/src/features/backup_targets/routes.rs new file mode 100644 index 0000000..2717892 --- /dev/null +++ b/apps/manager/src/features/backup_targets/routes.rs @@ -0,0 +1,219 @@ +use crate::features::backup_targets::envelope; +use crate::features::backup_targets::repo::{ + BackupTargetRepository, BackupTargetRow, CreateParams, +}; +use crate::AppState; +use axum::{extract::Path, http::StatusCode, response::IntoResponse, Extension, Json}; +use nexus_types::{BackupTarget, CreateBackupTargetRequest}; +use rand::RngCore; +use uuid::Uuid; + +fn row_to_wire(row: BackupTargetRow) -> BackupTarget { + BackupTarget { + id: row.id, + name: row.name, + endpoint: row.endpoint, + region: row.region, + bucket: row.bucket, + prefix: row.prefix, + access_key_id: row.access_key_id, + gc_hour: row.gc_hour as u8, + created_at: row.created_at, + deleted_at: row.deleted_at, + } +} + +#[derive(serde::Serialize, utoipa::ToSchema)] +pub struct BackupTargetListResponse { + pub items: Vec, +} + +pub async fn list(Extension(st): Extension) -> impl IntoResponse { + let repo = BackupTargetRepository::new(st.db.clone()); + match repo.list_active().await { + Ok(rows) => ( + StatusCode::OK, + Json(BackupTargetListResponse { + items: rows.into_iter().map(row_to_wire).collect(), + }), + ) + .into_response(), + Err(e) => { + tracing::error!("backup_targets list: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": "db"})), + ) + .into_response() + } + } +} + +pub async fn get_one( + Extension(st): Extension, + Path(id): Path, +) -> impl IntoResponse { + let repo = BackupTargetRepository::new(st.db.clone()); + match repo.get(id).await { + Ok(Some(row)) => (StatusCode::OK, Json(row_to_wire(row))).into_response(), + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({"error":"not found"})), + ) + .into_response(), + Err(e) => { + tracing::error!("backup_targets get: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub async fn create( + Extension(st): Extension, + Json(req): Json, +) -> impl IntoResponse { + let repo = BackupTargetRepository::new(st.db.clone()); + + let mut target_key = [0u8; 32]; + rand::thread_rng().fill_bytes(&mut target_key); + + let enc_secret = match envelope::wrap(req.secret_access_key.as_bytes()) { + Ok(b) => b, + Err(e) => { + tracing::error!("envelope wrap secret: {e:#}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"envelope"})), + ) + .into_response(); + } + }; + let enc_target = match envelope::wrap(&target_key) { + Ok(b) => b, + Err(e) => { + tracing::error!("envelope wrap target_key: {e:#}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"envelope"})), + ) + .into_response(); + } + }; + + match repo + .create(CreateParams { + name: &req.name, + endpoint: &req.endpoint, + region: req.region.as_deref(), + bucket: &req.bucket, + prefix: &req.prefix, + access_key_id: &req.access_key_id, + encrypted_secret_access_key: &enc_secret, + encrypted_target_key: &enc_target, + gc_hour: req.gc_hour as i16, + }) + .await + { + Ok(row) => (StatusCode::CREATED, Json(row_to_wire(row))).into_response(), + Err(sqlx::Error::Database(e)) if e.code().as_deref() == Some("23505") => ( + StatusCode::CONFLICT, + Json(serde_json::json!({"error":"name already exists"})), + ) + .into_response(), + Err(e) => { + tracing::error!("backup_targets create: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub async fn update( + Extension(_st): Extension, + Path(_id): Path, + Json(_req): Json, +) -> impl IntoResponse { + ( + StatusCode::NOT_IMPLEMENTED, + Json(serde_json::json!({"error":"update not in v1"})), + ) +} + +pub async fn soft_delete( + Extension(st): Extension, + Path(id): Path, +) -> impl IntoResponse { + let repo = BackupTargetRepository::new(st.db.clone()); + match repo.count_backups_for_target(id).await { + Ok(n) if n > 0 => ( + StatusCode::CONFLICT, + Json(serde_json::json!({ + "error": format!("target has {n} backups; delete them first"), + })), + ) + .into_response(), + Ok(_) => match repo.soft_delete(id).await { + Ok(()) => (StatusCode::NO_CONTENT, ()).into_response(), + Err(e) => { + tracing::error!("backup_targets soft_delete: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + }, + Err(e) => { + tracing::error!("backup_targets count: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub async fn trigger_gc( + Extension(st): Extension, + Path(id): Path, +) -> impl IntoResponse { + let target_repo = + crate::features::backup_targets::repo::BackupTargetRepository::new(st.db.clone()); + let target = match target_repo.get(id).await { + Ok(Some(t)) => t, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({"error":"not found"})), + ) + .into_response() + } + Err(e) => { + tracing::error!("backup_targets trigger_gc lookup: {e}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response(); + } + }; + let pool = st.db.clone(); + tokio::spawn(async move { + if let Err(e) = crate::features::backups::gc::run_gc(&pool, &target).await { + tracing::error!("ad-hoc GC failed: {e:#}"); + } + }); + ( + StatusCode::ACCEPTED, + Json(serde_json::json!({"queued": true})), + ) + .into_response() +} diff --git a/apps/manager/src/features/backups/gc.rs b/apps/manager/src/features/backups/gc.rs new file mode 100644 index 0000000..0d5ebd9 --- /dev/null +++ b/apps/manager/src/features/backups/gc.rs @@ -0,0 +1,253 @@ +//! Daily mark-and-sweep GC per backup_target. + +use crate::features::backup_targets::envelope; +use crate::features::backup_targets::repo::{BackupTargetRepository, BackupTargetRow}; +use anyhow::{Context, Result}; +use aws_sdk_s3::{ + config::{Builder as S3ConfBuilder, Region}, + types::{Delete, ObjectIdentifier}, + Client, +}; +use chrono::{TimeZone as _, Timelike, Utc}; +use nexus_backup::{decrypt_manifest, ChunkKey, Manifest}; +use sqlx::PgPool; +use std::collections::HashSet; + +pub async fn gc_loop(pool: PgPool) { + loop { + let now = Utc::now(); + // Sleep to the next minute boundary so we check every minute. + let secs_to_next_minute = (60 - now.second() as u64) % 60; + let sleep_for = if secs_to_next_minute == 0 { + 60 + } else { + secs_to_next_minute + }; + tokio::time::sleep(std::time::Duration::from_secs(sleep_for)).await; + + let now = Utc::now(); + let repo = BackupTargetRepository::new(pool.clone()); + match repo.list_active().await { + Ok(targets) => { + for t in targets { + if (t.gc_hour as u32) == now.hour() { + // Dedupe: only run once per target per gc_hour window. + let today_at_hour = now + .date_naive() + .and_hms_opt(t.gc_hour as u32, 0, 0) + .map(|dt| Utc.from_utc_datetime(&dt)) + .unwrap_or(now); + let existing: Option> = sqlx::query_scalar( + r#"SELECT started_at FROM backup_gc_run + WHERE target_id = $1 AND started_at >= $2 + ORDER BY started_at DESC LIMIT 1"#, + ) + .bind(t.id) + .bind(today_at_hour) + .fetch_optional(&pool) + .await + .ok() + .flatten(); + if existing.is_some() { + continue; + } + if let Err(e) = run_gc(&pool, &t).await { + tracing::error!(target=%t.name, "GC run failed: {e:#}"); + } + } + } + } + Err(e) => tracing::error!("gc_loop list_active: {e}"), + } + } +} + +pub async fn run_gc(pool: &PgPool, target: &BackupTargetRow) -> Result<()> { + let run_id: uuid::Uuid = sqlx::query_scalar( + r#"INSERT INTO backup_gc_run (target_id, status) VALUES ($1, 'running') RETURNING id"#, + ) + .bind(target.id) + .fetch_one(pool) + .await?; + + let result = sweep(target).await; + match result { + Ok((bytes, chunks)) => { + sqlx::query( + r#"UPDATE backup_gc_run SET status='completed', completed_at=now(), + bytes_freed=$1, chunks_deleted=$2 WHERE id = $3"#, + ) + .bind(bytes as i64) + .bind(chunks as i64) + .bind(run_id) + .execute(pool) + .await?; + tracing::info!(target=%target.name, bytes_freed=bytes, chunks_deleted=chunks, "GC complete"); + Ok(()) + } + Err(e) => { + sqlx::query( + r#"UPDATE backup_gc_run SET status='failed', completed_at=now(), error_message=$1 WHERE id=$2"#, + ) + .bind(e.to_string()) + .bind(run_id) + .execute(pool) + .await + .ok(); + Err(e) + } + } +} + +async fn sweep(target: &BackupTargetRow) -> Result<(u64, u64)> { + let secret = envelope::unwrap_to_string(&target.encrypted_secret_access_key)?; + let target_key = envelope::unwrap_to_array::<32>(&target.encrypted_target_key)?; + + let creds = aws_credential_types::Credentials::new( + &target.access_key_id, + &secret, + None, + None, + "nqrust-gc", + ); + let region = Region::new(target.region.clone().unwrap_or_else(|| "us-east-1".into())); + let s3_cfg = S3ConfBuilder::new() + .behavior_version_latest() + .endpoint_url(&target.endpoint) + .credentials_provider(creds) + .region(region) + .force_path_style(true) + .build(); + let client = Client::from_conf(s3_cfg); + + // 1. Mark: walk manifests, collect referenced chunk ids. + let prefix_manifests = if target.prefix.is_empty() { + "manifests/".to_string() + } else { + format!("{}/manifests/", target.prefix.trim_end_matches('/')) + }; + let mut referenced: HashSet<[u8; 32]> = HashSet::new(); + let mut continuation: Option = None; + loop { + let mut req = client + .list_objects_v2() + .bucket(&target.bucket) + .prefix(&prefix_manifests); + if let Some(c) = continuation.as_deref() { + req = req.continuation_token(c); + } + let resp = req.send().await.context("LIST manifests")?; + for obj in resp.contents() { + let Some(k) = obj.key() else { continue }; + let blob = client + .get_object() + .bucket(&target.bucket) + .key(k) + .send() + .await + .with_context(|| format!("GET {k}"))? + .body + .collect() + .await + .context("body collect")? + .into_bytes() + .to_vec(); + let chunk_key = ChunkKey::from_bytes(target_key); + let compressed = decrypt_manifest(&chunk_key, &blob).context("decrypt manifest")?; + let m: Manifest = + Manifest::deserialize_compressed(&compressed).context("deserialize manifest")?; + for c in m.chunks { + referenced.insert(c.chunk_id); + } + } + if resp.is_truncated().unwrap_or(false) { + continuation = resp.next_continuation_token().map(String::from); + } else { + break; + } + } + + // 2. Sweep: walk chunks, delete unreferenced ones older than 24h. + let prefix_chunks = if target.prefix.is_empty() { + "chunks/".to_string() + } else { + format!("{}/chunks/", target.prefix.trim_end_matches('/')) + }; + let cutoff = aws_smithy_types::DateTime::from_secs( + (Utc::now() - chrono::Duration::hours(24)).timestamp(), + ); + let mut bytes_freed: u64 = 0; + let mut chunks_deleted: u64 = 0; + + let mut continuation: Option = None; + loop { + let mut req = client + .list_objects_v2() + .bucket(&target.bucket) + .prefix(&prefix_chunks); + if let Some(c) = continuation.as_deref() { + req = req.continuation_token(c); + } + let resp = req.send().await.context("LIST chunks")?; + let mut to_delete: Vec = Vec::new(); + for obj in resp.contents() { + let Some(k) = obj.key() else { continue }; + let too_recent = obj.last_modified().map(|lm| lm > &cutoff).unwrap_or(true); + if too_recent { + continue; + } + let parts: Vec<&str> = k.rsplit('/').collect(); + if parts.is_empty() { + continue; + } + let id_hex = parts[0]; + if id_hex.len() != 64 { + continue; + } + let mut chunk_id = [0u8; 32]; + if hex::decode_to_slice(id_hex, &mut chunk_id).is_err() { + continue; + } + if referenced.contains(&chunk_id) { + continue; + } + bytes_freed += obj.size().unwrap_or(0) as u64; + chunks_deleted += 1; + if let Ok(oid) = ObjectIdentifier::builder().key(k).build() { + to_delete.push(oid); + } + if to_delete.len() == 1000 { + if let Ok(del) = Delete::builder() + .set_objects(Some(std::mem::take(&mut to_delete))) + .build() + { + client + .delete_objects() + .bucket(&target.bucket) + .delete(del) + .send() + .await + .context("DELETE chunks batch")?; + } + } + } + if !to_delete.is_empty() { + if let Ok(del) = Delete::builder().set_objects(Some(to_delete)).build() { + client + .delete_objects() + .bucket(&target.bucket) + .delete(del) + .send() + .await + .context("DELETE chunks final")?; + } + } + if resp.is_truncated().unwrap_or(false) { + continuation = resp.next_continuation_token().map(String::from); + } else { + break; + } + } + + Ok((bytes_freed, chunks_deleted)) +} diff --git a/apps/manager/src/features/backups/index_rebuild.rs b/apps/manager/src/features/backups/index_rebuild.rs new file mode 100644 index 0000000..22c1260 --- /dev/null +++ b/apps/manager/src/features/backups/index_rebuild.rs @@ -0,0 +1,122 @@ +//! Reconstruct the `backup` table from S3 manifests. +//! Run via: `manager backup index-rebuild --target `. + +use crate::features::backup_targets::envelope; +use crate::features::backup_targets::repo::BackupTargetRepository; +use anyhow::{Context, Result}; +use aws_sdk_s3::{ + config::{Builder, Region}, + Client, +}; +use chrono::TimeZone; +use nexus_backup::{decrypt_manifest, ChunkKey, Manifest}; +use sqlx::PgPool; +use uuid::Uuid; + +pub async fn run(pool: PgPool, target_id: Uuid) -> Result<()> { + let repo = BackupTargetRepository::new(pool.clone()); + let target = repo + .get(target_id) + .await? + .ok_or_else(|| anyhow::anyhow!("target {target_id} not found"))?; + + let secret = envelope::unwrap_to_string(&target.encrypted_secret_access_key)?; + let target_key = envelope::unwrap_to_array::<32>(&target.encrypted_target_key)?; + + let creds = aws_credential_types::Credentials::new( + &target.access_key_id, + &secret, + None, + None, + "nqrust-rebuild", + ); + let region = Region::new(target.region.clone().unwrap_or_else(|| "us-east-1".into())); + let s3_cfg = Builder::new() + .behavior_version_latest() + .endpoint_url(&target.endpoint) + .credentials_provider(creds) + .region(region) + .force_path_style(true) + .build(); + let client = Client::from_conf(s3_cfg); + + let prefix = if target.prefix.is_empty() { + "manifests/".to_string() + } else { + format!("{}/manifests/", target.prefix.trim_end_matches('/')) + }; + + let mut reconstructed = 0usize; + let mut skipped = 0usize; + let mut continuation: Option = None; + loop { + let mut req = client + .list_objects_v2() + .bucket(&target.bucket) + .prefix(&prefix); + if let Some(c) = continuation.as_deref() { + req = req.continuation_token(c); + } + let resp = req.send().await.context("LIST manifests")?; + for obj in resp.contents() { + let Some(k) = obj.key() else { continue }; + let blob = client + .get_object() + .bucket(&target.bucket) + .key(k) + .send() + .await + .with_context(|| format!("GET {k}"))? + .body + .collect() + .await? + .into_bytes() + .to_vec(); + let key = ChunkKey::from_bytes(target_key); + let compressed = decrypt_manifest(&key, &blob)?; + let m = Manifest::deserialize_compressed(&compressed)?; + let existed: Option = + sqlx::query_scalar(r#"SELECT id FROM backup WHERE id = $1"#) + .bind(m.backup_id) + .fetch_optional(&pool) + .await?; + if existed.is_some() { + skipped += 1; + continue; + } + let total_size: i64 = m.chunks.iter().map(|c| c.ciphertext_length as i64).sum(); + let chunk_count = m.chunks.len() as i64; + let created_at = chrono::Utc + .timestamp_opt(m.created_at_unix_seconds, 0) + .single() + .unwrap_or_else(chrono::Utc::now); + sqlx::query( + r#"INSERT INTO backup + (id, source_volume_id, source_snapshot_id, target_id, + manifest_object_key, size_bytes, unique_bytes, chunk_count, + status, created_at, completed_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'completed', $9, $9, now())"#, + ) + .bind(m.backup_id) + .bind(m.source_volume_id) + .bind(m.source_snapshot_id) + .bind(target_id) + .bind(k) + .bind(total_size) + .bind(0i64) + .bind(chunk_count) + .bind(created_at) + .execute(&pool) + .await + .ok(); + reconstructed += 1; + } + if resp.is_truncated().unwrap_or(false) { + continuation = resp.next_continuation_token().map(String::from); + } else { + break; + } + } + println!("index-rebuild: reconstructed {reconstructed}, skipped (already in DB) {skipped}"); + Ok(()) +} diff --git a/apps/manager/src/features/backups/mod.rs b/apps/manager/src/features/backups/mod.rs new file mode 100644 index 0000000..ded3b79 --- /dev/null +++ b/apps/manager/src/features/backups/mod.rs @@ -0,0 +1,10 @@ +pub mod gc; +pub mod index_rebuild; +pub mod reconciler; +pub mod repo; +pub mod routes; +pub mod scheduler; +pub mod service; +pub mod types; + +pub use routes::{router, volume_backup_router}; diff --git a/apps/manager/src/features/backups/reconciler.rs b/apps/manager/src/features/backups/reconciler.rs new file mode 100644 index 0000000..9b46b86 --- /dev/null +++ b/apps/manager/src/features/backups/reconciler.rs @@ -0,0 +1,29 @@ +//! Periodically marks `running` backups older than 24h as `failed`. + +use crate::features::backups::repo::BackupRepository; +use sqlx::PgPool; + +const STALE_MINUTES: i64 = 24 * 60; + +pub async fn reconcile_loop(pool: PgPool) { + loop { + tokio::time::sleep(std::time::Duration::from_secs(5 * 60)).await; + let repo = BackupRepository::new(pool.clone()); + match repo.list_stale_running(STALE_MINUTES).await { + Ok(rows) => { + for r in rows { + let _ = repo + .mark_failed( + r.id, + &format!( + "marked failed by reconciler: status was 'running' for >{STALE_MINUTES} minutes" + ), + ) + .await; + tracing::warn!(backup_id=%r.id, "reconciler aged stuck 'running' to 'failed'"); + } + } + Err(e) => tracing::error!("reconciler: {e}"), + } + } +} diff --git a/apps/manager/src/features/backups/repo.rs b/apps/manager/src/features/backups/repo.rs new file mode 100644 index 0000000..1518b74 --- /dev/null +++ b/apps/manager/src/features/backups/repo.rs @@ -0,0 +1,143 @@ +use chrono::{DateTime, Utc}; +use sqlx::PgPool; +use uuid::Uuid; + +#[derive(Clone)] +pub struct BackupRepository { + pool: PgPool, +} + +#[derive(Debug, Clone, sqlx::FromRow)] +pub struct BackupRow { + pub id: Uuid, + pub source_volume_id: Option, + #[allow(dead_code)] + pub source_snapshot_id: Option, + pub target_id: Uuid, + pub manifest_object_key: Option, + pub size_bytes: i64, + pub unique_bytes: i64, + pub chunk_count: i64, + pub status: String, + pub error_message: Option, + pub created_at: DateTime, + pub completed_at: Option>, + #[allow(dead_code)] + pub updated_at: DateTime, +} + +impl BackupRepository { + pub fn new(pool: PgPool) -> Self { + Self { pool } + } + + pub async fn insert_running( + &self, + source_volume_id: Uuid, + source_snapshot_id: Uuid, + target_id: Uuid, + ) -> sqlx::Result { + sqlx::query_as::<_, BackupRow>( + r#"INSERT INTO backup (source_volume_id, source_snapshot_id, target_id, status) + VALUES ($1, $2, $3, 'running') + RETURNING *"#, + ) + .bind(source_volume_id) + .bind(source_snapshot_id) + .bind(target_id) + .fetch_one(&self.pool) + .await + } + + pub async fn mark_completed( + &self, + id: Uuid, + manifest_object_key: &str, + size_bytes: i64, + unique_bytes: i64, + chunk_count: i64, + ) -> sqlx::Result<()> { + sqlx::query( + r#"UPDATE backup + SET status = 'completed', + manifest_object_key = $1, + size_bytes = $2, + unique_bytes = $3, + chunk_count = $4, + completed_at = now(), + updated_at = now() + WHERE id = $5"#, + ) + .bind(manifest_object_key) + .bind(size_bytes) + .bind(unique_bytes) + .bind(chunk_count) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn mark_failed(&self, id: Uuid, error: &str) -> sqlx::Result<()> { + sqlx::query( + r#"UPDATE backup + SET status = 'failed', error_message = $1, updated_at = now() + WHERE id = $2"#, + ) + .bind(error) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn get(&self, id: Uuid) -> sqlx::Result> { + sqlx::query_as::<_, BackupRow>(r#"SELECT * FROM backup WHERE id = $1"#) + .bind(id) + .fetch_optional(&self.pool) + .await + } + + pub async fn list_for_volume(&self, volume_id: Uuid) -> sqlx::Result> { + sqlx::query_as::<_, BackupRow>( + r#"SELECT * FROM backup WHERE source_volume_id = $1 ORDER BY created_at DESC"#, + ) + .bind(volume_id) + .fetch_all(&self.pool) + .await + } + + pub async fn list_completed_oldest_first( + &self, + volume_id: Uuid, + ) -> sqlx::Result> { + sqlx::query_as::<_, BackupRow>( + r#"SELECT * FROM backup WHERE source_volume_id = $1 AND status = 'completed' ORDER BY created_at ASC"#, + ) + .bind(volume_id) + .fetch_all(&self.pool) + .await + } + + pub async fn delete_row(&self, id: Uuid) -> sqlx::Result<()> { + sqlx::query("DELETE FROM backup WHERE id = $1") + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } + + pub async fn list_stale_running( + &self, + older_than_minutes: i64, + ) -> sqlx::Result> { + sqlx::query_as::<_, BackupRow>( + r#"SELECT * FROM backup + WHERE status = 'running' + AND updated_at < now() - make_interval(mins => $1)"#, + ) + .bind(older_than_minutes) + .fetch_all(&self.pool) + .await + } +} diff --git a/apps/manager/src/features/backups/routes.rs b/apps/manager/src/features/backups/routes.rs new file mode 100644 index 0000000..b5cb03a --- /dev/null +++ b/apps/manager/src/features/backups/routes.rs @@ -0,0 +1,235 @@ +use crate::features::backups::repo::{BackupRepository, BackupRow}; +use crate::features::backups::service; +use crate::AppState; +use axum::{ + extract::{Path, Query}, + http::StatusCode, + response::IntoResponse, + routing::{get, post}, + Extension, Json, Router, +}; +use nexus_types::{Backup, BackupStatus, CreateBackupRequest, RestoreRequest}; +use serde::Deserialize; +use uuid::Uuid; + +fn row_to_wire(row: BackupRow) -> Backup { + Backup { + id: row.id, + source_volume_id: row.source_volume_id, + target_id: row.target_id, + size_bytes: row.size_bytes, + unique_bytes: row.unique_bytes, + chunk_count: row.chunk_count, + status: match row.status.as_str() { + "running" => BackupStatus::Running, + "completed" => BackupStatus::Completed, + "failed" => BackupStatus::Failed, + "pruning" => BackupStatus::Pruning, + _ => BackupStatus::Failed, + }, + error_message: row.error_message, + created_at: row.created_at, + completed_at: row.completed_at, + } +} + +#[derive(Deserialize)] +pub struct ListQuery { + pub volume_id: Option, +} + +pub async fn list( + Extension(st): Extension, + Query(q): Query, +) -> impl IntoResponse { + let repo = BackupRepository::new(st.db.clone()); + let rows = if let Some(vid) = q.volume_id { + repo.list_for_volume(vid).await + } else { + sqlx::query_as::<_, BackupRow>(r#"SELECT * FROM backup ORDER BY created_at DESC LIMIT 200"#) + .fetch_all(&st.db) + .await + }; + match rows { + Ok(rs) => ( + StatusCode::OK, + Json(serde_json::json!({ + "items": rs.into_iter().map(row_to_wire).collect::>(), + })), + ) + .into_response(), + Err(e) => { + tracing::error!("backups list: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub async fn get_one( + Extension(st): Extension, + Path(id): Path, +) -> impl IntoResponse { + let repo = BackupRepository::new(st.db.clone()); + match repo.get(id).await { + Ok(Some(row)) => (StatusCode::OK, Json(row_to_wire(row))).into_response(), + Ok(None) => ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({"error":"not found"})), + ) + .into_response(), + Err(e) => { + tracing::error!("backups get: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub async fn create_for_volume( + Extension(st): Extension, + Path(volume_id): Path, + Json(req): Json, +) -> impl IntoResponse { + match service::create_backup(&st, volume_id, req.target_id).await { + Ok(id) => ( + StatusCode::CREATED, + Json(serde_json::json!({"backup_id": id})), + ) + .into_response(), + Err(e) => { + tracing::error!("create_backup: {e:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": e.to_string()})), + ) + .into_response() + } + } +} + +pub async fn restore( + Extension(st): Extension, + Path(backup_id): Path, + Json(req): Json, +) -> impl IntoResponse { + match service::restore_backup(&st, backup_id, req.target_backend_id).await { + Ok(volume_id) => ( + StatusCode::CREATED, + Json(serde_json::json!({"volume_id": volume_id})), + ) + .into_response(), + Err(e) => { + tracing::error!("restore_backup: {e:#}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error": e.to_string()})), + ) + .into_response() + } + } +} + +pub async fn delete_one( + Extension(st): Extension, + Path(id): Path, +) -> impl IntoResponse { + let repo = BackupRepository::new(st.db.clone()); + let backup = match repo.get(id).await { + Ok(Some(b)) => b, + Ok(None) => { + return ( + StatusCode::NOT_FOUND, + Json(serde_json::json!({"error":"not found"})), + ) + .into_response() + } + Err(e) => { + tracing::error!("backups delete lookup: {e}"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response(); + } + }; + + // Mark pruning so concurrent operations see the intent. + sqlx::query(r#"UPDATE backup SET status = 'pruning', updated_at = now() WHERE id = $1"#) + .bind(id) + .execute(&st.db) + .await + .ok(); + + // Delete the manifest from S3 first so GC can reclaim the chunks on its + // next pass. If S3 delete fails, log a warning but still drop the DB row; + // the rebuild tool can detect orphan manifests later. + if let Some(mkey) = backup.manifest_object_key.as_deref() { + if let Ok(Some(target)) = + crate::features::backup_targets::repo::BackupTargetRepository::new(st.db.clone()) + .get(backup.target_id) + .await + { + if let Ok(secret) = crate::features::backup_targets::envelope::unwrap_to_string( + &target.encrypted_secret_access_key, + ) { + let creds = aws_credential_types::Credentials::new( + &target.access_key_id, + &secret, + None, + None, + "nqrust-mgr-delete", + ); + let region = aws_sdk_s3::config::Region::new( + target.region.clone().unwrap_or_else(|| "us-east-1".into()), + ); + let s3_cfg = aws_sdk_s3::config::Builder::new() + .behavior_version_latest() + .endpoint_url(&target.endpoint) + .credentials_provider(creds) + .region(region) + .force_path_style(true) + .build(); + let client = aws_sdk_s3::Client::from_conf(s3_cfg); + if let Err(e) = client + .delete_object() + .bucket(&target.bucket) + .key(mkey) + .send() + .await + { + tracing::warn!(backup_id=%id, manifest=%mkey, "S3 manifest delete failed: {e:#}"); + } + } + } + } + + match repo.delete_row(id).await { + Ok(()) => (StatusCode::NO_CONTENT, ()).into_response(), + Err(e) => { + tracing::error!("backups delete row: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + +pub fn router() -> Router { + Router::new() + .route("/", get(list)) + .route("/:id", get(get_one).delete(delete_one)) + .route("/:id/restore", post(restore)) +} + +pub fn volume_backup_router() -> Router { + Router::new().route("/", post(create_for_volume)) +} diff --git a/apps/manager/src/features/backups/scheduler.rs b/apps/manager/src/features/backups/scheduler.rs new file mode 100644 index 0000000..8ce2609 --- /dev/null +++ b/apps/manager/src/features/backups/scheduler.rs @@ -0,0 +1,68 @@ +//! Per-volume cron scheduler. Wakes once a minute, checks every volume +//! that has backup_cron + backup_target_id set, dispatches a backup if due. + +use crate::AppState; +use chrono::Utc; +use cron::Schedule; +use std::str::FromStr; +use std::time::Duration; +use uuid::Uuid; + +pub async fn schedule_loop(state: AppState) { + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + if let Err(e) = tick(&state).await { + tracing::error!("scheduler tick: {e:#}"); + } + } +} + +type VolumeScheduleRow = ( + Uuid, + Option, + Option, + Option>, +); + +async fn tick(st: &AppState) -> anyhow::Result<()> { + let rows: Vec = sqlx::query_as( + r#"SELECT v.id, v.backup_cron, v.backup_target_id, + (SELECT MAX(created_at) FROM backup b WHERE b.source_volume_id = v.id) AS last_backup + FROM volume v + WHERE v.backup_cron IS NOT NULL AND v.backup_target_id IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM backup b2 + WHERE b2.source_volume_id = v.id AND b2.status = 'running' + )"#, + ) + .fetch_all(&st.db) + .await?; + + let now = Utc::now(); + for (volume_id, cron_str, target_id, last) in rows { + let (Some(cron_str), Some(target_id)) = (cron_str, target_id) else { + continue; + }; + let Ok(schedule) = Schedule::from_str(&cron_str) else { + tracing::warn!(volume_id=%volume_id, "invalid cron: {cron_str}"); + continue; + }; + let after = last.unwrap_or(now - chrono::Duration::days(365)); + if let Some(next_fire) = schedule.after(&after).next() { + if next_fire <= now { + tracing::info!(volume_id=%volume_id, "scheduler firing backup"); + let st_cl = st.clone(); + tokio::spawn(async move { + if let Err(e) = crate::features::backups::service::create_backup( + &st_cl, volume_id, target_id, + ) + .await + { + tracing::error!(volume_id=%volume_id, "scheduled backup failed: {e:#}"); + } + }); + } + } + } + Ok(()) +} diff --git a/apps/manager/src/features/backups/service.rs b/apps/manager/src/features/backups/service.rs new file mode 100644 index 0000000..7436338 --- /dev/null +++ b/apps/manager/src/features/backups/service.rs @@ -0,0 +1,270 @@ +use crate::features::backup_targets::envelope; +use crate::features::backup_targets::repo::BackupTargetRepository; +use crate::features::backups::repo::BackupRepository; +use crate::features::backups::types::{BackupReq, BackupTargetConfig, ChunkerParams, RestoreReq}; +use crate::features::storage::agent_rpc; +use crate::features::volumes::repo::VolumeRepository; +use crate::AppState; +use anyhow::{anyhow, Context, Result}; +use nexus_storage::{BackendInstanceId, VolumeHandle}; +use uuid::Uuid; + +pub async fn create_backup(st: &AppState, volume_id: Uuid, target_id: Uuid) -> Result { + let backup_repo = BackupRepository::new(st.db.clone()); + let target_repo = BackupTargetRepository::new(st.db.clone()); + let target_row = target_repo + .get(target_id) + .await? + .ok_or_else(|| anyhow!("target {target_id} not found"))?; + + // Resolve volume → backend → host. + let vol: (Uuid, String, i64, Option, Uuid) = sqlx::query_as( + r#"SELECT v.id, v.path, v.size_bytes, v.host_id, v.backend_id FROM volume v WHERE v.id = $1"#, + ) + .bind(volume_id) + .fetch_one(&st.db) + .await + .context("looking up volume")?; + let (vol_id, locator, size_bytes, host_id_opt, backend_id) = + (vol.0, vol.1, vol.2, vol.3, vol.4); + + let backend = st + .registry + .get(backend_id) + .ok_or_else(|| anyhow!("registry has no backend with id {backend_id}"))? + .clone(); + let host_id = host_id_opt.ok_or_else(|| { + anyhow!("volume has no home host (network-attached not supported by backup service yet)") + })?; + let host = st.hosts.get(host_id).await.context("getting host row")?; + + let volume_handle = VolumeHandle { + volume_id: vol_id, + backend_id: BackendInstanceId(backend_id), + backend_kind: backend.kind(), + locator, + size_bytes: size_bytes as u64, + }; + let snap_name = format!("backup-{}", Uuid::new_v4()); + let snap = backend + .snapshot(&volume_handle, &snap_name) + .await + .context("control-plane snapshot")?; + + let backup_row = backup_repo + .insert_running(volume_id, snap.snapshot_id, target_id) + .await?; + + let secret_access_key = envelope::unwrap_to_string(&target_row.encrypted_secret_access_key) + .context("decrypt secret_access_key")?; + let target_key = envelope::unwrap_to_array::<32>(&target_row.encrypted_target_key) + .context("decrypt target_key")?; + + let target_config = BackupTargetConfig { + endpoint: target_row.endpoint.clone(), + region: target_row.region.clone(), + bucket: target_row.bucket.clone(), + prefix: target_row.prefix.clone(), + access_key_id: target_row.access_key_id.clone(), + secret_access_key, + }; + + let req = BackupReq { + backup_id: backup_row.id, + snapshot: snap.clone(), + backend_kind: backend.kind(), + target: target_config, + encryption_key: target_key, + chunker_params: ChunkerParams::default(), + }; + + match agent_rpc::agent_backup(&host.addr, req).await { + Ok(resp) => { + backup_repo + .mark_completed( + backup_row.id, + &resp.manifest_object_key, + resp.bytes_written as i64, + resp.bytes_unique as i64, + resp.chunk_count as i64, + ) + .await?; + let _ = backend.delete_snapshot(snap).await; + let _ = enforce_retention(st, volume_id, &backup_repo).await; + Ok(backup_row.id) + } + Err(e) => { + backup_repo + .mark_failed(backup_row.id, &format!("{e:#}")) + .await?; + let _ = backend.delete_snapshot(snap).await; + Err(e) + } + } +} + +async fn enforce_retention( + st: &AppState, + volume_id: Uuid, + backup_repo: &BackupRepository, +) -> Result<()> { + let retain: Option = + sqlx::query_scalar(r#"SELECT backup_retain_count FROM volume WHERE id = $1"#) + .bind(volume_id) + .fetch_one(&st.db) + .await?; + let Some(retain) = retain else { + return Ok(()); + }; + if retain <= 0 { + return Ok(()); + } + + let mut completed = backup_repo.list_completed_oldest_first(volume_id).await?; + while completed.len() as i32 > retain { + let oldest = completed.remove(0); + // Best-effort delete the manifest from S3. + if let Some(mkey) = oldest.manifest_object_key.as_deref() { + if let Some(t) = BackupTargetRepository::new(st.db.clone()) + .get(oldest.target_id) + .await + .ok() + .flatten() + { + if let Ok(secret) = envelope::unwrap_to_string(&t.encrypted_secret_access_key) { + let creds = aws_credential_types::Credentials::new( + &t.access_key_id, + &secret, + None, + None, + "nqrust-mgr-prune", + ); + let region = aws_sdk_s3::config::Region::new( + t.region.clone().unwrap_or_else(|| "us-east-1".into()), + ); + let s3_cfg = aws_sdk_s3::config::Builder::new() + .behavior_version_latest() + .endpoint_url(&t.endpoint) + .credentials_provider(creds) + .region(region) + .force_path_style(true) + .build(); + let client = aws_sdk_s3::Client::from_conf(s3_cfg); + let _ = client + .delete_object() + .bucket(&t.bucket) + .key(mkey) + .send() + .await; + } + } + } + backup_repo.delete_row(oldest.id).await?; + } + Ok(()) +} + +pub async fn restore_backup( + st: &AppState, + backup_id: Uuid, + target_backend_id: Uuid, +) -> Result { + let backup_repo = BackupRepository::new(st.db.clone()); + let target_repo = BackupTargetRepository::new(st.db.clone()); + let backup = backup_repo + .get(backup_id) + .await? + .ok_or_else(|| anyhow!("backup {backup_id} not found"))?; + if backup.status != "completed" { + return Err(anyhow!( + "backup is in status '{}', expected 'completed'", + backup.status + )); + } + let manifest_key = backup + .manifest_object_key + .ok_or_else(|| anyhow!("backup has no manifest_object_key"))?; + let target_row = target_repo + .get(backup.target_id) + .await? + .ok_or_else(|| anyhow!("target {} no longer exists", backup.target_id))?; + + let backend = st + .registry + .get(target_backend_id) + .ok_or_else(|| anyhow!("registry has no backend {target_backend_id}"))? + .clone(); + let new_volume = backend + .provision(nexus_storage::CreateOpts { + name: format!("restore-{}", backup_id), + size_bytes: backup.size_bytes as u64, + description: Some(format!("restored from backup {backup_id}")), + }) + .await + .context("provision restore target")?; + + let kind_str = backend.kind().as_db_str(); + // Pick a host that supports the chosen backend. + let candidate_host_id: Option = { + let active = st.hosts.list_healthy().await?; + let mut chosen = None; + for h in active { + let kinds = st + .hosts + .supported_backend_kinds(h.id) + .await + .unwrap_or_default(); + if kinds.is_empty() || kinds.iter().any(|k| k == kind_str) { + chosen = Some(h.id); + break; + } + } + chosen + }; + let host_id = + candidate_host_id.ok_or_else(|| anyhow!("no host supports backend kind '{kind_str}'"))?; + let host = st.hosts.get(host_id).await?; + + let attached = agent_rpc::agent_attach(&host.addr, &new_volume).await?; + + let secret = envelope::unwrap_to_string(&target_row.encrypted_secret_access_key)?; + let target_key = envelope::unwrap_to_array::<32>(&target_row.encrypted_target_key)?; + + let req = RestoreReq { + target_volume: new_volume.clone(), + target_attached: attached.clone(), + manifest_object_key: manifest_key.clone(), + target: BackupTargetConfig { + endpoint: target_row.endpoint, + region: target_row.region, + bucket: target_row.bucket, + prefix: target_row.prefix, + access_key_id: target_row.access_key_id, + secret_access_key: secret, + }, + encryption_key: target_key, + }; + + match agent_rpc::agent_restore(&host.addr, req).await { + Ok(_) => { + let volume_repo = VolumeRepository::new(st.db.clone()); + let inserted = volume_repo + .create( + &format!("restore-{}", backup_id), + Some(&format!("Restored from backup {backup_id}")), + &new_volume.locator, + new_volume.size_bytes as i64, + "raw", + Some(host_id), + target_backend_id, + ) + .await?; + Ok(inserted.id) + } + Err(e) => { + let _ = agent_rpc::agent_detach(&host.addr, &new_volume, &attached).await; + let _ = backend.destroy(new_volume.clone()).await; + Err(e) + } + } +} diff --git a/apps/manager/src/features/backups/types.rs b/apps/manager/src/features/backups/types.rs new file mode 100644 index 0000000..04e3f52 --- /dev/null +++ b/apps/manager/src/features/backups/types.rs @@ -0,0 +1,66 @@ +//! RPC request/response types between manager and agent for backup ops. + +use nexus_storage::{AttachedPath, BackendKind, VolumeHandle, VolumeSnapshotHandle}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct BackupTargetConfig { + pub endpoint: String, + pub region: Option, + pub bucket: String, + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +pub struct ChunkerParams { + pub min_size: u32, + pub avg_size: u32, + pub max_size: u32, +} + +impl Default for ChunkerParams { + fn default() -> Self { + Self { + min_size: 4096, + avg_size: 65536, + max_size: 1048576, + } + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct BackupReq { + pub backup_id: Uuid, + pub snapshot: VolumeSnapshotHandle, + pub backend_kind: BackendKind, + pub target: BackupTargetConfig, + pub encryption_key: [u8; 32], + pub chunker_params: ChunkerParams, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct BackupResp { + pub manifest_object_key: String, + pub chunk_count: u64, + pub bytes_written: u64, + pub bytes_unique: u64, + pub duration_ms: u64, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct RestoreReq { + pub target_volume: VolumeHandle, + pub target_attached: AttachedPath, + pub manifest_object_key: String, + pub target: BackupTargetConfig, + pub encryption_key: [u8; 32], +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct RestoreResp { + pub bytes_written: u64, + pub duration_ms: u64, +} diff --git a/apps/manager/src/features/mod.rs b/apps/manager/src/features/mod.rs index f622fcf..5efcd0b 100644 --- a/apps/manager/src/features/mod.rs +++ b/apps/manager/src/features/mod.rs @@ -2,6 +2,8 @@ use crate::AppState; use axum::{Extension, Json, Router}; use serde::Serialize; +pub mod backup_targets; +pub mod backups; pub mod containers; pub mod functions; pub mod hosts; @@ -87,6 +89,9 @@ pub fn router(state: AppState) -> Router { .nest("/v1/metrics", metrics::router()) .nest("/v1/volumes", volumes::router()) .nest("/v1/storage_backends", storage_backends::router()) + .nest("/v1/backup_targets", backup_targets::router()) + .nest("/v1/backups", backups::router()) + .nest("/v1/volumes/:id/backup", backups::volume_backup_router()) // SSO public routes (no auth — these ARE the auth flow) .nest("/v1/sso", sso::public_router()) // SSO admin routes (auth + admin required) diff --git a/apps/manager/src/features/storage/agent_rpc.rs b/apps/manager/src/features/storage/agent_rpc.rs index 0bac287..95e776c 100644 --- a/apps/manager/src/features/storage/agent_rpc.rs +++ b/apps/manager/src/features/storage/agent_rpc.rs @@ -1,3 +1,4 @@ +use crate::features::backups::types::{BackupReq, BackupResp, RestoreReq, RestoreResp}; use anyhow::{anyhow, Context, Result}; use nexus_storage::{AttachedPath, BackendKind, VolumeHandle}; use reqwest::Client; @@ -114,3 +115,33 @@ pub async fn agent_resize2fs(host_addr: &str, attached: &AttachedPath) -> Result } Ok(()) } + +pub async fn agent_backup(host_addr: &str, req: BackupReq) -> Result { + let resp = Client::new() + .post(agent_url(host_addr, "/v1/storage/backup")) + .json(&req) + .send() + .await + .with_context(|| format!("POST /v1/storage/backup to {host_addr}"))?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow!("agent backup: {s}: {body}")); + } + Ok(resp.json::().await?) +} + +pub async fn agent_restore(host_addr: &str, req: RestoreReq) -> Result { + let resp = Client::new() + .post(agent_url(host_addr, "/v1/storage/restore")) + .json(&req) + .send() + .await + .with_context(|| format!("POST /v1/storage/restore to {host_addr}"))?; + if !resp.status().is_success() { + let s = resp.status(); + let body = resp.text().await.unwrap_or_default(); + return Err(anyhow!("agent restore: {s}: {body}")); + } + Ok(resp.json::().await?) +} diff --git a/apps/manager/src/features/volumes/mod.rs b/apps/manager/src/features/volumes/mod.rs index 9e6998b..cd145ac 100644 --- a/apps/manager/src/features/volumes/mod.rs +++ b/apps/manager/src/features/volumes/mod.rs @@ -1,5 +1,5 @@ use axum::{ - routing::{get, post}, + routing::{get, patch, post}, Router, }; @@ -12,4 +12,5 @@ pub fn router() -> Router { .route("/:id", get(routes::get).delete(routes::delete)) .route("/:id/attach", post(routes::attach)) .route("/:id/detach", post(routes::detach)) + .route("/:id/backup_schedule", patch(routes::patch_backup_schedule)) } diff --git a/apps/manager/src/features/volumes/routes.rs b/apps/manager/src/features/volumes/routes.rs index a2284b3..2ab21cd 100644 --- a/apps/manager/src/features/volumes/routes.rs +++ b/apps/manager/src/features/volumes/routes.rs @@ -10,6 +10,53 @@ use serde::{Deserialize, Serialize}; use tracing::error; use uuid::Uuid; +#[derive(serde::Deserialize, utoipa::ToSchema)] +pub struct PatchBackupScheduleRequest { + pub cron: Option, + pub retain_count: Option, + pub target_id: Option, +} + +pub async fn patch_backup_schedule( + Extension(st): Extension, + Path(id): Path, + Json(req): Json, +) -> impl IntoResponse { + use std::str::FromStr as _; + if let Some(c) = &req.cron { + if let Err(e) = cron::Schedule::from_str(c) { + return ( + StatusCode::BAD_REQUEST, + Json(serde_json::json!({"error": format!("invalid cron: {e}")})), + ) + .into_response(); + } + } + let res = sqlx::query( + r#"UPDATE volume SET backup_cron = COALESCE($1, backup_cron), + backup_retain_count = COALESCE($2, backup_retain_count), + backup_target_id = COALESCE($3, backup_target_id) + WHERE id = $4"#, + ) + .bind(req.cron) + .bind(req.retain_count) + .bind(req.target_id) + .bind(id) + .execute(&st.db) + .await; + match res { + Ok(_) => (StatusCode::NO_CONTENT, ()).into_response(), + Err(e) => { + tracing::error!("patch_backup_schedule: {e}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({"error":"db"})), + ) + .into_response() + } + } +} + #[derive(Debug, Deserialize)] pub struct CreateVolumeRequest { pub name: String, diff --git a/apps/manager/src/main.rs b/apps/manager/src/main.rs index 9438d18..3cfa74f 100644 --- a/apps/manager/src/main.rs +++ b/apps/manager/src/main.rs @@ -69,6 +69,26 @@ async fn main() -> anyhow::Result<()> { let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); tracing_subscriber::fmt().with_env_filter(filter).init(); + let args: Vec = std::env::args().collect(); + if args.len() >= 4 && args[1] == "backup" && args[2] == "index-rebuild" { + if args.len() != 5 || args[3] != "--target" { + eprintln!("usage: manager backup index-rebuild --target "); + std::process::exit(2); + } + let target_id: uuid::Uuid = match args[4].parse() { + Ok(u) => u, + Err(e) => { + eprintln!("bad uuid: {e}"); + std::process::exit(2); + } + }; + let database_url = + std::env::var("DATABASE_URL").map_err(|e| anyhow::anyhow!("DATABASE_URL: {e}"))?; + let pool = sqlx::PgPool::connect(&database_url).await?; + crate::features::backups::index_rebuild::run(pool, target_id).await?; + return Ok(()); + } + let db = PgPool::connect(&std::env::var("DATABASE_URL")?).await?; sqlx::migrate!("./migrations").run(&db).await?; @@ -240,6 +260,30 @@ async fn main() -> anyhow::Result<()> { }); } + // Backup GC loop: daily mark-and-sweep per target. + { + let pool = state.db.clone(); + tokio::spawn(async move { + crate::features::backups::gc::gc_loop(pool).await; + }); + } + + // Backup reconciler: ages stuck 'running' rows after 24h. + { + let pool = state.db.clone(); + tokio::spawn(async move { + crate::features::backups::reconciler::reconcile_loop(pool).await; + }); + } + + // Backup scheduler: per-volume cron-triggered backups. + { + let st = state.clone(); + tokio::spawn(async move { + crate::features::backups::scheduler::schedule_loop(st).await; + }); + } + let openapi = docs::ApiDoc::openapi(); if let Err(err) = docs::write_openapi_yaml(&openapi).await { warn!(error = ?err, "failed to write OpenAPI specification to disk"); diff --git a/apps/ui/app/(dashboard)/backup-targets/page.tsx b/apps/ui/app/(dashboard)/backup-targets/page.tsx new file mode 100644 index 0000000..5abad45 --- /dev/null +++ b/apps/ui/app/(dashboard)/backup-targets/page.tsx @@ -0,0 +1,27 @@ +"use client"; +import { useBackupTargets } from "@/lib/queries"; +import { BackupTargetForm } from "@/components/backup/backup-target-form"; + +export default function BackupTargetsPage() { + const { data: targets, isLoading } = useBackupTargets(); + return ( +
+

Backup targets

+ +
+

Configured targets

+ {isLoading &&

Loading…

} +
    + {(targets ?? []).map((t) => ( +
  • +
    {t.name}
    +
    + {t.endpoint} → s3://{t.bucket}/{t.prefix} +
    +
    • + ))} +
+
+
+ ); +} diff --git a/apps/ui/app/(dashboard)/volumes/[id]/page.tsx b/apps/ui/app/(dashboard)/volumes/[id]/page.tsx new file mode 100644 index 0000000..859f1b5 --- /dev/null +++ b/apps/ui/app/(dashboard)/volumes/[id]/page.tsx @@ -0,0 +1,136 @@ +"use client"; + +import { use } from "react"; +import Link from "next/link"; +import { useSearchParams } from "next/navigation"; +import { ArrowLeft, HardDrive, Loader2 } from "lucide-react"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Alert, AlertDescription } from "@/components/ui/alert"; +import { ReusableTabs, TabItem, TabContentItem } from "@/components/dashboard/tabs-new"; +import { VolumeBackupsTab } from "@/components/volume/volume-backups-tab"; +import { useVolume } from "@/lib/queries"; + +function getStatusColor(status: string) { + switch (status) { + case "available": + return "bg-green-500/10 text-green-700 border-green-200"; + case "attached": + return "bg-blue-500/10 text-blue-700 border-blue-200"; + case "creating": + return "bg-yellow-500/10 text-yellow-700 border-yellow-200"; + case "error": + return "bg-red-500/10 text-red-700 border-red-200"; + default: + return "bg-gray-500/10 text-gray-700 border-gray-200"; + } +} + +export default function VolumeDetailPage({ + params, +}: { + params: Promise<{ id: string }>; +}) { + const { id } = use(params); + const searchParams = useSearchParams(); + const tabParam = searchParams.get("tab"); + + const validTabs = ["overview", "backups"]; + const defaultTab = + tabParam && validTabs.includes(tabParam) ? tabParam : "overview"; + + const { data: volume, isLoading, error } = useVolume(id); + + const tabs: TabItem[] = [ + { value: "overview", label: "Overview" }, + { value: "backups", label: "Backups" }, + ]; + + const contents: TabContentItem[] = [ + { + value: "overview", + content: volume ? ( +
+
+
ID
+
{volume.id}
+
Name
+
{volume.name}
+ {volume.description && ( + <> +
Description
+
{volume.description}
+ + )} +
Status
+
+ + {volume.status} + +
+
Type
+
{volume.type}
+
Size
+
{volume.size_gb} GB
+
Created
+
{new Date(volume.created_at).toLocaleString()}
+
+
+ ) : null, + }, + { + value: "backups", + content: , + }, + ]; + + if (isLoading) { + return ( +
+ +
+ ); + } + + if (error || !volume) { + return ( +
+ + + Failed to load volume. + +
+ ); + } + + return ( +
+
+ +
+ +

{volume.name}

+ + {volume.status} + +
+
+ + +
+ ); +} diff --git a/apps/ui/components/backup/backup-list.tsx b/apps/ui/components/backup/backup-list.tsx new file mode 100644 index 0000000..325af1b --- /dev/null +++ b/apps/ui/components/backup/backup-list.tsx @@ -0,0 +1,57 @@ +"use client"; +import { useBackups } from "@/lib/queries"; +import { facadeApi } from "@/lib/api/facade"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Button } from "@/components/ui/button"; +import { useState } from "react"; +import { RestoreDialog } from "./restore-dialog"; + +export function BackupList({ volumeId }: { volumeId: string }) { + const { data: backups, isLoading } = useBackups(volumeId); + const qc = useQueryClient(); + const del = useMutation({ + mutationFn: (id: string) => facadeApi.deleteBackup(id), + onSuccess: () => qc.invalidateQueries({ queryKey: ["backups", volumeId] }), + }); + const [restoring, setRestoring] = useState(null); + + if (isLoading) return

Loading…

; + if (!backups?.length) return

No backups yet.

; + + return ( + <> + + + + + + + + + + + + {backups.map((b) => ( + + + + + + + + ))} + +
CreatedStatusSizeChunks
{new Date(b.created_at).toLocaleString()}{b.status}{(b.size_bytes / 1024 / 1024).toFixed(1)} MiB{b.chunk_count} + {b.status === "completed" && ( + + )} + +
+ {restoring && setRestoring(null)} />} + + ); +} diff --git a/apps/ui/components/backup/backup-schedule-editor.tsx b/apps/ui/components/backup/backup-schedule-editor.tsx new file mode 100644 index 0000000..4a24cb9 --- /dev/null +++ b/apps/ui/components/backup/backup-schedule-editor.tsx @@ -0,0 +1,83 @@ +"use client"; +import { useState } from "react"; +import { facadeApi } from "@/lib/api/facade"; +import { useBackupTargets } from "@/lib/queries"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; + +export function BackupScheduleEditor({ + volumeId, + current, +}: { + volumeId: string; + current?: { cron?: string; retain_count?: number; target_id?: string }; +}) { + const { data: targets } = useBackupTargets(); + const [cron, setCron] = useState(current?.cron ?? "0 2 * * *"); + const [retain, setRetain] = useState(current?.retain_count ?? 7); + const [target, setTarget] = useState(current?.target_id); + const qc = useQueryClient(); + const mut = useMutation({ + mutationFn: () => + facadeApi.patchBackupSchedule(volumeId, { + cron, + retain_count: retain, + target_id: target, + }), + onSuccess: () => qc.invalidateQueries({ queryKey: ["volumes", volumeId] }), + }); + return ( +
{ + e.preventDefault(); + mut.mutate(); + }} + className="space-y-3 max-w-md" + > +
+ + setCron(e.target.value)} + placeholder="0 2 * * *" + /> +
+
+ + setRetain(parseInt(e.target.value))} + /> +
+
+ + +
+ +
+ ); +} diff --git a/apps/ui/components/backup/backup-target-form.tsx b/apps/ui/components/backup/backup-target-form.tsx new file mode 100644 index 0000000..cfcca00 --- /dev/null +++ b/apps/ui/components/backup/backup-target-form.tsx @@ -0,0 +1,88 @@ +"use client"; +import { useState } from "react"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { facadeApi } from "@/lib/api/facade"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; + +export function BackupTargetForm({ onCreated }: { onCreated?: () => void }) { + const [name, setName] = useState(""); + const [endpoint, setEndpoint] = useState(""); + const [bucket, setBucket] = useState(""); + const [prefix, setPrefix] = useState(""); + const [accessKey, setAccessKey] = useState(""); + const [secretKey, setSecretKey] = useState(""); + const [region, setRegion] = useState("us-east-1"); + const qc = useQueryClient(); + const mut = useMutation({ + mutationFn: () => + facadeApi.createBackupTarget({ + name, + endpoint, + bucket, + prefix, + access_key_id: accessKey, + secret_access_key: secretKey, + region, + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["backup_targets"] }); + onCreated?.(); + }, + }); + return ( +
{ + e.preventDefault(); + mut.mutate(); + }} + className="space-y-3 max-w-md" + > +
+ + setName(e.target.value)} required /> +
+
+ + setEndpoint(e.target.value)} + required + placeholder="https://seaweedfs.local:8333" + /> +
+
+ + setRegion(e.target.value)} /> +
+
+ + setBucket(e.target.value)} required /> +
+
+ + setPrefix(e.target.value)} /> +
+
+ + setAccessKey(e.target.value)} required /> +
+
+ + setSecretKey(e.target.value)} + required + /> +
+ + {mut.error && ( +

{(mut.error as Error).message}

+ )} +
+ ); +} diff --git a/apps/ui/components/backup/restore-dialog.tsx b/apps/ui/components/backup/restore-dialog.tsx new file mode 100644 index 0000000..3ee839a --- /dev/null +++ b/apps/ui/components/backup/restore-dialog.tsx @@ -0,0 +1,71 @@ +"use client"; +import { useState } from "react"; +import { useStorageBackends } from "@/lib/queries"; +import { facadeApi } from "@/lib/api/facade"; +import { useMutation } from "@tanstack/react-query"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Label } from "@/components/ui/label"; + +export function RestoreDialog({ + backupId, + onClose, +}: { + backupId: string; + onClose: () => void; +}) { + const { data: backends } = useStorageBackends(); + const active = (backends ?? []).filter((b) => !b.deleted_at); + const [target, setTarget] = useState( + active.find((b) => b.is_default)?.id, + ); + const mut = useMutation({ + mutationFn: () => facadeApi.restoreBackup(backupId, target!), + onSuccess: () => onClose(), + }); + return ( + !o && onClose()}> + + + Restore backup to a new volume + +
+ + +
+ + + + + + + ); +} diff --git a/apps/ui/components/volume/volume-backups-tab.tsx b/apps/ui/components/volume/volume-backups-tab.tsx new file mode 100644 index 0000000..71a002b --- /dev/null +++ b/apps/ui/components/volume/volume-backups-tab.tsx @@ -0,0 +1,61 @@ +"use client"; +import { BackupList } from "@/components/backup/backup-list"; +import { BackupScheduleEditor } from "@/components/backup/backup-schedule-editor"; +import { useBackupTargets } from "@/lib/queries"; +import { facadeApi } from "@/lib/api/facade"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { Button } from "@/components/ui/button"; +import { useState } from "react"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Label } from "@/components/ui/label"; + +export function VolumeBackupsTab({ volumeId }: { volumeId: string }) { + const { data: targets } = useBackupTargets(); + const [target, setTarget] = useState(); + const qc = useQueryClient(); + const back = useMutation({ + mutationFn: () => facadeApi.createBackup(volumeId, target!), + onSuccess: () => qc.invalidateQueries({ queryKey: ["backups", volumeId] }), + }); + return ( +
+
+

Back up now

+
+
+ + +
+ +
+
+
+

Schedule

+ +
+
+

History

+ +
+
+ ); +} diff --git a/apps/ui/lib/api/facade.ts b/apps/ui/lib/api/facade.ts index be7611f..1acc1b3 100644 --- a/apps/ui/lib/api/facade.ts +++ b/apps/ui/lib/api/facade.ts @@ -101,6 +101,10 @@ import type { ContainerMetric, MetricsQueryParams, StorageBackendListResponse, + BackupTarget, + CreateBackupTargetRequest, + Backup, + BackupSchedule, } from "@/lib/types" /** @@ -915,6 +919,47 @@ export class FacadeApi { async activateLicenseFile(fileContent: string): Promise { return apiClient.post("/licensing/license/activate-file", { file_content: fileContent }); } + + // ============== + // Backup Targets + // ============== + + async listBackupTargets(): Promise<{ items: BackupTarget[] }> { + return apiClient.get<{ items: BackupTarget[] }>("/backup_targets"); + } + + async createBackupTarget(req: CreateBackupTargetRequest): Promise { + return apiClient.post("/backup_targets", req); + } + + async deleteBackupTarget(id: string): Promise { + return apiClient.delete(`/backup_targets/${id}`); + } + + // ============== + // Backups + // ============== + + async listBackups(volumeId?: string): Promise<{ items: Backup[] }> { + const q = volumeId ? `?volume_id=${volumeId}` : ""; + return apiClient.get<{ items: Backup[] }>(`/backups${q}`); + } + + async createBackup(volumeId: string, targetId: string): Promise<{ backup_id: string }> { + return apiClient.post<{ backup_id: string }>(`/volumes/${volumeId}/backup`, { target_id: targetId }); + } + + async restoreBackup(backupId: string, targetBackendId: string): Promise<{ volume_id: string }> { + return apiClient.post<{ volume_id: string }>(`/backups/${backupId}/restore`, { target_backend_id: targetBackendId }); + } + + async deleteBackup(backupId: string): Promise { + return apiClient.delete(`/backups/${backupId}`); + } + + async patchBackupSchedule(volumeId: string, req: Partial): Promise { + return apiClient.patch(`/volumes/${volumeId}/backup_schedule`, req); + } } // Export singleton instance diff --git a/apps/ui/lib/queries.ts b/apps/ui/lib/queries.ts index 8554da5..ef621b0 100644 --- a/apps/ui/lib/queries.ts +++ b/apps/ui/lib/queries.ts @@ -122,6 +122,10 @@ export const queryKeys = { // storage backends storageBackends: () => ["storage_backends"] as const, + + // backups + backupTargets: () => ["backup_targets"] as const, + backups: (vid?: string) => ["backups", vid ?? "all"] as const, } // Function Query @@ -1412,3 +1416,22 @@ export function useStorageBackends() { staleTime: 60_000, }); } + +// ============== +// Backups +// ============== + +export function useBackupTargets() { + return useQuery({ + queryKey: queryKeys.backupTargets(), + queryFn: async () => (await facadeApi.listBackupTargets()).items, + }); +} + +export function useBackups(volumeId?: string) { + return useQuery({ + queryKey: queryKeys.backups(volumeId), + queryFn: async () => (await facadeApi.listBackups(volumeId)).items, + refetchInterval: 5_000, // for in-progress backups + }); +} diff --git a/apps/ui/lib/types/index.ts b/apps/ui/lib/types/index.ts index 7e83c8e..1ff69d1 100644 --- a/apps/ui/lib/types/index.ts +++ b/apps/ui/lib/types/index.ts @@ -1085,3 +1085,52 @@ export interface StorageBackend { export interface StorageBackendListResponse { items: StorageBackend[]; } + +// ======================================== +// Backup Types +// ======================================== + +export type BackupStatus = "running" | "completed" | "failed" | "pruning"; + +export interface BackupTarget { + id: string; + name: string; + endpoint: string; + region?: string; + bucket: string; + prefix: string; + access_key_id: string; + gc_hour: number; + created_at: string; + deleted_at?: string | null; +} + +export interface CreateBackupTargetRequest { + name: string; + endpoint: string; + region?: string; + bucket: string; + prefix?: string; + access_key_id: string; + secret_access_key: string; + gc_hour?: number; +} + +export interface Backup { + id: string; + source_volume_id?: string; + target_id: string; + size_bytes: number; + unique_bytes: number; + chunk_count: number; + status: BackupStatus; + error_message?: string; + created_at: string; + completed_at?: string; +} + +export interface BackupSchedule { + cron: string; + retain_count: number; + target_id: string; +} diff --git a/crates/nexus-backup/Cargo.toml b/crates/nexus-backup/Cargo.toml new file mode 100644 index 0000000..697fc1a --- /dev/null +++ b/crates/nexus-backup/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "nexus-backup" +version = "0.1.0" +edition = "2021" + +[dependencies] +blake3 = "1" +chacha20poly1305 = { version = "0.10", features = ["alloc"] } +fastcdc = "3" +zstd = "0.13" +bincode = "1" +serde = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +uuid = { workspace = true } +hex = "0.4" +rand = "0.8" +async-trait = "0.1" + +[dev-dependencies] +tokio = { workspace = true } +proptest = "1" diff --git a/crates/nexus-backup/src/chunker.rs b/crates/nexus-backup/src/chunker.rs new file mode 100644 index 0000000..f552ab0 --- /dev/null +++ b/crates/nexus-backup/src/chunker.rs @@ -0,0 +1,155 @@ +use crate::error::BackupError; +use tokio::io::{AsyncRead, AsyncReadExt}; + +#[derive(Debug, Clone, Copy)] +pub struct ChunkerParams { + pub min_size: u32, + pub avg_size: u32, + pub max_size: u32, +} + +impl Default for ChunkerParams { + fn default() -> Self { + Self { + min_size: 4 * 1024, + avg_size: 64 * 1024, + max_size: 1024 * 1024, + } + } +} + +pub struct Chunk { + pub plaintext_offset: u64, + pub plaintext_length: u32, + pub plaintext_bytes: Vec, +} + +pub struct Chunker { + reader: R, + params: ChunkerParams, + buf: Vec, + offset: u64, + eof: bool, +} + +impl Chunker { + pub fn new(reader: R, params: ChunkerParams) -> Self { + Self { + reader, + params, + buf: Vec::with_capacity(params.max_size as usize * 2), + offset: 0, + eof: false, + } + } + + async fn fill_until(&mut self, target: usize) -> Result<(), BackupError> { + while self.buf.len() < target && !self.eof { + let mut tmp = vec![0u8; (target - self.buf.len()).max(64 * 1024)]; + let n = self.reader.read(&mut tmp).await?; + if n == 0 { + self.eof = true; + break; + } + tmp.truncate(n); + self.buf.extend_from_slice(&tmp); + } + Ok(()) + } + + pub async fn next_chunk(&mut self) -> Result, BackupError> { + self.fill_until(self.params.max_size as usize).await?; + if self.buf.is_empty() { + return Ok(None); + } + + let cdc = fastcdc::v2020::FastCDC::new( + &self.buf, + self.params.min_size, + self.params.avg_size, + self.params.max_size, + ); + let first = cdc.into_iter().next(); + let cut_at = match first { + Some(chunk_meta) => chunk_meta.length, + None => self.buf.len(), + }; + + let bytes: Vec = self.buf.drain(..cut_at).collect(); + let chunk = Chunk { + plaintext_offset: self.offset, + plaintext_length: bytes.len() as u32, + plaintext_bytes: bytes, + }; + self.offset += chunk.plaintext_length as u64; + Ok(Some(chunk)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::BufReader; + + fn deterministic_payload(size: usize) -> Vec { + let mut v = vec![0u8; size]; + let mut s: u64 = 0xdeadbeefu64; + for byte in v.iter_mut() { + s = s + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + *byte = (s >> 33) as u8; + } + v + } + + #[tokio::test] + async fn chunks_emit_in_order_and_cover_input() { + let payload = deterministic_payload(1_500_000); + let reader = BufReader::new(&payload[..]); + let mut c = Chunker::new(reader, ChunkerParams::default()); + let mut total = 0u64; + let mut last_offset: i64 = -1; + while let Some(chunk) = c.next_chunk().await.unwrap() { + assert!(chunk.plaintext_offset as i64 > last_offset); + assert_eq!(chunk.plaintext_bytes.len(), chunk.plaintext_length as usize); + assert_eq!(chunk.plaintext_offset, total); + total += chunk.plaintext_length as u64; + last_offset = chunk.plaintext_offset as i64; + } + assert_eq!(total, payload.len() as u64); + } + + #[tokio::test] + async fn deterministic_chunking_same_input() { + let payload = deterministic_payload(800_000); + let mut c1 = Chunker::new(BufReader::new(&payload[..]), ChunkerParams::default()); + let mut c2 = Chunker::new(BufReader::new(&payload[..]), ChunkerParams::default()); + + let mut h1 = Vec::new(); + let mut h2 = Vec::new(); + while let Some(chunk) = c1.next_chunk().await.unwrap() { + h1.push(blake3::hash(&chunk.plaintext_bytes)); + } + while let Some(chunk) = c2.next_chunk().await.unwrap() { + h2.push(blake3::hash(&chunk.plaintext_bytes)); + } + assert_eq!(h1, h2, "FastCDC must be deterministic for the same input"); + } + + #[tokio::test] + async fn empty_input_yields_no_chunks() { + let payload: Vec = Vec::new(); + let mut c = Chunker::new(BufReader::new(&payload[..]), ChunkerParams::default()); + assert!(c.next_chunk().await.unwrap().is_none()); + } + + #[tokio::test] + async fn small_input_yields_single_chunk() { + let payload = deterministic_payload(2 * 1024); + let mut c = Chunker::new(BufReader::new(&payload[..]), ChunkerParams::default()); + let chunk = c.next_chunk().await.unwrap().expect("one chunk"); + assert_eq!(chunk.plaintext_length as usize, payload.len()); + assert!(c.next_chunk().await.unwrap().is_none()); + } +} diff --git a/crates/nexus-backup/src/cipher.rs b/crates/nexus-backup/src/cipher.rs new file mode 100644 index 0000000..beab088 --- /dev/null +++ b/crates/nexus-backup/src/cipher.rs @@ -0,0 +1,155 @@ +use chacha20poly1305::{aead::Aead, KeyInit, XChaCha20Poly1305, XNonce}; + +use crate::error::BackupError; + +/// 32-byte XChaCha20-Poly1305 key. Per-target. Manager generates it, +/// encrypts with envelope key for storage, sends in-memory to the agent +/// during backup/restore RPC. +pub struct ChunkKey([u8; 32]); + +impl ChunkKey { + pub fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } +} + +impl Drop for ChunkKey { + fn drop(&mut self) { + for b in &mut self.0 { + unsafe { + std::ptr::write_volatile(b, 0); + } + } + } +} + +/// Convergent encryption: nonce derived from BLAKE3(plaintext) so identical +/// plaintexts encrypt to identical ciphertexts under the same key. Returns +/// the ciphertext (which already includes the Poly1305 tag). +pub fn encrypt_chunk(key: &ChunkKey, plaintext: &[u8]) -> Result, BackupError> { + let cipher = XChaCha20Poly1305::new(key.as_bytes().into()); + let plaintext_hash = blake3::hash(plaintext); + let nonce = XNonce::from_slice(&plaintext_hash.as_bytes()[..24]); + cipher + .encrypt(nonce, plaintext) + .map_err(|e| BackupError::Cipher(format!("encrypt: {e}"))) +} + +/// Decrypt a chunk. The caller must supply the original plaintext hash +/// (recovered from the manifest) so we can reconstruct the nonce. +/// Returns the plaintext on success, AuthFailed on tag mismatch. +pub fn decrypt_chunk( + key: &ChunkKey, + ciphertext: &[u8], + plaintext_hash: &[u8; 32], +) -> Result, BackupError> { + let cipher = XChaCha20Poly1305::new(key.as_bytes().into()); + let nonce = XNonce::from_slice(&plaintext_hash[..24]); + cipher + .decrypt(nonce, ciphertext) + .map_err(|_| BackupError::AuthFailed) +} + +/// Encrypt the manifest with a random nonce. Returns nonce-prepended +/// ciphertext: `[nonce(24) | ciphertext+tag]`. +pub fn encrypt_manifest(key: &ChunkKey, plaintext: &[u8]) -> Result, BackupError> { + use rand::RngCore; + let cipher = XChaCha20Poly1305::new(key.as_bytes().into()); + let mut nonce_bytes = [0u8; 24]; + rand::thread_rng().fill_bytes(&mut nonce_bytes); + let nonce = XNonce::from_slice(&nonce_bytes); + let ciphertext = cipher + .encrypt(nonce, plaintext) + .map_err(|e| BackupError::Cipher(format!("encrypt manifest: {e}")))?; + let mut out = Vec::with_capacity(24 + ciphertext.len()); + out.extend_from_slice(&nonce_bytes); + out.extend_from_slice(&ciphertext); + Ok(out) +} + +/// Inverse of `encrypt_manifest`: input is `[nonce(24) | ciphertext+tag]`. +pub fn decrypt_manifest(key: &ChunkKey, blob: &[u8]) -> Result, BackupError> { + if blob.len() < 24 { + return Err(BackupError::Cipher("manifest blob too short".into())); + } + let cipher = XChaCha20Poly1305::new(key.as_bytes().into()); + let nonce = XNonce::from_slice(&blob[..24]); + cipher + .decrypt(nonce, &blob[24..]) + .map_err(|_| BackupError::AuthFailed) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key() -> ChunkKey { + ChunkKey::from_bytes([0x42u8; 32]) + } + + #[test] + fn convergent_chunk_round_trip() { + let k = key(); + let plain = b"hello, backup pipeline"; + let plain_hash: [u8; 32] = *blake3::hash(plain).as_bytes(); + let cipher = encrypt_chunk(&k, plain).unwrap(); + let recovered = decrypt_chunk(&k, &cipher, &plain_hash).unwrap(); + assert_eq!(recovered, plain); + } + + #[test] + fn convergent_same_plaintext_same_ciphertext() { + let k = key(); + let plain = b"identical plaintext"; + let c1 = encrypt_chunk(&k, plain).unwrap(); + let c2 = encrypt_chunk(&k, plain).unwrap(); + assert_eq!(c1, c2, "convergent encryption must be deterministic"); + } + + #[test] + fn different_plaintext_different_ciphertext() { + let k = key(); + let c1 = encrypt_chunk(&k, b"alpha").unwrap(); + let c2 = encrypt_chunk(&k, b"bravo").unwrap(); + assert_ne!(c1, c2); + } + + #[test] + fn manifest_round_trip_with_random_nonce() { + let k = key(); + let plain = b"manifest payload bytes"; + let blob1 = encrypt_manifest(&k, plain).unwrap(); + let blob2 = encrypt_manifest(&k, plain).unwrap(); + assert_ne!( + blob1, blob2, + "manifest nonce must be random — successive encrypts differ" + ); + let r1 = decrypt_manifest(&k, &blob1).unwrap(); + assert_eq!(r1, plain); + } + + #[test] + fn tampered_chunk_fails_auth() { + let k = key(); + let plain = b"sensitive content"; + let plain_hash: [u8; 32] = *blake3::hash(plain).as_bytes(); + let mut cipher = encrypt_chunk(&k, plain).unwrap(); + cipher[0] ^= 0x01; + let err = decrypt_chunk(&k, &cipher, &plain_hash).unwrap_err(); + assert!(matches!(err, BackupError::AuthFailed)); + } + + #[test] + fn wrong_key_fails_auth() { + let k1 = key(); + let k2 = ChunkKey::from_bytes([0x99u8; 32]); + let plain = b"abc"; + let plain_hash: [u8; 32] = *blake3::hash(plain).as_bytes(); + let cipher = encrypt_chunk(&k1, plain).unwrap(); + let err = decrypt_chunk(&k2, &cipher, &plain_hash).unwrap_err(); + assert!(matches!(err, BackupError::AuthFailed)); + } +} diff --git a/crates/nexus-backup/src/error.rs b/crates/nexus-backup/src/error.rs new file mode 100644 index 0000000..2704866 --- /dev/null +++ b/crates/nexus-backup/src/error.rs @@ -0,0 +1,25 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum BackupError { + #[error("chunker: {0}")] + Chunker(String), + + #[error("cipher: {0}")] + Cipher(String), + + #[error("manifest: {0}")] + Manifest(String), + + #[error("authentication failed (Poly1305 MAC mismatch)")] + AuthFailed, + + #[error("manifest version mismatch: got {got}, expected {expected}")] + ManifestVersion { got: u32, expected: u32 }, + + #[error("io: {0}")] + Io(#[from] std::io::Error), + + #[error("other: {0}")] + Other(#[source] Box), +} diff --git a/crates/nexus-backup/src/lib.rs b/crates/nexus-backup/src/lib.rs new file mode 100644 index 0000000..d42c365 --- /dev/null +++ b/crates/nexus-backup/src/lib.rs @@ -0,0 +1,13 @@ +//! Pure-Rust backup transforms: FastCDC chunking, BLAKE3 hashing, +//! XChaCha20-Poly1305 convergent encryption, manifest serialization. +//! No I/O. Both manager and agent depend on this crate. + +pub mod chunker; +pub mod cipher; +pub mod error; +pub mod manifest; + +pub use chunker::{Chunk, Chunker, ChunkerParams}; +pub use cipher::{decrypt_chunk, decrypt_manifest, encrypt_chunk, encrypt_manifest, ChunkKey}; +pub use error::BackupError; +pub use manifest::{chunk_object_key, manifest_object_key, ChunkRef, Manifest, MANIFEST_VERSION}; diff --git a/crates/nexus-backup/src/manifest.rs b/crates/nexus-backup/src/manifest.rs new file mode 100644 index 0000000..7a1df33 --- /dev/null +++ b/crates/nexus-backup/src/manifest.rs @@ -0,0 +1,154 @@ +use crate::error::BackupError; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +pub const MANIFEST_VERSION: u32 = 1; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ChunkRef { + pub plaintext_offset: u64, + pub plaintext_length: u32, + pub plaintext_hash: [u8; 32], + pub chunk_id: [u8; 32], + pub ciphertext_length: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Manifest { + pub version: u32, + pub backup_id: Uuid, + pub source_volume_id: Uuid, + pub source_snapshot_id: Option, + pub total_plaintext_size: u64, + pub created_at_unix_seconds: i64, + pub chunks: Vec, +} + +impl Manifest { + pub fn serialize_compressed(&self) -> Result, BackupError> { + let bytes = + bincode::serialize(self).map_err(|e| BackupError::Manifest(format!("bincode: {e}")))?; + let compressed = zstd::stream::encode_all(&bytes[..], 3) + .map_err(|e| BackupError::Manifest(format!("zstd: {e}")))?; + Ok(compressed) + } + + pub fn deserialize_compressed(blob: &[u8]) -> Result { + let bytes = zstd::stream::decode_all(blob) + .map_err(|e| BackupError::Manifest(format!("zstd decode: {e}")))?; + let manifest: Manifest = bincode::deserialize(&bytes) + .map_err(|e| BackupError::Manifest(format!("bincode decode: {e}")))?; + if manifest.version != MANIFEST_VERSION { + return Err(BackupError::ManifestVersion { + got: manifest.version, + expected: MANIFEST_VERSION, + }); + } + Ok(manifest) + } +} + +pub fn chunk_object_key(prefix: &str, chunk_id: &[u8; 32]) -> String { + let hex = hex::encode(chunk_id); + if prefix.is_empty() { + format!("chunks/{}/{}", &hex[..2], hex) + } else { + format!( + "{}/chunks/{}/{}", + prefix.trim_end_matches('/'), + &hex[..2], + hex + ) + } +} + +pub fn manifest_object_key(prefix: &str, backup_id: &Uuid) -> String { + if prefix.is_empty() { + format!("manifests/{}.bin", backup_id) + } else { + format!( + "{}/manifests/{}.bin", + prefix.trim_end_matches('/'), + backup_id + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_manifest() -> Manifest { + Manifest { + version: MANIFEST_VERSION, + backup_id: Uuid::nil(), + source_volume_id: Uuid::nil(), + source_snapshot_id: None, + total_plaintext_size: 12345, + created_at_unix_seconds: 1735689600, + chunks: vec![ + ChunkRef { + plaintext_offset: 0, + plaintext_length: 4096, + plaintext_hash: [1u8; 32], + chunk_id: [2u8; 32], + ciphertext_length: 4128, + }, + ChunkRef { + plaintext_offset: 4096, + plaintext_length: 8192, + plaintext_hash: [3u8; 32], + chunk_id: [4u8; 32], + ciphertext_length: 8224, + }, + ], + } + } + + #[test] + fn manifest_round_trip() { + let m = sample_manifest(); + let blob = m.serialize_compressed().unwrap(); + let recovered = Manifest::deserialize_compressed(&blob).unwrap(); + assert_eq!(m, recovered); + } + + #[test] + fn manifest_version_mismatch_rejected() { + let mut m = sample_manifest(); + m.version = 999; + let blob = m.serialize_compressed().unwrap(); + let err = Manifest::deserialize_compressed(&blob).unwrap_err(); + assert!(matches!( + err, + BackupError::ManifestVersion { + got: 999, + expected: 1 + } + )); + } + + #[test] + fn chunk_key_format() { + let mut id = [0u8; 32]; + id[0] = 0xab; + id[1] = 0xcd; + let key = chunk_object_key("", &id); + assert!(key.starts_with("chunks/ab/abcd")); + let key2 = chunk_object_key("backup-prefix/", &id); + assert!(key2.starts_with("backup-prefix/chunks/ab/abcd")); + } + + #[test] + fn manifest_key_format() { + let id = Uuid::nil(); + assert_eq!( + manifest_object_key("", &id), + format!("manifests/{}.bin", id) + ); + assert_eq!( + manifest_object_key("p/", &id), + format!("p/manifests/{}.bin", id) + ); + } +} diff --git a/crates/nexus-storage/Cargo.toml b/crates/nexus-storage/Cargo.toml index 73e14eb..41828dc 100644 --- a/crates/nexus-storage/Cargo.toml +++ b/crates/nexus-storage/Cargo.toml @@ -10,6 +10,7 @@ thiserror = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } async-trait = "0.1" +tokio = { workspace = true } [dev-dependencies] tokio = { workspace = true } diff --git a/crates/nexus-storage/src/host.rs b/crates/nexus-storage/src/host.rs index 92ad9fa..fa2abc9 100644 --- a/crates/nexus-storage/src/host.rs +++ b/crates/nexus-storage/src/host.rs @@ -1,5 +1,5 @@ use crate::error::StorageError; -use crate::handle::{AttachedPath, VolumeHandle}; +use crate::handle::{AttachedPath, VolumeHandle, VolumeSnapshotHandle}; use crate::types::BackendKind; use async_trait::async_trait; use std::path::Path; @@ -33,4 +33,19 @@ pub trait HostBackend: Send + Sync { source: &Path, target_size_bytes: u64, ) -> Result<(), StorageError>; + + /// Open a snapshot for reading. Returns a stream of bytes representing + /// the volume contents at snapshot time. Used by the backup pipeline. + /// + /// Implementations: + /// - LocalFile: open the snapshot file from disk. + /// - Iscsi/TrueNasIscsi: attach the snapshot LUN read-only and return + /// a File handle over the block device. + /// + /// Returns `StorageError::NotSupported("read_snapshot")` if the backend + /// can't expose a snapshot for streaming reads. + async fn read_snapshot( + &self, + snap: &VolumeSnapshotHandle, + ) -> Result, StorageError>; } diff --git a/crates/nexus-types/src/lib.rs b/crates/nexus-types/src/lib.rs index d6adb3a..1b55b66 100644 --- a/crates/nexus-types/src/lib.rs +++ b/crates/nexus-types/src/lib.rs @@ -1747,3 +1747,88 @@ pub struct StorageBackend { #[serde(default, skip_serializing_if = "Option::is_none")] pub deleted_at: Option>, } + +// ── Backup pipeline ────────────────────────────────────────────────────── + +#[derive( + Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize, utoipa::ToSchema, +)] +#[serde(rename_all = "lowercase")] +pub enum BackupStatus { + Running, + Completed, + Failed, + Pruning, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct BackupTarget { + pub id: uuid::Uuid, + pub name: String, + pub endpoint: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub region: Option, + pub bucket: String, + #[serde(default)] + pub prefix: String, + pub access_key_id: String, + /// gc_hour 0-23 (UTC). + pub gc_hour: u8, + pub created_at: chrono::DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deleted_at: Option>, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct CreateBackupTargetRequest { + pub name: String, + pub endpoint: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub region: Option, + pub bucket: String, + #[serde(default)] + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, + #[serde(default = "default_backup_gc_hour")] + pub gc_hour: u8, +} + +fn default_backup_gc_hour() -> u8 { + 3 +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct Backup { + pub id: uuid::Uuid, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_volume_id: Option, + pub target_id: uuid::Uuid, + pub size_bytes: i64, + pub unique_bytes: i64, + pub chunk_count: i64, + pub status: BackupStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub error_message: Option, + pub created_at: chrono::DateTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub completed_at: Option>, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct BackupSchedule { + /// Standard 5-field cron expression in UTC. + pub cron: String, + pub retain_count: i32, + pub target_id: uuid::Uuid, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct CreateBackupRequest { + pub target_id: uuid::Uuid, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, utoipa::ToSchema)] +pub struct RestoreRequest { + pub target_backend_id: uuid::Uuid, +}