Validate entire archive

.github/workflows/github-actions.yml

@@ -1,5 +1,5 @@
-name: GitHub Action for blk-archive
-run-name: ${{ github.actor }} is running blk-archive unit tests 🚀
+name: GitHub Action for blk-stash
+run-name: ${{ github.actor }} is running blk-stash unit tests 🚀
 on: [push]
 jobs:
   Explore-GitHub-Actions:

Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "blk_archive"
+name = "blk_stash"
 version = "0.1.0"
 edition = "2021"
 
@@ -8,20 +8,24 @@ edition = "2021"
 [dependencies]
 anyhow = "1.0"
 atty = "0.2"
+bincode = "1.3"
 blake2 = "0.10"
 blake3 = "1"
 byteorder = "1.4"
 chrono = "0.4"
 clap = { version = "4.5.26", features = ["cargo", "env"] }
 cli = { path = "./cli" }
+crc = "3"
 devicemapper = { git = "https://github.com/stratis-storage/devicemapper-rs", branch = "master" }
 gearhash = "0.1.3"
 generic-array = "0.14"
+hex = "0.4"
 serde_json = "1.0.96"
 libc = "0.2"
 linked-hash-map = "0.5.6"
 lru = "0.12.5"
-nix = "0.29"
+memmap2 = "0.9"
+nix = { version = "0.29", features = ["process", "signal"] }
 nom = "7.1"
 num_enum = "0.7.3"
 rand = "0.8"
@@ -53,9 +57,21 @@ tempfile = "3.16"
 [features]
 skip_cleanup = []
 
+[[bin]]
+name = "slab-verify"
+path = "src/bin/slab_verify.rs"
+
+[[bin]]
+name = "stream-dump"
+path = "src/bin/stream_dump.rs"
+
+[[bin]]
+name = "slab-hexdump"
+path = "src/bin/slab_hexdump.rs"
+
 [profile.release]
 debug = 1
 
 [[bin]]
-name = "blk-archive"
+name = "blk-stash"
 path = "src/main.rs"

README.md

@@ -1,11 +1,14 @@
+
+![Project Logo](doc/assets/blkslash.png)
+
 # Introduction
 
-blk-archive is a tool for archiving block devices.
+blk-stash is a tool for archiving block devices.
 
 Archived data is deduplicated, compressed and stored in a directory
 within a file system.
 
-blk-archive is not a complete backup solution.  But it may well make an
+blk-stash is not a complete backup solution.  But it may well make an
 excellent first step in a backup process.  For instance you may wish to
 sync the archive directory to cloud storage, or write it to tape.
 
@@ -36,7 +39,7 @@ also use this tool.
 
 # Building
 
-blk-archive is written in Rust.  You will need to install a recent
+blk-stash is written in Rust.  You will need to install a recent
 tool chain.  I recommend using *rustup*.
 
 ### Build dependencies

blk-stash

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cargo run --bin blk-stash --release -- "$@"

build.rs

@@ -56,7 +56,7 @@ fn gen_man() {
     use std::fs::File;
     use std::path::Path;
 
-    let binary = "blk-archive";
+    let binary = "blk-stash";
     let ver = env!("CARGO_PKG_VERSION");
 
     let root_cmd = cli::build_cli();
@@ -69,7 +69,7 @@ fn gen_man() {
     let base = Path::new(&out_dir);
 
     // Generate main page
-    let main_path = base.join("blk-archive.1");
+    let main_path = base.join("blk-stash.1");
     let mut file = File::create(&main_path).unwrap();
 
     let mut root_man = Man::new(root_cmd.clone());

cli/src/lib.rs

@@ -1,4 +1,4 @@
-use clap::{command, Arg, ArgAction, ArgGroup, Command, ValueHint};
+use clap::{command, Arg, ArgAction, Command, ValueHint};
 
 use std::env;
 
@@ -9,7 +9,7 @@ pub fn build_cli() -> clap::Command {
         .short('a')
         .value_name("ARCHIVE")
         .num_args(1)
-        .env("BLK_ARCHIVE_DIR")
+        .env("BLK_STASH_DIR")
         .required(true)
         .help_heading("Required Options");
 
@@ -40,10 +40,10 @@ pub fn build_cli() -> clap::Command {
         .num_args(1)
         .help_heading("Optional Options");
 
-    command!("blk-archive")
+    command!("blk-stash")
         .version(env!("CARGO_PKG_VERSION"))
         .propagate_version(true)
-        .bin_name("blk-archive")
+        .bin_name("blk-stash")
         .disable_help_flag(true)
         .disable_version_flag(true)
         .subcommand_required(true)
@@ -119,37 +119,51 @@ pub fn build_cli() -> clap::Command {
                 .about("packs a stream into the archive")
                 .arg(
                     Arg::new("INPUT")
-                        .help("Specify a device or file to archive")
+                        .help("Specify one or more devices or files to archive (only one INPUT allowed with --delta-stream/--delta-device)")
                         .required(true)
                         .value_name("INPUT")
-                        .num_args(1)
+                        .num_args(1..)
                         .help_heading("Required Options"),
                 )
                 .arg(archive_arg.clone())
                 .arg(
                     Arg::new("DELTA_STREAM")
                         .help(
-                            "Specify the stream that contains an older version of this thin device",
+                            "Specify the stream that contains an older version of this thin device (requires --delta-device, only single INPUT allowed)",
                         )
                         .required(false)
                         .long("delta-stream")
                         .value_name("DELTA_STREAM")
                         .num_args(1)
+                        .requires("DELTA_DEVICE")
                         .help_heading("Optional Options"),
                 )
                 .arg(
                     Arg::new("DELTA_DEVICE")
                         .help(
-                            "Specify the device that contains an older version of this thin device",
+                            "Specify the device that contains an older version of this thin device (requires --delta-stream, only single INPUT allowed)",
                         )
                         .required(false)
                         .long("delta-device")
                         .value_name("DELTA_DEVICE")
                         .num_args(1)
+                        .requires("DELTA_STREAM")
                         .help_heading("Optional Options"),
                 )
                 .arg(data_cache_size.clone())
-                .arg(json.clone()),
+                .arg(json.clone())
+                .arg(
+                    Arg::new("SYNC_POINT_SECS")
+                        .help("Number of seconds before creating a sync point in the archive. Smaller \
+                              values allow you to restart a pack with less data needing to be added to \
+                              archive at the cost of slower pack times")
+                        .required(false)
+                        .long("sync-point-secs")
+                        .value_name("SYNC_POINT_SECS")
+                        .num_args(1)
+                        .default_value("15")
+                        .value_parser(clap::value_parser!(u64).range(1..)),
+                ),
         )
         .subcommand(
             Command::new("unpack")
@@ -179,25 +193,46 @@ pub fn build_cli() -> clap::Command {
         .about("verifies stream in the archive against the original file/dev or an internal blake3 hash")
         .arg(
             Arg::new("INPUT")
-                .help("Device or file containing the correct version of the data")
+                .help("Device or file containing the correct version of the data (only valid with --stream)")
                 .value_name("INPUT")
                 .value_hint(ValueHint::FilePath)
-                .num_args(1), // not required; enforced via group below
+                .num_args(1)
+                .conflicts_with("ALL"), // INPUT not allowed with --all
         )
         .arg(
             Arg::new("internal")
                 .long("internal")
-                .help("Verify using the archive's internally stored blake3 hash (no INPUT needed)")
+                .help("Verify using the archive's internally stored blake3 hash (implied with --all)")
                 .action(ArgAction::SetTrue),
         )
-        .group(
-            ArgGroup::new("verify-source")
-                .args(["INPUT", "internal"])
-                .required(true), // must choose exactly one
+        .arg(
+            Arg::new("ALL")
+                .help("Verify all streams in the archive using internal blake3 hashes")
+                .long("all")
+                .action(ArgAction::SetTrue)
+                .help_heading("Optional Options"),
+        )
+        .arg(
+            Arg::new("REPAIR")
+                .help("Attempt to repair any errors found (requires --all)")
+                .long("repair")
+                .action(ArgAction::SetTrue)
+                .requires("ALL")
+                .conflicts_with("STREAM")
+                .help_heading("Optional Options"),
+        )
+        .arg(
+            Arg::new("FORCE")
+                .help("Skip confirmation prompt when using --repair")
+                .long("force")
+                .short('f')
+                .action(ArgAction::SetTrue)
+                .requires("REPAIR")
+                .help_heading("Optional Options"),
         )
         .arg(data_cache_size.clone())
         .arg(archive_arg.clone())
-        .arg(stream_arg.clone())
+        .arg(stream_arg.clone().required(false).conflicts_with("ALL"))
         .arg(json.clone()),
 )
         .subcommand(