Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
893 changes: 667 additions & 226 deletions Cargo.lock

Large diffs are not rendered by default.

48 changes: 24 additions & 24 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,42 +1,42 @@
[package]
name = "kernelci-storage"
version = "0.2.0"
version = "0.2.1"
edition = "2021"

[dependencies]
async-trait = "0.1"
axum = { version = "0.7.9", features = ["tracing", "multipart", "macros"] }
axum = { version = "0.8.9", features = ["tracing", "multipart", "macros"] }
axum-server = { version = "0.8.0", features = ["rustls", "tls-rustls"] }
azure_blob_uploader = "0.1.4"
azure_storage = "0.21.0"
azure_storage_blobs = "0.21.0"
bytes = "1.9"
bytes = "1.11"
chksum-hash-sha2-512 = "0.0.1"
chrono = "0.4.39"
clap = { version = "4.5.23", features = ["derive"] }
chrono = "0.4.44"
clap = { version = "4.6.1", features = ["derive"] }
fs2 = "0.4.3"
futures = "0.3.31"
futures-util = "0.3.31"
headers = "0.4.0"
futures = "0.3.32"
futures-util = "0.3.32"
headers = "0.4.1"
hex = "0.4.3"
http-body-util = "0.1.2"
http-body-util = "0.1.3"
jsonwebtoken = { version = "10", features = ["rust_crypto"] }
rand = "0.8.5"
reqwest = { version = "0.12.9", features = ["blocking"] }
rustls = "0.23.20"
serde = { version = "1.0.216", features = ["derive"] }
sysinfo = { version = "0.35.2", features = ["serde"] }
tempfile = "3.14.0"
tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros"] }
tokio-util = "0.7.13"
rand = "0.10.1"
reqwest = { version = "0.13.4", features = ["blocking"] }
rustls = "0.23.40"
serde = { version = "1.0.228", features = ["derive"] }
sysinfo = { version = "0.39.3", features = ["serde"] }
tempfile = "3.27.0"
tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros"] }
tokio-util = "0.7.18"
toml = "1.1.2"
tower = "0.5.2"
tower-http = { version = "0.6.2", features = ["trace", "fs", "limit"] }
tracing = "0.1.41"
tracing-subscriber = "0.3.19"
tower = "0.5.3"
tower-http = { version = "0.6.11", features = ["trace", "fs", "limit"] }
tracing = "0.1.44"
tracing-subscriber = "0.3.23"

[dev-dependencies]
reqwest = { version = "0.12.9", features = ["blocking", "multipart"] }
reqwest = { version = "0.13.4", features = ["blocking", "multipart"] }
jsonwebtoken = { version = "10", features = ["rust_crypto"] }
tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros", "time", "process"] }
tempfile = "3.14.0"
tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "time", "process"] }
tempfile = "3.27.0"
49 changes: 25 additions & 24 deletions src/azure.rs
Original file line number Diff line number Diff line change
Expand Up @@ -83,29 +83,6 @@ fn get_azure_credentials(name: &str) -> AzureConfig {
}
}

#[cfg(test)]
mod tests {
use super::normalize_sas_token;

#[test]
fn sas_token_is_left_empty() {
assert_eq!(normalize_sas_token(""), "");
assert_eq!(normalize_sas_token(" "), "");
}

#[test]
fn sas_token_is_left_intact_when_prefixed() {
assert_eq!(normalize_sas_token("?sv=1"), "?sv=1");
assert_eq!(normalize_sas_token(" ?sv=1 "), "?sv=1");
}

#[test]
fn sas_token_is_prefixed_when_missing_question_mark() {
assert_eq!(normalize_sas_token("sv=1"), "?sv=1");
assert_eq!(normalize_sas_token(" sv=1 "), "?sv=1");
}
}

#[allow(dead_code)]
fn calculate_checksum(filename: &String, data: &[u8]) {
let hash = sha2_512::default().update(data).finalize();
Expand Down Expand Up @@ -618,7 +595,8 @@ impl super::Driver for AzureDriver {
cont_type: String,
owner_email: Option<String>,
) -> (String, usize) {
let (_status, size) = write_file_to_blob_streaming(filename.clone(), data, cont_type, owner_email).await;
let (_status, size) =
write_file_to_blob_streaming(filename.clone(), data, cont_type, owner_email).await;
(filename, size)
}
async fn tag_file(
Expand All @@ -639,3 +617,26 @@ impl super::Driver for AzureDriver {
Vec::new()
}
}

#[cfg(test)]
mod tests {
use super::normalize_sas_token;

#[test]
fn sas_token_is_left_empty() {
assert_eq!(normalize_sas_token(""), "");
assert_eq!(normalize_sas_token(" "), "");
}

#[test]
fn sas_token_is_left_intact_when_prefixed() {
assert_eq!(normalize_sas_token("?sv=1"), "?sv=1");
assert_eq!(normalize_sas_token(" ?sv=1 "), "?sv=1");
}

#[test]
fn sas_token_is_prefixed_when_missing_question_mark() {
assert_eq!(normalize_sas_token("sv=1"), "?sv=1");
assert_eq!(normalize_sas_token(" sv=1 "), "?sv=1");
}
}
46 changes: 25 additions & 21 deletions src/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,14 @@ fn get_storage_file_path(filename: &str) -> PathBuf {
let safe_name = filename.trim_start_matches('/');
let full = storage_path.join(safe_name);
// Defense-in-depth: verify resolved path stays within storage root
let canonical_storage = storage_path.canonicalize().unwrap_or_else(|_| storage_path.to_path_buf());
let canonical_storage = storage_path
.canonicalize()
.unwrap_or_else(|_| storage_path.to_path_buf());
// Use the parent directory for canonicalization since the file may not exist yet
let parent = full.parent().unwrap_or(&full);
let canonical_parent = parent.canonicalize().unwrap_or_else(|_| parent.to_path_buf());
let canonical_parent = parent
.canonicalize()
.unwrap_or_else(|_| parent.to_path_buf());
if !canonical_parent.starts_with(&canonical_storage) {
// Fall back to storage root to prevent escape
storage_path.join(Path::new(safe_name).file_name().unwrap_or_default())
Expand Down Expand Up @@ -265,26 +269,24 @@ fn list_files_in_local(directory: String) -> Vec<String> {

fn collect_files_recursive(path: &Path, base_path: &Path, files: &mut Vec<String>) {
if let Ok(entries) = fs::read_dir(path) {
for entry in entries {
if let Ok(entry) = entry {
let entry_path = entry.path();

// Skip metadata directory
if entry_path
.file_name()
.map_or(false, |name| name == ".metadata")
{
continue;
}
for entry in entries.flatten() {
let entry_path = entry.path();

// Skip metadata directory
if entry_path
.file_name()
.is_some_and(|name| name == ".metadata")
{
continue;
}

if entry_path.is_file() {
// Get relative path from storage root
if let Ok(relative_path) = entry_path.strip_prefix(base_path) {
files.push(relative_path.to_string_lossy().to_string());
}
} else if entry_path.is_dir() {
collect_files_recursive(&entry_path, base_path, files);
if entry_path.is_file() {
// Get relative path from storage root
if let Ok(relative_path) = entry_path.strip_prefix(base_path) {
files.push(relative_path.to_string_lossy().to_string());
}
} else if entry_path.is_dir() {
collect_files_recursive(&entry_path, base_path, files);
}
}
}
Expand Down Expand Up @@ -352,7 +354,9 @@ impl super::Driver for LocalDriver {
let fname = filename.clone();
match tokio::task::spawn_blocking(move || {
write_file_to_local(fname, data, cont_type, owner_email)
}).await {
})
.await
{
Ok(Ok(_)) => filename,
Ok(Err(e)) => {
eprintln!("Local storage write error: {}", e);
Expand Down
4 changes: 2 additions & 2 deletions src/logging.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ pub fn verbose_enabled() -> bool {
}

fn env_verbose_enabled() -> bool {
std::env::var(ENV_DEBUG).map_or(false, |v| !v.is_empty())
|| std::env::var(ENV_VERBOSE).map_or(false, |v| !v.is_empty())
std::env::var(ENV_DEBUG).is_ok_and(|v| !v.is_empty())
|| std::env::var(ENV_VERBOSE).is_ok_and(|v| !v.is_empty())
}

#[macro_export]
Expand Down
Loading