From 0cc46fa7b461aa62c0787ede5fde629845aa02d0 Mon Sep 17 00:00:00 2001 From: Simon <266044650+Simon9870@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:49:25 +0000 Subject: [PATCH 1/3] Update ls command to recurse all objects if -r flag is provided with no bucket This brings rc more in line with mc's behaviour, specifically when using the recursive flag. No other behaviour changes other than if you provide the -r flag with an alias but no bucket specified. Previously this would have listed just the buckets (same as without the flag), now it lists all objects on the server. --- crates/cli/src/commands/ls.rs | 196 ++++++++++++++++++++++++++++------ 1 file changed, 163 insertions(+), 33 deletions(-) diff --git a/crates/cli/src/commands/ls.rs b/crates/cli/src/commands/ls.rs index 4e0b072..2c86be7 100644 --- a/crates/cli/src/commands/ls.rs +++ b/crates/cli/src/commands/ls.rs @@ -6,6 +6,7 @@ use clap::Args; use rc_core::{AliasManager, ListOptions, ObjectInfo, ObjectStore as _, RemotePath}; use rc_s3::S3Client; use serde::Serialize; +use std::collections::HashMap; use crate::exit_code::ExitCode; use crate::output::{Formatter, OutputConfig}; @@ -90,9 +91,14 @@ pub async fn execute(args: LsArgs, output_config: OutputConfig) -> ExitCode { } }; - // If no bucket specified, list buckets if bucket.is_none() { - return list_buckets(&client, &formatter, args.summarize).await; + if args.recursive { + // If no bucket specified & recursive flag is provided, list all objects on the server + return list_all_objects(&client, alias_name, &formatter, args.summarize).await; + } else { + // Else no bucket specified, list buckets + return list_buckets(&client, &formatter, args.summarize).await; + } } let bucket = bucket.unwrap(); @@ -160,38 +166,14 @@ async fn list_objects( ..Default::default() }; - let mut all_items = Vec::new(); - let mut continuation_token: Option = None; - let mut is_truncated; - - // Paginate through all results - loop { - let opts = ListOptions { - continuation_token: continuation_token.clone(), - ..options.clone() - }; - - match client.list_objects(path, opts).await { - Ok(result) => { - all_items.extend(result.items); - is_truncated = result.truncated; - continuation_token = result.continuation_token.clone(); - - if !result.truncated { - break; - } + let (all_items, is_truncated, continuation_token) = + match list_objects_with_paging(client, path, &options).await { + Ok(r) => r, + Err((message, exit_code)) => { + formatter.error(&message); + return exit_code; } - Err(e) => { - let err_str = e.to_string(); - if err_str.contains("NotFound") || err_str.contains("NoSuchBucket") { - formatter.error(&format!("Bucket not found: {}", path.bucket)); - return ExitCode::NotFound; - } - formatter.error(&format!("Failed to list objects: {e}")); - return ExitCode::NetworkError; - } - } - } + }; // Calculate summary let total_objects = all_items.iter().filter(|i| !i.is_dir).count(); @@ -246,6 +228,154 @@ async fn list_objects( ExitCode::Success } +async fn list_all_objects( + client: &S3Client, + alias: String, + formatter: &Formatter, + summarize: bool, +) -> ExitCode { + let buckets = match client.list_buckets().await { + Ok(buckets) => buckets, + Err(e) => { + formatter.error(&format!("Failed to list buckets: {e}")); + return ExitCode::NetworkError; + } + }; + + let options = ListOptions { + recursive: true, + max_keys: Some(1000), + ..Default::default() + }; + + let mut all_items: HashMap<&str, Vec> = HashMap::new(); + let mut is_truncated = false; + let mut continuation_token: Option = None; + + // List all objects in each bucket + for bucket in &buckets { + let path = &RemotePath::new(&alias, &bucket.key, ""); + let new_items: Vec; + + (new_items, is_truncated, continuation_token) = + match list_objects_with_paging(client, path, &options).await { + Ok(r) => r, + Err((message, exit_code)) => { + formatter.error(&message); + return exit_code; + } + }; + + all_items.entry(&bucket.key).or_default().extend(new_items); + } + + // Calculate summary + let total_objects = all_items.values().flatten().filter(|i| !i.is_dir).count(); + let total_size = all_items + .values() + .flatten() + .filter_map(|i| i.size_bytes) + .sum(); + + if formatter.is_json() { + let output = LsOutput { + items: all_items.into_values().flatten().collect(), + truncated: is_truncated, + continuation_token, + summary: if summarize { + Some(Summary { + total_objects, + total_size_bytes: total_size, + total_size_human: humansize::format_size(total_size as u64, humansize::BINARY), + }) + } else { + None + }, + }; + formatter.json(&output); + } else { + for (bucket_name, objects) in all_items { + for item in objects { + let date = item + .last_modified + .map(|d| d.strftime("%Y-%m-%d %H:%M:%S").to_string()) + .unwrap_or_else(|| " ".to_string()); + let styled_date = formatter.style_date(&format!("[{date}]")); + + if item.is_dir { + let styled_size = formatter.style_size(&format!("{:>10}", "0B")); + let styled_name = + formatter.style_dir(&format!("{}/{}", bucket_name, &item.key)); + formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); + } else { + let size = item.size_human.clone().unwrap_or_else(|| "0 B".to_string()); + let styled_size = formatter.style_size(&format!("{:>10}", size)); + let styled_name = + formatter.style_file(&format!("{}/{}", bucket_name, &item.key)); + formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); + } + } + } + + if summarize { + let total_size_human = humansize::format_size(total_size as u64, humansize::BINARY); + formatter.println(&format!( + "\nTotal: {} objects, {}", + formatter.style_size(&total_objects.to_string()), + formatter.style_size(&total_size_human) + )); + } + } + + ExitCode::Success +} + +// List objects using pagination, returns (Vec, is_truncated, Option) +async fn list_objects_with_paging( + client: &S3Client, + path: &RemotePath, + options: &ListOptions, +) -> Result<(Vec, bool, Option), (String, ExitCode)> { + let mut all_items = Vec::new(); + let mut is_truncated; + let mut continuation_token = None; + + // Paginate through all results + loop { + let opts = ListOptions { + continuation_token: continuation_token.clone(), + ..options.clone() + }; + + match client.list_objects(path, opts).await { + Ok(result) => { + all_items.extend(result.items); + is_truncated = result.truncated; + continuation_token = result.continuation_token.clone(); + + if !result.truncated { + break; + } + } + Err(e) => { + let err_str = e.to_string(); + if err_str.contains("NotFound") || err_str.contains("NoSuchBucket") { + return Err(( + format!("Bucket not found: {}", path.bucket), + ExitCode::NotFound, + )); + } + return Err(( + format!("Failed to list objects: {e}"), + ExitCode::NetworkError, + )); + } + } + } + + Ok((all_items, is_truncated, continuation_token)) +} + /// Parse ls path into (alias, bucket, prefix) fn parse_ls_path(path: &str) -> Result<(String, Option, Option), String> { let path = path.trim_end_matches('/'); From 7af524a8941ec641daa705bd1970948bf727fd4e Mon Sep 17 00:00:00 2001 From: Simon <266044650+Simon9870@users.noreply.github.com> Date: Fri, 20 Mar 2026 09:53:43 +0000 Subject: [PATCH 2/3] Update listing all objects to have deterministic ordering Iterating over HashMap in the text output path (for (bucket_name, objects) in all_items) produces a non-deterministic bucket order between runs, which can make CLI output harder to read and test. If you keep the bucket grouping, consider using an order-preserving structure (e.g., Vec in list_buckets() order, or BTreeMap) so output is stable. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Simon <266044650+Simon9870@users.noreply.github.com> --- crates/cli/src/commands/ls.rs | 44 ++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/crates/cli/src/commands/ls.rs b/crates/cli/src/commands/ls.rs index 2c86be7..cbd9c90 100644 --- a/crates/cli/src/commands/ls.rs +++ b/crates/cli/src/commands/ls.rs @@ -294,25 +294,31 @@ async fn list_all_objects( }; formatter.json(&output); } else { - for (bucket_name, objects) in all_items { - for item in objects { - let date = item - .last_modified - .map(|d| d.strftime("%Y-%m-%d %H:%M:%S").to_string()) - .unwrap_or_else(|| " ".to_string()); - let styled_date = formatter.style_date(&format!("[{date}]")); - - if item.is_dir { - let styled_size = formatter.style_size(&format!("{:>10}", "0B")); - let styled_name = - formatter.style_dir(&format!("{}/{}", bucket_name, &item.key)); - formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); - } else { - let size = item.size_human.clone().unwrap_or_else(|| "0 B".to_string()); - let styled_size = formatter.style_size(&format!("{:>10}", size)); - let styled_name = - formatter.style_file(&format!("{}/{}", bucket_name, &item.key)); - formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); + // Ensure deterministic bucket order in text output + let mut bucket_names: Vec<&str> = all_items.keys().copied().collect(); + bucket_names.sort_unstable(); + + for bucket_name in bucket_names { + if let Some(objects) = all_items.get(bucket_name) { + for item in objects { + let date = item + .last_modified + .map(|d| d.strftime("%Y-%m-%d %H:%M:%S").to_string()) + .unwrap_or_else(|| " ".to_string()); + let styled_date = formatter.style_date(&format!("[{date}]")); + + if item.is_dir { + let styled_size = formatter.style_size(&format!("{:>10}", "0B")); + let styled_name = + formatter.style_dir(&format!("{}/{}", bucket_name, &item.key)); + formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); + } else { + let size = item.size_human.clone().unwrap_or_else(|| "0 B".to_string()); + let styled_size = formatter.style_size(&format!("{:>10}", size)); + let styled_name = + formatter.style_file(&format!("{}/{}", bucket_name, &item.key)); + formatter.println(&format!("{styled_date} {styled_size} {styled_name}")); + } } } } From df2f33851596ab7440da850f090c6a0621718681 Mon Sep 17 00:00:00 2001 From: Simon <266044650+Simon9870@users.noreply.github.com> Date: Fri, 20 Mar 2026 10:16:24 +0000 Subject: [PATCH 3/3] list_all_objects JSON output prefix object keys with bucket name --- crates/cli/src/commands/ls.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/cli/src/commands/ls.rs b/crates/cli/src/commands/ls.rs index cbd9c90..1676099 100644 --- a/crates/cli/src/commands/ls.rs +++ b/crates/cli/src/commands/ls.rs @@ -279,7 +279,15 @@ async fn list_all_objects( if formatter.is_json() { let output = LsOutput { - items: all_items.into_values().flatten().collect(), + items: all_items + .into_iter() + .flat_map(|(bucket, objects)| { + objects.into_iter().map(move |mut obj| { + obj.key = format!("{}/{}", bucket, obj.key); + obj + }) + }) + .collect(), truncated: is_truncated, continuation_token, summary: if summarize {