From e519229e1ca25328693090de255eb17376132f6f Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Mon, 18 May 2026 21:01:10 +0200 Subject: [PATCH 1/6] fix(services/pcloud): use digest authentication --- core/Cargo.lock | 1 + core/services/pcloud/Cargo.toml | 1 + core/services/pcloud/src/core.rs | 241 ++++++++++++++++++++----------- core/services/pcloud/src/docs.md | 2 +- 4 files changed, 162 insertions(+), 83 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index c5e953b134f2..0d84d067cd8d 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -7272,6 +7272,7 @@ dependencies = [ "opendal-core", "serde", "serde_json", + "sha1", "tokio", ] diff --git a/core/services/pcloud/Cargo.toml b/core/services/pcloud/Cargo.toml index 2dbe3a32a375..ed016b7125ea 100644 --- a/core/services/pcloud/Cargo.toml +++ b/core/services/pcloud/Cargo.toml @@ -37,6 +37,7 @@ log = { workspace = true } opendal-core = { path = "../../core", version = "0.57.0", default-features = false } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha1 = "0.10.6" [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/core/services/pcloud/src/core.rs b/core/services/pcloud/src/core.rs index 35703a8ca9c5..8c3693827724 100644 --- a/core/services/pcloud/src/core.rs +++ b/core/services/pcloud/src/core.rs @@ -26,6 +26,8 @@ use http::header; use opendal_core::raw::*; use opendal_core::*; use serde::Deserialize; +use sha1::Digest; +use sha1::Sha1; use super::error::PcloudError; use super::error::parse_error; @@ -59,19 +61,67 @@ impl PcloudCore { pub async fn send(&self, req: Request) -> Result> { self.info.http_client().send(req).await } + + async fn build_url(&self, method: &str, query: String) -> Result { + let auth_query = self.digest_auth_query().await?; + + if query.is_empty() { + Ok(format!("{}/{method}?{auth_query}", self.endpoint)) + } else { + Ok(format!("{}/{method}?{query}&{auth_query}", self.endpoint)) + } + } + + async fn digest_auth_query(&self) -> Result { + let digest = self.get_digest().await?; + let passworddigest = build_passworddigest(&self.username, &self.password, &digest); + + Ok(format!( + "username={}&digest={}&passworddigest={}", + percent_encode_path(&self.username), + percent_encode_path(&digest), + percent_encode_path(&passworddigest) + )) + } + + async fn get_digest(&self) -> Result { + let req = Request::get(format!("{}/getdigest", self.endpoint)) + .body(Buffer::new()) + .map_err(new_request_build_error)?; + + let resp = self.send(req).await?; + + match resp.status() { + StatusCode::OK => { + let bs = resp.into_body(); + let resp: GetDigestResponse = + serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?; + + if resp.result != 0 { + return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}"))); + } + + let debug = format!("{resp:?}"); + match resp.digest { + Some(digest) => Ok(digest), + None => Err(Error::new(ErrorKind::Unexpected, debug)), + } + } + _ => Err(parse_error(resp)), + } + } } impl PcloudCore { pub async fn get_file_link(&self, path: &str) -> Result { let path = build_abs_path(&self.root, path); - let url = format!( - "{}/getfilelink?path=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&path), - self.username, - self.password - ); + let url = self + .build_url( + "getfilelink", + format!("path=/{}", percent_encode_path(&path)), + ) + .await?; let req = Request::get(url); @@ -158,13 +208,12 @@ impl PcloudCore { } pub async fn create_folder_if_not_exists(&self, path: &str) -> Result> { - let url = format!( - "{}/createfolderifnotexists?path=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(path), - self.username, - self.password - ); + let url = self + .build_url( + "createfolderifnotexists", + format!("path=/{}", percent_encode_path(path)), + ) + .await?; let req = Request::post(url); @@ -181,14 +230,16 @@ impl PcloudCore { let from = build_abs_path(&self.root, from); let to = build_abs_path(&self.root, to); - let url = format!( - "{}/renamefile?path=/{}&topath=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&from), - percent_encode_path(&to), - self.username, - self.password - ); + let url = self + .build_url( + "renamefile", + format!( + "path=/{}&topath=/{}", + percent_encode_path(&from), + percent_encode_path(&to) + ), + ) + .await?; let req = Request::post(url); @@ -204,14 +255,16 @@ impl PcloudCore { pub async fn rename_folder(&self, from: &str, to: &str) -> Result> { let from = build_abs_path(&self.root, from); let to = build_abs_path(&self.root, to); - let url = format!( - "{}/renamefolder?path=/{}&topath=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&from), - percent_encode_path(&to), - self.username, - self.password - ); + let url = self + .build_url( + "renamefolder", + format!( + "path=/{}&topath=/{}", + percent_encode_path(&from), + percent_encode_path(&to) + ), + ) + .await?; let req = Request::post(url); @@ -227,13 +280,12 @@ impl PcloudCore { pub async fn delete_folder(&self, path: &str) -> Result> { let path = build_abs_path(&self.root, path); - let url = format!( - "{}/deletefolder?path=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&path), - self.username, - self.password - ); + let url = self + .build_url( + "deletefolder", + format!("path=/{}", percent_encode_path(&path)), + ) + .await?; let req = Request::post(url); @@ -249,13 +301,12 @@ impl PcloudCore { pub async fn delete_file(&self, path: &str) -> Result> { let path = build_abs_path(&self.root, path); - let url = format!( - "{}/deletefile?path=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&path), - self.username, - self.password - ); + let url = self + .build_url( + "deletefile", + format!("path=/{}", percent_encode_path(&path)), + ) + .await?; let req = Request::post(url); @@ -272,14 +323,16 @@ impl PcloudCore { let from = build_abs_path(&self.root, from); let to = build_abs_path(&self.root, to); - let url = format!( - "{}/copyfile?path=/{}&topath=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&from), - percent_encode_path(&to), - self.username, - self.password - ); + let url = self + .build_url( + "copyfile", + format!( + "path=/{}&topath=/{}", + percent_encode_path(&from), + percent_encode_path(&to) + ), + ) + .await?; let req = Request::post(url); @@ -296,14 +349,16 @@ impl PcloudCore { let from = build_abs_path(&self.root, from); let to = build_abs_path(&self.root, to); - let url = format!( - "{}/copyfolder?path=/{}&topath=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(&from), - percent_encode_path(&to), - self.username, - self.password - ); + let url = self + .build_url( + "copyfolder", + format!( + "path=/{}&topath=/{}", + percent_encode_path(&from), + percent_encode_path(&to) + ), + ) + .await?; let req = Request::post(url); @@ -321,13 +376,9 @@ impl PcloudCore { let path = path.trim_end_matches('/'); - let url = format!( - "{}/stat?path=/{}&username={}&password={}", - self.endpoint, - percent_encode_path(path), - self.username, - self.password - ); + let url = self + .build_url("stat", format!("path=/{}", percent_encode_path(path))) + .await?; let req = Request::post(url); @@ -345,14 +396,16 @@ impl PcloudCore { let (name, path) = (get_basename(&path), get_parent(&path).trim_end_matches('/')); - let url = format!( - "{}/uploadfile?path=/{}&filename={}&username={}&password={}", - self.endpoint, - percent_encode_path(path), - percent_encode_path(name), - self.username, - self.password - ); + let url = self + .build_url( + "uploadfile", + format!( + "path=/{}&filename={}", + percent_encode_path(path), + percent_encode_path(name) + ), + ) + .await?; let req = Request::put(url); @@ -372,13 +425,9 @@ impl PcloudCore { let path = path.trim_end_matches('/'); - let url = format!( - "{}/listfolder?path={}&username={}&password={}", - self.endpoint, - percent_encode_path(path), - self.username, - self.password - ); + let url = self + .build_url("listfolder", format!("path={}", percent_encode_path(path))) + .await?; let req = Request::get(url); @@ -424,6 +473,21 @@ pub(super) fn parse_list_metadata(content: ListMetadata) -> Result { Ok(md) } +fn build_passworddigest(username: &str, password: &str, digest: &str) -> String { + let username_hash = sha1_hex(username.to_lowercase().as_bytes()); + sha1_hex(format!("{password}{username_hash}{digest}").as_bytes()) +} + +fn sha1_hex(input: &[u8]) -> String { + format!("{:x}", Sha1::digest(input)) +} + +#[derive(Debug, Deserialize)] +pub struct GetDigestResponse { + pub result: u64, + pub digest: Option, +} + #[derive(Debug, Deserialize)] pub struct GetFileLinkResponse { pub result: u64, @@ -458,3 +522,16 @@ pub struct ListMetadata { pub size: Option, pub contents: Option>, } + +#[cfg(test)] +mod tests { + use super::build_passworddigest; + + #[test] + fn build_passworddigest_lowercases_username() { + assert_eq!( + build_passworddigest("Alice@example.com", "s3cr3t", "abc123"), + "994e8926e4d573e614f3f56de6449ad81288cc87" + ); + } +} diff --git a/core/services/pcloud/src/docs.md b/core/services/pcloud/src/docs.md index 821688316576..73e66e0ccf98 100644 --- a/core/services/pcloud/src/docs.md +++ b/core/services/pcloud/src/docs.md @@ -17,7 +17,7 @@ This service can be used to: - `root`: Set the work directory for backend - `endpoint`: Pcloud bucket name - `username` Pcloud username -- `password` Pcloud password +- `password` Pcloud password used for pCloud digest authentication You can refer to [`PcloudBuilder`]'s docs for more information From b7897de4917ee73fdd815bfc2dc8822e3c0e6d5d Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Mon, 18 May 2026 21:24:59 +0200 Subject: [PATCH 2/6] feat(services/pcloud): support recursive listing --- core/services/pcloud/src/backend.rs | 5 +- core/services/pcloud/src/core.rs | 32 +++++- core/services/pcloud/src/lister.rs | 153 +++++++++++++++++++++++++--- 3 files changed, 172 insertions(+), 18 deletions(-) diff --git a/core/services/pcloud/src/backend.rs b/core/services/pcloud/src/backend.rs index 7e6d1898da69..8ef37290705e 100644 --- a/core/services/pcloud/src/backend.rs +++ b/core/services/pcloud/src/backend.rs @@ -155,6 +155,7 @@ impl Builder for PcloudBuilder { copy: true, list: true, + list_with_recursive: true, shared: true, @@ -257,8 +258,8 @@ impl Access for PcloudBackend { )) } - async fn list(&self, path: &str, _args: OpList) -> Result<(RpList, Self::Lister)> { - let l = PcloudLister::new(self.core.clone(), path); + async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Lister)> { + let l = PcloudLister::new(self.core.clone(), path, args.recursive()); Ok((RpList::default(), oio::PageLister::new(l))) } diff --git a/core/services/pcloud/src/core.rs b/core/services/pcloud/src/core.rs index 8c3693827724..2469c21c5aee 100644 --- a/core/services/pcloud/src/core.rs +++ b/core/services/pcloud/src/core.rs @@ -418,15 +418,20 @@ impl PcloudCore { self.send(req).await } - pub async fn list_folder(&self, path: &str) -> Result> { + pub async fn list_folder(&self, path: &str, recursive: bool) -> Result> { let path = build_abs_path(&self.root, path); let path = normalize_root(&path); let path = path.trim_end_matches('/'); + let mut query = format!("path={}", percent_encode_path(path)); + if recursive { + query.push_str("&recursive=1"); + } + let url = self - .build_url("listfolder", format!("path={}", percent_encode_path(path))) + .build_url("listfolder", query) .await?; let req = Request::get(url); @@ -457,7 +462,7 @@ pub(super) fn parse_stat_metadata(content: StatMetadata) -> Result { Ok(md) } -pub(super) fn parse_list_metadata(content: ListMetadata) -> Result { +pub(super) fn parse_list_metadata(content: &ListMetadata) -> Result { let mut md = if content.isfolder { Metadata::new(EntryMode::DIR) } else { @@ -516,7 +521,8 @@ pub struct ListFolderResponse { #[derive(Debug, Deserialize)] pub struct ListMetadata { - pub path: String, + pub path: Option, + pub name: String, pub modified: String, pub isfolder: bool, pub size: Option, @@ -525,7 +531,9 @@ pub struct ListMetadata { #[cfg(test)] mod tests { + use super::ListMetadata; use super::build_passworddigest; + use super::parse_list_metadata; #[test] fn build_passworddigest_lowercases_username() { @@ -534,4 +542,20 @@ mod tests { "994e8926e4d573e614f3f56de6449ad81288cc87" ); } + + #[test] + fn parse_list_metadata_preserves_file_size() { + let md = parse_list_metadata(&ListMetadata { + path: Some("/repo/data/00/file".to_string()), + name: "file".to_string(), + modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), + isfolder: false, + size: Some(123), + contents: None, + }) + .expect("metadata should parse"); + + assert_eq!(md.content_length(), 123); + assert!(md.is_file()); + } } diff --git a/core/services/pcloud/src/lister.rs b/core/services/pcloud/src/lister.rs index 8cd8cf766c0c..d417e7cf7d3d 100644 --- a/core/services/pcloud/src/lister.rs +++ b/core/services/pcloud/src/lister.rs @@ -29,20 +29,52 @@ pub struct PcloudLister { core: Arc, path: String, + recursive: bool, } impl PcloudLister { - pub(super) fn new(core: Arc, path: &str) -> Self { + pub(super) fn new(core: Arc, path: &str, recursive: bool) -> Self { PcloudLister { core, path: path.to_string(), + recursive, } } } +fn append_entries( + entries: &mut std::collections::VecDeque, + root: &str, + parent_path: &str, + content: ListMetadata, + recursive: bool, +) -> Result<()> { + let mut path = content + .path + .clone() + .unwrap_or_else(|| format!("{parent_path}/{}", content.name)); + if content.isfolder { + path.push('/'); + } + + let md = parse_list_metadata(&content)?; + let path = build_rel_path(root, &path); + entries.push_back(oio::Entry::new(&path, md)); + + if recursive { + if let Some(contents) = content.contents { + for child in contents { + append_entries(entries, root, path.trim_end_matches('/'), child, true)?; + } + } + } + + Ok(()) +} + impl oio::PageList for PcloudLister { async fn next_page(&self, ctx: &mut oio::PageContext) -> Result<()> { - let resp = self.core.list_folder(&self.path).await?; + let resp = self.core.list_folder(&self.path, self.recursive).await?; let status = resp.status(); @@ -64,18 +96,16 @@ impl oio::PageList for PcloudLister { } if let Some(metadata) = resp.metadata { + let parent_path = metadata.path.as_deref().unwrap_or(&self.path); if let Some(contents) = metadata.contents { for content in contents { - let path = if content.isfolder { - format!("{}/", content.path.clone()) - } else { - content.path.clone() - }; - - let md = parse_list_metadata(content)?; - let path = build_rel_path(&self.core.root, &path); - - ctx.entries.push_back(oio::Entry::new(&path, md)) + append_entries( + &mut ctx.entries, + &self.core.root, + parent_path, + content, + self.recursive, + )?; } } @@ -92,3 +122,102 @@ impl oio::PageList for PcloudLister { } } } + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use opendal_core::EntryMode; + + use super::append_entries; + use super::ListMetadata; + + fn file(path: &str, size: u64) -> ListMetadata { + ListMetadata { + path: Some(path.to_string()), + name: path.rsplit('/').next().unwrap_or(path).to_string(), + modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), + isfolder: false, + size: Some(size), + contents: None, + } + } + + fn dir(path: &str, contents: Vec) -> ListMetadata { + ListMetadata { + path: Some(path.to_string()), + name: path.rsplit('/').next().unwrap_or(path).to_string(), + modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), + isfolder: true, + size: None, + contents: Some(contents), + } + } + + #[test] + fn append_entries_flattens_recursive_contents() { + let mut entries = VecDeque::new(); + + append_entries( + &mut entries, + "/repo/", + "/repo", + dir( + "/repo/data/00", + vec![file("/repo/data/00/pack1", 11), dir("/repo/data/00/sub", vec![file("/repo/data/00/sub/pack2", 22)])], + ), + true, + ) + .expect("entries should flatten"); + + let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); + assert_eq!(paths, vec!["data/00/", "data/00/pack1", "data/00/sub/", "data/00/sub/pack2"]); + assert_eq!(entries[0].mode(), EntryMode::DIR); + assert_eq!(entries[1].mode(), EntryMode::FILE); + assert_eq!(entries[3].mode(), EntryMode::FILE); + } + + #[test] + fn append_entries_keeps_non_recursive_listing_shallow() { + let mut entries = VecDeque::new(); + + append_entries( + &mut entries, + "/repo/", + "/repo", + dir( + "/repo/data/00", + vec![file("/repo/data/00/pack1", 11), dir("/repo/data/00/sub", vec![file("/repo/data/00/sub/pack2", 22)])], + ), + false, + ) + .expect("entries should flatten"); + + let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); + assert_eq!(paths, vec!["data/00/"]); + } + + #[test] + fn append_entries_rebuilds_missing_child_paths() { + let mut entries = VecDeque::new(); + + append_entries( + &mut entries, + "/repo/", + "/repo/keys", + ListMetadata { + path: None, + name: "file1".to_string(), + modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), + isfolder: false, + size: Some(363), + contents: None, + }, + true, + ) + .expect("entries should rebuild missing child paths"); + + let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); + assert_eq!(paths, vec!["keys/file1"]); + } +} From e98f274155e2945caf32f72665ae077a0a57f5fe Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Tue, 19 May 2026 08:32:32 +0200 Subject: [PATCH 3/6] fix(services/pcloud): keep absolute paths in recursive listing --- core/services/pcloud/src/lister.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/core/services/pcloud/src/lister.rs b/core/services/pcloud/src/lister.rs index d417e7cf7d3d..e51ad6af8ece 100644 --- a/core/services/pcloud/src/lister.rs +++ b/core/services/pcloud/src/lister.rs @@ -49,22 +49,28 @@ fn append_entries( content: ListMetadata, recursive: bool, ) -> Result<()> { - let mut path = content + let mut absolute_path = content .path .clone() .unwrap_or_else(|| format!("{parent_path}/{}", content.name)); if content.isfolder { - path.push('/'); + absolute_path.push('/'); } let md = parse_list_metadata(&content)?; - let path = build_rel_path(root, &path); - entries.push_back(oio::Entry::new(&path, md)); + let relative_path = build_rel_path(root, &absolute_path); + entries.push_back(oio::Entry::new(&relative_path, md)); if recursive { if let Some(contents) = content.contents { for child in contents { - append_entries(entries, root, path.trim_end_matches('/'), child, true)?; + append_entries( + entries, + root, + absolute_path.trim_end_matches('/'), + child, + true, + )?; } } } From bc35c51243d50eaf7c37099b3387d56b61dbd8c7 Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Tue, 19 May 2026 15:39:13 +0200 Subject: [PATCH 4/6] perf(services/pcloud): cache ids and download links --- core/services/pcloud/src/backend.rs | 74 +++++++----- core/services/pcloud/src/core.rs | 171 +++++++++++++++++++++++++--- core/services/pcloud/src/deleter.rs | 6 + core/services/pcloud/src/lister.rs | 24 ++++ core/services/pcloud/src/writer.rs | 2 + 5 files changed, 233 insertions(+), 44 deletions(-) diff --git a/core/services/pcloud/src/backend.rs b/core/services/pcloud/src/backend.rs index 8ef37290705e..101b72a0f278 100644 --- a/core/services/pcloud/src/backend.rs +++ b/core/services/pcloud/src/backend.rs @@ -135,40 +135,42 @@ impl Builder for PcloudBuilder { .with_context("service", PCLOUD_SCHEME)), }?; - Ok(PcloudBackend { - core: Arc::new(PcloudCore { - info: { - let am = AccessorInfo::default(); - am.set_scheme(PCLOUD_SCHEME) - .set_root(&root) - .set_native_capability(Capability { - stat: true, + let info = { + let am = AccessorInfo::default(); + am.set_scheme(PCLOUD_SCHEME) + .set_root(&root) + .set_native_capability(Capability { + stat: true, + + create_dir: true, - create_dir: true, + read: true, - read: true, + write: true, - write: true, + delete: true, + rename: true, + copy: true, - delete: true, - rename: true, - copy: true, + list: true, + list_with_recursive: true, - list: true, - list_with_recursive: true, + shared: true, - shared: true, + ..Default::default() + }); - ..Default::default() - }); + am.into() + }; - am.into() - }, + Ok(PcloudBackend { + core: Arc::new(PcloudCore::new( + info, root, - endpoint: self.config.endpoint.clone(), + self.config.endpoint.clone(), username, password, - }), + )), }) } } @@ -184,7 +186,6 @@ impl Access for PcloudBackend { type Writer = PcloudWriters; type Lister = oio::PageLister; type Deleter = oio::OneShotDeleter; - type Copier = (); fn info(&self) -> Arc { self.core.info.clone() @@ -214,6 +215,9 @@ impl Access for PcloudBackend { } if let Some(md) = resp.metadata { + if let Some(file_id) = md.fileid { + self.core.cache_file_id(path, file_id); + } let md = parse_stat_metadata(md); return md.map(RpStat::new); } @@ -263,13 +267,7 @@ impl Access for PcloudBackend { Ok((RpList::default(), oio::PageLister::new(l))) } - async fn copy( - &self, - from: &str, - to: &str, - _args: OpCopy, - _opts: OpCopier, - ) -> Result<(RpCopy, Self::Copier)> { + async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result { self.core.ensure_dir_exists(to).await?; let resp = if from.ends_with('/') { @@ -293,6 +291,12 @@ impl Access for PcloudBackend { return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}"))); } + if from.ends_with('/') { + self.core.invalidate_path_prefix_cache(to); + } else { + self.core.invalidate_path_cache(to); + } + Ok((RpCopy::default(), ())) } _ => Err(parse_error(resp)), @@ -323,6 +327,14 @@ impl Access for PcloudBackend { return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}"))); } + if from.ends_with('/') { + self.core.invalidate_path_prefix_cache(from); + self.core.invalidate_path_prefix_cache(to); + } else { + self.core.invalidate_path_cache(from); + self.core.invalidate_path_cache(to); + } + Ok(RpRename::default()) } _ => Err(parse_error(resp)), diff --git a/core/services/pcloud/src/core.rs b/core/services/pcloud/src/core.rs index 2469c21c5aee..525207da11b2 100644 --- a/core/services/pcloud/src/core.rs +++ b/core/services/pcloud/src/core.rs @@ -15,8 +15,10 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::sync::Arc; +use std::sync::RwLock; use bytes::Buf; use http::Request; @@ -32,7 +34,6 @@ use sha1::Sha1; use super::error::PcloudError; use super::error::parse_error; -#[derive(Clone)] pub struct PcloudCore { pub info: Arc, @@ -44,6 +45,14 @@ pub struct PcloudCore { pub username: String, /// The password of this backend. pub password: String, + file_ids: RwLock>, + download_links: RwLock>, +} + +#[derive(Debug, Clone)] +struct CachedDownloadLink { + url: String, + expires_at: Timestamp, } impl Debug for PcloudCore { @@ -57,11 +66,116 @@ impl Debug for PcloudCore { } impl PcloudCore { + pub fn new( + info: Arc, + root: String, + endpoint: String, + username: String, + password: String, + ) -> Self { + Self { + info, + root, + endpoint, + username, + password, + file_ids: RwLock::new(HashMap::new()), + download_links: RwLock::new(HashMap::new()), + } + } + #[inline] pub async fn send(&self, req: Request) -> Result> { self.info.http_client().send(req).await } + fn normalize_cached_path(&self, path: &str) -> String { + let path = if path.starts_with('/') { + path.to_string() + } else { + build_rooted_abs_path(&self.root, path) + }; + + if path == "/" { + path + } else { + path.trim_end_matches('/').to_string() + } + } + + fn cached_file_id(&self, path: &str) -> Option { + let path = self.normalize_cached_path(path); + self.file_ids + .read() + .expect("file id cache lock poisoned") + .get(&path) + .copied() + } + + fn cached_download_link(&self, path: &str) -> Option { + let path = self.normalize_cached_path(path); + let now = Timestamp::now(); + let mut links = self + .download_links + .write() + .expect("download link cache lock poisoned"); + + match links.get(&path) { + Some(link) if link.expires_at > now => Some(link.url.clone()), + Some(_) => { + links.remove(&path); + None + } + None => None, + } + } + + fn cache_download_link(&self, path: &str, url: String, expires: &str) -> Result<()> { + let path = self.normalize_cached_path(path); + let expires_at = Timestamp::parse_rfc2822(expires)?; + + self.download_links + .write() + .expect("download link cache lock poisoned") + .insert(path, CachedDownloadLink { url, expires_at }); + + Ok(()) + } + + pub fn cache_file_id(&self, path: &str, file_id: u64) { + let path = self.normalize_cached_path(path); + self.file_ids + .write() + .expect("file id cache lock poisoned") + .insert(path, file_id); + } + + pub fn invalidate_path_cache(&self, path: &str) { + let path = self.normalize_cached_path(path); + + self.file_ids + .write() + .expect("file id cache lock poisoned") + .remove(&path); + self.download_links + .write() + .expect("download link cache lock poisoned") + .remove(&path); + } + + pub fn invalidate_path_prefix_cache(&self, path: &str) { + let prefix = format!("{}/", self.normalize_cached_path(path).trim_end_matches('/')); + + self.file_ids + .write() + .expect("file id cache lock poisoned") + .retain(|entry_path, _| !entry_path.starts_with(&prefix)); + self.download_links + .write() + .expect("download link cache lock poisoned") + .retain(|entry_path, _| !entry_path.starts_with(&prefix)); + } + async fn build_url(&self, method: &str, query: String) -> Result { let auth_query = self.digest_auth_query().await?; @@ -113,15 +227,8 @@ impl PcloudCore { } impl PcloudCore { - pub async fn get_file_link(&self, path: &str) -> Result { - let path = build_abs_path(&self.root, path); - - let url = self - .build_url( - "getfilelink", - format!("path=/{}", percent_encode_path(&path)), - ) - .await?; + async fn get_file_link_by_query(&self, path: &str, query: String) -> Result { + let url = self.build_url("getfilelink", query).await?; let req = Request::get(url); @@ -140,7 +247,7 @@ impl PcloudCore { let resp: GetFileLinkResponse = serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?; let result = resp.result; - if result == 2010 || result == 2055 || result == 2002 { + if result == 2009 || result == 2010 || result == 2055 || result == 2002 { return Err(Error::new(ErrorKind::NotFound, format!("{resp:?}"))); } if result != 0 { @@ -148,9 +255,13 @@ impl PcloudCore { } if let Some(hosts) = resp.hosts { - if let Some(path) = resp.path { + if let Some(link_path) = resp.path { if !hosts.is_empty() { - return Ok(format!("https://{}{}", hosts[0], path)); + let url = format!("https://{}{}", hosts[0], link_path); + if let Some(expires) = resp.expires.as_deref() { + self.cache_download_link(path, url.clone(), expires)?; + } + return Ok(url); } } } @@ -160,6 +271,36 @@ impl PcloudCore { } } + pub async fn get_file_link(&self, path: &str) -> Result { + let path = self.normalize_cached_path(path); + + if let Some(url) = self.cached_download_link(&path) { + return Ok(url); + } + + if let Some(file_id) = self.cached_file_id(&path) { + match self + .get_file_link_by_query(&path, format!("fileid={file_id}")) + .await + { + Ok(url) => return Ok(url), + Err(err) if err.kind() == ErrorKind::NotFound => { + self.invalidate_path_cache(&path); + } + Err(err) => return Err(err), + } + } + + self.get_file_link_by_query( + &path, + format!( + "path=/{}", + percent_encode_path(path.trim_start_matches('/')) + ), + ) + .await + } + pub async fn download(&self, url: &str, range: BytesRange) -> Result> { let req = Request::get(url); @@ -497,6 +638,7 @@ pub struct GetDigestResponse { pub struct GetFileLinkResponse { pub result: u64, pub path: Option, + pub expires: Option, pub hosts: Option>, } @@ -510,6 +652,7 @@ pub struct StatResponse { pub struct StatMetadata { pub modified: String, pub isfolder: bool, + pub fileid: Option, pub size: Option, } @@ -525,6 +668,7 @@ pub struct ListMetadata { pub name: String, pub modified: String, pub isfolder: bool, + pub fileid: Option, pub size: Option, pub contents: Option>, } @@ -550,6 +694,7 @@ mod tests { name: "file".to_string(), modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), isfolder: false, + fileid: Some(42), size: Some(123), contents: None, }) diff --git a/core/services/pcloud/src/deleter.rs b/core/services/pcloud/src/deleter.rs index 255d6f933be2..22ce64de539d 100644 --- a/core/services/pcloud/src/deleter.rs +++ b/core/services/pcloud/src/deleter.rs @@ -58,6 +58,12 @@ impl oio::OneShotDelete for PcloudDeleter { return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}"))); } + if path.ends_with('/') { + self.core.invalidate_path_prefix_cache(&path); + } else { + self.core.invalidate_path_cache(&path); + } + Ok(()) } _ => Err(parse_error(resp)), diff --git a/core/services/pcloud/src/lister.rs b/core/services/pcloud/src/lister.rs index e51ad6af8ece..44d4f5df3b27 100644 --- a/core/services/pcloud/src/lister.rs +++ b/core/services/pcloud/src/lister.rs @@ -44,6 +44,7 @@ impl PcloudLister { fn append_entries( entries: &mut std::collections::VecDeque, + core: &PcloudCore, root: &str, parent_path: &str, content: ListMetadata, @@ -55,6 +56,8 @@ fn append_entries( .unwrap_or_else(|| format!("{parent_path}/{}", content.name)); if content.isfolder { absolute_path.push('/'); + } else if let Some(file_id) = content.fileid { + core.cache_file_id(&absolute_path, file_id); } let md = parse_list_metadata(&content)?; @@ -66,6 +69,7 @@ fn append_entries( for child in contents { append_entries( entries, + core, root, absolute_path.trim_end_matches('/'), child, @@ -107,6 +111,7 @@ impl oio::PageList for PcloudLister { for content in contents { append_entries( &mut ctx.entries, + &self.core, &self.core.root, parent_path, content, @@ -132,11 +137,24 @@ impl oio::PageList for PcloudLister { #[cfg(test)] mod tests { use std::collections::VecDeque; + use std::sync::Arc; use opendal_core::EntryMode; + use opendal_core::raw::AccessorInfo; use super::append_entries; use super::ListMetadata; + use super::PcloudCore; + + fn core() -> PcloudCore { + PcloudCore::new( + Arc::new(AccessorInfo::default()), + "/repo/".to_string(), + "https://api.pcloud.com".to_string(), + "user@example.com".to_string(), + "secret".to_string(), + ) + } fn file(path: &str, size: u64) -> ListMetadata { ListMetadata { @@ -144,6 +162,7 @@ mod tests { name: path.rsplit('/').next().unwrap_or(path).to_string(), modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), isfolder: false, + fileid: Some(1), size: Some(size), contents: None, } @@ -155,6 +174,7 @@ mod tests { name: path.rsplit('/').next().unwrap_or(path).to_string(), modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), isfolder: true, + fileid: None, size: None, contents: Some(contents), } @@ -166,6 +186,7 @@ mod tests { append_entries( &mut entries, + &core(), "/repo/", "/repo", dir( @@ -189,6 +210,7 @@ mod tests { append_entries( &mut entries, + &core(), "/repo/", "/repo", dir( @@ -209,6 +231,7 @@ mod tests { append_entries( &mut entries, + &core(), "/repo/", "/repo/keys", ListMetadata { @@ -216,6 +239,7 @@ mod tests { name: "file1".to_string(), modified: "Mon, 18 May 2026 18:00:17 +0000".to_string(), isfolder: false, + fileid: Some(1), size: Some(363), contents: None, }, diff --git a/core/services/pcloud/src/writer.rs b/core/services/pcloud/src/writer.rs index 091d5b9a96d1..4b6ac5709967 100644 --- a/core/services/pcloud/src/writer.rs +++ b/core/services/pcloud/src/writer.rs @@ -58,6 +58,8 @@ impl oio::OneShotWrite for PcloudWriter { return Err(Error::new(ErrorKind::Unexpected, format!("{resp:?}"))); } + self.core.invalidate_path_cache(&self.path); + Ok(Metadata::default()) } _ => Err(parse_error(resp)), From 327af1c79e5e3e41e98f947e7e75177aeb2dc724 Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Mon, 25 May 2026 07:11:26 +0200 Subject: [PATCH 5/6] fix(services/pcloud): update copy access signature --- core/Cargo.lock | 2 +- core/services/pcloud/src/backend.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 0d84d067cd8d..182d68363ba0 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -7272,7 +7272,7 @@ dependencies = [ "opendal-core", "serde", "serde_json", - "sha1", + "sha1 0.10.6", "tokio", ] diff --git a/core/services/pcloud/src/backend.rs b/core/services/pcloud/src/backend.rs index 101b72a0f278..1bb6509d2cfe 100644 --- a/core/services/pcloud/src/backend.rs +++ b/core/services/pcloud/src/backend.rs @@ -186,6 +186,7 @@ impl Access for PcloudBackend { type Writer = PcloudWriters; type Lister = oio::PageLister; type Deleter = oio::OneShotDeleter; + type Copier = (); fn info(&self) -> Arc { self.core.info.clone() @@ -267,7 +268,13 @@ impl Access for PcloudBackend { Ok((RpList::default(), oio::PageLister::new(l))) } - async fn copy(&self, from: &str, to: &str, _args: OpCopy) -> Result { + async fn copy( + &self, + from: &str, + to: &str, + _args: OpCopy, + _opts: OpCopier, + ) -> Result<(RpCopy, Self::Copier)> { self.core.ensure_dir_exists(to).await?; let resp = if from.ends_with('/') { From 76ebc9377eb6befee950be694ef651283db8f629 Mon Sep 17 00:00:00 2001 From: mro68 <86678134+mro68@users.noreply.github.com> Date: Mon, 25 May 2026 09:57:22 +0200 Subject: [PATCH 6/6] style(services/pcloud): format files with rustfmt --- core/services/pcloud/src/core.rs | 9 ++++--- core/services/pcloud/src/lister.rs | 43 +++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/core/services/pcloud/src/core.rs b/core/services/pcloud/src/core.rs index 525207da11b2..42e68ee343bf 100644 --- a/core/services/pcloud/src/core.rs +++ b/core/services/pcloud/src/core.rs @@ -164,7 +164,10 @@ impl PcloudCore { } pub fn invalidate_path_prefix_cache(&self, path: &str) { - let prefix = format!("{}/", self.normalize_cached_path(path).trim_end_matches('/')); + let prefix = format!( + "{}/", + self.normalize_cached_path(path).trim_end_matches('/') + ); self.file_ids .write() @@ -571,9 +574,7 @@ impl PcloudCore { query.push_str("&recursive=1"); } - let url = self - .build_url("listfolder", query) - .await?; + let url = self.build_url("listfolder", query).await?; let req = Request::get(url); diff --git a/core/services/pcloud/src/lister.rs b/core/services/pcloud/src/lister.rs index 44d4f5df3b27..842a52adebf7 100644 --- a/core/services/pcloud/src/lister.rs +++ b/core/services/pcloud/src/lister.rs @@ -142,9 +142,9 @@ mod tests { use opendal_core::EntryMode; use opendal_core::raw::AccessorInfo; - use super::append_entries; use super::ListMetadata; use super::PcloudCore; + use super::append_entries; fn core() -> PcloudCore { PcloudCore::new( @@ -191,14 +191,31 @@ mod tests { "/repo", dir( "/repo/data/00", - vec![file("/repo/data/00/pack1", 11), dir("/repo/data/00/sub", vec![file("/repo/data/00/sub/pack2", 22)])], + vec![ + file("/repo/data/00/pack1", 11), + dir( + "/repo/data/00/sub", + vec![file("/repo/data/00/sub/pack2", 22)], + ), + ], ), true, ) .expect("entries should flatten"); - let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); - assert_eq!(paths, vec!["data/00/", "data/00/pack1", "data/00/sub/", "data/00/sub/pack2"]); + let paths: Vec<_> = entries + .iter() + .map(|entry| entry.path().to_string()) + .collect(); + assert_eq!( + paths, + vec![ + "data/00/", + "data/00/pack1", + "data/00/sub/", + "data/00/sub/pack2" + ] + ); assert_eq!(entries[0].mode(), EntryMode::DIR); assert_eq!(entries[1].mode(), EntryMode::FILE); assert_eq!(entries[3].mode(), EntryMode::FILE); @@ -215,13 +232,22 @@ mod tests { "/repo", dir( "/repo/data/00", - vec![file("/repo/data/00/pack1", 11), dir("/repo/data/00/sub", vec![file("/repo/data/00/sub/pack2", 22)])], + vec![ + file("/repo/data/00/pack1", 11), + dir( + "/repo/data/00/sub", + vec![file("/repo/data/00/sub/pack2", 22)], + ), + ], ), false, ) .expect("entries should flatten"); - let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); + let paths: Vec<_> = entries + .iter() + .map(|entry| entry.path().to_string()) + .collect(); assert_eq!(paths, vec!["data/00/"]); } @@ -247,7 +273,10 @@ mod tests { ) .expect("entries should rebuild missing child paths"); - let paths: Vec<_> = entries.iter().map(|entry| entry.path().to_string()).collect(); + let paths: Vec<_> = entries + .iter() + .map(|entry| entry.path().to_string()) + .collect(); assert_eq!(paths, vec!["keys/file1"]); } }