From 4a8cc128aab564d5fe03528a904a3b767c4ce2eb Mon Sep 17 00:00:00 2001 From: Adrian Borup Date: Wed, 3 Sep 2025 13:26:21 +0200 Subject: [PATCH 1/4] Add method to retrieve all users in Firebase Auth --- src/auth/mod.rs | 118 ++++++++++++++++++++++++++++++++++++++--- src/auth/models/mod.rs | 7 +++ 2 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/auth/mod.rs b/src/auth/mod.rs index 882cddc..d1021dc 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -5,7 +5,7 @@ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{ auth::{ error::AuthApiErrorResponse, - models::{UpdateUserBody, UpdateUserValues}, + models::{BatchGetResponse, UpdateUserBody, UpdateUserValues}, }, error::FirebaseError, ServiceAccount, @@ -26,6 +26,7 @@ pub struct FirebaseAuthClient { api_url: String, user_token_manager: UserTokenManager, api_auth_token_manager: ApiAuthTokenManager, + project_id: String, } impl FirebaseAuthClient { @@ -36,6 +37,7 @@ impl FirebaseAuthClient { .context("Failed to create HTTP client")?; let credential_manager = ApiAuthTokenManager::new(service_account.clone()); + let project_id = service_account.project_id.clone(); let token_handler = UserTokenManager::new(service_account, client.clone()); Ok(Self { @@ -43,6 +45,7 @@ impl FirebaseAuthClient { client, api_url: "https://identitytoolkit.googleapis.com/v1".to_string(), api_auth_token_manager: credential_manager, + project_id, }) } @@ -50,21 +53,36 @@ impl FirebaseAuthClient { format!("{}{}", self.api_url, path.as_ref()) } - /// Creates a new `POST` request builder with the `Authorization` header set - /// to an authorized admin access token. - async fn auth_post( - &self, - url: impl AsRef, - ) -> Result { + fn project_url(&self, path: impl AsRef) -> String { + format!( + "{}/projects/{}{}", + self.api_url, + &self.project_id, + path.as_ref() + ) + } + + async fn get_access_token(&self) -> Result { let access_token = self .api_auth_token_manager .get_access_token() .await .map_err(|e| { - tracing::error!("Failed to get access token: {}", e); + tracing::error!("Failed to get access token: {e}"); e })?; + Ok(access_token) + } + + /// Creates a new `POST` request builder with the `Authorization` header set + /// to an authorized admin access token. + async fn auth_post( + &self, + url: impl AsRef, + ) -> Result { + let access_token = self.get_access_token().await?; + let builder = self .client .post(url.as_ref()) @@ -73,6 +91,22 @@ impl FirebaseAuthClient { Ok(builder) } + /// Creates a new `GET` request builder with the `Authorization` header set + /// to an authorized admin access token. + async fn auth_get( + &self, + url: impl AsRef, + ) -> Result { + let access_token = self.get_access_token().await?; + + let builder = self + .client + .get(url.as_ref()) + .header("Authorization", format!("Bearer {access_token}")); + + Ok(builder) + } + /// Decodes an ID token and returns its claims. Only succeeds if the token /// is valid. The token is valid if it: /// @@ -306,6 +340,74 @@ impl FirebaseAuthClient { Ok(user) } + #[tracing::instrument(name = "Get all users", skip_all)] + pub async fn get_all_users(&self) -> Result, FirebaseError> { + let base_url = self.project_url("/accounts:batchGet"); + + fn make_pagination_url( + base_url: &str, + max_results: usize, + next_page_token: Option<&str>, + ) -> String { + format!( + "{base_url}?maxResults={max_results}{}", + next_page_token + .map(|token| format!("&nextPageToken={token}")) + .unwrap_or_else(|| { "".to_string() }) + ) + } + + // Potential future work: make the requests concurrently by using a binary search + // methodology. The "next page token" seems to be a user ID, but testing shows that + // the ID does not need to exist - the results are just users after that ID + // (lexicographically). + // + // So we could do something like: + // - Two initial requests: + // 1) no token + // 2) a request with next page token `a` (halfway through the alphabet of upper + + // lowercase letters) + // For example, find lower and upper bounds for user IDs and divide & conquer all + // subranges until they start overlapping. + // + // Keep "guessing" and combining results until we have all users. Would be slower + // for small sets, but for large sets of users it could cut down on the sequential + // requests significantly. + + let mut all_users = Vec::new(); + let mut next_page_token = None; + loop { + let url = make_pagination_url(&base_url, 1000, next_page_token.as_deref()); + + let res = self + .auth_get(url) + .await? + .header("Content-Type", "application/json") + .send() + .await + .context("Failed to send get all users request")?; + + if !res.status().is_success() { + return Err(response_error("Failed to get all users", res).await); + } + + let res_body: BatchGetResponse = + res.json().await.context("Failed to read response JSON")?; + + if let Some(mut users) = res_body.users { + all_users.append(&mut users); + } + + next_page_token = res_body.next_page_token.map(|t| t.to_string()); + + if next_page_token.is_none() { + break; + } + } + + Ok(all_users) + } + /// Creates a new user in Firebase Auth using the email/password provider. /// /// # Examples diff --git a/src/auth/models/mod.rs b/src/auth/models/mod.rs index 3d007de..c51a966 100644 --- a/src/auth/models/mod.rs +++ b/src/auth/models/mod.rs @@ -11,6 +11,13 @@ pub(crate) struct GetAccountInfoResponse { pub users: Option>, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct BatchGetResponse { + pub users: Option>, + pub next_page_token: Option, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct User { From 0d674d4b908d027271b6e5acf63992f3b8141ab4 Mon Sep 17 00:00:00 2001 From: Adrian Borup Date: Wed, 3 Sep 2025 15:29:11 +0200 Subject: [PATCH 2/4] Add concurrent version of get_all_users --- src/auth/mod.rs | 135 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 96 insertions(+), 39 deletions(-) diff --git a/src/auth/mod.rs b/src/auth/mod.rs index d1021dc..49ac194 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use anyhow::Context; use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; @@ -340,8 +342,30 @@ impl FirebaseAuthClient { Ok(user) } + /// Retrieve all users in Firebase Auth. By default, this will send multiple batch requests to + /// the Firebase Auth API sequentially until all users have been retrieved. If you have + /// thousands of users, this can take a while. See + /// [`get_all_users_concurrently`](Self::get_all_users_concurrently) if you want these requests + /// to be sent concurrently by `N` workers. #[tracing::instrument(name = "Get all users", skip_all)] pub async fn get_all_users(&self) -> Result, FirebaseError> { + self.get_all_users_concurrently(1).await + } + + /// Retrieve all users in Firebase Auth using `num_workers` concurrent workers. See + /// [`get_all_users`](Self::get_all_users) for more info. + /// + /// The number of workers should be between 1 and 62. If you pass a number outside + /// this range, it will be clamped. + /// + /// Keep in mind that, at the time of writing, the Identity Toolkit API limit is 500 + /// requests/second per service account. This method will fire off multiple requests + /// simultaneously. See the up-to-date limits in the [official docs](https://firebase.google.com/docs/auth/limits). + #[tracing::instrument(name = "Get all users concurrently", skip_all)] + pub async fn get_all_users_concurrently( + &self, + num_workers: u8, + ) -> Result, FirebaseError> { let base_url = self.project_url("/accounts:batchGet"); fn make_pagination_url( @@ -357,54 +381,87 @@ impl FirebaseAuthClient { ) } - // Potential future work: make the requests concurrently by using a binary search - // methodology. The "next page token" seems to be a user ID, but testing shows that - // the ID does not need to exist - the results are just users after that ID - // (lexicographically). - // - // So we could do something like: - // - Two initial requests: - // 1) no token - // 2) a request with next page token `a` (halfway through the alphabet of upper + - // lowercase letters) - // For example, find lower and upper bounds for user IDs and divide & conquer all - // subranges until they start overlapping. - // - // Keep "guessing" and combining results until we have all users. Would be slower - // for small sets, but for large sets of users it could cut down on the sequential - // requests significantly. - - let mut all_users = Vec::new(); - let mut next_page_token = None; - loop { - let url = make_pagination_url(&base_url, 1000, next_page_token.as_deref()); - - let res = self - .auth_get(url) - .await? - .header("Content-Type", "application/json") - .send() - .await - .context("Failed to send get all users request")?; + let legal = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + .chars() + .collect::>(); - if !res.status().is_success() { - return Err(response_error("Failed to get all users", res).await); - } + let num_workers = (num_workers as usize).clamp(1, legal.len()); - let res_body: BatchGetResponse = - res.json().await.context("Failed to read response JSON")?; + let chunk_size = legal.len() / num_workers; + let mut ranges = Vec::new(); - if let Some(mut users) = res_body.users { - all_users.append(&mut users); + for i in 0..num_workers { + let min = legal[i * chunk_size]; + let max = if i < num_workers - 1 { + legal[(i + 1) * chunk_size] + } else { + (*legal.last().unwrap() as u8 + 1) as char + }; + + ranges.push((min, max)); + } + + async fn run_worker( + client: &FirebaseAuthClient, + range: (char, char), + base_url: &str, + ) -> Result, FirebaseError> { + const MAX_RESULTS: usize = 1000; + let (min, max) = (range.0.to_string(), range.1.to_string()); + + let mut all_users = Vec::new(); + let mut next_page_token = Some(min); + loop { + let url = make_pagination_url(base_url, MAX_RESULTS, next_page_token.as_deref()); + + let res = client + .auth_get(url) + .await? + .header("Content-Type", "application/json") + .send() + .await + .context("Failed to send get all users request")?; + + if !res.status().is_success() { + return Err(response_error("Failed to get all users", res).await); + } + + let res_body: BatchGetResponse = + res.json().await.context("Failed to read response JSON")?; + + if let Some(mut users) = res_body.users { + all_users.append(&mut users); + } + + next_page_token = res_body.next_page_token.map(|t| t.to_string()); + + if next_page_token.is_none() || next_page_token.as_deref() >= Some(&max) { + break; + } } - next_page_token = res_body.next_page_token.map(|t| t.to_string()); + Ok(all_users) + } + + let futures = ranges + .into_iter() + .map(|range| run_worker(self, range, &base_url)); - if next_page_token.is_none() { - break; + let worker_results = futures::future::try_join_all(futures).await?; + let mut deduped = HashMap::new(); + + for users in worker_results { + for user in users { + deduped.insert(user.uid.clone(), user); } } + let keys = deduped.keys().cloned().collect::>(); + let all_users = keys + .iter() + .map(|k| deduped.remove(k).unwrap()) + .collect::>(); + Ok(all_users) } From 2b586c82a8ec2d30fd7e1fa3ce2efce04f5a9be6 Mon Sep 17 00:00:00 2001 From: Adrian Borup Date: Tue, 9 Sep 2025 12:06:11 +0200 Subject: [PATCH 3/4] Fix formatting --- src/auth/credential/api_auth_token.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/auth/credential/api_auth_token.rs b/src/auth/credential/api_auth_token.rs index ce272dd..d54b599 100644 --- a/src/auth/credential/api_auth_token.rs +++ b/src/auth/credential/api_auth_token.rs @@ -52,9 +52,7 @@ impl ApiAuthTokenManager { "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Ajwt-bearer&assertion={jwt}" ); - let url = format!( - "https://{GOOGLE_AUTH_TOKEN_HOST}{GOOGLE_AUTH_TOKEN_PATH}" - ); + let url = format!("https://{GOOGLE_AUTH_TOKEN_HOST}{GOOGLE_AUTH_TOKEN_PATH}"); let res = self .http_client From 84683b288d3042ce1bbdc3faded5218476ab10ba Mon Sep 17 00:00:00 2001 From: Adrian Borup Date: Tue, 9 Sep 2025 12:40:25 +0200 Subject: [PATCH 4/4] Simplify value extraction from hashmap --- src/auth/mod.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/auth/mod.rs b/src/auth/mod.rs index 49ac194..90d5a43 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -448,19 +448,15 @@ impl FirebaseAuthClient { .map(|range| run_worker(self, range, &base_url)); let worker_results = futures::future::try_join_all(futures).await?; - let mut deduped = HashMap::new(); + let mut deduped = HashMap::new(); for users in worker_results { for user in users { deduped.insert(user.uid.clone(), user); } } - let keys = deduped.keys().cloned().collect::>(); - let all_users = keys - .iter() - .map(|k| deduped.remove(k).unwrap()) - .collect::>(); + let all_users = deduped.into_values().collect(); Ok(all_users) }