From 76db44766d254b36788706d9bfde78198125d0cd Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 3 Mar 2026 04:05:27 +0000 Subject: [PATCH 01/89] Add AzureBlobFileSystem placeholder, verify devtools::document() behaves correctly. --- r/NAMESPACE | 1 + r/R/filesystem.R | 17 +++++++++++++++++ r/man/FileSystem.Rd | 1 + 3 files changed, 19 insertions(+) diff --git a/r/NAMESPACE b/r/NAMESPACE index f74034c965b7..802c3000efca 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -183,6 +183,7 @@ S3method(vec_ptype_full,arrow_fixed_size_list) S3method(vec_ptype_full,arrow_large_list) S3method(vec_ptype_full,arrow_list) export(Array) +export(AzureBlobFileSystem) export(Buffer) export(BufferOutputStream) export(BufferReader) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 99c09c40dc3b..0fc10c4702ec 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -645,6 +645,23 @@ GcsFileSystem$create <- function(anonymous = FALSE, retry_limit_seconds = 15, .. fs___GcsFileSystem__Make(anonymous, options) } +#' @usage NULL +#' @format NULL +#' @rdname FileSystem +#' @importFrom utils modifyList +#' @export +AzureBlobFileSystem <- R6Class( + "AzureBlobFileSystem", + inherit = FileSystem, + active = list( + region = function() fs___S3FileSystem__region(self) + ) +) + +AzureBlobFileSystem$test <- function(msg) { + sprintf("Hello, %s", msg) +} + #' @usage NULL #' @format NULL #' @rdname FileSystem diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index 83e7fc652616..4ff80e26220e 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -6,6 +6,7 @@ \alias{LocalFileSystem} \alias{S3FileSystem} \alias{GcsFileSystem} +\alias{AzureBlobFileSystem} \alias{SubTreeFileSystem} \title{FileSystem classes} \description{ From 943cbfed1ff4e41fb1734a1b955bf2e22ec44888 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 02:59:07 +0000 Subject: [PATCH 02/89] add simple test function to work through codegen.R --- r/R/arrowExports.R | 4 ++++ r/src/arrowExports.cpp | 9 +++++++++ r/src/filesystem.cpp | 15 +++++++++++++++ r/tmp.md | 11 +++++++++++ 4 files changed, 39 insertions(+) create mode 100644 r/tmp.md diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 22e66e2243ec..842fc3e2e526 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,6 +1416,10 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } +azurefs_is_functional_test <- function(input_string) { + .Call(`_arrow_azurefs_is_functional_test`, input_string) +} + fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 8a67d0acd89d..c642fd235528 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,6 +3646,14 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp +bool azurefs_is_functional_test(std::string input_string); +extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ +BEGIN_CPP11 + arrow::r::Input::type input_string(input_string_sexp); + return cpp11::as_sexp(azurefs_is_functional_test(input_string)); +END_CPP11 +} +// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6096,6 +6104,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, + { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 9324a13ce0f4..f0b1d9d5bfd7 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,6 +381,21 @@ void FinalizeS3() { #endif } +#if defined(ARROW_R_WITH_AZUREFS) + +#include + +// [[arrow::export]] +bool azurefs_is_functional_test(std::string input_string) { + // This just proves we can pass data in and out of the guarded block + if (input_string == "hello") { + return true; + } + return false; +} + +#endif + #if defined(ARROW_R_WITH_GCS) #include diff --git a/r/tmp.md b/r/tmp.md new file mode 100644 index 000000000000..872728514064 --- /dev/null +++ b/r/tmp.md @@ -0,0 +1,11 @@ +# Temporary development notes + +> TODO: Remove this before we open a PR to upstream arrow library. + +## Using codegen.R + +1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` + +2. Rscript `data-raw/codegen.R` + +The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From 5a40fdea75d29174cdafff089f5c45c42dce0b51 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 03:56:16 +0000 Subject: [PATCH 03/89] temporarily "force" build DARROW_R_WITH_AZUREFS build flag. We'll want to use the proper build flag before submitting the PR but for now this successfully links the C++ binding to the azurefs_is_functional_test function. --- r/configure | 3 +++ r/tmp.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/r/configure b/r/configure index 9e92eb6b47f2..72524e7d8954 100755 --- a/r/configure +++ b/r/configure @@ -359,6 +359,9 @@ add_feature_flags () { if arrow_built_with ARROW_S3; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_S3" fi + if arrow_built_with ARROW_AZURE || [ "$ARROW_R_WITH_AZUREFS" = "true" ]; then + PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZUREFS" + fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" fi diff --git a/r/tmp.md b/r/tmp.md index 872728514064..4ee07eadea58 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -9,3 +9,5 @@ 2. Rscript `data-raw/codegen.R` The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. + +**Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. From 4681b41c8064d4d60df35e97597b22b3e5b7b2f0 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 4 Mar 2026 21:26:48 -0500 Subject: [PATCH 04/89] Added c++ stub --- r/R/filesystem.R | 20 ++++++++++++++------ r/src/filesystem.cpp | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 0fc10c4702ec..3af19b7b3312 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -652,14 +652,22 @@ GcsFileSystem$create <- function(anonymous = FALSE, retry_limit_seconds = 15, .. #' @export AzureBlobFileSystem <- R6Class( "AzureBlobFileSystem", - inherit = FileSystem, - active = list( - region = function() fs___S3FileSystem__region(self) - ) + inherit = FileSystem ) -AzureBlobFileSystem$test <- function(msg) { - sprintf("Hello, %s", msg) +# TODO: +AzureBlobFileSystem$create <- function(...) { + fs___AzureFileSystem__Make(...) +} + +# TODO: +az_bucket <- function(bucket, ...) { + assert_that(is.string(bucket)) + args <- list2(...) + + fs <- exec(AzureFileSystem$create, !!!args) + + SubTreeFileSystem$create(bucket, fs) } #' @usage NULL diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f0b1d9d5bfd7..9299bbbeff81 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -253,7 +253,7 @@ std::shared_ptr fs___SubTreeFileSystem__base_fs( // [[arrow::export]] std::string fs___SubTreeFileSystem__base_path( const std::shared_ptr& file_system) { - return file_system->base_path(); + // return file_system->base_path(); } // Forward declaration - defined in the ARROW_R_WITH_S3 block below. @@ -541,3 +541,19 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr } #endif + +// TODO: +#if defined(ARROW_R_WITH_AZURE) +#include + +// [[azure::export]] +std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options) { + fs::AzureOptions azure_opts; + azure_opts = fs::AzureOptions::Defaults(); + + auto io_context = MainRThread::GetInstance().CancellableIOContext(); + return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); + +} + +#endif \ No newline at end of file From 5769649ff46e7ac61268fca48620b434aee706ad Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 4 Mar 2026 21:41:43 -0500 Subject: [PATCH 05/89] Updated codegen --- r/R/arrowExports.R | 4 ++++ r/data-raw/codegen.R | 2 +- r/src/arrowExports.cpp | 26 ++++++++++++++++++++++++++ r/src/filesystem.cpp | 2 +- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 842fc3e2e526..82099ec73a6b 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1428,6 +1428,10 @@ fs___GcsFileSystem__options <- function(fs) { .Call(`_arrow_fs___GcsFileSystem__options`, fs) } +fs___AzureFileSystem__Make <- function(options) { + .Call(`_arrow_fs___AzureFileSystem__Make`, options) +} + io___Readable__Read <- function(x, nbytes) { .Call(`_arrow_io___Readable__Read`, x, nbytes) } diff --git a/r/data-raw/codegen.R b/r/data-raw/codegen.R index 9acfef109c56..8a78ba7ecaac 100644 --- a/r/data-raw/codegen.R +++ b/r/data-raw/codegen.R @@ -30,7 +30,7 @@ # Ensure that all machines are sorting the same way invisible(Sys.setlocale("LC_COLLATE", "C")) -features <- c("acero", "dataset", "substrait", "parquet", "s3", "gcs", "json") +features <- c("acero", "dataset", "substrait", "parquet", "s3", "gcs", "azure", "json") suppressPackageStartupMessages({ library(decor) diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index c642fd235528..9955ceb7544f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3684,6 +3684,21 @@ extern "C" SEXP _arrow_fs___GcsFileSystem__options(SEXP fs_sexp){ } #endif +// filesystem.cpp +#if defined(ARROW_R_WITH_AZURE) +std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options); +extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ +BEGIN_CPP11 + arrow::r::Input::type options(options_sexp); + return cpp11::as_sexp(fs___AzureFileSystem__Make(options)); +END_CPP11 +} +#else +extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ + Rf_error("Cannot call fs___AzureFileSystem__Make(). See https://arrow.apache.org/docs/r/articles/install.html for help installing Arrow C++ libraries. "); +} +#endif + // io.cpp std::shared_ptr io___Readable__Read(const std::shared_ptr& x, int64_t nbytes); extern "C" SEXP _arrow_io___Readable__Read(SEXP x_sexp, SEXP nbytes_sexp){ @@ -5733,6 +5748,15 @@ return Rf_ScalarLogical( #endif ); } +extern "C" SEXP _azure_available() { +return Rf_ScalarLogical( +#if defined(ARROW_R_WITH_AZURE) + TRUE +#else + FALSE +#endif +); +} extern "C" SEXP _json_available() { return Rf_ScalarLogical( #if defined(ARROW_R_WITH_JSON) @@ -5749,6 +5773,7 @@ static const R_CallMethodDef CallEntries[] = { { "_parquet_available", (DL_FUNC)& _parquet_available, 0 }, { "_s3_available", (DL_FUNC)& _s3_available, 0 }, { "_gcs_available", (DL_FUNC)& _gcs_available, 0 }, + { "_azure_available", (DL_FUNC)& _azure_available, 0 }, { "_json_available", (DL_FUNC)& _json_available, 0 }, { "_arrow_is_arrow_altrep", (DL_FUNC) &_arrow_is_arrow_altrep, 1}, { "_arrow_test_arrow_altrep_set_string_elt", (DL_FUNC) &_arrow_test_arrow_altrep_set_string_elt, 3}, @@ -6107,6 +6132,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, + { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, { "_arrow_io___Readable__Read", (DL_FUNC) &_arrow_io___Readable__Read, 2}, { "_arrow_io___InputStream__Close", (DL_FUNC) &_arrow_io___InputStream__Close, 1}, { "_arrow_io___OutputStream__Close", (DL_FUNC) &_arrow_io___OutputStream__Close, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 9299bbbeff81..394b97434132 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -556,4 +556,4 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti } -#endif \ No newline at end of file +#endif From 2dbf5c701b9020e980411cd57632df2eec238a7c Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 4 Mar 2026 21:44:11 -0500 Subject: [PATCH 06/89] Added a comment --- r/src/filesystem.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 394b97434132..1ad7239b54c1 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -542,7 +542,8 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr #endif -// TODO: +// TODO: Write the Rcpp function to interface with the AzureFileSystem class in +// arrow/filesystem/azurefs.h. #if defined(ARROW_R_WITH_AZURE) #include From 936ee78e30b648d26eb9ac5dde916be561e7b2f8 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:10 +0000 Subject: [PATCH 07/89] cleanup azurefs test function code --- r/R/arrowExports.R | 4 ---- r/src/arrowExports.cpp | 9 --------- r/src/filesystem.cpp | 15 --------------- 3 files changed, 28 deletions(-) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 82099ec73a6b..dac7b4609c56 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,10 +1416,6 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } -azurefs_is_functional_test <- function(input_string) { - .Call(`_arrow_azurefs_is_functional_test`, input_string) -} - fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 9955ceb7544f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,14 +3646,6 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp -bool azurefs_is_functional_test(std::string input_string); -extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ -BEGIN_CPP11 - arrow::r::Input::type input_string(input_string_sexp); - return cpp11::as_sexp(azurefs_is_functional_test(input_string)); -END_CPP11 -} -// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6129,7 +6121,6 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, - { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 1ad7239b54c1..34de54976940 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,21 +381,6 @@ void FinalizeS3() { #endif } -#if defined(ARROW_R_WITH_AZUREFS) - -#include - -// [[arrow::export]] -bool azurefs_is_functional_test(std::string input_string) { - // This just proves we can pass data in and out of the guarded block - if (input_string == "hello") { - return true; - } - return false; -} - -#endif - #if defined(ARROW_R_WITH_GCS) #include From 778c60621b9bc79139b177051f62ca0e431c1f41 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:31 +0000 Subject: [PATCH 08/89] document instructions to start local azurite container --- r/tmp.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/r/tmp.md b/r/tmp.md index 4ee07eadea58..8b145cfb03bd 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -11,3 +11,8 @@ The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. **Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. + +## Using Azurite + +`docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) + From 83ea378cd792133e5fb9171098ee4cf496440767 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:49:37 +0000 Subject: [PATCH 09/89] add arrow_with_azure helper following convention for s3/gcp --- r/NAMESPACE | 1 + r/R/arrow-info.R | 10 ++++++++++ r/man/arrow_info.Rd | 3 +++ 3 files changed, 14 insertions(+) diff --git a/r/NAMESPACE b/r/NAMESPACE index 802c3000efca..8e5f2d5aaa09 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -283,6 +283,7 @@ export(arrow_available) export(arrow_info) export(arrow_table) export(arrow_with_acero) +export(arrow_with_azure) export(arrow_with_dataset) export(arrow_with_gcs) export(arrow_with_json) diff --git a/r/R/arrow-info.R b/r/R/arrow-info.R index 699f94dcbdb5..91b46788aab2 100644 --- a/r/R/arrow-info.R +++ b/r/R/arrow-info.R @@ -46,6 +46,7 @@ arrow_info <- function() { json = arrow_with_json(), s3 = arrow_with_s3(), gcs = arrow_with_gcs(), + azure = arrow_with_azure(), utf8proc = "utf8_upper" %in% compute_funcs, re2 = "replace_substring_regex" %in% compute_funcs, vapply(tolower(names(CompressionType)[-1]), codec_is_available, logical(1)) @@ -128,6 +129,15 @@ arrow_with_gcs <- function() { }) } +#' @rdname arrow_info +#' @export +arrow_with_azure <- function() { + tryCatch(.Call(`_azure_available`), error = function(e) { + return(FALSE) + }) +} + + #' @rdname arrow_info #' @export arrow_with_json <- function() { diff --git a/r/man/arrow_info.Rd b/r/man/arrow_info.Rd index a839d3ba8fd2..4e6d12c46cbe 100644 --- a/r/man/arrow_info.Rd +++ b/r/man/arrow_info.Rd @@ -9,6 +9,7 @@ \alias{arrow_with_parquet} \alias{arrow_with_s3} \alias{arrow_with_gcs} +\alias{arrow_with_azure} \alias{arrow_with_json} \title{Report information on the package's capabilities} \usage{ @@ -28,6 +29,8 @@ arrow_with_s3() arrow_with_gcs() +arrow_with_azure() + arrow_with_json() } \value{ From 33de88cc653028d4533ba5017edabfbe978c5f65 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 23:36:19 +0000 Subject: [PATCH 10/89] add ARROW_AZURE flag to nixlibs.R --- r/tools/nixlibs.R | 1 + 1 file changed, 1 insertion(+) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index dd230c3764ff..780422ab500c 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -606,6 +606,7 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- c( env_var_list, ARROW_S3 = Sys.getenv("ARROW_S3", "ON"), + # ARROW_AZURE = Sys.getenv("ARROW_AZURE", "ON"), # ARROW_GCS = Sys.getenv("ARROW_GCS", "ON"), ARROW_WITH_ZSTD = Sys.getenv("ARROW_WITH_ZSTD", "ON") ) From 38e44923b81533d892e34d62cf4709476312f505 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 23:53:09 +0000 Subject: [PATCH 11/89] debug first argument check Note: I think what's happening is that the build of Arrow I install doesn't have the ARROW_AZURE flag enabled. When I "force" the environment variable I get an error like "fs___AzureFileSystem__Make not found". --- r/src/arrowExports.cpp | 4 ++-- r/src/filesystem.cpp | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..c0cd8aa3b71f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3677,7 +3677,7 @@ extern "C" SEXP _arrow_fs___GcsFileSystem__options(SEXP fs_sexp){ #endif // filesystem.cpp -#if defined(ARROW_R_WITH_AZURE) +#if defined(ARROW_R_WITH_AZUREFS) std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options); extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ BEGIN_CPP11 @@ -5742,7 +5742,7 @@ return Rf_ScalarLogical( } extern "C" SEXP _azure_available() { return Rf_ScalarLogical( -#if defined(ARROW_R_WITH_AZURE) +#if defined(ARROW_R_WITH_AZUREFS) TRUE #else FALSE diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 34de54976940..d36e88ae341b 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -529,13 +529,17 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr // TODO: Write the Rcpp function to interface with the AzureFileSystem class in // arrow/filesystem/azurefs.h. -#if defined(ARROW_R_WITH_AZURE) +#if defined(ARROW_R_WITH_AZUREFS) #include // [[azure::export]] std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options) { fs::AzureOptions azure_opts; - azure_opts = fs::AzureOptions::Defaults(); + + // Set account name + if (!Rf_isNull(options["account_name"])) { + azure_opts.account_name = cpp11::as_cpp(options["account_name"]); + } auto io_context = MainRThread::GetInstance().CancellableIOContext(); return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); From 8ba0eaab02dd9a6b8707e0dab142127841f1cf2f Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 10 Mar 2026 22:29:03 -0400 Subject: [PATCH 12/89] Renamed R6 class correctly --- r/NAMESPACE | 2 +- r/R/filesystem.R | 6 +++--- r/man/FileSystem.Rd | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/r/NAMESPACE b/r/NAMESPACE index 8e5f2d5aaa09..7bc446410d8e 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -183,7 +183,7 @@ S3method(vec_ptype_full,arrow_fixed_size_list) S3method(vec_ptype_full,arrow_large_list) S3method(vec_ptype_full,arrow_list) export(Array) -export(AzureBlobFileSystem) +export(AzureFileSystem) export(Buffer) export(BufferOutputStream) export(BufferReader) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 3af19b7b3312..302fc43e3267 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -650,13 +650,13 @@ GcsFileSystem$create <- function(anonymous = FALSE, retry_limit_seconds = 15, .. #' @rdname FileSystem #' @importFrom utils modifyList #' @export -AzureBlobFileSystem <- R6Class( - "AzureBlobFileSystem", +AzureFileSystem <- R6Class( + "AzureFileSystem", inherit = FileSystem ) # TODO: -AzureBlobFileSystem$create <- function(...) { +AzureFileSystem$create <- function(...) { fs___AzureFileSystem__Make(...) } diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index 4ff80e26220e..eeccda31b04c 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -6,7 +6,7 @@ \alias{LocalFileSystem} \alias{S3FileSystem} \alias{GcsFileSystem} -\alias{AzureBlobFileSystem} +\alias{AzureFileSystem} \alias{SubTreeFileSystem} \title{FileSystem classes} \description{ From cbbd65430ffdc560acc05ee484f8543b1b98b161 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 10 Mar 2026 23:06:45 -0400 Subject: [PATCH 13/89] Added endpoint + key, token, and default authentication --- r/R/filesystem.R | 33 ++++++++++++++++++++++++++++++++- r/src/filesystem.cpp | 26 ++++++++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 302fc43e3267..92be0bf9f6e2 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -657,10 +657,41 @@ AzureFileSystem <- R6Class( # TODO: AzureFileSystem$create <- function(...) { + options <- list(...) + valid_opts <- c( + "account_name", + "account_key", + "blob_storage_authority", + "blob_storage_scheme", + "client_id", + "client_secret", + "dfs_storage_authority", + "dfs_storage_scheme", + "sas_token", + "tenant_id" + ) + + invalid_opts <- setdiff(names(options), valid_opts) + if (length(invalid_opts)) { + stop( + "Invalid options for AzureFileSystem: ", + oxford_paste(invalid_opts), + call. = FALSE + ) + } + if (!is.null(options$account_key) && !is.null(options$sas_token)) { + stop( + "Cannot specify both `account_key` and `sas_token`", + call. = FALSE + ) + } + # TODO: Validate combinations of tenant id/client id/client secret before + # handing off to C++. + fs___AzureFileSystem__Make(...) } -# TODO: +# TODO: Probably shouldn't be called bucket. az_bucket <- function(bucket, ...) { assert_that(is.string(bucket)) args <- list2(...) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index d36e88ae341b..6d523cf6139b 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -535,15 +535,37 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr // [[azure::export]] std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options) { fs::AzureOptions azure_opts; - + // Set account name if (!Rf_isNull(options["account_name"])) { azure_opts.account_name = cpp11::as_cpp(options["account_name"]); } + if (!Rf_isNull(options["blob_storage_authority"])) { + azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); + } + if (!Rf_isNull(options["dfs_storage_authority"])) { + azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); + } + if (!Rf_isNull(options["blob_storage_schema"])) { + azure_opts.blob_storage_schema = cpp11::as_cpp(options["blob_storage_schema"]); + } + if (!Rf_isNull(options["dfs_storage_schema"])) { + azure_opts.dfs_storage_schema = cpp11::as_cpp(options["dfs_storage_schema"]); + } + + // TODO: Deal with different combinations of tenant id/client id/client secret. + if (!Rf_isNull(options["account_key"])) { + azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); + } else if (!Rf_isNull(options["sas_token"])) { + azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); + } else { + azure_opts.ConfigureDefaultCredential(); + } + auto io_context = MainRThread::GetInstance().CancellableIOContext(); return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); - + } #endif From 878a450aaef9af6a0d738cdf62bec1354a45b0b1 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 11 Mar 2026 00:11:14 -0400 Subject: [PATCH 14/89] Finished logical for AzureFileSystem to match pyarrow --- r/R/filesystem.R | 26 +++++++++++++++++++++----- r/src/filesystem.cpp | 25 ++++++++++++++++--------- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 92be0bf9f6e2..1d97ad29ac91 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -655,7 +655,6 @@ AzureFileSystem <- R6Class( inherit = FileSystem ) -# TODO: AzureFileSystem$create <- function(...) { options <- list(...) valid_opts <- c( @@ -679,19 +678,36 @@ AzureFileSystem$create <- function(...) { call. = FALSE ) } - if (!is.null(options$account_key) && !is.null(options$sas_token)) { + if (is.null(options$account_name)) { + stop("Missing `account_name`", call. = FALSE) + } + if (!is.null(options$tenant_id) || !is.null(options$client_id) || !is.null(options$client_secret)) { + if (is.null(options$client_id)) { + stop( + "`client_id` must be given with `tenant_id` and `client_secret`", + call. = FALSE + ) + } + if (sum(is.null(options$tenant_id), is.null(options$client_secret)) == 1) { + stop( + "Provide only `client_id` to authenticate with ", + "Managed Identity Credential, or provide `client_id`, `tenant_id`, ", + "and`client_secret` to authenticate with Client Secret Credential", + call. = FALSE + ) + } + } else if (!is.null(options$account_key) && !is.null(options$sas_token)) { stop( "Cannot specify both `account_key` and `sas_token`", call. = FALSE ) } - # TODO: Validate combinations of tenant id/client id/client secret before - # handing off to C++. - fs___AzureFileSystem__Make(...) + fs___AzureFileSystem__Make(options) } # TODO: Probably shouldn't be called bucket. +# TODO: Add documentation. az_bucket <- function(bucket, ...) { assert_that(is.string(bucket)) args <- list2(...) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 6d523cf6139b..f44370593feb 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -537,9 +537,7 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti fs::AzureOptions azure_opts; // Set account name - if (!Rf_isNull(options["account_name"])) { - azure_opts.account_name = cpp11::as_cpp(options["account_name"]); - } + azure_opts.account_name = cpp11::as_cpp(options["account_name"]); if (!Rf_isNull(options["blob_storage_authority"])) { azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); @@ -547,15 +545,24 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti if (!Rf_isNull(options["dfs_storage_authority"])) { azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); } - if (!Rf_isNull(options["blob_storage_schema"])) { - azure_opts.blob_storage_schema = cpp11::as_cpp(options["blob_storage_schema"]); + if (!Rf_isNull(options["blob_storage_scheme"])) { + azure_opts.blob_storage_scheme = cpp11::as_cpp(options["blob_storage_scheme"]); } - if (!Rf_isNull(options["dfs_storage_schema"])) { - azure_opts.dfs_storage_schema = cpp11::as_cpp(options["dfs_storage_schema"]); + if (!Rf_isNull(options["dfs_storage_scheme"])) { + azure_opts.dfs_storage_scheme = cpp11::as_cpp(options["dfs_storage_scheme"]); } - // TODO: Deal with different combinations of tenant id/client id/client secret. - if (!Rf_isNull(options["account_key"])) { + if (!Rf_isNull(options["client_id"])) { + if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { + azure_opts.ConfigureManagedIdentityCredential(cpp11::as_cpp(options["client_id"])); + } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { + azure_opts.ConfigureClientSecretCredential( + cpp11::as_cpp(options["tenant_id"]), + cpp11::as_cpp(options["client_id"]), + cpp11::as_cpp(options["client_secret"]) + ); + } + } else if (!Rf_isNull(options["account_key"])) { azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); } else if (!Rf_isNull(options["sas_token"])) { azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); From 9134cb7a9c15ffbad863d968cbbcefda53dbe24c Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:30:47 +0000 Subject: [PATCH 15/89] standardize on ARROW_R_WITH_AZURE instead of ARROW_R_WITH_AZUREFS --- r/src/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f44370593feb..ec4331203b3d 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -529,7 +529,7 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr // TODO: Write the Rcpp function to interface with the AzureFileSystem class in // arrow/filesystem/azurefs.h. -#if defined(ARROW_R_WITH_AZUREFS) +#if defined(ARROW_R_WITH_AZURE) #include // [[azure::export]] From 0ac57a8a74130ea58b6e9c45c1370be09967773a Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:31:08 +0000 Subject: [PATCH 16/89] standardize on ARROW_R_WITH_AZURE --- r/src/arrowExports.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index c0cd8aa3b71f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3677,7 +3677,7 @@ extern "C" SEXP _arrow_fs___GcsFileSystem__options(SEXP fs_sexp){ #endif // filesystem.cpp -#if defined(ARROW_R_WITH_AZUREFS) +#if defined(ARROW_R_WITH_AZURE) std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options); extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ BEGIN_CPP11 @@ -5742,7 +5742,7 @@ return Rf_ScalarLogical( } extern "C" SEXP _azure_available() { return Rf_ScalarLogical( -#if defined(ARROW_R_WITH_AZUREFS) +#if defined(ARROW_R_WITH_AZURE) TRUE #else FALSE From e8fcd43d924a0de50e00f91ef96897f5fb124df0 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:31:28 +0000 Subject: [PATCH 17/89] Turn on ARROW_AZURE flag in nixlibs.R --- r/tools/nixlibs.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 780422ab500c..a06deaa949f9 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -606,7 +606,7 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- c( env_var_list, ARROW_S3 = Sys.getenv("ARROW_S3", "ON"), - # ARROW_AZURE = Sys.getenv("ARROW_AZURE", "ON"), + ARROW_AZURE = Sys.getenv("ARROW_AZURE", "ON"), # ARROW_GCS = Sys.getenv("ARROW_GCS", "ON"), ARROW_WITH_ZSTD = Sys.getenv("ARROW_WITH_ZSTD", "ON") ) From 8ffc149431374ef69319d771a107a540b4d44433 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:33:33 +0000 Subject: [PATCH 18/89] drop temporary arrow env var hack --- r/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure b/r/configure index 72524e7d8954..b417a4dd04b4 100755 --- a/r/configure +++ b/r/configure @@ -359,7 +359,7 @@ add_feature_flags () { if arrow_built_with ARROW_S3; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_S3" fi - if arrow_built_with ARROW_AZURE || [ "$ARROW_R_WITH_AZUREFS" = "true" ]; then + if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZUREFS" fi if arrow_built_with ARROW_GCS; then From 517750d9322613e5abc4a786c72d5dc5873a95b0 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:39:10 +0000 Subject: [PATCH 19/89] temporary documentation of what I've tried so far --- r/tmp.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/r/tmp.md b/r/tmp.md index 8b145cfb03bd..6712928d7bd2 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -16,3 +16,73 @@ The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` `docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) +## Build troubleshooting continued + +```bash +export ARROW_HOME=/workspaces/arrow/dist + +cmake \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_COMPUTE=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_EXTRA_ERROR_CONTEXT=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_INSTALL_NAME_RPATH=OFF \ + -DARROW_JEMALLOC=ON \ + -DARROW_JSON=ON \ + -DARROW_PARQUET=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_AZURE=ON \ + .. + + +# Try building from source via R with the relevant env vars set for feature flags. + +# Core Build Settings +export LIBARROW_MINIMAL=false +export FORCE_BUNDLED_BUILD=true +export ARROW_HOME=$ARROW_HOME +export BOOST_SOURCE=BUNDLED + +# Feature Toggles +export ARROW_COMPUTE=ON +export ARROW_CSV=ON +export ARROW_DATASET=ON +export ARROW_EXTRA_ERROR_CONTEXT=ON +export ARROW_FILESYSTEM=ON +export ARROW_JEMALLOC=ON +export ARROW_JSON=ON +export ARROW_PARQUET=ON +export ARROW_AZURE=ON + +# Visibility into build +export ARROW_R_DEV=TRUE + +# Use multiple available cores +export MAKEFLAGS="-j8" + +# Compression Codecs +export ARROW_WITH_SNAPPY=ON +export ARROW_WITH_ZLIB=ON + +# Library Linkage +export ARROW_BUILD_STATIC=ON +export ARROW_BUILD_SHARED=OFF + +# For R-specific behavior (replaces CMAKE_INSTALL_LIBDIR=lib) +export LIBARROW_BINARY=false + +export EXTRA_CMAKE_FLAGS="-DARROW_INSTALL_NAME_RPATH=OFF -DARROW_AZURE=ON -DCMAKE_SHARED_LINKER_FLAGS=-lxml2" + +export PKG_CONFIG_PATH="/usr/lib/x86_64-linux-gnu/pkgconfig" +export LDFLAGS=$(pkg-config --libs libxml-2.0) +export PKG_LIBS=$(pkg-config --libs libxml-2.0) + +# export LIBARROW_EXTERNAL_LIBDIR=/workspaces/arrow/r/libarrow + +R CMD INSTALL . --preclean + +``` \ No newline at end of file From 8bf5b0566908f276ebd8e31eb68be65ae3597e79 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 03:04:09 +0000 Subject: [PATCH 20/89] Add TODO note in configure script to remove hard-coded link flags --- r/configure | 5 +++-- r/tmp.md | 10 +++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/r/configure b/r/configure index b417a4dd04b4..69b2a392a8f6 100755 --- a/r/configure +++ b/r/configure @@ -271,7 +271,8 @@ set_pkg_vars_with_pc () { PKG_CFLAGS="`${PKG_CONFIG} --cflags ${pkg_config_names}` $PKG_CFLAGS" PKG_CFLAGS="$PKG_CFLAGS $PKG_CFLAGS_FEATURES" PKG_LIBS=`${PKG_CONFIG} --libs-only-l --libs-only-other ${pkg_config_names}` - PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES" + # TODO: Figure out how to pass these link flags properly, need this temporarily to get R to link properly. + PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" PKG_DIRS=`${PKG_CONFIG} --libs-only-L ${pkg_config_names}` } @@ -360,7 +361,7 @@ add_feature_flags () { PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_S3" fi if arrow_built_with ARROW_AZURE; then - PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZUREFS" + PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" diff --git a/r/tmp.md b/r/tmp.md index 6712928d7bd2..c7c7118779ba 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -38,13 +38,21 @@ cmake \ -DARROW_AZURE=ON \ .. +cmake --build . --target install -j8 + + +R -e 'install.packages("remotes"); remotes::install_deps(dependencies = TRUE)' + +R CMD INSTALL --no-multiarch . + +# ---------------------- + # Try building from source via R with the relevant env vars set for feature flags. # Core Build Settings export LIBARROW_MINIMAL=false export FORCE_BUNDLED_BUILD=true -export ARROW_HOME=$ARROW_HOME export BOOST_SOURCE=BUNDLED # Feature Toggles From cc7994038810f340b78ac7b54d1bef6282fd5f4c Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sat, 14 Mar 2026 18:44:09 +0000 Subject: [PATCH 21/89] initial filesystem tests --- r/tests/testthat/test-azure.R | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 r/tests/testthat/test-azure.R diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R new file mode 100644 index 000000000000..361ea474005a --- /dev/null +++ b/r/tests/testthat/test-azure.R @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +skip_if_not_available("azure") +# TODO: Add local azurite install to setup script +# skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") + +# TODO: Start azurite from the test code instead of relying on it to be already running externally. + +# Use default azurite credentials, +# see https://learn.microsoft.com/en-us/azure/storage/common/storage-connect-azurite?tabs=blob-storage +azurite_account_name <- "devstoreaccount1" +# Note that this is a well-known default credential for local development on Azurite. +azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +azurite_blob_host <- "127.0.0.1" +azurite_blob_port <- "10000" +azurite_blob_storage_authority <- sprintf("%s:%s",azurite_blob_host, azurite_blob_port) +azurite_blob_storage_scheme <- "http" + +# Helper functions for Azure URIs and paths +azure_uri <- function(...) { + template <- "az://%s:%s@%s?scheme=http&blob_endpoint=localhost%s%s" + # URL encode the account key because it contains reserved characters + encoded_key <- curl::curl_escape(azurite_account_key) + sprintf(template, azurite_account_name, encoded_key, azure_path(...), "%3A", azurite_blob_port) +} + +azure_path <- azure_path <- function(...) { + # 'now' is the container name (following the convention in the s3 tests). + paste(now, ..., sep = "/") +} + +fs <- AzureFileSystem$create( + account_name="devstoreaccount1", + account_key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + blob_storage_authority="127.0.0.1:10000", + blob_storage_scheme="http" +) + +now <- as.character(as.numeric(Sys.time())) +fs$CreateDir(now) +# Clean up when we're all done +withr::defer(fs$DeleteDir(now)) + +# (1) Run default filesystem tests on azure filesystem +test_filesystem("azure", fs, azure_path, azure_uri) From b5d0703426ccdeefe62f2fec569f1ae93fe78895 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 02:25:21 +0000 Subject: [PATCH 22/89] uncomment line 256 of filesystem.cpp Note: this was causing a seg fault when anything that inherits from FileSystem ries to access the base_path property. Uncommenting the return line and rebuilding the R package resolved this issue. --- r/src/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index ec4331203b3d..e78962b10492 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -253,7 +253,7 @@ std::shared_ptr fs___SubTreeFileSystem__base_fs( // [[arrow::export]] std::string fs___SubTreeFileSystem__base_path( const std::shared_ptr& file_system) { - // return file_system->base_path(); + return file_system->base_path(); } // Forward declaration - defined in the ARROW_R_WITH_S3 block below. From 96e3744f26f0704af4c7b2fbaabfc2aaf0e96308 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 02:27:20 +0000 Subject: [PATCH 23/89] checkpoint: resolved segfault error At this point, I was able to call write_feather(example_data, fs$path("test/test.feather")) successfully against Azurite. Committing progress before I make any further changes. --- r/tests/testthat/test-azure.R | 65 ++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 361ea474005a..b2d6c2550902 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +library(arrow) skip_if_not_available("azure") # TODO: Add local azurite install to setup script # skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") @@ -26,35 +26,68 @@ skip_if_not_available("azure") azurite_account_name <- "devstoreaccount1" # Note that this is a well-known default credential for local development on Azurite. azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -azurite_blob_host <- "127.0.0.1" +azurite_blob_host <- "host.docker.internal" azurite_blob_port <- "10000" azurite_blob_storage_authority <- sprintf("%s:%s",azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" # Helper functions for Azure URIs and paths azure_uri <- function(...) { - template <- "az://%s:%s@%s?scheme=http&blob_endpoint=localhost%s%s" + endpoint <- sprintf("%s%s%s", azurite_blob_host, "%3A", azurite_blob_port) + template <- "abfs://%s:%s@%s?endpoint=%s" # URL encode the account key because it contains reserved characters encoded_key <- curl::curl_escape(azurite_account_key) - sprintf(template, azurite_account_name, encoded_key, azure_path(...), "%3A", azurite_blob_port) + sprintf(template, azurite_account_name, encoded_key, azure_path(...), endpoint) } - -azure_path <- azure_path <- function(...) { - # 'now' is the container name (following the convention in the s3 tests). - paste(now, ..., sep = "/") +azure_path <- function(...) { + # 'dir' is the container name (following the convention in the s3 tests). + paste(dir, ..., sep = "/") } fs <- AzureFileSystem$create( - account_name="devstoreaccount1", - account_key="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", - blob_storage_authority="127.0.0.1:10000", - blob_storage_scheme="http" + account_name=azurite_account_name, + account_key=azurite_account_key, + blob_storage_authority=azurite_blob_storage_authority, + blob_storage_scheme=azurite_blob_storage_scheme +) + +fs2 <- arrow:::az_bucket( + bucket="test", + account_name=azurite_account_name, + account_key=azurite_account_key, + blob_storage_authority=azurite_blob_storage_authority, + blob_storage_scheme=azurite_blob_storage_scheme ) -now <- as.character(as.numeric(Sys.time())) -fs$CreateDir(now) +# TODO: Factor these into tests once finished debugging. + +# (1) CreateDir and DeleteDir work correctly +dir <- "test" +fs$CreateDir(dir) # Clean up when we're all done withr::defer(fs$DeleteDir(now)) -# (1) Run default filesystem tests on azure filesystem -test_filesystem("azure", fs, azure_path, azure_uri) +# (XX) Run default filesystem tests on azure filesystem +# TODO: As far as I can tell, there is no way to pass an Azurite URI to write_feather +#test_filesystem("azure", fs, azure_path, azure_uri) + +example_data <- tibble::tibble( + int = c(1:3, NA_integer_, 5:10), + dbl = c(1:8, NA, 10) + 0.1, + dbl2 = rep(5, 10), + lgl = sample(c(TRUE, FALSE, NA), 10, replace = TRUE), + false = logical(10), + chr = letters[c(1:5, NA, 7:10)], + fct = factor(letters[c(1:4, NA, NA, 7:10)]) +) + +# Verify that write file operation works +write_feather(example_data, azure_uri("test.feather")) + +encoded_key <- curl::curl_escape(azurite_account_key) +encoded_key +write_feather(example_data, sprintf("abfs://devstoreaccount1:%s@127.0.0.1:10000/test/test.feather", encoded_key)) + +write_feather(example_data, "az://test@devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq%2FK1SZFPTOtr%2FKBHBeksoGMGw%3D%3D@127.0.0.1:10000/test/test.feather") + + From d365ac35f0faf6f97a17671e6d7c6a162ef96e21 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 04:16:09 +0000 Subject: [PATCH 24/89] skip test_filesystem tests that rely on being able to connect directly with URI with Azure. --- r/tests/testthat/helper-filesystems.R | 135 ++++++++++++++------------ 1 file changed, 74 insertions(+), 61 deletions(-) diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 7b37abf764b0..4755bbbc9de1 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -25,12 +25,18 @@ #' returns a URI containing the filesystem scheme (e.g. 's3://', 'gs://'), the #' absolute path, and any necessary connection options as URL query parameters. test_filesystem <- function(name, fs, path_formatter, uri_formatter) { - # NOTE: it's important that we label these tests with name of filesystem so + # NOTE 1: it's important that we label these tests with name of filesystem so # that we can differentiate the different calls to these test in the output. - test_that(sprintf("read/write Feather on %s using URIs", name), { - write_feather(example_data, uri_formatter("test.feather")) - expect_identical(read_feather(uri_formatter("test.feather")), example_data) - }) + + # NOTE 2: as far as I can tell, Azure doesn't support passing a URI directly + # like we can do in S3/GCS. Skipping any tests that rely on this feature + # for name == "azure". + if (name != "azure") { + test_that(sprintf("read/write Feather on %s using URIs", name), { + write_feather(example_data, uri_formatter("test.feather")) + expect_identical(read_feather(uri_formatter("test.feather")), example_data) + }) + } test_that(sprintf("read/write Feather on %s using Filesystem", name), { write_feather(example_data, fs$path(path_formatter("test2.feather"))) @@ -71,12 +77,14 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { example_data ) }) - - test_that(sprintf("read/write Parquet on %s", name), { - skip_if_not_available("parquet") - write_parquet(example_data, fs$path(path_formatter("test.parquet"))) - expect_identical(read_parquet(uri_formatter("test.parquet")), example_data) - }) + + if (name != "azure") { + test_that(sprintf("read/write Parquet on %s", name), { + skip_if_not_available("parquet") + write_parquet(example_data, fs$path(path_formatter("test.parquet"))) + expect_identical(read_parquet(uri_formatter("test.parquet")), example_data) + }) + } if (arrow_with_dataset()) { make_temp_dir <- function() { @@ -85,39 +93,41 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { normalizePath(path, winslash = "/") } - test_that(sprintf("open_dataset with an %s file (not directory) URI", name), { - skip_if_not_available("parquet") - expect_identical( - open_dataset(uri_formatter("test.parquet")) |> collect() |> arrange(int), - example_data |> arrange(int) - ) - }) - - test_that(sprintf("open_dataset with vector of %s file URIs", name), { - expect_identical( - open_dataset( - c(uri_formatter("test.feather"), uri_formatter("test2.feather")), - format = "feather" - ) |> - arrange(int) |> - collect(), - rbind(example_data, example_data) |> arrange(int) - ) - }) - - test_that(sprintf("open_dataset errors if passed URIs mixing %s and local fs", name), { - td <- make_temp_dir() - expect_error( - open_dataset( - c( - uri_formatter("test.feather"), - paste0("file://", file.path(td, "fake.feather")) + if (name != "azure") { + test_that(sprintf("open_dataset with an %s file (not directory) URI", name), { + skip_if_not_available("parquet") + expect_identical( + open_dataset(uri_formatter("test.parquet")) |> collect() |> arrange(int), + example_data |> arrange(int) + ) + }) + + test_that(sprintf("open_dataset with vector of %s file URIs", name), { + expect_identical( + open_dataset( + c(uri_formatter("test.feather"), uri_formatter("test2.feather")), + format = "feather" + ) |> + arrange(int) |> + collect(), + rbind(example_data, example_data) |> arrange(int) + ) + }) + + test_that(sprintf("open_dataset errors if passed URIs mixing %s and local fs", name), { + td <- make_temp_dir() + expect_error( + open_dataset( + c( + uri_formatter("test.feather"), + paste0("file://", file.path(td, "fake.feather")) + ), + format = "feather" ), - format = "feather" - ), - "Vectors of URIs for different file systems are not supported" - ) - }) + "Vectors of URIs for different file systems are not supported" + ) + }) + } # Dataset test setup, cf. test-dataset.R first_date <- lubridate::ymd_hms("2015-04-29 03:12:39") @@ -167,24 +177,27 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { write_dataset(ds, fs$path(path_formatter("new_dataset_dir"))) expect_length(fs$ls(path_formatter("new_dataset_dir")), 1) }) - + if (name != "azure") { + test_that(sprintf("copy files with %s", name), { + td <- make_temp_dir() + copy_files(uri_formatter("hive_dir"), td) + expect_length(dir(td), 2) + ds <- open_dataset(td) + expect_identical( + ds |> select(int, dbl, lgl) |> collect() |> arrange(int), + rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) + ) + }) + } test_that(sprintf("copy files with %s", name), { - td <- make_temp_dir() - copy_files(uri_formatter("hive_dir"), td) - expect_length(dir(td), 2) - ds <- open_dataset(td) - expect_identical( - ds |> select(int, dbl, lgl) |> collect() |> arrange(int), - rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) - ) - - # Let's copy the other way and use a SubTreeFileSystem rather than URI - copy_files(td, fs$path(path_formatter("hive_dir2"))) - ds2 <- open_dataset(fs$path(path_formatter("hive_dir2"))) - expect_identical( - ds2 |> select(int, dbl, lgl) |> collect() |> arrange(int), - rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) - ) - }) + td <- make_temp_dir() + copy_files(fs$path(path_formatter("hive_dir")), td) + copy_files(td, fs$path(path_formatter("hive_dir2"))) + ds2 <- open_dataset(fs$path(path_formatter("hive_dir2"))) + expect_identical( + ds2 |> select(int, dbl, lgl) |> collect() |> arrange(int), + rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) + ) + }) } # if(arrow_with_dataset()) } From f274b46d2c9dc2702d30e888de523cf58d236d4f Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 04:20:26 +0000 Subject: [PATCH 25/89] Add most test cases from test_filesystem and recreate a couple that were skipped because of the URI issue. --- r/tests/testthat/test-azure.R | 67 ++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index b2d6c2550902..88b781ed7648 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -14,8 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -library(arrow) + skip_if_not_available("azure") + +# test_filesystem requires dplyr +library(dplyr) + # TODO: Add local azurite install to setup script # skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") @@ -39,6 +43,7 @@ azure_uri <- function(...) { encoded_key <- curl::curl_escape(azurite_account_key) sprintf(template, azurite_account_name, encoded_key, azure_path(...), endpoint) } + azure_path <- function(...) { # 'dir' is the container name (following the convention in the s3 tests). paste(dir, ..., sep = "/") @@ -51,25 +56,20 @@ fs <- AzureFileSystem$create( blob_storage_scheme=azurite_blob_storage_scheme ) -fs2 <- arrow:::az_bucket( - bucket="test", - account_name=azurite_account_name, - account_key=azurite_account_key, - blob_storage_authority=azurite_blob_storage_authority, - blob_storage_scheme=azurite_blob_storage_scheme -) - -# TODO: Factor these into tests once finished debugging. - # (1) CreateDir and DeleteDir work correctly dir <- "test" fs$CreateDir(dir) # Clean up when we're all done -withr::defer(fs$DeleteDir(now)) +withr::defer(fs$DeleteDir(dir)) -# (XX) Run default filesystem tests on azure filesystem -# TODO: As far as I can tell, there is no way to pass an Azurite URI to write_feather -#test_filesystem("azure", fs, azure_path, azure_uri) +# (2) Run default filesystem tests on azure filesystem + +# TODO: As far as I can tell, there is no way to pass an Azurite URI to write_feather, +# so some of the test_filesystem tests can't be run with AzureFilesystem. Some tests +# below cover some of the skipped cases in test_filesystem. +test_filesystem("azure", fs, azure_path, azure_uri) + +# (3) Test write/read parquet example_data <- tibble::tibble( int = c(1:3, NA_integer_, 5:10), @@ -81,13 +81,38 @@ example_data <- tibble::tibble( fct = factor(letters[c(1:4, NA, NA, 7:10)]) ) -# Verify that write file operation works -write_feather(example_data, azure_uri("test.feather")) +test_that("read/write Parquet on azure", { + skip_if_not_available("parquet") + write_parquet(example_data, fs$path(azure_path("test.parquet"))) + expect_identical(read_parquet(fs$path(azure_path("test.parquet"))), example_data) +}) + +# (4) open_dataset with a vector of azure file paths + +# TODO: I couldn't pass a vector of paths similar to the original test in +# test_filesystem, but you can pass a folder containing many files. +write_feather(example_data, fs$path(azure_path("openmulti/dataset1.feather"))) +write_feather(example_data, fs$path(azure_path("openmulti/dataset2.feather"))) + +open_multi_fs = arrow:::az_bucket( + bucket=azure_path("openmulti"), + account_name=azurite_account_name, + account_key=azurite_account_key, + blob_storage_authority=azurite_blob_storage_authority, + blob_storage_scheme=azurite_blob_storage_scheme +) -encoded_key <- curl::curl_escape(azurite_account_key) -encoded_key -write_feather(example_data, sprintf("abfs://devstoreaccount1:%s@127.0.0.1:10000/test/test.feather", encoded_key)) +test_that("open_dataset with AzureFileSystem folder", { + expect_identical( + open_dataset( + open_multi_fs, + format = "feather" + ) |> + arrange(int) |> + collect(), + rbind(example_data, example_data) |> arrange(int) + ) +}) -write_feather(example_data, "az://test@devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq%2FK1SZFPTOtr%2FKBHBeksoGMGw%3D%3D@127.0.0.1:10000/test/test.feather") From 4e707341024575f3c489c360476bb98224ea63e3 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 04:26:30 +0000 Subject: [PATCH 26/89] rename az_bucket to az_container --- r/R/filesystem.R | 3 +-- r/tests/testthat/test-azure.R | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 1d97ad29ac91..e6dbd5c3fecc 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -706,9 +706,8 @@ AzureFileSystem$create <- function(...) { fs___AzureFileSystem__Make(options) } -# TODO: Probably shouldn't be called bucket. # TODO: Add documentation. -az_bucket <- function(bucket, ...) { +az_container <- function(bucket, ...) { assert_that(is.string(bucket)) args <- list2(...) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 88b781ed7648..2f7c5e7be4a4 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -94,7 +94,7 @@ test_that("read/write Parquet on azure", { write_feather(example_data, fs$path(azure_path("openmulti/dataset1.feather"))) write_feather(example_data, fs$path(azure_path("openmulti/dataset2.feather"))) -open_multi_fs = arrow:::az_bucket( +open_multi_fs = arrow:::az_container( bucket=azure_path("openmulti"), account_name=azurite_account_name, account_key=azurite_account_key, From 89e9adc8a6c04f6a0111c414a71fd87b9565dae2 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 21:34:12 +0000 Subject: [PATCH 27/89] check that azurite is installed as precondition for test-azure.R script. Switch over to locally hosted azurite instead of local docker container azurite. --- r/tests/testthat/test-azure.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 2f7c5e7be4a4..ac5785702669 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -21,7 +21,7 @@ skip_if_not_available("azure") library(dplyr) # TODO: Add local azurite install to setup script -# skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") +skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") # TODO: Start azurite from the test code instead of relying on it to be already running externally. @@ -30,7 +30,7 @@ library(dplyr) azurite_account_name <- "devstoreaccount1" # Note that this is a well-known default credential for local development on Azurite. azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -azurite_blob_host <- "host.docker.internal" +azurite_blob_host <- "127.0.0.1" azurite_blob_port <- "10000" azurite_blob_storage_authority <- sprintf("%s:%s",azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" From 9bf81a57f83e48c30e5c749db743f819ecadd131 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 22:22:32 +0000 Subject: [PATCH 28/89] add setup code to start azurite from the test-azure.R script, then kill the azurite process in a cleanup step. --- r/tests/testthat/test-azure.R | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index ac5785702669..85a448e2f074 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -20,11 +20,9 @@ skip_if_not_available("azure") # test_filesystem requires dplyr library(dplyr) -# TODO: Add local azurite install to setup script +# This test script depends on ./ci/scripts/install_azurite.sh skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") -# TODO: Start azurite from the test code instead of relying on it to be already running externally. - # Use default azurite credentials, # see https://learn.microsoft.com/en-us/azure/storage/common/storage-connect-azurite?tabs=blob-storage azurite_account_name <- "devstoreaccount1" @@ -35,6 +33,14 @@ azurite_blob_port <- "10000" azurite_blob_storage_authority <- sprintf("%s:%s",azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" +pid_azurite <- sys::exec_background( + "azurite", + c("azurite", "--inMemoryPersistence", "--blobHost", azurite_blob_host), + std_out = FALSE +) +# Kill azurite background process once tests have finished running. +withr::defer(tools::pskill(pid_azurite)) + # Helper functions for Azure URIs and paths azure_uri <- function(...) { endpoint <- sprintf("%s%s%s", azurite_blob_host, "%3A", azurite_blob_port) @@ -64,9 +70,10 @@ withr::defer(fs$DeleteDir(dir)) # (2) Run default filesystem tests on azure filesystem -# TODO: As far as I can tell, there is no way to pass an Azurite URI to write_feather, -# so some of the test_filesystem tests can't be run with AzureFilesystem. Some tests -# below cover some of the skipped cases in test_filesystem. +# TODO: As far as I can tell, there is no way to pass an Azurite URI to write_feather +# (or any other read/write helper), so some of the test_filesystem tests can't be run +# with AzureFilesystem. Some tests below cover some of the skipped cases in +# test_filesystem. test_filesystem("azure", fs, azure_path, azure_uri) # (3) Test write/read parquet From 835879bde5e07a367e8c975594cdd18bcf47c04b Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 22:35:02 +0000 Subject: [PATCH 29/89] run air formatter --- r/tests/testthat/helper-filesystems.R | 22 +++++++++++----------- r/tests/testthat/test-azure.R | 23 ++++++++++------------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/r/tests/testthat/helper-filesystems.R b/r/tests/testthat/helper-filesystems.R index 4755bbbc9de1..9fba086a18e3 100644 --- a/r/tests/testthat/helper-filesystems.R +++ b/r/tests/testthat/helper-filesystems.R @@ -27,7 +27,7 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { # NOTE 1: it's important that we label these tests with name of filesystem so # that we can differentiate the different calls to these test in the output. - + # NOTE 2: as far as I can tell, Azure doesn't support passing a URI directly # like we can do in S3/GCS. Skipping any tests that rely on this feature # for name == "azure". @@ -77,7 +77,7 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { example_data ) }) - + if (name != "azure") { test_that(sprintf("read/write Parquet on %s", name), { skip_if_not_available("parquet") @@ -190,14 +190,14 @@ test_filesystem <- function(name, fs, path_formatter, uri_formatter) { }) } test_that(sprintf("copy files with %s", name), { - td <- make_temp_dir() - copy_files(fs$path(path_formatter("hive_dir")), td) - copy_files(td, fs$path(path_formatter("hive_dir2"))) - ds2 <- open_dataset(fs$path(path_formatter("hive_dir2"))) - expect_identical( - ds2 |> select(int, dbl, lgl) |> collect() |> arrange(int), - rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) - ) - }) + td <- make_temp_dir() + copy_files(fs$path(path_formatter("hive_dir")), td) + copy_files(td, fs$path(path_formatter("hive_dir2"))) + ds2 <- open_dataset(fs$path(path_formatter("hive_dir2"))) + expect_identical( + ds2 |> select(int, dbl, lgl) |> collect() |> arrange(int), + rbind(df1[, c("int", "dbl", "lgl")], df2[, c("int", "dbl", "lgl")]) |> arrange(int) + ) + }) } # if(arrow_with_dataset()) } diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 85a448e2f074..e0bc627551a1 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -30,7 +30,7 @@ azurite_account_name <- "devstoreaccount1" azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" azurite_blob_host <- "127.0.0.1" azurite_blob_port <- "10000" -azurite_blob_storage_authority <- sprintf("%s:%s",azurite_blob_host, azurite_blob_port) +azurite_blob_storage_authority <- sprintf("%s:%s", azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" pid_azurite <- sys::exec_background( @@ -56,10 +56,10 @@ azure_path <- function(...) { } fs <- AzureFileSystem$create( - account_name=azurite_account_name, - account_key=azurite_account_key, - blob_storage_authority=azurite_blob_storage_authority, - blob_storage_scheme=azurite_blob_storage_scheme + account_name = azurite_account_name, + account_key = azurite_account_key, + blob_storage_authority = azurite_blob_storage_authority, + blob_storage_scheme = azurite_blob_storage_scheme ) # (1) CreateDir and DeleteDir work correctly @@ -102,11 +102,11 @@ write_feather(example_data, fs$path(azure_path("openmulti/dataset1.feather"))) write_feather(example_data, fs$path(azure_path("openmulti/dataset2.feather"))) open_multi_fs = arrow:::az_container( - bucket=azure_path("openmulti"), - account_name=azurite_account_name, - account_key=azurite_account_key, - blob_storage_authority=azurite_blob_storage_authority, - blob_storage_scheme=azurite_blob_storage_scheme + bucket = azure_path("openmulti"), + account_name = azurite_account_name, + account_key = azurite_account_key, + blob_storage_authority = azurite_blob_storage_authority, + blob_storage_scheme = azurite_blob_storage_scheme ) test_that("open_dataset with AzureFileSystem folder", { @@ -120,6 +120,3 @@ test_that("open_dataset with AzureFileSystem folder", { rbind(example_data, example_data) |> arrange(int) ) }) - - - From cf1a3b5f7973244b16bf19211b48c3c6b714562c Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 23:04:34 +0000 Subject: [PATCH 30/89] add documentation to az_container. rebuild docs with devtools::document(). rename argument in az_container to container_path instead of bucket --- r/NAMESPACE | 1 + r/R/filesystem.R | 28 ++++++++++++++++++++++++---- r/man/az_container.Rd | 34 ++++++++++++++++++++++++++++++++++ r/tests/testthat/test-azure.R | 2 +- 4 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 r/man/az_container.Rd diff --git a/r/NAMESPACE b/r/NAMESPACE index 7bc446410d8e..46c29b3e9370 100644 --- a/r/NAMESPACE +++ b/r/NAMESPACE @@ -297,6 +297,7 @@ export(as_data_type) export(as_record_batch) export(as_record_batch_reader) export(as_schema) +export(az_container) export(binary) export(bool) export(boolean) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index e6dbd5c3fecc..e06a7c259765 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -706,14 +706,34 @@ AzureFileSystem$create <- function(...) { fs___AzureFileSystem__Make(options) } -# TODO: Add documentation. -az_container <- function(bucket, ...) { - assert_that(is.string(bucket)) +#' Connect to an Azure Blob Storage container +#' +#' `az_conainer` is a convenience function to create an `AzureFileSystem` object +#' that provides a file system interface for blob storage containers in an Azure +#' Storage Account. +#' +#' @param container_path string Container name or path +#' @param ... Additional connection options, passed to `AzureFileSystem$create()` +#' +#' @return A `SubTreeFileSystem` containing an `AzureFileSystem` and the container's +#' relative path. Note that this function's success does not guarantee that you +#' are authorized to access the container's contents. +#' @examplesIf FALSE +#' container_fs <- az_container( +#' container_path = "arrow-datasets", +#' account_name = azurite_account_name, +#' account_key = azurite_account_key, +#' blob_storage_authority = azurite_blob_storage_authority, +#' blob_storage_scheme = azurite_blob_storage_scheme +#' ) +#' @export +az_container <- function(container_path, ...) { + assert_that(is.string(container_path)) args <- list2(...) fs <- exec(AzureFileSystem$create, !!!args) - SubTreeFileSystem$create(bucket, fs) + SubTreeFileSystem$create(container_path, fs) } #' @usage NULL diff --git a/r/man/az_container.Rd b/r/man/az_container.Rd new file mode 100644 index 000000000000..da074b337b1b --- /dev/null +++ b/r/man/az_container.Rd @@ -0,0 +1,34 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/filesystem.R +\name{az_container} +\alias{az_container} +\title{Connect to an Azure Blob Storage container} +\usage{ +az_container(container_path, ...) +} +\arguments{ +\item{container_path}{string Container name or path} + +\item{...}{Additional connection options, passed to \code{AzureFileSystem$create()}} +} +\value{ +A \code{SubTreeFileSystem} containing an \code{AzureFileSystem} and the container's +relative path. Note that this function's success does not guarantee that you +are authorized to access the container's contents. +} +\description{ +\code{az_conainer} is a convenience function to create an \code{AzureFileSystem} object +that provides a file system interface for blob storage containers in an Azure +Storage Account. +} +\examples{ +\dontshow{if (FALSE) withAutoprint(\{ # examplesIf} +container_fs <- az_container( + container_path = "arrow-datasets", + account_name = azurite_account_name, + account_key = azurite_account_key, + blob_storage_authority = azurite_blob_storage_authority, + blob_storage_scheme = azurite_blob_storage_scheme +) +\dontshow{\}) # examplesIf} +} diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index e0bc627551a1..30e1647f649f 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -102,7 +102,7 @@ write_feather(example_data, fs$path(azure_path("openmulti/dataset1.feather"))) write_feather(example_data, fs$path(azure_path("openmulti/dataset2.feather"))) open_multi_fs = arrow:::az_container( - bucket = azure_path("openmulti"), + container_path = azure_path("openmulti"), account_name = azurite_account_name, account_key = azurite_account_key, blob_storage_authority = azurite_blob_storage_authority, From 321de88d1b2f1926d548445c3c48cb0aa37ca2c8 Mon Sep 17 00:00:00 2001 From: Steve Martin <62676717+marberts@users.noreply.github.com> Date: Tue, 17 Mar 2026 16:11:36 -0400 Subject: [PATCH 31/89] docs: Updated documentation for AzureFileSystem and updated vignette (#1) --- r/R/filesystem.R | 47 ++++++++++++++++++++++++++++--------- r/man/FileSystem.Rd | 31 +++++++++++++++++++++++++ r/man/az_container.Rd | 4 ++-- r/src/filesystem.cpp | 2 -- r/vignettes/fs.Rmd | 54 +++++++++++++++++++++++++++++-------------- 5 files changed, 106 insertions(+), 32 deletions(-) diff --git a/r/R/filesystem.R b/r/R/filesystem.R index e06a7c259765..863051334692 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -189,6 +189,31 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F #' - `default_metadata`: default metadata to write in new objects. #' - `project_id`: the project to use for creating buckets. #' +#' `AzureFileSystem$create()` takes following required argument: +#' +#' - `account_name`: Azure Blob Storage account name. +#' +#' `AzureFileSystem$create()` takes following optional arguments: +#' +#' - `account_key`: Account key of the storage account. Cannot be used with +#' `sas_token`. +#' - `blob_storage_authority`: Hostname of the blob service, defaulting to +#' `"blob.core.windows.net"`. +#' - `blob_storage_scheme`: Either `"http"` or `"https"` (the default). +#' - `client_id`: The client/application ID for Azure Active Directory +#' authentication. If used with `client_secret` and `tenant_id` then it is the +#' application ID for a registered Azure AD application. Otherwise, it is the +#' client ID of a user-assigned managed identity. +#' - `client_secret`: Client secret for Azure Active Directory authentication. +#' Must be provided with both `client_id` and `tenant_id`. +#' - `dfs_storage_authority`: Hostname of the data lake (gen 2) service, +#' defaulting to `"dfs.core.windows.net"`. +#' - `dfs_storage_scheme`: Either `"http"` or `"https"` (the default). +#' - `sas_token`: Shared access signature (SAS) token for the storage account. +#' Cannot be used with `account key`. +#' - `tenant_id`: Tenant ID for Azure Active Directory authentication. Must +#' be provided with both `client_id` and `client_secret`. +#' #' @section Methods: #' #' - `path(x)`: Create a `SubTreeFileSystem` from the current `FileSystem` @@ -253,6 +278,10 @@ FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = F #' (the default), 'ERROR', 'WARN', 'INFO', 'DEBUG' (recommended), 'TRACE', and #' 'OFF'. #' +#' On `AzureFileSystem`, passing no arguments for authentication uses the +#' `AzureDefaultCredential` for authentication, so that several authentication +#' types are tried until one succeeds. +#' #' @usage NULL #' @format NULL #' @docType class @@ -655,10 +684,9 @@ AzureFileSystem <- R6Class( inherit = FileSystem ) -AzureFileSystem$create <- function(...) { +AzureFileSystem$create <- function(account_name, ...) { options <- list(...) valid_opts <- c( - "account_name", "account_key", "blob_storage_authority", "blob_storage_scheme", @@ -678,9 +706,6 @@ AzureFileSystem$create <- function(...) { call. = FALSE ) } - if (is.null(options$account_name)) { - stop("Missing `account_name`", call. = FALSE) - } if (!is.null(options$tenant_id) || !is.null(options$client_id) || !is.null(options$client_secret)) { if (is.null(options$client_id)) { stop( @@ -703,18 +728,18 @@ AzureFileSystem$create <- function(...) { ) } - fs___AzureFileSystem__Make(options) + fs___AzureFileSystem__Make(c(account_name = account_name, options)) } #' Connect to an Azure Blob Storage container -#' +#' #' `az_conainer` is a convenience function to create an `AzureFileSystem` object #' that provides a file system interface for blob storage containers in an Azure #' Storage Account. -#' -#' @param container_path string Container name or path -#' @param ... Additional connection options, passed to `AzureFileSystem$create()` -#' +#' +#' @param container_path string Container name or path. +#' @param ... Additional connection options, passed to `AzureFileSystem$create()`. +#' #' @return A `SubTreeFileSystem` containing an `AzureFileSystem` and the container's #' relative path. Note that this function's success does not guarantee that you #' are authorized to access the container's contents. diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index eeccda31b04c..0cca6d3767d8 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -90,6 +90,33 @@ the filesystem encounters errors. Default is 15 seconds. \item \code{default_metadata}: default metadata to write in new objects. \item \code{project_id}: the project to use for creating buckets. } + +\code{AzureFileSystem$create()} takes following required argument: +\itemize{ +\item \code{account_name}: Azure Blob Storage account name. +} + +\code{AzureFileSystem$create()} takes following optional arguments: +\itemize{ +\item \code{account_key}: Account key of the storage account. Cannot be used with +\code{sas_token}. +\item \code{blob_storage_authority}: Hostname of the blob service, defaulting to +\code{"blob.core.windows.net"}. +\item \code{blob_storage_scheme}: Either \code{"http"} or \code{"https"} (the default). +\item \code{client_id}: The client/application ID for Azure Active Directory +authentication. If used with \code{client_secret} and \code{tenant_id} then it is the +application ID for a registered Azure AD application. Otherwise, it is the +client ID of a user-assigned managed identity. +\item \code{client_secret}: Client secret for Azure Active Directory authentication. +Must be provided with both \code{client_id} and \code{tenant_id}. +\item \code{dfs_storage_authority}: Hostname of the data lake (gen 2) service, +defaulting to \code{"dfs.core.windows.net"}. +\item \code{dfs_storage_scheme}: Either \code{"http"} or \code{"https"} (the default). +\item \code{sas_token}: Shared access signature (SAS) token for the storage account. +Cannot be used with \verb{account key}. +\item \code{tenant_id}: Tenant ID for Azure Active Directory authentication. Must +be provided with both \code{client_id} and \code{client_secret}. +} } \section{Methods}{ @@ -162,5 +189,9 @@ environment variable \code{ARROW_S3_LOG_LEVEL} (e.g., to running any code that interacts with S3. Possible values include 'FATAL' (the default), 'ERROR', 'WARN', 'INFO', 'DEBUG' (recommended), 'TRACE', and 'OFF'. + +On \code{AzureFileSystem}, passing no arguments for authentication uses the +\code{AzureDefaultCredential} for authentication, so that several authentication +types are tried until one succeeds. } diff --git a/r/man/az_container.Rd b/r/man/az_container.Rd index da074b337b1b..a749b4a4e188 100644 --- a/r/man/az_container.Rd +++ b/r/man/az_container.Rd @@ -7,9 +7,9 @@ az_container(container_path, ...) } \arguments{ -\item{container_path}{string Container name or path} +\item{container_path}{string Container name or path.} -\item{...}{Additional connection options, passed to \code{AzureFileSystem$create()}} +\item{...}{Additional connection options, passed to \code{AzureFileSystem$create()}.} } \value{ A \code{SubTreeFileSystem} containing an \code{AzureFileSystem} and the container's diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index e78962b10492..eefc4c46b5d8 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -527,8 +527,6 @@ cpp11::list fs___GcsFileSystem__options(const std::shared_ptr #endif -// TODO: Write the Rcpp function to interface with the AzureFileSystem class in -// arrow/filesystem/azurefs.h. #if defined(ARROW_R_WITH_AZURE) #include diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd index 52652ad7e9ed..4c2138f693f8 100644 --- a/r/vignettes/fs.Rmd +++ b/r/vignettes/fs.Rmd @@ -1,29 +1,30 @@ --- -title: "Using cloud storage (S3, GCS)" +title: "Using cloud storage (S3, GCS, Azure)" description: > Learn how to work with data sets stored in an - Amazon S3 bucket or on Google Cloud Storage + Amazon S3 bucket, on Google Cloud Storage, or on Azure output: rmarkdown::html_vignette --- -Working with data stored in cloud storage systems like [Amazon Simple Storage Service](https://docs.aws.amazon.com/s3/) (S3) and [Google Cloud Storage](https://cloud.google.com/storage/docs) (GCS) is a very common task. Because of this, the Arrow C++ library provides a toolkit aimed to make it as simple to work with cloud storage as it is to work with the local filesystem. +Working with data stored in cloud storage systems like [Amazon Simple Storage Service](https://docs.aws.amazon.com/s3/) (S3), [Google Cloud Storage](https://cloud.google.com/storage/docs) (GCS), and [Microsoft Azure](https://azure.microsoft.com) is a very common task. Because of this, the Arrow C++ library provides a toolkit aimed to make it as simple to work with cloud storage as it is to work with the local filesystem. -To make this work, the Arrow C++ library contains a general-purpose interface for file systems, and the arrow package exposes this interface to R users. For instance, if you want to you can create a `LocalFileSystem` object that allows you to interact with the local file system in the usual ways: copying, moving, and deleting files, obtaining information about files and folders, and so on (see `help("FileSystem", package = "arrow")` for details). In general you probably don't need this functionality because you already have tools for working with your local file system, but this interface becomes much more useful in the context of remote file systems. Currently there is a specific implementation for Amazon S3 provided by the `S3FileSystem` class, and another one for Google Cloud Storage provided by `GcsFileSystem`. +To make this work, the Arrow C++ library contains a general-purpose interface for file systems, and the arrow package exposes this interface to R users. For instance, if you want to you can create a `LocalFileSystem` object that allows you to interact with the local file system in the usual ways: copying, moving, and deleting files, obtaining information about files and folders, and so on (see `help("FileSystem", package = "arrow")` for details). In general you probably don't need this functionality because you already have tools for working with your local file system, but this interface becomes much more useful in the context of remote file systems. Currently there is a specific implementation for Amazon S3 provided by the `S3FileSystem` class, one for Google Cloud Storage provided by `GcsFileSystem`, and another for Microsoft Azure provided by the `AzureFileSystem` class. -This article provides an overview of working with both S3 and GCS data using the Arrow toolkit. +This article provides an overview of working with S3, GCS, and Azure data using the Arrow toolkit. -## S3 and GCS support +## S3, GCS, and Azure support -Before you start, make sure that your arrow installation has support for S3 and/or GCS enabled. You can check whether support is enabled via helper functions: +Before you start, make sure that your arrow installation has support for S3, GCS, and/or Azure enabled. You can check whether support is enabled via helper functions: ```r arrow_with_s3() arrow_with_gcs() +arrow_with_azure() ``` If these return `TRUE` then the relevant support is enabled. -CRAN builds of arrow include S3 support but not GCS support. If you need GCS support, you can install arrow with full features using one of the following methods: +CRAN builds of arrow include S3 and Azure support but not GCS support. If you need GCS support, you can install arrow with full features using one of the following methods: ```r # Option 1: Install from R-universe @@ -36,15 +37,15 @@ Sys.setenv("NOT_CRAN" = "true") install.packages("arrow", type = "source") ``` -On Linux, S3 and GCS support is not always enabled by default when installing from source, and there are additional system requirements involved. See the [installation article](./install.html) for details. +On Linux, S3, GCS, and Azure support is not always enabled by default when installing from source, and there are additional system requirements involved. See the [installation article](./install.html) for details. ## Connecting to cloud storage One way of working with filesystems is to create `?FileSystem` objects. `?S3FileSystem` objects can be created with the `s3_bucket()` function, which automatically detects the bucket's AWS region. Similarly, `?GcsFileSystem` objects -can be created with the `gs_bucket()` function. The resulting -`FileSystem` will consider paths relative to the bucket's path (so for example +can be created with the `gs_bucket()` function and `?AzureFileSystem` objects can be created with the `az_container()` function. The resulting +`FileSystem` will consider paths relative to the bucket/container's path (so for example you don't need to prefix the bucket path when listing a directory). With a `FileSystem` object, you can point to specific files in it with the `$path()` method @@ -52,7 +53,7 @@ and pass the result to file readers and writers (`read_parquet()`, `write_feathe Often the reason users work with cloud storage in real world analysis is to access large data sets. An example of this is discussed in the [datasets article](./dataset.html), but new users may prefer to work with a much smaller data set while learning how the arrow cloud storage interface works. To that end, the examples in this article rely on a multi-file Parquet dataset that stores a copy of the `diamonds` data made available through the [`ggplot2`](https://ggplot2.tidyverse.org/) package, documented in `help("diamonds", package = "ggplot2")`. The cloud storage version of this data set consists of 5 Parquet files totaling less than 1MB in size. -The diamonds data set is hosted on both S3 and GCS, in a bucket named `arrow-datasets`. To create an S3FileSystem object that refers to that bucket, use the following command: +The diamonds data set is hosted on both S3 and GCS, in a bucket named `arrow-datasets`. To create an `S3FileSystem` object that refers to that bucket, use the following command: ```r bucket <- s3_bucket("arrow-datasets") @@ -147,7 +148,7 @@ june2019 <- SubTreeFileSystem$create("s3://arrow-datasets/nyc-taxi/year=2019/mon ## Connecting directly with a URI -In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()` and `gs_bucket()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_feather()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3 or GCS. The format of an S3 URI is as follows: +In most use cases, the easiest and most natural way to connect to cloud storage in arrow is to use the FileSystem objects returned by `s3_bucket()`, `gs_bucket()`, and `az_container()`, especially when multiple file operations are required. However, in some cases you may want to download a file directly by specifying the URI. This is permitted by arrow, and functions like `read_parquet()`, `write_feather()`, `open_dataset()` etc will all accept URIs to cloud resources hosted on S3, GCS, or Azure. The format of an S3 URI is as follows: ``` s3://[access_key:secret_key@]bucket/path[?region=] @@ -160,6 +161,12 @@ gs://[access_key:secret_key@]bucket/path gs://anonymous@bucket/path ``` +For Azure, the URI format looks like this: + +``` +abfs://container@account_name.dfs.core.windows.net/path +``` + For example, the Parquet file storing the "good cut" diamonds that we downloaded earlier in the article is available on both S3 and CGS. The relevant URIs are as follows: ```r @@ -258,6 +265,21 @@ df <- read_parquet("gs://anonymous@arrow-datasets/diamonds/cut=Good/part-0.parqu +### Azure Authentication + +By default, `AzureFileSystem$create()` and `az_container()` use the [DefaultAzureCredential]( https://github.com/Azure/azure-sdk-for-cpp/blob/main/sdk/identity/azure-identity/README.md#defaultazurecredential) for authentication. This will try several different types of authentication, using the first one that succeeds. Like with GCS, a simple way to authenticate with Azure is to first use [Azure CLI](https://learn.microsoft.com/en-us/cli/azure/?view=azure-cli-latest) to login and setup default credentials: + +``` +az login +``` + +It is possible to use other forms of authentication with Azure when calling `AzureFileSystem$create()` and `az_container()`. + +- Passing `client_id` on its own will use [`ManagedIdentityCredential`](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) to authenticate. +- Passing `client_id` with `tenant_id` and `client_secret` will use [`ClientSecretCredential`](https://learn.microsoft.com/en-us/entra/identity-platform/app-objects-and-service-principals?tabs=browser) to authenticate. +- Passing `sas_token` will use a shared access signature (SAS) token for the storage account. +- Passing `account_key` will use the account key for the storage account. + ## Using a proxy server If you need to use a proxy server to connect to an S3 bucket, you can provide @@ -329,10 +351,8 @@ variables, you can set environment variable `AWS_EC2_METADATA_DISABLED` to Sys.setenv(AWS_EC2_METADATA_DISABLED = TRUE) ``` - ## Further reading -- To learn more about `FileSystem` classes, including `S3FileSystem` and `GcsFileSystem`, see `help("FileSystem", package = "arrow")`. -- To see a data analysis example that relies on data hosted on cloud storage, see the [dataset article](./dataset.html). - +- To learn more about `FileSystem` classes, including `S3FileSystem`, `GcsFileSystem`, and `AzureFileSystem`, see `help("FileSystem", package = "arrow")`. +- To see a data analysis example that relies on data hosted on cloud storage, see the [dataset article](./dataset.html). From bab125a4387264fe5bf3527c528c97ab1cf463e3 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 17 Mar 2026 22:12:07 -0400 Subject: [PATCH 32/89] Updated installation vignettes to include Azure --- r/vignettes/developers/setup.Rmd | 2 ++ r/vignettes/install.Rmd | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd index d13fc53db1ee..2f9e4d220f89 100644 --- a/r/vignettes/developers/setup.Rmd +++ b/r/vignettes/developers/setup.Rmd @@ -155,6 +155,7 @@ To enable optional features including: S3 support, an alternative memory allocat -DARROW_GCS=ON \ -DARROW_MIMALLOC=ON \ -DARROW_S3=ON \ + -DARROW_AZURE=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ @@ -228,6 +229,7 @@ cmake \ -DARROW_MIMALLOC=ON \ -DARROW_PARQUET=ON \ -DARROW_S3=ON \ + -DARROW_AZURE=ON \ -DARROW_WITH_BROTLI=ON \ -DARROW_WITH_BZ2=ON \ -DARROW_WITH_LZ4=ON \ diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd index a058975ccf19..01955d6fdc72 100644 --- a/r/vignettes/install.Rmd +++ b/r/vignettes/install.Rmd @@ -32,13 +32,13 @@ exception, as it ships with gcc 4.8. ### Libraries -Optional support for reading from cloud storage--AWS S3 and -Google Cloud Storage (GCS)--requires additional system dependencies: +Optional support for reading from cloud storage--AWS S3, +Google Cloud Storage (GCS), and Azure--requires additional system dependencies: * CURL: install `libcurl-devel` (rpm) or `libcurl4-openssl-dev` (deb) * OpenSSL >= 3.0: install `openssl-devel` (rpm) or `libssl-dev` (deb) -The prebuilt binaries come with S3 and GCS support enabled, so you will need to meet these system requirements in order to use them. If you're building everything from source, the install script will check for the presence of these dependencies and turn off S3 and GCS support in the build if the prerequisites are not met--installation will succeed but without S3 or GCS functionality. If afterwards you install the missing system requirements, you'll need to reinstall the package in order to enable S3 and GCS support. +The prebuilt binaries come with S3, GCS, and Azure support enabled, so you will need to meet these system requirements in order to use them. If you're building everything from source, the install script will check for the presence of these dependencies and turn off S3, GCS, and Azure support in the build if the prerequisites are not met--installation will succeed but without S3, GCS, or Azure functionality. If afterwards you install the missing system requirements, you'll need to reinstall the package in order to enable S3, GCS, and Azure support. ## Install release version (easy way) @@ -99,9 +99,9 @@ install.packages("arrow") This installs the source version of the R package, but during the installation process will check for compatible libarrow binaries that we host and use those if available. If no binary is available or can't be found, then this option falls back onto method 2 below (full source build), but setting the environment variable results in a more fully-featured build than default. -The libarrow binaries include support for AWS S3 and GCS, so they require the +The libarrow binaries include support for AWS S3, GCS, and Azure, so they require the libcurl and openssl libraries installed separately, as noted above. -If you don't have these installed, the libarrow binary won't be used, and you will fall back to the full source build (with S3 and GCS support disabled). +If you don't have these installed, the libarrow binary won't be used, and you will fall back to the full source build (with S3, GCS, and Azure support disabled). If the internet access of your computer doesn't allow downloading the libarrow binaries (e.g. if access is limited to CRAN), you can first identify the right source and version by trying to install on the offline computer: @@ -204,19 +204,19 @@ information about dependencies and minimum versions. If downloading dependencies at build time is not an option, as when building on a system that is disconnected or behind a firewall, there are a few options. See "Offline builds" below. -#### Dependencies for S3 and GCS support +#### Dependencies for S3, GCS, and Azure support -Support for working with data in S3 and GCS is not enabled in the default +Support for working with data in S3, GCS, and Azure is not enabled in the default source build, and it has additional system requirements as described above. To enable it, set the environment variable `LIBARROW_MINIMAL=false` or `NOT_CRAN=true` to choose the full-featured build, or more selectively set -`ARROW_S3=ON` and/or `ARROW_GCS=ON`. +`ARROW_S3=ON`, `ARROW_GCS=ON`, and/or `ARROW_AZURE=ON`. -When either feature is enabled, the install script will check for the presence -of the required dependencies, and if the prerequisites are met, it will turn -off S3 and GCS support--installation will succeed but without S3 or GCS +When one of these features is enabled, the install script will check for the presence +of the required dependencies, and if the prerequisites are not met, it will turn +off S3, GCS, and Azure support--installation will succeed but without S3, GCS, or Azure functionality. If afterwards you install the missing system requirements, -you'll need to reinstall the package in order to enable S3 and GCS support. +you'll need to reinstall the package in order to enable S3, GCS, and Azure support. ### Advanced configuration @@ -233,6 +233,7 @@ default values are shown below. | ---| --- | :-: | | `ARROW_S3` | S3 support (if dependencies are met)* | `OFF` | | `ARROW_GCS` | GCS support (if dependencies are met)* | `OFF` | +| `ARROW_Azure` | Azure support (if dependencies are met)* | `OFF` | | `ARROW_JEMALLOC` | The `jemalloc` memory allocator | `ON` | | `ARROW_MIMALLOC` | The `mimalloc` memory allocator | `ON` | | `ARROW_PARQUET` | | `ON` | From e93645f1f9f37ddce5db992d1b87a9dc55abb970 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 17 Mar 2026 22:59:48 -0400 Subject: [PATCH 33/89] Updated install scripts --- r/configure | 2 +- r/configure.win | 5 ++++- r/inst/build_arrow_static.sh | 1 + r/tools/nixlibs.R | 10 ++++++++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/r/configure b/r/configure index 69b2a392a8f6..973234ac6658 100755 --- a/r/configure +++ b/r/configure @@ -366,7 +366,7 @@ add_feature_flags () { if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" fi - if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3; then + if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3 || arrow_built_with ARROW_AZURE; then # If pkg-config is available it will handle this for us automatically SSL_LIBS_WITHOUT_PC="-lcurl -lssl -lcrypto" fi diff --git a/r/configure.win b/r/configure.win index 16c5ec1bee8d..67fcaf2feb44 100755 --- a/r/configure.win +++ b/r/configure.win @@ -187,10 +187,13 @@ add_feature_flags () { if arrow_built_with ARROW_S3; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_S3" fi + if arrow_built_with ARROW_AZURE; then + PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" fi - if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3; then + if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3 || arrow_built_with ARROW_AZURE; then # If pkg-config is available it will handle this for us automatically SSL_LIBS_WITHOUT_PC="-lcurl -lssl -lcrypto" fi diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 349531b75fd9..190e5d5fe150 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -84,6 +84,7 @@ ${CMAKE_WRAPPER} ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -Dlz4_SOURCE=${lz4_SOURCE:-} \ -DARROW_FILESYSTEM=ON \ -DARROW_GCS=${ARROW_GCS:-OFF} \ + -DARROW_AZURE=${ARROW_AZURE:-OFF} \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-$ARROW_DEFAULT_PARAM} \ -DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \ -DARROW_JSON=${ARROW_JSON:-ON} \ diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index a06deaa949f9..86c4938d3ff9 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -802,6 +802,7 @@ turn_off_all_optional_features <- function(env_var_list) { "ARROW_DATASET" = "OFF", # depends on parquet "ARROW_S3" = "OFF", "ARROW_GCS" = "OFF", + "ARROW_AZURE" = "OFF", "ARROW_WITH_GOOGLE_CLOUD_CPP" = "OFF", "ARROW_WITH_NLOHMANN_JSON" = "OFF", "ARROW_SUBSTRAIT" = "OFF", @@ -923,13 +924,15 @@ with_wasm_support <- function(env_var_list) { with_cloud_support <- function(env_var_list) { arrow_s3 <- is_feature_requested("ARROW_S3", env_var_list) arrow_gcs <- is_feature_requested("ARROW_GCS", env_var_list) + arrow_azure <- is_feature_requested("ARROW_AZURE", env_var_list) - if (arrow_s3 || arrow_gcs) { - # User wants S3 or GCS support. + if (arrow_s3 || arrow_gcs || arrow_azure) { + # User wants S3 or GCS or Azure support. # Make sure that we have curl and openssl system libs feats <- c( if (arrow_s3) "S3", if (arrow_gcs) "GCS" + if (arrow_azure) "AZURE" ) start_msg <- paste(feats, collapse = "/") off_flags <- paste("ARROW_", feats, "=OFF", sep = "", collapse = " and ") @@ -944,16 +947,19 @@ with_cloud_support <- function(env_var_list) { print_warning("requires libcurl-devel (rpm) or libcurl4-openssl-dev (deb)") arrow_s3 <- FALSE arrow_gcs <- FALSE + arrow_azure <- FALSE } else if (!cmake_find_package("OpenSSL", "1.0.2", env_var_list)) { print_warning("requires version >= 1.0.2 of openssl-devel (rpm), libssl-dev (deb), or openssl (brew)") arrow_s3 <- FALSE arrow_gcs <- FALSE + arrow_azure <- FALSE } } # Update the build flags env_var_list <- replace(env_var_list, "ARROW_S3", ifelse(arrow_s3, "ON", "OFF")) replace(env_var_list, "ARROW_GCS", ifelse(arrow_gcs, "ON", "OFF")) + replace(env_var_list, "ARROW_AZURE", ifelse(arrow_azure, "ON", "OFF")) } cmake_find_package <- function(pkg, version = NULL, env_var_list) { From 6f70268df7c4e34e3194f6d59b35d4b64f5cc845 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 18 Mar 2026 23:25:07 +0000 Subject: [PATCH 34/89] add tests for valid and invalid combinations of options to AzureFileSystem$create method --- r/tests/testthat/test-azure.R | 123 +++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 30e1647f649f..671b72a4627a 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -101,7 +101,7 @@ test_that("read/write Parquet on azure", { write_feather(example_data, fs$path(azure_path("openmulti/dataset1.feather"))) write_feather(example_data, fs$path(azure_path("openmulti/dataset2.feather"))) -open_multi_fs = arrow:::az_container( +open_multi_fs <- arrow:::az_container( container_path = azure_path("openmulti"), account_name = azurite_account_name, account_key = azurite_account_key, @@ -120,3 +120,124 @@ test_that("open_dataset with AzureFileSystem folder", { rbind(example_data, example_data) |> arrange(int) ) }) + +# (5) Check that multiple valid combinations of options can be used to +# instantiate AzureFileSystem. + +fs1 <- AzureFileSystem$create(account_name = "fake-account-name") +expect_s3_class(fs1, "AzureFileSystem") + +fs2 <- AzureFileSystem$create(account_name = "fake-account-name", account_key = "fakeaccountkey") +expect_s3_class(fs2, "AzureFileSystem") + + +fs3 <- AzureFileSystem$create( + account_name = "fake-account", account_key = "fakeaccount", + blob_storage_authority = "fake-blob-authority", + dfs_storage_authority = "fake-dfs-authority", + blob_storage_scheme = "https", + dfs_storage_scheme = "https" +) +expect_s3_class(fs3, "AzureFileSystem") + +fs4 <- AzureFileSystem$create( + account_name = "fake-account-name", + sas_token = "fakesastoken" +) +expect_s3_class(fs4, "AzureFileSystem") + +fs5 <- AzureFileSystem$create( + account_name = "fake-account-name", + tenant_id = "fake-tenant-id", + client_id = "fake-client-id", + client_secret = "fake-client-secret" +) +expect_s3_class(fs5, "AzureFileSystem") + +fs6 <- AzureFileSystem$create( + account_name = "fake-account-name", + client_id = "fake-client-id" +) +expect_s3_class(fs6, "AzureFileSystem") + +# (6) Check that invalid argument combinations are caught upfront +# with appropriate error message. + +error_msg_1 <- "`client_id` must be given with `tenant_id` and `client_secret`" +error_msg_2 <- "Provide only `client_id` to authenticate with Managed Identity Credential, or provide `client_id`, `tenant_id`, and`client_secret` to authenticate with Client Secret Credential" + +test_that("client_id must be specified with account_name and tenant_id", { + expect_error( + AzureFileSystem$create( + account_name = "fake-account-name", + tenant_id = "fake-tenant-id" + ), + error_msg_1, + fixed = TRUE + ) +}) + +test_that("client_id must be specified with account_name and client_secret", { + expect_error( + AzureFileSystem$create( + account_name = "fake-account-name", + client_secret = "fake-client-secret" + ), + error_msg_1, + fixed = TRUE + ) +}) + +test_that("client_secret must not be provided with client_id", { + expect_error( + AzureFileSystem$create( + account_name = "fake-account-name", + client_id = "fake-client-id", + client_secret = "fake-client-secret" + ), + error_msg_2, + fixed = TRUE + ) +}) + +test_that("client_id must be specified with account_name, tenant_id, and client_secret", { + expect_error( + AzureFileSystem$create( + account_name = "fake-account-name", + tenant_id = "fake-tenant-id", + client_secret = "fake-client-secret" + ), + error_msg_1, + fixed = TRUE + ) +}) + + +test_that("client_id must be provided alone or with tenant_id and client_secret", { + expect_error( + AzureFileSystem$create( + account_name = "fake-account-name", + tenant_id = "fake-tenant-id", + client_id = "fake-client-id" + ), + error_msg_2, + fixed = TRUE + ) +}) + +test_that("cannot specify both account_key and sas_token", { + expect_error( + AzureFileSystem$create(account_name='fake-account-name', account_key='fakeaccount', + sas_token='fakesastoken'), + "Cannot specify both `account_key` and `sas_token`", + fixed = TRUE + ) +}) + +test_that("at a minimum account_name must be passed", { + expect_error( + AzureFileSystem$create(), + "Missing `account_name`", + fixed = TRUE + ) +}) From 1c886cd964638994bd4fca35e507c4c4303aa2e1 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 18 Mar 2026 21:24:05 -0400 Subject: [PATCH 35/89] Ran pre-commit hooks --- r/_pkgdown.yml | 3 ++- r/src/filesystem.cpp | 26 +++++++++++++++----------- r/tests/testthat/test-azure.R | 12 ++++++++---- r/tools/nixlibs.R | 2 +- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/r/_pkgdown.yml b/r/_pkgdown.yml index 57e99648ec55..c3f68b221602 100644 --- a/r/_pkgdown.yml +++ b/r/_pkgdown.yml @@ -261,10 +261,11 @@ reference: - title: File systems desc: > - Functions for working with files on S3 and GCS + Functions for working with files on S3, GCS, and Azure contents: - s3_bucket - gs_bucket + - az_container - copy_files - title: Flight diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index eefc4c46b5d8..57a0a7183d95 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -538,30 +538,35 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti azure_opts.account_name = cpp11::as_cpp(options["account_name"]); if (!Rf_isNull(options["blob_storage_authority"])) { - azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); + azure_opts.blob_storage_authority = + cpp11::as_cpp(options["blob_storage_authority"]); } if (!Rf_isNull(options["dfs_storage_authority"])) { - azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); + azure_opts.dfs_storage_authority = + cpp11::as_cpp(options["dfs_storage_authority"]); } if (!Rf_isNull(options["blob_storage_scheme"])) { - azure_opts.blob_storage_scheme = cpp11::as_cpp(options["blob_storage_scheme"]); + azure_opts.blob_storage_scheme = + cpp11::as_cpp(options["blob_storage_scheme"]); } if (!Rf_isNull(options["dfs_storage_scheme"])) { - azure_opts.dfs_storage_scheme = cpp11::as_cpp(options["dfs_storage_scheme"]); + azure_opts.dfs_storage_scheme = + cpp11::as_cpp(options["dfs_storage_scheme"]); } if (!Rf_isNull(options["client_id"])) { if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureManagedIdentityCredential(cpp11::as_cpp(options["client_id"])); + azure_opts.ConfigureManagedIdentityCredential( + cpp11::as_cpp(options["client_id"])); } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { azure_opts.ConfigureClientSecretCredential( - cpp11::as_cpp(options["tenant_id"]), - cpp11::as_cpp(options["client_id"]), - cpp11::as_cpp(options["client_secret"]) - ); + cpp11::as_cpp(options["tenant_id"]), + cpp11::as_cpp(options["client_id"]), + cpp11::as_cpp(options["client_secret"])); } } else if (!Rf_isNull(options["account_key"])) { - azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); + azure_opts.ConfigureAccountKeyCredential( + cpp11::as_cpp(options["account_key"])); } else if (!Rf_isNull(options["sas_token"])) { azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); } else { @@ -570,7 +575,6 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti auto io_context = MainRThread::GetInstance().CancellableIOContext(); return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); - } #endif diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 671b72a4627a..44721ba1ef66 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -132,7 +132,8 @@ expect_s3_class(fs2, "AzureFileSystem") fs3 <- AzureFileSystem$create( - account_name = "fake-account", account_key = "fakeaccount", + account_name = "fake-account", + account_key = "fakeaccount", blob_storage_authority = "fake-blob-authority", dfs_storage_authority = "fake-dfs-authority", blob_storage_scheme = "https", @@ -164,7 +165,7 @@ expect_s3_class(fs6, "AzureFileSystem") # with appropriate error message. error_msg_1 <- "`client_id` must be given with `tenant_id` and `client_secret`" -error_msg_2 <- "Provide only `client_id` to authenticate with Managed Identity Credential, or provide `client_id`, `tenant_id`, and`client_secret` to authenticate with Client Secret Credential" +error_msg_2 <- "Provide only `client_id` to authenticate with Managed Identity Credential, or provide `client_id`, `tenant_id`, and`client_secret` to authenticate with Client Secret Credential" # nolint test_that("client_id must be specified with account_name and tenant_id", { expect_error( @@ -227,8 +228,11 @@ test_that("client_id must be provided alone or with tenant_id and client_secret" test_that("cannot specify both account_key and sas_token", { expect_error( - AzureFileSystem$create(account_name='fake-account-name', account_key='fakeaccount', - sas_token='fakesastoken'), + AzureFileSystem$create( + account_name = "fake-account-name", + account_key = "fakeaccount", + sas_token = "fakesastoken" + ), "Cannot specify both `account_key` and `sas_token`", fixed = TRUE ) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 86c4938d3ff9..d7e54b2491a8 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -931,7 +931,7 @@ with_cloud_support <- function(env_var_list) { # Make sure that we have curl and openssl system libs feats <- c( if (arrow_s3) "S3", - if (arrow_gcs) "GCS" + if (arrow_gcs) "GCS", if (arrow_azure) "AZURE" ) start_msg <- paste(feats, collapse = "/") From 0e87a5d90a07425f746431a4abf6c05b9aef27a2 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 18 Mar 2026 23:13:55 -0400 Subject: [PATCH 36/89] Removed tmp.md --- r/tmp.md | 96 -------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 r/tmp.md diff --git a/r/tmp.md b/r/tmp.md deleted file mode 100644 index c7c7118779ba..000000000000 --- a/r/tmp.md +++ /dev/null @@ -1,96 +0,0 @@ -# Temporary development notes - -> TODO: Remove this before we open a PR to upstream arrow library. - -## Using codegen.R - -1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` - -2. Rscript `data-raw/codegen.R` - -The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. - -**Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. - -## Using Azurite - -`docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) - -## Build troubleshooting continued - -```bash -export ARROW_HOME=/workspaces/arrow/dist - -cmake \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_COMPUTE=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_EXTRA_ERROR_CONTEXT=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_INSTALL_NAME_RPATH=OFF \ - -DARROW_JEMALLOC=ON \ - -DARROW_JSON=ON \ - -DARROW_PARQUET=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_AZURE=ON \ - .. - -cmake --build . --target install -j8 - - -R -e 'install.packages("remotes"); remotes::install_deps(dependencies = TRUE)' - -R CMD INSTALL --no-multiarch . - -# ---------------------- - - -# Try building from source via R with the relevant env vars set for feature flags. - -# Core Build Settings -export LIBARROW_MINIMAL=false -export FORCE_BUNDLED_BUILD=true -export BOOST_SOURCE=BUNDLED - -# Feature Toggles -export ARROW_COMPUTE=ON -export ARROW_CSV=ON -export ARROW_DATASET=ON -export ARROW_EXTRA_ERROR_CONTEXT=ON -export ARROW_FILESYSTEM=ON -export ARROW_JEMALLOC=ON -export ARROW_JSON=ON -export ARROW_PARQUET=ON -export ARROW_AZURE=ON - -# Visibility into build -export ARROW_R_DEV=TRUE - -# Use multiple available cores -export MAKEFLAGS="-j8" - -# Compression Codecs -export ARROW_WITH_SNAPPY=ON -export ARROW_WITH_ZLIB=ON - -# Library Linkage -export ARROW_BUILD_STATIC=ON -export ARROW_BUILD_SHARED=OFF - -# For R-specific behavior (replaces CMAKE_INSTALL_LIBDIR=lib) -export LIBARROW_BINARY=false - -export EXTRA_CMAKE_FLAGS="-DARROW_INSTALL_NAME_RPATH=OFF -DARROW_AZURE=ON -DCMAKE_SHARED_LINKER_FLAGS=-lxml2" - -export PKG_CONFIG_PATH="/usr/lib/x86_64-linux-gnu/pkgconfig" -export LDFLAGS=$(pkg-config --libs libxml-2.0) -export PKG_LIBS=$(pkg-config --libs libxml-2.0) - -# export LIBARROW_EXTERNAL_LIBDIR=/workspaces/arrow/r/libarrow - -R CMD INSTALL . --preclean - -``` \ No newline at end of file From 143a129ed663d15ca80f64193e1c659371668496 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Mar 2026 14:30:44 +0000 Subject: [PATCH 37/89] wrap credential configuration methods with StopIfNotOk --- r/src/filesystem.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 57a0a7183d95..d29cf5f30508 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -556,21 +556,21 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti if (!Rf_isNull(options["client_id"])) { if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureManagedIdentityCredential( - cpp11::as_cpp(options["client_id"])); + StopIfNotOk(azure_opts.ConfigureManagedIdentityCredential( + cpp11::as_cpp(options["client_id"]))); } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureClientSecretCredential( + StopIfNotOk(azure_opts.ConfigureClientSecretCredential( cpp11::as_cpp(options["tenant_id"]), cpp11::as_cpp(options["client_id"]), - cpp11::as_cpp(options["client_secret"])); + cpp11::as_cpp(options["client_secret"]))); } } else if (!Rf_isNull(options["account_key"])) { - azure_opts.ConfigureAccountKeyCredential( - cpp11::as_cpp(options["account_key"])); + StopIfNotOk(azure_opts.ConfigureAccountKeyCredential( + cpp11::as_cpp(options["account_key"]))); } else if (!Rf_isNull(options["sas_token"])) { - azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); + StopIfNotOk(azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"]))); } else { - azure_opts.ConfigureDefaultCredential(); + StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } auto io_context = MainRThread::GetInstance().CancellableIOContext(); From dbb33a479f1ea2506e0ab86db328d440060ece89 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Mar 2026 14:53:05 +0000 Subject: [PATCH 38/89] move link flags to arrow_built_with ARROW_AZURE block in configure script --- r/configure | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/configure b/r/configure index 973234ac6658..7642b9b9dd3f 100755 --- a/r/configure +++ b/r/configure @@ -271,8 +271,7 @@ set_pkg_vars_with_pc () { PKG_CFLAGS="`${PKG_CONFIG} --cflags ${pkg_config_names}` $PKG_CFLAGS" PKG_CFLAGS="$PKG_CFLAGS $PKG_CFLAGS_FEATURES" PKG_LIBS=`${PKG_CONFIG} --libs-only-l --libs-only-other ${pkg_config_names}` - # TODO: Figure out how to pass these link flags properly, need this temporarily to get R to link properly. - PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" + PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES" PKG_DIRS=`${PKG_CONFIG} --libs-only-L ${pkg_config_names}` } @@ -362,6 +361,7 @@ add_feature_flags () { fi if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" From 9137668e0588b594e427deaec8d4f62c66bc8a14 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Mar 2026 14:53:44 +0000 Subject: [PATCH 39/89] fix error message to check in test for empty call to AzureFileSystem$create() --- r/tests/testthat/test-azure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 44721ba1ef66..378444791981 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -241,7 +241,7 @@ test_that("cannot specify both account_key and sas_token", { test_that("at a minimum account_name must be passed", { expect_error( AzureFileSystem$create(), - "Missing `account_name`", + 'argument "account_name" is missing, with no default', fixed = TRUE ) }) From 450181658ecbcaea366fb6ca3dcb738a1e4f093e Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Mon, 23 Mar 2026 21:01:14 -0400 Subject: [PATCH 40/89] Updated docs to include libxml2 --- r/DESCRIPTION | 3 ++- r/src/filesystem.cpp | 3 ++- r/tools/nixlibs.R | 5 ++++- r/vignettes/developers/setup.Rmd | 6 +++--- r/vignettes/install.Rmd | 3 ++- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/r/DESCRIPTION b/r/DESCRIPTION index ce32fa9abe25..f682de189e72 100644 --- a/r/DESCRIPTION +++ b/r/DESCRIPTION @@ -28,7 +28,8 @@ URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/ BugReports: https://github.com/apache/arrow/issues Encoding: UTF-8 Language: en-US -SystemRequirements: C++20; for AWS S3 support on Linux, libcurl and openssl (optional); +SystemRequirements: C++20; for AWS S3 support on Linux, libcurl and openssl, and + libxml2 for Azure (optional); cmake >= 3.26 (build-time only, and only for full source build) Biarch: true Imports: diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index d29cf5f30508..fc5d6c0466e6 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -568,7 +568,8 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti StopIfNotOk(azure_opts.ConfigureAccountKeyCredential( cpp11::as_cpp(options["account_key"]))); } else if (!Rf_isNull(options["sas_token"])) { - StopIfNotOk(azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"]))); + StopIfNotOk(azure_opts.ConfigureSASCredential( + cpp11::as_cpp(options["sas_token"]))); } else { StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index d7e54b2491a8..47bc9a722ddb 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -953,12 +953,15 @@ with_cloud_support <- function(env_var_list) { arrow_s3 <- FALSE arrow_gcs <- FALSE arrow_azure <- FALSE + } else if (!cmake_find_package("libxml2", NULL, env_var_list)) { + print_warning("requires libxml2-devel (rpm), or libxml2-dev (deb), libxml2 (brew)") + arrow_azure <- FALSE } } # Update the build flags env_var_list <- replace(env_var_list, "ARROW_S3", ifelse(arrow_s3, "ON", "OFF")) - replace(env_var_list, "ARROW_GCS", ifelse(arrow_gcs, "ON", "OFF")) + env_var_list <- replace(env_var_list, "ARROW_GCS", ifelse(arrow_gcs, "ON", "OFF")) replace(env_var_list, "ARROW_AZURE", ifelse(arrow_azure, "ON", "OFF")) } diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd index 2f9e4d220f89..4a08dbd770b9 100644 --- a/r/vignettes/developers/setup.Rmd +++ b/r/vignettes/developers/setup.Rmd @@ -71,18 +71,18 @@ There are five major steps to the process. ### Step 1 - Install dependencies -When building libarrow, by default, system dependencies will be used if suitable versions are found. If system dependencies are not present, libarrow will build them during its own build process. The only dependencies that you need to install _outside_ of the build process are [cmake](https://cmake.org/) (for configuring the build) and [openssl](https://www.openssl.org/) if you are building with S3 support. +When building libarrow, by default, system dependencies will be used if suitable versions are found. If system dependencies are not present, libarrow will build them during its own build process. The only dependencies that you need to install _outside_ of the build process are [cmake](https://cmake.org/) (for configuring the build), [openssl](https://www.openssl.org/) and [curl](https://curl.se/libcurl/) if you are building with S3 and GCS support, and [libxml2](https://gitlab.gnome.org/GNOME/libxml2/-/wikis/home) if you're building with Azure support. For a faster build, you may choose to pre-install more C++ library dependencies (such as [lz4](http://lz4.github.io/lz4/), [zstd](https://facebook.github.io/zstd/), etc.) on the system so that they don't need to be built from source in the libarrow build. #### Ubuntu ```{bash, save=run & ubuntu} -sudo apt install -y cmake libcurl4-openssl-dev libssl-dev +sudo apt install -y cmake libcurl4-openssl-dev libssl-dev libxml2-dev ``` #### macOS ```{bash, save=run & macos} -brew install cmake openssl +brew install cmake openssl libxml2 ``` ### Step 2 - Configure the libarrow build diff --git a/r/vignettes/install.Rmd b/r/vignettes/install.Rmd index 01955d6fdc72..099ed6d3580c 100644 --- a/r/vignettes/install.Rmd +++ b/r/vignettes/install.Rmd @@ -37,6 +37,7 @@ Google Cloud Storage (GCS), and Azure--requires additional system dependencies: * CURL: install `libcurl-devel` (rpm) or `libcurl4-openssl-dev` (deb) * OpenSSL >= 3.0: install `openssl-devel` (rpm) or `libssl-dev` (deb) +* libxml2 (Azure only): install `libxml2-devel` (rpm) or `libxml2-dev` (deb) The prebuilt binaries come with S3, GCS, and Azure support enabled, so you will need to meet these system requirements in order to use them. If you're building everything from source, the install script will check for the presence of these dependencies and turn off S3, GCS, and Azure support in the build if the prerequisites are not met--installation will succeed but without S3, GCS, or Azure functionality. If afterwards you install the missing system requirements, you'll need to reinstall the package in order to enable S3, GCS, and Azure support. @@ -100,7 +101,7 @@ install.packages("arrow") This installs the source version of the R package, but during the installation process will check for compatible libarrow binaries that we host and use those if available. If no binary is available or can't be found, then this option falls back onto method 2 below (full source build), but setting the environment variable results in a more fully-featured build than default. The libarrow binaries include support for AWS S3, GCS, and Azure, so they require the -libcurl and openssl libraries installed separately, as noted above. +libcurl and openssl libraries installed separately (along with libxml2 for Azure), as noted above. If you don't have these installed, the libarrow binary won't be used, and you will fall back to the full source build (with S3, GCS, and Azure support disabled). If the internet access of your computer doesn't allow downloading the libarrow binaries (e.g. if access is limited to CRAN), you can first identify the right source and version by trying to install on the offline computer: From e2654af2f438409456ff12eea1b43bd328298e0f Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 02:59:07 +0000 Subject: [PATCH 41/89] add simple test function to work through codegen.R --- r/R/arrowExports.R | 4 ++++ r/src/arrowExports.cpp | 9 +++++++++ r/src/filesystem.cpp | 15 +++++++++++++++ r/tmp.md | 11 +++++++++++ 4 files changed, 39 insertions(+) create mode 100644 r/tmp.md diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index dac7b4609c56..82099ec73a6b 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,6 +1416,10 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } +azurefs_is_functional_test <- function(input_string) { + .Call(`_arrow_azurefs_is_functional_test`, input_string) +} + fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..9955ceb7544f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,6 +3646,14 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp +bool azurefs_is_functional_test(std::string input_string); +extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ +BEGIN_CPP11 + arrow::r::Input::type input_string(input_string_sexp); + return cpp11::as_sexp(azurefs_is_functional_test(input_string)); +END_CPP11 +} +// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6121,6 +6129,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, + { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index fc5d6c0466e6..f74f4c2e0193 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,6 +381,21 @@ void FinalizeS3() { #endif } +#if defined(ARROW_R_WITH_AZUREFS) + +#include + +// [[arrow::export]] +bool azurefs_is_functional_test(std::string input_string) { + // This just proves we can pass data in and out of the guarded block + if (input_string == "hello") { + return true; + } + return false; +} + +#endif + #if defined(ARROW_R_WITH_GCS) #include diff --git a/r/tmp.md b/r/tmp.md new file mode 100644 index 000000000000..872728514064 --- /dev/null +++ b/r/tmp.md @@ -0,0 +1,11 @@ +# Temporary development notes + +> TODO: Remove this before we open a PR to upstream arrow library. + +## Using codegen.R + +1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` + +2. Rscript `data-raw/codegen.R` + +The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From 6e20676adf4427945f20c996b7b480e465dd12e7 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:10 +0000 Subject: [PATCH 42/89] cleanup azurefs test function code --- r/R/arrowExports.R | 4 ---- r/src/arrowExports.cpp | 9 --------- r/src/filesystem.cpp | 15 --------------- 3 files changed, 28 deletions(-) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 82099ec73a6b..dac7b4609c56 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,10 +1416,6 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } -azurefs_is_functional_test <- function(input_string) { - .Call(`_arrow_azurefs_is_functional_test`, input_string) -} - fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 9955ceb7544f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,14 +3646,6 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp -bool azurefs_is_functional_test(std::string input_string); -extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ -BEGIN_CPP11 - arrow::r::Input::type input_string(input_string_sexp); - return cpp11::as_sexp(azurefs_is_functional_test(input_string)); -END_CPP11 -} -// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6129,7 +6121,6 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, - { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f74f4c2e0193..fc5d6c0466e6 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,21 +381,6 @@ void FinalizeS3() { #endif } -#if defined(ARROW_R_WITH_AZUREFS) - -#include - -// [[arrow::export]] -bool azurefs_is_functional_test(std::string input_string) { - // This just proves we can pass data in and out of the guarded block - if (input_string == "hello") { - return true; - } - return false; -} - -#endif - #if defined(ARROW_R_WITH_GCS) #include From d09817be5ccecb2603e76b0a6cffce41b6fa6465 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 24 Mar 2026 21:18:48 -0400 Subject: [PATCH 43/89] Updated filesystem.cpp --- r/src/filesystem.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index fc5d6c0466e6..08cb564aadcc 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -37,9 +37,9 @@ const char* r6_class_name::get( return "S3FileSystem"; } else if (type_name == "gcs") { return "GcsFileSystem"; + } else if (type_name == "abfs") { + return "AzureFileSystem"; // Uncomment these once R6 classes for these filesystems are added - // } else if (type_name == "abfs") { - // return "AzureBlobFileSystem"; // } else if (type_name == "hdfs") { // return "HadoopFileSystem"; } else if (type_name == "subtree") { From fa6cc9e631135f0acb3b70730cdef63fc0e1a381 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 24 Mar 2026 22:43:00 -0400 Subject: [PATCH 44/89] Updated build scripts --- r/configure | 2 +- r/configure.win | 10 ++++++++-- r/inst/build_arrow_static.sh | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/r/configure b/r/configure index 7642b9b9dd3f..8724f3eaf991 100755 --- a/r/configure +++ b/r/configure @@ -361,7 +361,7 @@ add_feature_flags () { fi if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" - PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" + PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" diff --git a/r/configure.win b/r/configure.win index 67fcaf2feb44..6ffc29ca4510 100755 --- a/r/configure.win +++ b/r/configure.win @@ -67,6 +67,7 @@ function configure_binaries() { # pkg-config --libs libcurl GCS_LIBS="-lcurl -lnormaliz -lssh2 -lgdi32 -lssl -lcrypto -lcrypt32 -lwldap32 \ -lz -lws2_32 -lnghttp2 -ldbghelp" + AZURE_LIBS="-lcurl -lssl -lxml2" # Set the right flags to point to and enable arrow/parquet if [ -d "windows/r-libarrow-windows-x86_64-$VERSION" ]; then @@ -94,8 +95,8 @@ function configure_binaries() { # S3, GCS, and re2 support only for Rtools40 (i.e. R >= 4.0) "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 if [ $? -eq 0 ]; then - PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3 -DARROW_R_WITH_GCS" - PKG_LIBS="${PKG_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" + PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3 -DARROW_R_WITH_GCS -DARROW_R_WITH_AZURE" + PKG_LIBS="${PKG_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS} ${AZURE_LIBS}" else # It seems that order matters PKG_LIBS="${PKG_LIBS} -lws2_32" @@ -189,6 +190,7 @@ add_feature_flags () { fi if arrow_built_with ARROW_AZURE; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" @@ -295,6 +297,10 @@ function configure_dev() { PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_GCS" fi + if [ $(cmake_option ARROW_AZURE) -eq 1 ]; then + PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_AZUREf" + fi + if [ $(cmake_option ARROW_JSON) -eq 1 ]; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_JSON" fi diff --git a/r/inst/build_arrow_static.sh b/r/inst/build_arrow_static.sh index 190e5d5fe150..e7f453bb64e4 100755 --- a/r/inst/build_arrow_static.sh +++ b/r/inst/build_arrow_static.sh @@ -84,7 +84,7 @@ ${CMAKE_WRAPPER} ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \ -Dlz4_SOURCE=${lz4_SOURCE:-} \ -DARROW_FILESYSTEM=ON \ -DARROW_GCS=${ARROW_GCS:-OFF} \ - -DARROW_AZURE=${ARROW_AZURE:-OFF} \ + -DARROW_AZURE=${ARROW_AZURE:-$ARROW_DEFAULT_PARAM} \ -DARROW_JEMALLOC=${ARROW_JEMALLOC:-$ARROW_DEFAULT_PARAM} \ -DARROW_MIMALLOC=${ARROW_MIMALLOC:-ON} \ -DARROW_JSON=${ARROW_JSON:-ON} \ From cf1050aaf0494ff35495c1580e521266a395e7e7 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 25 Mar 2026 10:09:06 -0400 Subject: [PATCH 45/89] Update configure.win: fix typo --- r/configure.win | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/configure.win b/r/configure.win index 6ffc29ca4510..f562208b609c 100755 --- a/r/configure.win +++ b/r/configure.win @@ -298,7 +298,7 @@ function configure_dev() { fi if [ $(cmake_option ARROW_AZURE) -eq 1 ]; then - PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_AZUREf" + PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_AZURE" fi if [ $(cmake_option ARROW_JSON) -eq 1 ]; then From e71257c53bf48694c62b1aa6f3d4c9d3b46f34e9 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Thu, 26 Mar 2026 20:27:23 -0400 Subject: [PATCH 46/89] Updated CI setup scripts for windows --- ci/scripts/PKGBUILD | 2 ++ ci/scripts/r_windows_build.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 6a66d8686724..0a48aaa826e8 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,6 +26,7 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build + "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" @@ -122,6 +123,7 @@ build() { -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ -DARROW_S3=ON \ + -DARROW_AZURE=ON \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index ef9c58f6afca..0834492547e7 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -66,7 +66,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -74,7 +74,7 @@ if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/lib/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -82,7 +82,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact From 51e01d479c1d0043f05c22c5b19085a41c9ef696 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Thu, 26 Mar 2026 21:45:02 -0400 Subject: [PATCH 47/89] Trying PKGBUILD and windows build script again --- ci/scripts/PKGBUILD | 2 +- ci/scripts/r_windows_build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 0a48aaa826e8..545ecabb7dff 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,7 +26,7 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build - "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure + "${MINGW_PACKAGE_PREFIX}-xml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index 0834492547e7..b183f67ea4c5 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -66,7 +66,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -74,7 +74,7 @@ if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/lib/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -82,7 +82,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,libxml2}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact From e0a0a17b43dc49cda2b3ff092c46171adccf3afd Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Thu, 26 Mar 2026 22:23:41 -0400 Subject: [PATCH 48/89] Updated PKGBUILD --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 545ecabb7dff..0a48aaa826e8 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,7 +26,7 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build - "${MINGW_PACKAGE_PREFIX}-xml2" # for azure + "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" From 7be0a70dd0e9caa16c304ca2a980f8d68a315fdc Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 29 Mar 2026 00:05:22 +0000 Subject: [PATCH 49/89] debug ci: add azure dependencies that CI tries to build from source. The hypothesis is that the errors in the build logs are caused by incompatibilities between WIL (Windows Implementation Library) and MinGW GCC, so install them directly instead of trying to build from source. --- ci/scripts/PKGBUILD | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 0a48aaa826e8..00043e188860 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -33,7 +33,12 @@ depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" "${MINGW_PACKAGE_PREFIX}-zstd" - "${MINGW_PACKAGE_PREFIX}-brotli") + "${MINGW_PACKAGE_PREFIX}-brotli" + "${MINGW_PACKAGE_PREFIX}-azure-core-cpp" # for azure + "${MINGW_PACKAGE_PREFIX}-azure-identity-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-blobs-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-common-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-files-datalake-cpp") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") From 0c3ff5888a844e61281b96efc657d502ee131a70 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 1 Apr 2026 00:02:28 -0400 Subject: [PATCH 50/89] Removed tmp.md --- r/tmp.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 r/tmp.md diff --git a/r/tmp.md b/r/tmp.md deleted file mode 100644 index 872728514064..000000000000 --- a/r/tmp.md +++ /dev/null @@ -1,11 +0,0 @@ -# Temporary development notes - -> TODO: Remove this before we open a PR to upstream arrow library. - -## Using codegen.R - -1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` - -2. Rscript `data-raw/codegen.R` - -The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From 7d1aad94a736f159588ecd2586c39b48ce63454a Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sat, 4 Apr 2026 00:23:28 -0400 Subject: [PATCH 51/89] add install script to build azure-sdk-for-cpp dependencies from source. Notes: - As far as I can tell, using vcpkg (even with mingw triplets) does not install azure-sdk-cpp dependencies correctly. E.g., `vcpkg install azure-storage-blobs-cpp:x64-mingw-dynamic --triplet x64-mingw-dynamic --host-triplet x64-mingw-dynamic` fails. - All azure dependencies EXCEPT azure-identity build correctly from source. As far as I can tell, azure-identity can't practically be built with MinGW at this time. - Remove the azure dependencies from PKGBUILD as these cannot be installed with pacman. --- ci/scripts/PKGBUILD | 7 +-- .../install_azure_sdk_cpp_from_source.sh | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 ci/scripts/install_azure_sdk_cpp_from_source.sh diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 00043e188860..0a48aaa826e8 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -33,12 +33,7 @@ depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" "${MINGW_PACKAGE_PREFIX}-zstd" - "${MINGW_PACKAGE_PREFIX}-brotli" - "${MINGW_PACKAGE_PREFIX}-azure-core-cpp" # for azure - "${MINGW_PACKAGE_PREFIX}-azure-identity-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-blobs-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-common-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-files-datalake-cpp") + "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") diff --git a/ci/scripts/install_azure_sdk_cpp_from_source.sh b/ci/scripts/install_azure_sdk_cpp_from_source.sh new file mode 100644 index 000000000000..9333939f7017 --- /dev/null +++ b/ci/scripts/install_azure_sdk_cpp_from_source.sh @@ -0,0 +1,48 @@ +pushd /tmp && git clone https://github.com/Azure/azure-sdk-for-cpp.git && cd azure-sdk-for-cpp + +cd .. && rm -rf build && mkdir build && cd build + +# Rust dependency setup +pacman -S mingw-w64-ucrt-x86_64-rust + +RUSTC_PATH=$(which rustc | sed 's/\\/\//g') + +# Install extra dependencies needed to build azure-sdk-for-cpp from source +vcpkg install azure-macro-utils-c:x64-mingw-dynamic \ + nlohmann-json:x64-mingw-dynamic \ + opentelemetry-cpp:x64-mingw-dynamic \ + wil:x64-mingw-dynamic \ + --triplet x64-mingw-dynamic \ + --host-triplet x64-mingw-dynamic + +# TODO: tidy this up to run in CI context, paths are probably wrong as this was for local debugging. +cmake .. -G "Ninja" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_MAKE_PROGRAM=$(cygpath -m $(which ninja)) \ + -DRust_COMPILER=$(cygpath -m $(which rustc)) \ + -DCMAKE_PREFIX_PATH=$(cygpath -m "../../vcpkg/installed/x64-mingw-dynamic") \ + -Dopentelemetry-cpp_DIR=$(cygpath -m "../../vcpkg/installed/x64-mingw-dynamic/share/opentelemetry-cpp/opentelemetry-cpp-config.cmake") \ + -DVCPKG_TARGET_TRIPLET=x64-mingw-dynamic \ + -DVCPKG_HOST_TRIPLET=x64-mingw-dynamic \ + -DVCPKG_MANIFEST_INSTALL=ON \ + -DVCPKG_MANIFEST_FEATURES="core" \ + -DAZURE_SDK_DISABLE_AUTO_VCPKG=ON \ + -DBUILD_TRANSPORT_CURL=ON \ + -DBUILD_TRANSPORT_WINHTTP=OFF \ + -DSTORAGE_XML_BACKEND=Libxml2 \ + -DBUILD_TESTING=OFF \ + -DRUN_UNITTESTS=OFF \ + -DVCPKG_MANIFEST_MODE=OFF \ + -DWARNINGS_AS_ERRORS=OFF \ + -DCMAKE_CXX_FLAGS="-include cstring -include cstdint -include winsock2.h -DWS_XML_STRING_NULL={0,NULL} -DSIO_IDEAL_SEND_BACKLOG_QUERY=0x48000005 -DNOMINMAX=1 -D_WIN32_WINNT=0x0A00" + +ninja azure-core +# Azure Identity package seems to be incompatible with MinGW. +# Everything else works but azure-identity has many errors. +# ninja azure-identity +ninja azure-storage-blobs +ninja azure-storage-common +ninja azure-storage-files-datalake + +# Go back to the directory that called this script +popd From 8d1c8415e37e899041f04334843cd72692800052 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 02:59:07 +0000 Subject: [PATCH 52/89] add simple test function to work through codegen.R --- r/R/arrowExports.R | 4 ++++ r/src/arrowExports.cpp | 9 +++++++++ r/src/filesystem.cpp | 15 +++++++++++++++ r/tmp.md | 11 +++++++++++ 4 files changed, 39 insertions(+) create mode 100644 r/tmp.md diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index dac7b4609c56..82099ec73a6b 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,6 +1416,10 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } +azurefs_is_functional_test <- function(input_string) { + .Call(`_arrow_azurefs_is_functional_test`, input_string) +} + fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..9955ceb7544f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,6 +3646,14 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp +bool azurefs_is_functional_test(std::string input_string); +extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ +BEGIN_CPP11 + arrow::r::Input::type input_string(input_string_sexp); + return cpp11::as_sexp(azurefs_is_functional_test(input_string)); +END_CPP11 +} +// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6121,6 +6129,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, + { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 08cb564aadcc..8dc2b8dc645c 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,6 +381,21 @@ void FinalizeS3() { #endif } +#if defined(ARROW_R_WITH_AZUREFS) + +#include + +// [[arrow::export]] +bool azurefs_is_functional_test(std::string input_string) { + // This just proves we can pass data in and out of the guarded block + if (input_string == "hello") { + return true; + } + return false; +} + +#endif + #if defined(ARROW_R_WITH_GCS) #include diff --git a/r/tmp.md b/r/tmp.md new file mode 100644 index 000000000000..872728514064 --- /dev/null +++ b/r/tmp.md @@ -0,0 +1,11 @@ +# Temporary development notes + +> TODO: Remove this before we open a PR to upstream arrow library. + +## Using codegen.R + +1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` + +2. Rscript `data-raw/codegen.R` + +The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From d3ce08ddf356b7cd39d9e0203a160bc2de808c9d Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 03:56:16 +0000 Subject: [PATCH 53/89] temporarily "force" build DARROW_R_WITH_AZUREFS build flag. We'll want to use the proper build flag before submitting the PR but for now this successfully links the C++ binding to the azurefs_is_functional_test function. --- r/tmp.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/r/tmp.md b/r/tmp.md index 872728514064..4ee07eadea58 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -9,3 +9,5 @@ 2. Rscript `data-raw/codegen.R` The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. + +**Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. From 68c01e0d38380cbcf8204d4b26d6f1437f7b97e2 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 4 Mar 2026 21:26:48 -0500 Subject: [PATCH 54/89] Added c++ stub --- r/src/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 8dc2b8dc645c..ff841f491327 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -253,7 +253,7 @@ std::shared_ptr fs___SubTreeFileSystem__base_fs( // [[arrow::export]] std::string fs___SubTreeFileSystem__base_path( const std::shared_ptr& file_system) { - return file_system->base_path(); + // return file_system->base_path(); } // Forward declaration - defined in the ARROW_R_WITH_S3 block below. From 7d29231cec7b1253cf057dd0dec340f3d3071419 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:10 +0000 Subject: [PATCH 55/89] cleanup azurefs test function code --- r/R/arrowExports.R | 4 ---- r/src/arrowExports.cpp | 9 --------- r/src/filesystem.cpp | 15 --------------- 3 files changed, 28 deletions(-) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 82099ec73a6b..dac7b4609c56 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,10 +1416,6 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } -azurefs_is_functional_test <- function(input_string) { - .Call(`_arrow_azurefs_is_functional_test`, input_string) -} - fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 9955ceb7544f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,14 +3646,6 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp -bool azurefs_is_functional_test(std::string input_string); -extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ -BEGIN_CPP11 - arrow::r::Input::type input_string(input_string_sexp); - return cpp11::as_sexp(azurefs_is_functional_test(input_string)); -END_CPP11 -} -// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6129,7 +6121,6 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, - { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index ff841f491327..2934c4094845 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,21 +381,6 @@ void FinalizeS3() { #endif } -#if defined(ARROW_R_WITH_AZUREFS) - -#include - -// [[arrow::export]] -bool azurefs_is_functional_test(std::string input_string) { - // This just proves we can pass data in and out of the guarded block - if (input_string == "hello") { - return true; - } - return false; -} - -#endif - #if defined(ARROW_R_WITH_GCS) #include From 3892e4cf210e8b66a860052dcf53de95021409b2 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:31 +0000 Subject: [PATCH 56/89] document instructions to start local azurite container --- r/tmp.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/r/tmp.md b/r/tmp.md index 4ee07eadea58..8b145cfb03bd 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -11,3 +11,8 @@ The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. **Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. + +## Using Azurite + +`docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) + From f7c049f1b3d9b44c6d458bbe31acde6ec4a3c68a Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 23:53:09 +0000 Subject: [PATCH 57/89] debug first argument check Note: I think what's happening is that the build of Arrow I install doesn't have the ARROW_AZURE flag enabled. When I "force" the environment variable I get an error like "fs___AzureFileSystem__Make not found". --- r/src/arrowExports.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..c0cd8aa3b71f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3677,7 +3677,7 @@ extern "C" SEXP _arrow_fs___GcsFileSystem__options(SEXP fs_sexp){ #endif // filesystem.cpp -#if defined(ARROW_R_WITH_AZURE) +#if defined(ARROW_R_WITH_AZUREFS) std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options); extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ BEGIN_CPP11 @@ -5742,7 +5742,7 @@ return Rf_ScalarLogical( } extern "C" SEXP _azure_available() { return Rf_ScalarLogical( -#if defined(ARROW_R_WITH_AZURE) +#if defined(ARROW_R_WITH_AZUREFS) TRUE #else FALSE From bc9e0d92e9156aa13dec739cc618e287fd97309a Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Tue, 10 Mar 2026 23:06:45 -0400 Subject: [PATCH 58/89] Added endpoint + key, token, and default authentication --- r/src/filesystem.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 2934c4094845..f0fd648fa0e1 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -574,6 +574,28 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } + if (!Rf_isNull(options["blob_storage_authority"])) { + azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); + } + if (!Rf_isNull(options["dfs_storage_authority"])) { + azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); + } + if (!Rf_isNull(options["blob_storage_schema"])) { + azure_opts.blob_storage_schema = cpp11::as_cpp(options["blob_storage_schema"]); + } + if (!Rf_isNull(options["dfs_storage_schema"])) { + azure_opts.dfs_storage_schema = cpp11::as_cpp(options["dfs_storage_schema"]); + } + + // TODO: Deal with different combinations of tenant id/client id/client secret. + if (!Rf_isNull(options["account_key"])) { + azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); + } else if (!Rf_isNull(options["sas_token"])) { + azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); + } else { + azure_opts.ConfigureDefaultCredential(); + } + auto io_context = MainRThread::GetInstance().CancellableIOContext(); return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); } From 9931e26358029a4efbaf75485c30853b04d19fc4 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 11 Mar 2026 00:11:14 -0400 Subject: [PATCH 59/89] Finished logical for AzureFileSystem to match pyarrow --- r/src/filesystem.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f0fd648fa0e1..d83ab0fec721 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -580,15 +580,24 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti if (!Rf_isNull(options["dfs_storage_authority"])) { azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); } - if (!Rf_isNull(options["blob_storage_schema"])) { - azure_opts.blob_storage_schema = cpp11::as_cpp(options["blob_storage_schema"]); + if (!Rf_isNull(options["blob_storage_scheme"])) { + azure_opts.blob_storage_scheme = cpp11::as_cpp(options["blob_storage_scheme"]); } - if (!Rf_isNull(options["dfs_storage_schema"])) { - azure_opts.dfs_storage_schema = cpp11::as_cpp(options["dfs_storage_schema"]); + if (!Rf_isNull(options["dfs_storage_scheme"])) { + azure_opts.dfs_storage_scheme = cpp11::as_cpp(options["dfs_storage_scheme"]); } - // TODO: Deal with different combinations of tenant id/client id/client secret. - if (!Rf_isNull(options["account_key"])) { + if (!Rf_isNull(options["client_id"])) { + if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { + azure_opts.ConfigureManagedIdentityCredential(cpp11::as_cpp(options["client_id"])); + } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { + azure_opts.ConfigureClientSecretCredential( + cpp11::as_cpp(options["tenant_id"]), + cpp11::as_cpp(options["client_id"]), + cpp11::as_cpp(options["client_secret"]) + ); + } + } else if (!Rf_isNull(options["account_key"])) { azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); } else if (!Rf_isNull(options["sas_token"])) { azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); From a39c227c61d789e2108156ef0ef2d164c8977c72 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:31:08 +0000 Subject: [PATCH 60/89] standardize on ARROW_R_WITH_AZURE --- r/src/arrowExports.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index c0cd8aa3b71f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3677,7 +3677,7 @@ extern "C" SEXP _arrow_fs___GcsFileSystem__options(SEXP fs_sexp){ #endif // filesystem.cpp -#if defined(ARROW_R_WITH_AZUREFS) +#if defined(ARROW_R_WITH_AZURE) std::shared_ptr fs___AzureFileSystem__Make(cpp11::list options); extern "C" SEXP _arrow_fs___AzureFileSystem__Make(SEXP options_sexp){ BEGIN_CPP11 @@ -5742,7 +5742,7 @@ return Rf_ScalarLogical( } extern "C" SEXP _azure_available() { return Rf_ScalarLogical( -#if defined(ARROW_R_WITH_AZUREFS) +#if defined(ARROW_R_WITH_AZURE) TRUE #else FALSE From 8f3df6839ef74de00369eb9448bb7c6da39ebd8c Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 01:39:10 +0000 Subject: [PATCH 61/89] temporary documentation of what I've tried so far --- r/tmp.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/r/tmp.md b/r/tmp.md index 8b145cfb03bd..6712928d7bd2 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -16,3 +16,73 @@ The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` `docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) +## Build troubleshooting continued + +```bash +export ARROW_HOME=/workspaces/arrow/dist + +cmake \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DARROW_COMPUTE=ON \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ + -DARROW_EXTRA_ERROR_CONTEXT=ON \ + -DARROW_FILESYSTEM=ON \ + -DARROW_INSTALL_NAME_RPATH=OFF \ + -DARROW_JEMALLOC=ON \ + -DARROW_JSON=ON \ + -DARROW_PARQUET=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_AZURE=ON \ + .. + + +# Try building from source via R with the relevant env vars set for feature flags. + +# Core Build Settings +export LIBARROW_MINIMAL=false +export FORCE_BUNDLED_BUILD=true +export ARROW_HOME=$ARROW_HOME +export BOOST_SOURCE=BUNDLED + +# Feature Toggles +export ARROW_COMPUTE=ON +export ARROW_CSV=ON +export ARROW_DATASET=ON +export ARROW_EXTRA_ERROR_CONTEXT=ON +export ARROW_FILESYSTEM=ON +export ARROW_JEMALLOC=ON +export ARROW_JSON=ON +export ARROW_PARQUET=ON +export ARROW_AZURE=ON + +# Visibility into build +export ARROW_R_DEV=TRUE + +# Use multiple available cores +export MAKEFLAGS="-j8" + +# Compression Codecs +export ARROW_WITH_SNAPPY=ON +export ARROW_WITH_ZLIB=ON + +# Library Linkage +export ARROW_BUILD_STATIC=ON +export ARROW_BUILD_SHARED=OFF + +# For R-specific behavior (replaces CMAKE_INSTALL_LIBDIR=lib) +export LIBARROW_BINARY=false + +export EXTRA_CMAKE_FLAGS="-DARROW_INSTALL_NAME_RPATH=OFF -DARROW_AZURE=ON -DCMAKE_SHARED_LINKER_FLAGS=-lxml2" + +export PKG_CONFIG_PATH="/usr/lib/x86_64-linux-gnu/pkgconfig" +export LDFLAGS=$(pkg-config --libs libxml-2.0) +export PKG_LIBS=$(pkg-config --libs libxml-2.0) + +# export LIBARROW_EXTERNAL_LIBDIR=/workspaces/arrow/r/libarrow + +R CMD INSTALL . --preclean + +``` \ No newline at end of file From 110649014a4f1bede487239290842a7f71144762 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Mar 2026 03:04:09 +0000 Subject: [PATCH 62/89] Add TODO note in configure script to remove hard-coded link flags --- r/configure | 3 ++- r/tmp.md | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/r/configure b/r/configure index 8724f3eaf991..98099823a9e3 100755 --- a/r/configure +++ b/r/configure @@ -271,7 +271,8 @@ set_pkg_vars_with_pc () { PKG_CFLAGS="`${PKG_CONFIG} --cflags ${pkg_config_names}` $PKG_CFLAGS" PKG_CFLAGS="$PKG_CFLAGS $PKG_CFLAGS_FEATURES" PKG_LIBS=`${PKG_CONFIG} --libs-only-l --libs-only-other ${pkg_config_names}` - PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES" + # TODO: Figure out how to pass these link flags properly, need this temporarily to get R to link properly. + PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" PKG_DIRS=`${PKG_CONFIG} --libs-only-L ${pkg_config_names}` } diff --git a/r/tmp.md b/r/tmp.md index 6712928d7bd2..c7c7118779ba 100644 --- a/r/tmp.md +++ b/r/tmp.md @@ -38,13 +38,21 @@ cmake \ -DARROW_AZURE=ON \ .. +cmake --build . --target install -j8 + + +R -e 'install.packages("remotes"); remotes::install_deps(dependencies = TRUE)' + +R CMD INSTALL --no-multiarch . + +# ---------------------- + # Try building from source via R with the relevant env vars set for feature flags. # Core Build Settings export LIBARROW_MINIMAL=false export FORCE_BUNDLED_BUILD=true -export ARROW_HOME=$ARROW_HOME export BOOST_SOURCE=BUNDLED # Feature Toggles From e784d6196e984c5cf245736ca3a6c885b6c87344 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 02:25:21 +0000 Subject: [PATCH 63/89] uncomment line 256 of filesystem.cpp Note: this was causing a seg fault when anything that inherits from FileSystem ries to access the base_path property. Uncommenting the return line and rebuilding the R package resolved this issue. --- r/src/filesystem.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index d83ab0fec721..5352c2f1a621 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -253,7 +253,7 @@ std::shared_ptr fs___SubTreeFileSystem__base_fs( // [[arrow::export]] std::string fs___SubTreeFileSystem__base_path( const std::shared_ptr& file_system) { - // return file_system->base_path(); + return file_system->base_path(); } // Forward declaration - defined in the ARROW_R_WITH_S3 block below. From 17fc26e8b364e896fb1db2c6ca4274dce674e5b2 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 02:27:20 +0000 Subject: [PATCH 64/89] checkpoint: resolved segfault error At this point, I was able to call write_feather(example_data, fs$path("test/test.feather")) successfully against Azurite. Committing progress before I make any further changes. --- r/tests/testthat/test-azure.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 378444791981..3702b455a071 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +library(arrow) skip_if_not_available("azure") # test_filesystem requires dplyr @@ -28,7 +28,7 @@ skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") azurite_account_name <- "devstoreaccount1" # Note that this is a well-known default credential for local development on Azurite. azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -azurite_blob_host <- "127.0.0.1" +azurite_blob_host <- "host.docker.internal" azurite_blob_port <- "10000" azurite_blob_storage_authority <- sprintf("%s:%s", azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" From cb86452ed28849bc1986c2c68f70d5bfda6b585b Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 04:20:26 +0000 Subject: [PATCH 65/89] Add most test cases from test_filesystem and recreate a couple that were skipped because of the URI issue. --- r/tests/testthat/test-azure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index 3702b455a071..bccc6a9d56bc 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -library(arrow) + skip_if_not_available("azure") # test_filesystem requires dplyr From 69065d62f84e28c565c2a9eaa22741d78cb633d8 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 15 Mar 2026 21:34:12 +0000 Subject: [PATCH 66/89] check that azurite is installed as precondition for test-azure.R script. Switch over to locally hosted azurite instead of local docker container azurite. --- r/tests/testthat/test-azure.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/r/tests/testthat/test-azure.R b/r/tests/testthat/test-azure.R index bccc6a9d56bc..378444791981 100644 --- a/r/tests/testthat/test-azure.R +++ b/r/tests/testthat/test-azure.R @@ -28,7 +28,7 @@ skip_if_not(nzchar(Sys.which("azurite")), message = "azurite is not installed.") azurite_account_name <- "devstoreaccount1" # Note that this is a well-known default credential for local development on Azurite. azurite_account_key <- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" -azurite_blob_host <- "host.docker.internal" +azurite_blob_host <- "127.0.0.1" azurite_blob_port <- "10000" azurite_blob_storage_authority <- sprintf("%s:%s", azurite_blob_host, azurite_blob_port) azurite_blob_storage_scheme <- "http" From a6301e1fda52df7c3a4aa0f61ff5b2822b799e6c Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 18 Mar 2026 21:24:05 -0400 Subject: [PATCH 67/89] Ran pre-commit hooks --- r/src/filesystem.cpp | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 5352c2f1a621..4987516eb6ef 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -578,27 +578,31 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); } if (!Rf_isNull(options["dfs_storage_authority"])) { - azure_opts.dfs_storage_authority = cpp11::as_cpp(options["dfs_storage_authority"]); + azure_opts.dfs_storage_authority = + cpp11::as_cpp(options["dfs_storage_authority"]); } if (!Rf_isNull(options["blob_storage_scheme"])) { - azure_opts.blob_storage_scheme = cpp11::as_cpp(options["blob_storage_scheme"]); + azure_opts.blob_storage_scheme = + cpp11::as_cpp(options["blob_storage_scheme"]); } if (!Rf_isNull(options["dfs_storage_scheme"])) { - azure_opts.dfs_storage_scheme = cpp11::as_cpp(options["dfs_storage_scheme"]); + azure_opts.dfs_storage_scheme = + cpp11::as_cpp(options["dfs_storage_scheme"]); } if (!Rf_isNull(options["client_id"])) { if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureManagedIdentityCredential(cpp11::as_cpp(options["client_id"])); + azure_opts.ConfigureManagedIdentityCredential( + cpp11::as_cpp(options["client_id"])); } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { azure_opts.ConfigureClientSecretCredential( - cpp11::as_cpp(options["tenant_id"]), - cpp11::as_cpp(options["client_id"]), - cpp11::as_cpp(options["client_secret"]) - ); + cpp11::as_cpp(options["tenant_id"]), + cpp11::as_cpp(options["client_id"]), + cpp11::as_cpp(options["client_secret"])); } } else if (!Rf_isNull(options["account_key"])) { - azure_opts.ConfigureAccountKeyCredential(cpp11::as_cpp(options["account_key"])); + azure_opts.ConfigureAccountKeyCredential( + cpp11::as_cpp(options["account_key"])); } else if (!Rf_isNull(options["sas_token"])) { azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); } else { From 1d50661eb7110c892988c248f01b6ec9c6459b34 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 18 Mar 2026 23:13:55 -0400 Subject: [PATCH 68/89] Removed tmp.md --- r/tmp.md | 96 -------------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 r/tmp.md diff --git a/r/tmp.md b/r/tmp.md deleted file mode 100644 index c7c7118779ba..000000000000 --- a/r/tmp.md +++ /dev/null @@ -1,96 +0,0 @@ -# Temporary development notes - -> TODO: Remove this before we open a PR to upstream arrow library. - -## Using codegen.R - -1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` - -2. Rscript `data-raw/codegen.R` - -The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. - -**Note**: at the moment we need to run `export ARROW_R_WITH_AZUREFS=true` before `R CMD INSTALL .` to export the environment variable that "forces" the Azure build flag. - -## Using Azurite - -`docker run -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite` (see README.md in https://github.com/Azure/Azurite) - -## Build troubleshooting continued - -```bash -export ARROW_HOME=/workspaces/arrow/dist - -cmake \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DARROW_COMPUTE=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DARROW_EXTRA_ERROR_CONTEXT=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_INSTALL_NAME_RPATH=OFF \ - -DARROW_JEMALLOC=ON \ - -DARROW_JSON=ON \ - -DARROW_PARQUET=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_AZURE=ON \ - .. - -cmake --build . --target install -j8 - - -R -e 'install.packages("remotes"); remotes::install_deps(dependencies = TRUE)' - -R CMD INSTALL --no-multiarch . - -# ---------------------- - - -# Try building from source via R with the relevant env vars set for feature flags. - -# Core Build Settings -export LIBARROW_MINIMAL=false -export FORCE_BUNDLED_BUILD=true -export BOOST_SOURCE=BUNDLED - -# Feature Toggles -export ARROW_COMPUTE=ON -export ARROW_CSV=ON -export ARROW_DATASET=ON -export ARROW_EXTRA_ERROR_CONTEXT=ON -export ARROW_FILESYSTEM=ON -export ARROW_JEMALLOC=ON -export ARROW_JSON=ON -export ARROW_PARQUET=ON -export ARROW_AZURE=ON - -# Visibility into build -export ARROW_R_DEV=TRUE - -# Use multiple available cores -export MAKEFLAGS="-j8" - -# Compression Codecs -export ARROW_WITH_SNAPPY=ON -export ARROW_WITH_ZLIB=ON - -# Library Linkage -export ARROW_BUILD_STATIC=ON -export ARROW_BUILD_SHARED=OFF - -# For R-specific behavior (replaces CMAKE_INSTALL_LIBDIR=lib) -export LIBARROW_BINARY=false - -export EXTRA_CMAKE_FLAGS="-DARROW_INSTALL_NAME_RPATH=OFF -DARROW_AZURE=ON -DCMAKE_SHARED_LINKER_FLAGS=-lxml2" - -export PKG_CONFIG_PATH="/usr/lib/x86_64-linux-gnu/pkgconfig" -export LDFLAGS=$(pkg-config --libs libxml-2.0) -export PKG_LIBS=$(pkg-config --libs libxml-2.0) - -# export LIBARROW_EXTERNAL_LIBDIR=/workspaces/arrow/r/libarrow - -R CMD INSTALL . --preclean - -``` \ No newline at end of file From 263b46239b2e22fb535e918b6c55400947fec147 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Mar 2026 14:30:44 +0000 Subject: [PATCH 69/89] wrap credential configuration methods with StopIfNotOk --- r/src/filesystem.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 4987516eb6ef..4653c2761631 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -595,18 +595,18 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti azure_opts.ConfigureManagedIdentityCredential( cpp11::as_cpp(options["client_id"])); } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureClientSecretCredential( + StopIfNotOk(azure_opts.ConfigureClientSecretCredential( cpp11::as_cpp(options["tenant_id"]), cpp11::as_cpp(options["client_id"]), - cpp11::as_cpp(options["client_secret"])); + cpp11::as_cpp(options["client_secret"]))); } } else if (!Rf_isNull(options["account_key"])) { - azure_opts.ConfigureAccountKeyCredential( - cpp11::as_cpp(options["account_key"])); + StopIfNotOk(azure_opts.ConfigureAccountKeyCredential( + cpp11::as_cpp(options["account_key"]))); } else if (!Rf_isNull(options["sas_token"])) { - azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"])); + StopIfNotOk(azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"]))); } else { - azure_opts.ConfigureDefaultCredential(); + StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } auto io_context = MainRThread::GetInstance().CancellableIOContext(); From cb1943182f184b32c85a09ec6862e1cfef0e842b Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Mar 2026 14:53:05 +0000 Subject: [PATCH 70/89] move link flags to arrow_built_with ARROW_AZURE block in configure script --- r/configure | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/r/configure b/r/configure index 98099823a9e3..8724f3eaf991 100755 --- a/r/configure +++ b/r/configure @@ -271,8 +271,7 @@ set_pkg_vars_with_pc () { PKG_CFLAGS="`${PKG_CONFIG} --cflags ${pkg_config_names}` $PKG_CFLAGS" PKG_CFLAGS="$PKG_CFLAGS $PKG_CFLAGS_FEATURES" PKG_LIBS=`${PKG_CONFIG} --libs-only-l --libs-only-other ${pkg_config_names}` - # TODO: Figure out how to pass these link flags properly, need this temporarily to get R to link properly. - PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES -lcurl -lxml2 -lssl" + PKG_LIBS="$PKG_LIBS $PKG_LIBS_FEATURES" PKG_DIRS=`${PKG_CONFIG} --libs-only-L ${pkg_config_names}` } From 348628190bb774f0367f92fb841622b8bba8ccb1 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 02:59:07 +0000 Subject: [PATCH 71/89] add simple test function to work through codegen.R --- r/R/arrowExports.R | 4 ++++ r/src/arrowExports.cpp | 9 +++++++++ r/src/filesystem.cpp | 15 +++++++++++++++ r/tmp.md | 11 +++++++++++ 4 files changed, 39 insertions(+) create mode 100644 r/tmp.md diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index dac7b4609c56..82099ec73a6b 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,6 +1416,10 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } +azurefs_is_functional_test <- function(input_string) { + .Call(`_arrow_azurefs_is_functional_test`, input_string) +} + fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 4b47e635f7ec..9955ceb7544f 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,6 +3646,14 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp +bool azurefs_is_functional_test(std::string input_string); +extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ +BEGIN_CPP11 + arrow::r::Input::type input_string(input_string_sexp); + return cpp11::as_sexp(azurefs_is_functional_test(input_string)); +END_CPP11 +} +// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6121,6 +6129,7 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, + { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 4653c2761631..f40342560a62 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,6 +381,21 @@ void FinalizeS3() { #endif } +#if defined(ARROW_R_WITH_AZUREFS) + +#include + +// [[arrow::export]] +bool azurefs_is_functional_test(std::string input_string) { + // This just proves we can pass data in and out of the guarded block + if (input_string == "hello") { + return true; + } + return false; +} + +#endif + #if defined(ARROW_R_WITH_GCS) #include diff --git a/r/tmp.md b/r/tmp.md new file mode 100644 index 000000000000..872728514064 --- /dev/null +++ b/r/tmp.md @@ -0,0 +1,11 @@ +# Temporary development notes + +> TODO: Remove this before we open a PR to upstream arrow library. + +## Using codegen.R + +1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` + +2. Rscript `data-raw/codegen.R` + +The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From 2bc10bada51fbe00db3bc964818f6fe125743453 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Mar 2026 21:16:10 +0000 Subject: [PATCH 72/89] cleanup azurefs test function code --- r/R/arrowExports.R | 4 ---- r/src/arrowExports.cpp | 9 --------- r/src/filesystem.cpp | 15 --------------- 3 files changed, 28 deletions(-) diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R index 82099ec73a6b..dac7b4609c56 100644 --- a/r/R/arrowExports.R +++ b/r/R/arrowExports.R @@ -1416,10 +1416,6 @@ FinalizeS3 <- function() { invisible(.Call(`_arrow_FinalizeS3`)) } -azurefs_is_functional_test <- function(input_string) { - .Call(`_arrow_azurefs_is_functional_test`, input_string) -} - fs___GcsFileSystem__Make <- function(anonymous, options) { .Call(`_arrow_fs___GcsFileSystem__Make`, anonymous, options) } diff --git a/r/src/arrowExports.cpp b/r/src/arrowExports.cpp index 9955ceb7544f..4b47e635f7ec 100644 --- a/r/src/arrowExports.cpp +++ b/r/src/arrowExports.cpp @@ -3646,14 +3646,6 @@ BEGIN_CPP11 END_CPP11 } // filesystem.cpp -bool azurefs_is_functional_test(std::string input_string); -extern "C" SEXP _arrow_azurefs_is_functional_test(SEXP input_string_sexp){ -BEGIN_CPP11 - arrow::r::Input::type input_string(input_string_sexp); - return cpp11::as_sexp(azurefs_is_functional_test(input_string)); -END_CPP11 -} -// filesystem.cpp #if defined(ARROW_R_WITH_GCS) std::shared_ptr fs___GcsFileSystem__Make(bool anonymous, cpp11::list options); extern "C" SEXP _arrow_fs___GcsFileSystem__Make(SEXP anonymous_sexp, SEXP options_sexp){ @@ -6129,7 +6121,6 @@ static const R_CallMethodDef CallEntries[] = { { "_arrow_fs___S3FileSystem__create", (DL_FUNC) &_arrow_fs___S3FileSystem__create, 18}, { "_arrow_fs___S3FileSystem__region", (DL_FUNC) &_arrow_fs___S3FileSystem__region, 1}, { "_arrow_FinalizeS3", (DL_FUNC) &_arrow_FinalizeS3, 0}, - { "_arrow_azurefs_is_functional_test", (DL_FUNC) &_arrow_azurefs_is_functional_test, 1}, { "_arrow_fs___GcsFileSystem__Make", (DL_FUNC) &_arrow_fs___GcsFileSystem__Make, 2}, { "_arrow_fs___GcsFileSystem__options", (DL_FUNC) &_arrow_fs___GcsFileSystem__options, 1}, { "_arrow_fs___AzureFileSystem__Make", (DL_FUNC) &_arrow_fs___AzureFileSystem__Make, 1}, diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index f40342560a62..4653c2761631 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -381,21 +381,6 @@ void FinalizeS3() { #endif } -#if defined(ARROW_R_WITH_AZUREFS) - -#include - -// [[arrow::export]] -bool azurefs_is_functional_test(std::string input_string) { - // This just proves we can pass data in and out of the guarded block - if (input_string == "hello") { - return true; - } - return false; -} - -#endif - #if defined(ARROW_R_WITH_GCS) #include From 169ca190609ba60b975aa2513a8736fae14e4650 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Thu, 26 Mar 2026 21:45:02 -0400 Subject: [PATCH 73/89] Trying PKGBUILD and windows build script again --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 0a48aaa826e8..545ecabb7dff 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,7 +26,7 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build - "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure + "${MINGW_PACKAGE_PREFIX}-xml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" From d39eef45c29939fb2fd4e9d08b559fbe537e6719 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Thu, 26 Mar 2026 22:23:41 -0400 Subject: [PATCH 74/89] Updated PKGBUILD --- ci/scripts/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 545ecabb7dff..0a48aaa826e8 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,7 +26,7 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build - "${MINGW_PACKAGE_PREFIX}-xml2" # for azure + "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" From fb850f16ae894556a2cba966245f66db3d653784 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 29 Mar 2026 00:05:22 +0000 Subject: [PATCH 75/89] debug ci: add azure dependencies that CI tries to build from source. The hypothesis is that the errors in the build logs are caused by incompatibilities between WIL (Windows Implementation Library) and MinGW GCC, so install them directly instead of trying to build from source. --- ci/scripts/PKGBUILD | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 0a48aaa826e8..00043e188860 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -33,7 +33,12 @@ depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" "${MINGW_PACKAGE_PREFIX}-zstd" - "${MINGW_PACKAGE_PREFIX}-brotli") + "${MINGW_PACKAGE_PREFIX}-brotli" + "${MINGW_PACKAGE_PREFIX}-azure-core-cpp" # for azure + "${MINGW_PACKAGE_PREFIX}-azure-identity-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-blobs-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-common-cpp" + "${MINGW_PACKAGE_PREFIX}-azure-storage-files-datalake-cpp") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") From 7619f01672283089ab9e7b517e7062370257a082 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Wed, 1 Apr 2026 00:02:28 -0400 Subject: [PATCH 76/89] Removed tmp.md --- r/tmp.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 r/tmp.md diff --git a/r/tmp.md b/r/tmp.md deleted file mode 100644 index 872728514064..000000000000 --- a/r/tmp.md +++ /dev/null @@ -1,11 +0,0 @@ -# Temporary development notes - -> TODO: Remove this before we open a PR to upstream arrow library. - -## Using codegen.R - -1. Install repo dependencies in `arrow/r`: `install.packages("remotes")`, then `remotes::install_deps(dependencies = TRUE)` - -2. Rscript `data-raw/codegen.R` - -The second step auto-generates stubs in `arrowExports.R` and `arrowExports.cpp` based on which C++ functions have `// [[arrow::export]]` comments above them. From ff68542d1016f33042ac7241720a464aa2fb053a Mon Sep 17 00:00:00 2001 From: Steve Martin <62676717+marberts@users.noreply.github.com> Date: Sun, 26 Apr 2026 23:27:43 -0400 Subject: [PATCH 77/89] Removed windows build and added macos --- ci/scripts/PKGBUILD | 8 +------- r/R/filesystem.R | 2 ++ r/tools/nixlibs.R | 7 ++++++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 00043e188860..9306d1105e78 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -26,19 +26,13 @@ url="https://arrow.apache.org/" license=("Apache-2.0") depends=("${MINGW_PACKAGE_PREFIX}-bzip2" "${MINGW_PACKAGE_PREFIX}-curl" # for google-cloud-cpp bundled build - "${MINGW_PACKAGE_PREFIX}-libxml2" # for azure "${MINGW_PACKAGE_PREFIX}-libutf8proc" "${MINGW_PACKAGE_PREFIX}-re2" "${MINGW_PACKAGE_PREFIX}-snappy" "${MINGW_PACKAGE_PREFIX}-zlib" "${MINGW_PACKAGE_PREFIX}-lz4" "${MINGW_PACKAGE_PREFIX}-zstd" - "${MINGW_PACKAGE_PREFIX}-brotli" - "${MINGW_PACKAGE_PREFIX}-azure-core-cpp" # for azure - "${MINGW_PACKAGE_PREFIX}-azure-identity-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-blobs-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-common-cpp" - "${MINGW_PACKAGE_PREFIX}-azure-storage-files-datalake-cpp") + "${MINGW_PACKAGE_PREFIX}-brotli") makedepends=("${MINGW_PACKAGE_PREFIX}-ccache" "${MINGW_PACKAGE_PREFIX}-gcc") options=("staticlibs" "strip" "!buildflags") diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 863051334692..064f6438fa59 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -706,6 +706,8 @@ AzureFileSystem$create <- function(account_name, ...) { call. = FALSE ) } + # The c++ codes assumes that the various combinations of authentication methods + # have been validated in this function. if (!is.null(options$tenant_id) || !is.null(options$client_id) || !is.null(options$client_secret)) { if (is.null(options$client_id)) { stop( diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 47bc9a722ddb..93fca598edd4 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -617,6 +617,11 @@ build_libarrow <- function(src_dir, dst_dir) { env_var_list <- c(env_var_list, ARROW_MIMALLOC = Sys.getenv("ARROW_MIMALLOC", "OFF")) } + if (on_windows) { + # Disable azure on windows due to issues building azure c++ sdk with mingw. + env_var_list <- c(env_var_list, ARROW_AZURE = Sys.getenv("ARROW_AZURE", "OFF")) + } + env_var_list <- with_cloud_support(env_var_list) env_var_list <- with_wasm_support(env_var_list) @@ -928,7 +933,7 @@ with_cloud_support <- function(env_var_list) { if (arrow_s3 || arrow_gcs || arrow_azure) { # User wants S3 or GCS or Azure support. - # Make sure that we have curl and openssl system libs + # Make sure that we have curl, openssl, and libxml2 system libs feats <- c( if (arrow_s3) "S3", if (arrow_gcs) "GCS", From 9ea60df6fcb1a7de8d4339795e926722584cd486 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Sun, 26 Apr 2026 23:44:06 -0400 Subject: [PATCH 78/89] azure off for windows ci --- ci/scripts/PKGBUILD | 2 +- ci/scripts/r_windows_build.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD index 9306d1105e78..ff6615b91966 100644 --- a/ci/scripts/PKGBUILD +++ b/ci/scripts/PKGBUILD @@ -122,7 +122,7 @@ build() { -DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \ -DARROW_PARQUET=ON \ -DARROW_S3=ON \ - -DARROW_AZURE=ON \ + -DARROW_AZURE=OFF \ -DARROW_SNAPPY_USE_SHARED=OFF \ -DARROW_USE_GLOG=OFF \ -DARROW_UTF8PROC_USE_SHARED=OFF \ diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh index b183f67ea4c5..ef9c58f6afca 100755 --- a/ci/scripts/r_windows_build.sh +++ b/ci/scripts/r_windows_build.sh @@ -66,7 +66,7 @@ if [ -d mingw64/lib/ ]; then # Move the 64-bit versions of libarrow into the expected location mv mingw64/lib/*.a $DST_DIR/lib/x64 # These are from https://dl.bintray.com/rtools/mingw{32,64}/ - cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/x64 + cp $MSYS_LIB_DIR/mingw64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64 fi # Same for the 32-bit versions @@ -74,7 +74,7 @@ if [ -d mingw32/lib/ ]; then ls $MSYS_LIB_DIR/mingw32/lib/ mkdir -p $DST_DIR/lib/i386 mv mingw32/lib/*.a $DST_DIR/lib/i386 - cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/i386 + cp $MSYS_LIB_DIR/mingw32/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/i386 fi # Do the same also for ucrt64 @@ -82,7 +82,7 @@ if [ -d ucrt64/lib/ ]; then ls $MSYS_LIB_DIR/ucrt64/lib/ mkdir -p $DST_DIR/lib/x64-ucrt mv ucrt64/lib/*.a $DST_DIR/lib/x64-ucrt - cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2,xml2}.a $DST_DIR/lib/x64-ucrt + cp $MSYS_LIB_DIR/ucrt64/lib/lib{snappy,zstd,lz4,brotli*,bz2,crypto,curl,ss*,utf8proc,re2,nghttp2}.a $DST_DIR/lib/x64-ucrt fi # Create build artifact From 5a3eed4b4e623601cef8374cc5d8060f7936b2a9 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Mon, 27 Apr 2026 22:10:48 -0400 Subject: [PATCH 79/89] Disable azure in configure.win --- r/configure.win | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/r/configure.win b/r/configure.win index f562208b609c..7315013f29a4 100755 --- a/r/configure.win +++ b/r/configure.win @@ -67,7 +67,7 @@ function configure_binaries() { # pkg-config --libs libcurl GCS_LIBS="-lcurl -lnormaliz -lssh2 -lgdi32 -lssl -lcrypto -lcrypt32 -lwldap32 \ -lz -lws2_32 -lnghttp2 -ldbghelp" - AZURE_LIBS="-lcurl -lssl -lxml2" + # AZURE_LIBS="-lcurl -lssl -lxml2" # Set the right flags to point to and enable arrow/parquet if [ -d "windows/r-libarrow-windows-x86_64-$VERSION" ]; then @@ -95,8 +95,8 @@ function configure_binaries() { # S3, GCS, and re2 support only for Rtools40 (i.e. R >= 4.0) "${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" -e 'R.version$major >= 4' | grep TRUE >/dev/null 2>&1 if [ $? -eq 0 ]; then - PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3 -DARROW_R_WITH_GCS -DARROW_R_WITH_AZURE" - PKG_LIBS="${PKG_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS} ${AZURE_LIBS}" + PKG_CFLAGS="${PKG_CFLAGS} -DARROW_R_WITH_S3 -DARROW_R_WITH_GCS" # -DARROW_R_WITH_AZURE + PKG_LIBS="${PKG_LIBS} -lre2 ${AWS_LIBS} ${GCS_LIBS}" # ${AZURE_LIBS} else # It seems that order matters PKG_LIBS="${PKG_LIBS} -lws2_32" @@ -188,14 +188,14 @@ add_feature_flags () { if arrow_built_with ARROW_S3; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_S3" fi - if arrow_built_with ARROW_AZURE; then - PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" - PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" - fi + # if arrow_built_with ARROW_AZURE; then + # PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_AZURE" + # PKG_LIBS_FEATURES="$PKG_LIBS_FEATURES -lxml2" + # fi if arrow_built_with ARROW_GCS; then PKG_CFLAGS_FEATURES="$PKG_CFLAGS_FEATURES -DARROW_R_WITH_GCS" fi - if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3 || arrow_built_with ARROW_AZURE; then + if arrow_built_with ARROW_GCS || arrow_built_with ARROW_S3; then # If pkg-config is available it will handle this for us automatically SSL_LIBS_WITHOUT_PC="-lcurl -lssl -lcrypto" fi @@ -297,9 +297,9 @@ function configure_dev() { PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_GCS" fi - if [ $(cmake_option ARROW_AZURE) -eq 1 ]; then - PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_AZURE" - fi + # if [ $(cmake_option ARROW_AZURE) -eq 1 ]; then + # PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_AZURE" + # fi if [ $(cmake_option ARROW_JSON) -eq 1 ]; then PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_JSON" From 8f5d9ba44813f05979bb045c286e8d36fbe3d5ed Mon Sep 17 00:00:00 2001 From: Collin Brown <8021046+Collinbrown95@users.noreply.github.com> Date: Sat, 9 May 2026 11:28:52 -0400 Subject: [PATCH 80/89] Add placeholder comment to flag present issues with MinGW and the Azure C++ SDK --- r/man/FileSystem.Rd | 3 +++ 1 file changed, 3 insertions(+) diff --git a/r/man/FileSystem.Rd b/r/man/FileSystem.Rd index 0cca6d3767d8..0ecb391549f6 100644 --- a/r/man/FileSystem.Rd +++ b/r/man/FileSystem.Rd @@ -193,5 +193,8 @@ to running any code that interacts with S3. Possible values include 'FATAL' On \code{AzureFileSystem}, passing no arguments for authentication uses the \code{AzureDefaultCredential} for authentication, so that several authentication types are tried until one succeeds. + +\code{AzureFileSystem} is not presently supported on Windows due to upstream compatibility +issues between the Azure C++ SDK and the MinGW toolchain. } From addcfa558139a24605d1fda96504215b91bc188c Mon Sep 17 00:00:00 2001 From: Collin Brown <8021046+Collinbrown95@users.noreply.github.com> Date: Sun, 10 May 2026 12:27:24 -0400 Subject: [PATCH 81/89] remove debugging script to install azure sdk from source --- .../install_azure_sdk_cpp_from_source.sh | 48 ------------------- 1 file changed, 48 deletions(-) delete mode 100644 ci/scripts/install_azure_sdk_cpp_from_source.sh diff --git a/ci/scripts/install_azure_sdk_cpp_from_source.sh b/ci/scripts/install_azure_sdk_cpp_from_source.sh deleted file mode 100644 index 9333939f7017..000000000000 --- a/ci/scripts/install_azure_sdk_cpp_from_source.sh +++ /dev/null @@ -1,48 +0,0 @@ -pushd /tmp && git clone https://github.com/Azure/azure-sdk-for-cpp.git && cd azure-sdk-for-cpp - -cd .. && rm -rf build && mkdir build && cd build - -# Rust dependency setup -pacman -S mingw-w64-ucrt-x86_64-rust - -RUSTC_PATH=$(which rustc | sed 's/\\/\//g') - -# Install extra dependencies needed to build azure-sdk-for-cpp from source -vcpkg install azure-macro-utils-c:x64-mingw-dynamic \ - nlohmann-json:x64-mingw-dynamic \ - opentelemetry-cpp:x64-mingw-dynamic \ - wil:x64-mingw-dynamic \ - --triplet x64-mingw-dynamic \ - --host-triplet x64-mingw-dynamic - -# TODO: tidy this up to run in CI context, paths are probably wrong as this was for local debugging. -cmake .. -G "Ninja" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_MAKE_PROGRAM=$(cygpath -m $(which ninja)) \ - -DRust_COMPILER=$(cygpath -m $(which rustc)) \ - -DCMAKE_PREFIX_PATH=$(cygpath -m "../../vcpkg/installed/x64-mingw-dynamic") \ - -Dopentelemetry-cpp_DIR=$(cygpath -m "../../vcpkg/installed/x64-mingw-dynamic/share/opentelemetry-cpp/opentelemetry-cpp-config.cmake") \ - -DVCPKG_TARGET_TRIPLET=x64-mingw-dynamic \ - -DVCPKG_HOST_TRIPLET=x64-mingw-dynamic \ - -DVCPKG_MANIFEST_INSTALL=ON \ - -DVCPKG_MANIFEST_FEATURES="core" \ - -DAZURE_SDK_DISABLE_AUTO_VCPKG=ON \ - -DBUILD_TRANSPORT_CURL=ON \ - -DBUILD_TRANSPORT_WINHTTP=OFF \ - -DSTORAGE_XML_BACKEND=Libxml2 \ - -DBUILD_TESTING=OFF \ - -DRUN_UNITTESTS=OFF \ - -DVCPKG_MANIFEST_MODE=OFF \ - -DWARNINGS_AS_ERRORS=OFF \ - -DCMAKE_CXX_FLAGS="-include cstring -include cstdint -include winsock2.h -DWS_XML_STRING_NULL={0,NULL} -DSIO_IDEAL_SEND_BACKLOG_QUERY=0x48000005 -DNOMINMAX=1 -D_WIN32_WINNT=0x0A00" - -ninja azure-core -# Azure Identity package seems to be incompatible with MinGW. -# Everything else works but azure-identity has many errors. -# ninja azure-identity -ninja azure-storage-blobs -ninja azure-storage-common -ninja azure-storage-files-datalake - -# Go back to the directory that called this script -popd From b58fcc5918d2bf08083382cd26224eb7e9b90a1c Mon Sep 17 00:00:00 2001 From: Steve Martin <62676717+marberts@users.noreply.github.com> Date: Mon, 11 May 2026 22:07:48 -0400 Subject: [PATCH 82/89] Updated vignettes to reflect Azure off on Windows --- r/tools/nixlibs.R | 22 ++++++++++++---------- r/vignettes/developers/binary_features.Rmd | 14 ++++++++++---- r/vignettes/fs.Rmd | 2 +- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index 93fca598edd4..a92e8eb1610b 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -934,15 +934,17 @@ with_cloud_support <- function(env_var_list) { if (arrow_s3 || arrow_gcs || arrow_azure) { # User wants S3 or GCS or Azure support. # Make sure that we have curl, openssl, and libxml2 system libs - feats <- c( - if (arrow_s3) "S3", - if (arrow_gcs) "GCS", - if (arrow_azure) "AZURE" - ) - start_msg <- paste(feats, collapse = "/") - off_flags <- paste("ARROW_", feats, "=OFF", sep = "", collapse = " and ") - print_warning <- function(msg) { - # Utility to assemble warning message in the console + # Utility to assemble warning message in the console + print_warning <- function( + msg, + feats = c( + if (arrow_s3) "S3", + if (arrow_gcs) "GCS", + if (arrow_azure) "AZURE" + ), + start_msg = paste(feats, collapse = "/") + ) { + off_flags <- paste("ARROW_", feats, "=OFF", sep = "", collapse = " and ") cat("**** ", start_msg, " support ", msg, "; building with ", off_flags, "\n") } @@ -959,7 +961,7 @@ with_cloud_support <- function(env_var_list) { arrow_gcs <- FALSE arrow_azure <- FALSE } else if (!cmake_find_package("libxml2", NULL, env_var_list)) { - print_warning("requires libxml2-devel (rpm), or libxml2-dev (deb), libxml2 (brew)") + print_warning("requires libxml2-devel (rpm), or libxml2-dev (deb), libxml2 (brew)", "AZURE") arrow_azure <- FALSE } } diff --git a/r/vignettes/developers/binary_features.Rmd b/r/vignettes/developers/binary_features.Rmd index ed6c7180f5b1..fa242e8f73eb 100644 --- a/r/vignettes/developers/binary_features.Rmd +++ b/r/vignettes/developers/binary_features.Rmd @@ -31,11 +31,11 @@ users with a fully-featured experience out of the box. ### Current binary feature set -| Platform | S3 | GCS | Configured in | +| Platform | S3 | GCS | Azure | Configured in | |----------|----|----|---------------| -| macOS (ARM64, x86_64) | ON | ON | `dev/tasks/r/github.packages.yml` | -| Windows | ON | ON | `ci/scripts/PKGBUILD` | -| Linux (x86_64) | ON | ON | `compose.yaml` (`ubuntu-cpp-static`) | +| macOS (ARM64, x86_64) | ON | ON | ON | `dev/tasks/r/github.packages.yml` | +| Windows | ON | ON | OFF | `ci/scripts/PKGBUILD` | +| Linux (x86_64) | ON | ON | ON | `compose.yaml` (`ubuntu-cpp-static`) | ### Exceptions to our build defaults @@ -48,6 +48,9 @@ our prebuilt binaries because: user machines 3. **Parity across platforms** - users get the same features regardless of OS +Azure is always set to OFF for Windows because of a low-level incompatibility +with MinGW. The `azure-identity-cpp` SDK for Azure relies on the Windows +Implementation Library (WIL), and this lacks stable support for MinGW. ## Feature configuration in source builds of libarrow @@ -85,12 +88,15 @@ When `LIBARROW_MINIMAL=false`, the following additional features are enabled | Feature | CMake Flag | Default | |---------|------------|---------| | S3 | `ARROW_S3` | `$ARROW_DEFAULT_PARAM` | +| Azure | `ARROW_AZURE` | `$ARROW_DEFAULT_PARAM` | | Jemalloc | `ARROW_JEMALLOC` | `$ARROW_DEFAULT_PARAM` | | Brotli | `ARROW_WITH_BROTLI` | `$ARROW_DEFAULT_PARAM` | | BZ2 | `ARROW_WITH_BZ2` | `$ARROW_DEFAULT_PARAM` | | Zlib | `ARROW_WITH_ZLIB` | `$ARROW_DEFAULT_PARAM` | | Zstd | `ARROW_WITH_ZSTD` | `$ARROW_DEFAULT_PARAM` | +Note that `ARROW_AZURE` is always OFF on Windows. + ### Features that require explicit opt-in GCS (Google Cloud Storage) is **always off by default**, even when diff --git a/r/vignettes/fs.Rmd b/r/vignettes/fs.Rmd index 4c2138f693f8..cb981ef5e130 100644 --- a/r/vignettes/fs.Rmd +++ b/r/vignettes/fs.Rmd @@ -37,7 +37,7 @@ Sys.setenv("NOT_CRAN" = "true") install.packages("arrow", type = "source") ``` -On Linux, S3, GCS, and Azure support is not always enabled by default when installing from source, and there are additional system requirements involved. See the [installation article](./install.html) for details. +On Linux, S3, GCS, and Azure support is not always enabled by default when installing from source, and there are additional system requirements involved. See the [installation article](./install.html) for details. Note that it is not currently possible to work with Azure on Windows. ## Connecting to cloud storage From d5cd943b81b2339df3bcd3de59a92c021396a85a Mon Sep 17 00:00:00 2001 From: Steve Martin <62676717+marberts@users.noreply.github.com> Date: Mon, 22 Jun 2026 21:09:10 -0400 Subject: [PATCH 83/89] Update github.packages.yml From 211c1fc64b8f04278441030ea5b7c8d03da1539b Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Fri, 26 Jun 2026 22:23:13 -0400 Subject: [PATCH 84/89] Fixing lint in filesystem.cpp --- r/src/filesystem.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 4653c2761631..003e3a7131cf 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -575,7 +575,8 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti } if (!Rf_isNull(options["blob_storage_authority"])) { - azure_opts.blob_storage_authority = cpp11::as_cpp(options["blob_storage_authority"]); + azure_opts.blob_storage_authority = + cpp11::as_cpp(options["blob_storage_authority"]); } if (!Rf_isNull(options["dfs_storage_authority"])) { azure_opts.dfs_storage_authority = @@ -604,7 +605,8 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti StopIfNotOk(azure_opts.ConfigureAccountKeyCredential( cpp11::as_cpp(options["account_key"]))); } else if (!Rf_isNull(options["sas_token"])) { - StopIfNotOk(azure_opts.ConfigureSASCredential(cpp11::as_cpp(options["sas_token"]))); + StopIfNotOk(azure_opts.ConfigureSASCredential( + cpp11::as_cpp(options["sas_token"]))); } else { StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } From c7dd3e54a8aeeccf9868b0dfe62a9cc40ceaf2fe Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Fri, 26 Jun 2026 23:18:55 -0400 Subject: [PATCH 85/89] Fixed warning in cpp code --- r/src/filesystem.cpp | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 003e3a7131cf..08cb564aadcc 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -574,43 +574,6 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti StopIfNotOk(azure_opts.ConfigureDefaultCredential()); } - if (!Rf_isNull(options["blob_storage_authority"])) { - azure_opts.blob_storage_authority = - cpp11::as_cpp(options["blob_storage_authority"]); - } - if (!Rf_isNull(options["dfs_storage_authority"])) { - azure_opts.dfs_storage_authority = - cpp11::as_cpp(options["dfs_storage_authority"]); - } - if (!Rf_isNull(options["blob_storage_scheme"])) { - azure_opts.blob_storage_scheme = - cpp11::as_cpp(options["blob_storage_scheme"]); - } - if (!Rf_isNull(options["dfs_storage_scheme"])) { - azure_opts.dfs_storage_scheme = - cpp11::as_cpp(options["dfs_storage_scheme"]); - } - - if (!Rf_isNull(options["client_id"])) { - if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { - azure_opts.ConfigureManagedIdentityCredential( - cpp11::as_cpp(options["client_id"])); - } else if (!Rf_isNull(options["tenant_id"]) && !Rf_isNull(options["client_secret"])) { - StopIfNotOk(azure_opts.ConfigureClientSecretCredential( - cpp11::as_cpp(options["tenant_id"]), - cpp11::as_cpp(options["client_id"]), - cpp11::as_cpp(options["client_secret"]))); - } - } else if (!Rf_isNull(options["account_key"])) { - StopIfNotOk(azure_opts.ConfigureAccountKeyCredential( - cpp11::as_cpp(options["account_key"]))); - } else if (!Rf_isNull(options["sas_token"])) { - StopIfNotOk(azure_opts.ConfigureSASCredential( - cpp11::as_cpp(options["sas_token"]))); - } else { - StopIfNotOk(azure_opts.ConfigureDefaultCredential()); - } - auto io_context = MainRThread::GetInstance().CancellableIOContext(); return ValueOrStop(fs::AzureFileSystem::Make(azure_opts, io_context)); } From 24c918bb0bdcae308008ad61ae810a85bc7774d3 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Sat, 27 Jun 2026 09:14:57 -0400 Subject: [PATCH 86/89] Added build info for azure to tasks for mac + a couple code comments --- dev/tasks/r/github.packages.yml | 872 ++++++++++++++++---------------- r/R/filesystem.R | 2 +- r/src/filesystem.cpp | 2 +- 3 files changed, 439 insertions(+), 437 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index c640629a8d03..0153a9f646b9 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -1,435 +1,437 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{% set is_fork = macros.is_fork %} - -{{ macros.github_header() }} - -jobs: - source: - # This job will change the version to either the custom_version param or YMD format. - # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) - name: Source Package - runs-on: ubuntu-latest - outputs: - pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - - name: Save Version - id: save-version - shell: bash - run: | - echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT - - - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - - name: Build R source package - shell: bash - run: | - cd arrow/r - # Copy in the Arrow C++ source - make sync-cpp - R CMD build --no-build-vignettes . - - - name: Upload package artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg__src__contrib - path: arrow/r/arrow_*.tar.gz - - macos-cpp: - name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: macos-15-intel, arch: "x86_64" } - - { runs_on: macos-14, arch: "arm64" } - env: - PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} - PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow(action_v="3")|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - name: Install Deps - run: | - brew install sccache ninja - brew install openssl@3.0 - - name: Build libarrow - shell: bash - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - MACOSX_DEPLOYMENT_TARGET: "11.6" - ARROW_GCS: ON - ARROW_DEPENDENCY_SOURCE: BUNDLED - CMAKE_GENERATOR: Ninja - LIBARROW_MINIMAL: false - run: | - sccache --start-server - export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3.0)" - cd arrow - r/inst/build_arrow_static.sh - - name: Bundle libarrow - shell: bash - run: | - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - linux-cpp: - name: C++ Binary Linux {{ '${{ matrix.arch }}' }} - runs-on: {{ '${{ matrix.runs-on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - include: - - arch: x86_64 - runs-on: ubuntu-latest - ubuntu: "22.04" - - arch: arm64 - runs-on: ubuntu-24.04-arm - ubuntu: "22.04" - env: - PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} - PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - {{ macros.github_install_archery()|indent }} - - name: Build libarrow - shell: bash - env: - ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64/v8' }}" }} - UBUNTU: {{ '${{ matrix.ubuntu }}' }} - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - source arrow/ci/scripts/util_enable_core_dumps.sh - archery docker run ubuntu-cpp-static - - name: Bundle libarrow - shell: bash - run: | - # These files were created by the docker user so we have to chown them - sudo chown -R $USER:$USER arrow/r/libarrow - - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - windows-cpp: - name: C++ Binary Windows RTools (40 only) - needs: source - runs-on: windows-latest - env: - PKG_ID: r-libarrow-windows-x86_64 - PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - - run: git config --global core.autocrlf false - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - uses: r-lib/actions/setup-r@v2 - with: - rtools-version: 40 - r-version: "4.0" - Ncpus: 2 - - name: Install sccache - shell: bash - run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache - - name: Build Arrow C++ with rtools40 - shell: bash - env: - ARROW_HOME: "arrow" - MINGW_ARCH: "ucrt64" - RTOOLS_VERSION: 40 - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: arrow/ci/scripts/r_windows_build.sh - - name: Create Checksum - shell: bash - run: | - cd build - sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - r-packages: - needs: [source, windows-cpp, macos-cpp] - name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: 'windows-latest', name: "Windows"} - - { runs_on: macos-15-intel, name: "macOS x86_64"} - - { runs_on: macos-14, name: "macOS arm64" } - r_version: [oldrel, release] - steps: - - uses: r-lib/actions/setup-r@v2 - with: - r-version: {{ '${{ matrix.r_version }}' }} - {{ macros.github_setup_local_r_repo(false, true, true)|indent }} - - name: Prepare Dependency Installation - shell: bash - run: | - tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION - - name: Install dependencies - uses: r-lib/actions/setup-r-dependencies@v2 - with: - working-directory: 'arrow' - extra-packages: cpp11 - - name: Set CRAN like openssl - if: contains(matrix.platform.name, 'macOS') - # The -E forwards the GITHUB_* envvars - shell: sudo -E Rscript {0} - run: | - # get the mac-recipes version of openssl from CRAN - source("https://mac.R-project.org/bin/install.R") - install.libs("openssl") - - # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. - if (Sys.info()[["machine"]] == "arm64"){ - cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } else { - cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } - - name: Build Binary - id: build - shell: Rscript {0} - env: - NOT_CRAN: "false" # actions/setup-r sets this implicitly - ARROW_R_DEV: "true" - LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated - LIBARROW_BUILD: "false" - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - run: | - on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" - - # Build - Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) - INSTALL_opts <- "--build" - if (!on_windows) { - # Windows doesn't support the --strip arg - INSTALL_opts <- c(INSTALL_opts, "--strip") - } - - # always remove arrow (mainly for self-hosted runners) - try(remove.packages("arrow"), silent = TRUE) - - cat("Install arrow from dev repo.\n") - install.packages( - "arrow", - type = "source", - # The sub is necessary to prevent an error on windows. - repos = sub("file://", "file:", getOption("arrow.dev_repo")),, - INSTALL_opts = INSTALL_opts - ) - - - # Test - library(arrow) - arrow_info() - read_parquet(system.file("v0.7.1.parquet", package = "arrow")) - - # encode contrib.url for artifact name - cmd <- paste0( - "path=", - gsub( - "/", "__", - contrib.url("", type = "binary") - ), - "\n" - ) - cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) - - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg{{ '${{ steps.build.outputs.path }}' }} - path: arrow_* - test-linux-binary: - needs: [source, linux-cpp] - name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} - runs-on: {{ '${{ matrix.config.runner }}' }} - container: {{ '${{ matrix.config.image }}' }} - strategy: - fail-fast: false - matrix: - config: - # If libarrow_binary is unset, we're testing that we're automatically - # choosing a binary on this OS. If libarrow_binary is TRUE, we're on - # an OS that is not in the allowlist, so we have to opt-in to use the - # binary. Other env vars used in r_docker_configure.sh can be added - # here and wired up in the later steps. - # x86_64 tests - - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) - # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-24.04 - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-latest"} - # ARM64 tests - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-24.04-arm"} - steps: - # Get the arrow checkout just for the docker config scripts - # Don't need submodules for this (hence false arg to macro): they fail on - # actions/checkout for some reason in this context - {{ macros.github_checkout_arrow(1, false, '3')|indent }} - - - name: Install system requirements - env: - ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh - shell: bash - run: | - # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) - echo "${RPREFIX}/bin" >> $GITHUB_PATH - export PATH="${RPREFIX}/bin:${PATH}" - cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh - {{ macros.github_setup_local_r_repo(true, false)|indent }} - - name: Install arrow from our repo - env: - ARROW_R_DEV: "TRUE" - LIBARROW_BUILD: "FALSE" - LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - shell: bash - run: | - Rscript -e ' - {{ macros.github_test_r_src_pkg()|indent(8) }} - ' - test-source: - needs: source - name: Test {{ '${{ matrix.platform.name }}' }} source build - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - {runs_on: "ubuntu-latest", name: "Linux"} - - {runs_on: "macos-15-intel" , name: "macOS"} - steps: - - name: Install R - uses: r-lib/actions/setup-r@v2 - {{ macros.github_setup_local_r_repo(false, false)|indent }} - {{ macros.github_checkout_arrow(action_v="3")|indent }} - - name: Install sccache - if: matrix.platform.name == 'Linux' - shell: bash - run: | - arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - - name: Install R package system dependencies (Linux) - if: matrix.platform.name == 'Linux' - run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev - - name: Install R package system dependencies (macOS) - if: matrix.platform.name == 'macOS' - run: brew install sccache openssl curl libuv - - name: Remove arrow/ - run: | - rm -rf arrow/ - - name: Enable parallel build - run: | - cores=`nproc || sysctl -n hw.logicalcpu` - echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV - - name: Install arrow source package - env: - # Test source build so be sure not to download a binary - LIBARROW_BINARY: "FALSE" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - - name: Retry with verbosity if that failed - if: failure() - env: - LIBARROW_BINARY: "FALSE" - ARROW_R_DEV: "TRUE" - CMAKE_FIND_DEBUG_MODE: "ON" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - upload-binaries: - # Only upload binaries if all tests pass. - needs: [r-packages, test-source, test-linux-binary] - name: Upload artifacts - runs-on: ubuntu-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Install R - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - name: Move libarrow artifacts - run: | - mkdir -p binaries/ - mv artifacts/r-libarrow-*/* binaries/ - - name: Rename artifacts - shell: Rscript {0} - run: | - file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) - new_names <- file.path("binaries", sub("/", "__", file_paths)) - dir.create("binaries", showWarnings = FALSE) - file.copy(file.path("artifacts", file_paths), new_names) - - {{ macros.github_upload_releases("binaries/r-*")|indent }} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{% set is_fork = macros.is_fork %} + +{{ macros.github_header() }} + +jobs: + source: + # This job will change the version to either the custom_version param or YMD format. + # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) + name: Source Package + runs-on: ubuntu-latest + outputs: + pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} + - name: Save Version + id: save-version + shell: bash + run: | + echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT + + - uses: r-lib/actions/setup-r@v2 + with: + install-r: true + + - name: Build R source package + shell: bash + run: | + cd arrow/r + # Copy in the Arrow C++ source + make sync-cpp + R CMD build --no-build-vignettes . + + - name: Upload package artifact + uses: actions/upload-artifact@v4 + with: + name: r-pkg__src__contrib + path: arrow/r/arrow_*.tar.gz + + macos-cpp: + name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + needs: source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-15-intel, arch: "x86_64" } + - { runs_on: macos-14, arch: "arm64" } + env: + PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} + PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + {{ macros.github_checkout_arrow(action_v="3")|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + - name: Install Deps + run: | + brew install sccache ninja + brew install openssl@3.0 + brew install libxml2 + - name: Build libarrow + shell: bash + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} + MACOSX_DEPLOYMENT_TARGET: "11.6" + ARROW_GCS: ON + ARROW_AZURE: ON + ARROW_DEPENDENCY_SOURCE: BUNDLED + CMAKE_GENERATOR: Ninja + LIBARROW_MINIMAL: false + run: | + sccache --start-server + export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3.0)" + cd arrow + r/inst/build_arrow_static.sh + - name: Bundle libarrow + shell: bash + run: | + cd arrow/r/libarrow/dist + zip -r $PKG_FILE lib/ include/ + - name: Create Checksum + shell: bash + run: | + cd arrow/r/libarrow/dist + shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../../../../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + linux-cpp: + name: C++ Binary Linux {{ '${{ matrix.arch }}' }} + runs-on: {{ '${{ matrix.runs-on }}' }} + needs: source + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runs-on: ubuntu-latest + ubuntu: "22.04" + - arch: arm64 + runs-on: ubuntu-24.04-arm + ubuntu: "22.04" + env: + PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} + PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + {{ macros.github_install_archery()|indent }} + - name: Build libarrow + shell: bash + env: + ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64/v8' }}" }} + UBUNTU: {{ '${{ matrix.ubuntu }}' }} + {{ macros.github_set_sccache_envvars()|indent(8) }} + run: | + source arrow/ci/scripts/util_enable_core_dumps.sh + archery docker run ubuntu-cpp-static + - name: Bundle libarrow + shell: bash + run: | + # These files were created by the docker user so we have to chown them + sudo chown -R $USER:$USER arrow/r/libarrow + + cd arrow/r/libarrow/dist + zip -r $PKG_FILE lib/ include/ + - name: Create Checksum + shell: bash + run: | + cd arrow/r/libarrow/dist + shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../../../../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + windows-cpp: + name: C++ Binary Windows RTools (40 only) + needs: source + runs-on: windows-latest + env: + PKG_ID: r-libarrow-windows-x86_64 + PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + - run: git config --global core.autocrlf false + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + - uses: r-lib/actions/setup-r@v2 + with: + rtools-version: 40 + r-version: "4.0" + Ncpus: 2 + - name: Install sccache + shell: bash + run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + - name: Build Arrow C++ with rtools40 + shell: bash + env: + ARROW_HOME: "arrow" + MINGW_ARCH: "ucrt64" + RTOOLS_VERSION: 40 + {{ macros.github_set_sccache_envvars()|indent(8) }} + run: arrow/ci/scripts/r_windows_build.sh + - name: Create Checksum + shell: bash + run: | + cd build + sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + r-packages: + needs: [source, windows-cpp, macos-cpp] + name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: 'windows-latest', name: "Windows"} + - { runs_on: macos-15-intel, name: "macOS x86_64"} + - { runs_on: macos-14, name: "macOS arm64" } + r_version: [oldrel, release] + steps: + - uses: r-lib/actions/setup-r@v2 + with: + r-version: {{ '${{ matrix.r_version }}' }} + {{ macros.github_setup_local_r_repo(false, true, true)|indent }} + - name: Prepare Dependency Installation + shell: bash + run: | + tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION + - name: Install dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: 'arrow' + extra-packages: cpp11 + - name: Set CRAN like openssl + if: contains(matrix.platform.name, 'macOS') + # The -E forwards the GITHUB_* envvars + shell: sudo -E Rscript {0} + run: | + # get the mac-recipes version of openssl from CRAN + source("https://mac.R-project.org/bin/install.R") + install.libs("openssl") + + # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. + if (Sys.info()[["machine"]] == "arm64"){ + cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) + } else { + cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) + } + - name: Build Binary + id: build + shell: Rscript {0} + env: + NOT_CRAN: "false" # actions/setup-r sets this implicitly + ARROW_R_DEV: "true" + LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated + LIBARROW_BUILD: "false" + ARROW_R_ENFORCE_CHECKSUM: "true" + ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" + run: | + on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" + + # Build + Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) + INSTALL_opts <- "--build" + if (!on_windows) { + # Windows doesn't support the --strip arg + INSTALL_opts <- c(INSTALL_opts, "--strip") + } + + # always remove arrow (mainly for self-hosted runners) + try(remove.packages("arrow"), silent = TRUE) + + cat("Install arrow from dev repo.\n") + install.packages( + "arrow", + type = "source", + # The sub is necessary to prevent an error on windows. + repos = sub("file://", "file:", getOption("arrow.dev_repo")),, + INSTALL_opts = INSTALL_opts + ) + + + # Test + library(arrow) + arrow_info() + read_parquet(system.file("v0.7.1.parquet", package = "arrow")) + + # encode contrib.url for artifact name + cmd <- paste0( + "path=", + gsub( + "/", "__", + contrib.url("", type = "binary") + ), + "\n" + ) + cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) + + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: r-pkg{{ '${{ steps.build.outputs.path }}' }} + path: arrow_* + test-linux-binary: + needs: [source, linux-cpp] + name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} + runs-on: {{ '${{ matrix.config.runner }}' }} + container: {{ '${{ matrix.config.image }}' }} + strategy: + fail-fast: false + matrix: + config: + # If libarrow_binary is unset, we're testing that we're automatically + # choosing a binary on this OS. If libarrow_binary is TRUE, we're on + # an OS that is not in the allowlist, so we have to opt-in to use the + # binary. Other env vars used in r_docker_configure.sh can be added + # here and wired up in the later steps. + # x86_64 tests + - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} + # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) + # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} + - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-24.04 + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.4-noble", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.5-noble", runner: "ubuntu-latest"} + # ARM64 tests + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.4-noble", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.5-noble", runner: "ubuntu-24.04-arm"} + steps: + # Get the arrow checkout just for the docker config scripts + # Don't need submodules for this (hence false arg to macro): they fail on + # actions/checkout for some reason in this context + {{ macros.github_checkout_arrow(1, false, '3')|indent }} + + - name: Install system requirements + env: + ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh + shell: bash + run: | + # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) + echo "${RPREFIX}/bin" >> $GITHUB_PATH + export PATH="${RPREFIX}/bin:${PATH}" + cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh + {{ macros.github_setup_local_r_repo(true, false)|indent }} + - name: Install arrow from our repo + env: + ARROW_R_DEV: "TRUE" + LIBARROW_BUILD: "FALSE" + LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} + ARROW_R_ENFORCE_CHECKSUM: "true" + ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" + shell: bash + run: | + Rscript -e ' + {{ macros.github_test_r_src_pkg()|indent(8) }} + ' + test-source: + needs: source + name: Test {{ '${{ matrix.platform.name }}' }} source build + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + strategy: + fail-fast: false + matrix: + platform: + - {runs_on: "ubuntu-latest", name: "Linux"} + - {runs_on: "macos-15-intel" , name: "macOS"} + steps: + - name: Install R + uses: r-lib/actions/setup-r@v2 + {{ macros.github_setup_local_r_repo(false, false)|indent }} + {{ macros.github_checkout_arrow(action_v="3")|indent }} + - name: Install sccache + if: matrix.platform.name == 'Linux' + shell: bash + run: | + arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + - name: Install R package system dependencies (Linux) + if: matrix.platform.name == 'Linux' + run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev + - name: Install R package system dependencies (macOS) + if: matrix.platform.name == 'macOS' + run: brew install sccache openssl curl libuv + - name: Remove arrow/ + run: | + rm -rf arrow/ + - name: Enable parallel build + run: | + cores=`nproc || sysctl -n hw.logicalcpu` + echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV + - name: Install arrow source package + env: + # Test source build so be sure not to download a binary + LIBARROW_BINARY: "FALSE" + {{ macros.github_set_sccache_envvars()|indent(8) }} + shell: Rscript {0} + run: | + {{ macros.github_test_r_src_pkg()|indent(8) }} + + - name: Retry with verbosity if that failed + if: failure() + env: + LIBARROW_BINARY: "FALSE" + ARROW_R_DEV: "TRUE" + CMAKE_FIND_DEBUG_MODE: "ON" + {{ macros.github_set_sccache_envvars()|indent(8) }} + shell: Rscript {0} + run: | + {{ macros.github_test_r_src_pkg()|indent(8) }} + + upload-binaries: + # Only upload binaries if all tests pass. + needs: [r-packages, test-source, test-linux-binary] + name: Upload artifacts + runs-on: ubuntu-latest + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Install R + uses: r-lib/actions/setup-r@v2 + with: + install-r: true + - name: Move libarrow artifacts + run: | + mkdir -p binaries/ + mv artifacts/r-libarrow-*/* binaries/ + - name: Rename artifacts + shell: Rscript {0} + run: | + file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) + new_names <- file.path("binaries", sub("/", "__", file_paths)) + dir.create("binaries", showWarnings = FALSE) + file.copy(file.path("artifacts", file_paths), new_names) + + {{ macros.github_upload_releases("binaries/r-*")|indent }} diff --git a/r/R/filesystem.R b/r/R/filesystem.R index 064f6438fa59..abbc9cd6c5bf 100644 --- a/r/R/filesystem.R +++ b/r/R/filesystem.R @@ -706,7 +706,7 @@ AzureFileSystem$create <- function(account_name, ...) { call. = FALSE ) } - # The c++ codes assumes that the various combinations of authentication methods + # The c++ code assumes that the various combinations of authentication methods # have been validated in this function. if (!is.null(options$tenant_id) || !is.null(options$client_id) || !is.null(options$client_secret)) { if (is.null(options$client_id)) { diff --git a/r/src/filesystem.cpp b/r/src/filesystem.cpp index 08cb564aadcc..aa9e4b186cf9 100644 --- a/r/src/filesystem.cpp +++ b/r/src/filesystem.cpp @@ -553,7 +553,7 @@ std::shared_ptr fs___AzureFileSystem__Make(cpp11::list opti azure_opts.dfs_storage_scheme = cpp11::as_cpp(options["dfs_storage_scheme"]); } - + // Validation of the different auth paths happens in the R code. if (!Rf_isNull(options["client_id"])) { if (Rf_isNull(options["tenant_id"]) && Rf_isNull(options["client_secret"])) { StopIfNotOk(azure_opts.ConfigureManagedIdentityCredential( From a8ad6a9b1410a765e7f76bf4f1bc4f9f543c0c65 Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Sat, 27 Jun 2026 14:17:28 -0400 Subject: [PATCH 87/89] Fixed line endings on github.packages yaml --- dev/tasks/r/github.packages.yml | 874 ++++++++++++++++---------------- 1 file changed, 437 insertions(+), 437 deletions(-) diff --git a/dev/tasks/r/github.packages.yml b/dev/tasks/r/github.packages.yml index 0153a9f646b9..feeac3dba47b 100644 --- a/dev/tasks/r/github.packages.yml +++ b/dev/tasks/r/github.packages.yml @@ -1,437 +1,437 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{% set is_fork = macros.is_fork %} - -{{ macros.github_header() }} - -jobs: - source: - # This job will change the version to either the custom_version param or YMD format. - # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) - name: Source Package - runs-on: ubuntu-latest - outputs: - pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} - - name: Save Version - id: save-version - shell: bash - run: | - echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT - - - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - - name: Build R source package - shell: bash - run: | - cd arrow/r - # Copy in the Arrow C++ source - make sync-cpp - R CMD build --no-build-vignettes . - - - name: Upload package artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg__src__contrib - path: arrow/r/arrow_*.tar.gz - - macos-cpp: - name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: macos-15-intel, arch: "x86_64" } - - { runs_on: macos-14, arch: "arm64" } - env: - PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} - PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow(action_v="3")|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - name: Install Deps - run: | - brew install sccache ninja - brew install openssl@3.0 - brew install libxml2 - - name: Build libarrow - shell: bash - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - MACOSX_DEPLOYMENT_TARGET: "11.6" - ARROW_GCS: ON - ARROW_AZURE: ON - ARROW_DEPENDENCY_SOURCE: BUNDLED - CMAKE_GENERATOR: Ninja - LIBARROW_MINIMAL: false - run: | - sccache --start-server - export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3.0)" - cd arrow - r/inst/build_arrow_static.sh - - name: Bundle libarrow - shell: bash - run: | - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - linux-cpp: - name: C++ Binary Linux {{ '${{ matrix.arch }}' }} - runs-on: {{ '${{ matrix.runs-on }}' }} - needs: source - strategy: - fail-fast: false - matrix: - include: - - arch: x86_64 - runs-on: ubuntu-latest - ubuntu: "22.04" - - arch: arm64 - runs-on: ubuntu-24.04-arm - ubuntu: "22.04" - env: - PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} - PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - {{ macros.github_install_archery()|indent }} - - name: Build libarrow - shell: bash - env: - ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64/v8' }}" }} - UBUNTU: {{ '${{ matrix.ubuntu }}' }} - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - source arrow/ci/scripts/util_enable_core_dumps.sh - archery docker run ubuntu-cpp-static - - name: Bundle libarrow - shell: bash - run: | - # These files were created by the docker user so we have to chown them - sudo chown -R $USER:$USER arrow/r/libarrow - - cd arrow/r/libarrow/dist - zip -r $PKG_FILE lib/ include/ - - name: Create Checksum - shell: bash - run: | - cd arrow/r/libarrow/dist - shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../../../../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - windows-cpp: - name: C++ Binary Windows RTools (40 only) - needs: source - runs-on: windows-latest - env: - PKG_ID: r-libarrow-windows-x86_64 - PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip - steps: - - run: git config --global core.autocrlf false - {{ macros.github_checkout_arrow()|indent }} - {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} - - uses: r-lib/actions/setup-r@v2 - with: - rtools-version: 40 - r-version: "4.0" - Ncpus: 2 - - name: Install sccache - shell: bash - run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache - - name: Build Arrow C++ with rtools40 - shell: bash - env: - ARROW_HOME: "arrow" - MINGW_ARCH: "ucrt64" - RTOOLS_VERSION: 40 - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: arrow/ci/scripts/r_windows_build.sh - - name: Create Checksum - shell: bash - run: | - cd build - sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 - mv ${PKG_FILE}{,.sha512} ../ - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: {{ '${{ env.PKG_ID }}' }} - path: | - {{ '${{ env.PKG_FILE }}' }} - {{ '${{ env.PKG_FILE }}' }}.sha512 - - r-packages: - needs: [source, windows-cpp, macos-cpp] - name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - { runs_on: 'windows-latest', name: "Windows"} - - { runs_on: macos-15-intel, name: "macOS x86_64"} - - { runs_on: macos-14, name: "macOS arm64" } - r_version: [oldrel, release] - steps: - - uses: r-lib/actions/setup-r@v2 - with: - r-version: {{ '${{ matrix.r_version }}' }} - {{ macros.github_setup_local_r_repo(false, true, true)|indent }} - - name: Prepare Dependency Installation - shell: bash - run: | - tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION - - name: Install dependencies - uses: r-lib/actions/setup-r-dependencies@v2 - with: - working-directory: 'arrow' - extra-packages: cpp11 - - name: Set CRAN like openssl - if: contains(matrix.platform.name, 'macOS') - # The -E forwards the GITHUB_* envvars - shell: sudo -E Rscript {0} - run: | - # get the mac-recipes version of openssl from CRAN - source("https://mac.R-project.org/bin/install.R") - install.libs("openssl") - - # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. - if (Sys.info()[["machine"]] == "arm64"){ - cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } else { - cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) - } - - name: Build Binary - id: build - shell: Rscript {0} - env: - NOT_CRAN: "false" # actions/setup-r sets this implicitly - ARROW_R_DEV: "true" - LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated - LIBARROW_BUILD: "false" - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - run: | - on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" - - # Build - Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) - INSTALL_opts <- "--build" - if (!on_windows) { - # Windows doesn't support the --strip arg - INSTALL_opts <- c(INSTALL_opts, "--strip") - } - - # always remove arrow (mainly for self-hosted runners) - try(remove.packages("arrow"), silent = TRUE) - - cat("Install arrow from dev repo.\n") - install.packages( - "arrow", - type = "source", - # The sub is necessary to prevent an error on windows. - repos = sub("file://", "file:", getOption("arrow.dev_repo")),, - INSTALL_opts = INSTALL_opts - ) - - - # Test - library(arrow) - arrow_info() - read_parquet(system.file("v0.7.1.parquet", package = "arrow")) - - # encode contrib.url for artifact name - cmd <- paste0( - "path=", - gsub( - "/", "__", - contrib.url("", type = "binary") - ), - "\n" - ) - cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) - - - name: Upload binary artifact - uses: actions/upload-artifact@v4 - with: - name: r-pkg{{ '${{ steps.build.outputs.path }}' }} - path: arrow_* - test-linux-binary: - needs: [source, linux-cpp] - name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} - runs-on: {{ '${{ matrix.config.runner }}' }} - container: {{ '${{ matrix.config.image }}' }} - strategy: - fail-fast: false - matrix: - config: - # If libarrow_binary is unset, we're testing that we're automatically - # choosing a binary on this OS. If libarrow_binary is TRUE, we're on - # an OS that is not in the allowlist, so we have to opt-in to use the - # binary. Other env vars used in r_docker_configure.sh can be added - # here and wired up in the later steps. - # x86_64 tests - - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) - # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} - - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-24.04 - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-latest"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-latest"} - # ARM64 tests - - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.4-noble", runner: "ubuntu-24.04-arm"} - - {image: "posit/r-base:4.5-noble", runner: "ubuntu-24.04-arm"} - steps: - # Get the arrow checkout just for the docker config scripts - # Don't need submodules for this (hence false arg to macro): they fail on - # actions/checkout for some reason in this context - {{ macros.github_checkout_arrow(1, false, '3')|indent }} - - - name: Install system requirements - env: - ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh - shell: bash - run: | - # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) - echo "${RPREFIX}/bin" >> $GITHUB_PATH - export PATH="${RPREFIX}/bin:${PATH}" - cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh - {{ macros.github_setup_local_r_repo(true, false)|indent }} - - name: Install arrow from our repo - env: - ARROW_R_DEV: "TRUE" - LIBARROW_BUILD: "FALSE" - LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} - ARROW_R_ENFORCE_CHECKSUM: "true" - ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" - shell: bash - run: | - Rscript -e ' - {{ macros.github_test_r_src_pkg()|indent(8) }} - ' - test-source: - needs: source - name: Test {{ '${{ matrix.platform.name }}' }} source build - runs-on: {{ '${{ matrix.platform.runs_on }}' }} - strategy: - fail-fast: false - matrix: - platform: - - {runs_on: "ubuntu-latest", name: "Linux"} - - {runs_on: "macos-15-intel" , name: "macOS"} - steps: - - name: Install R - uses: r-lib/actions/setup-r@v2 - {{ macros.github_setup_local_r_repo(false, false)|indent }} - {{ macros.github_checkout_arrow(action_v="3")|indent }} - - name: Install sccache - if: matrix.platform.name == 'Linux' - shell: bash - run: | - arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin - - name: Install R package system dependencies (Linux) - if: matrix.platform.name == 'Linux' - run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev - - name: Install R package system dependencies (macOS) - if: matrix.platform.name == 'macOS' - run: brew install sccache openssl curl libuv - - name: Remove arrow/ - run: | - rm -rf arrow/ - - name: Enable parallel build - run: | - cores=`nproc || sysctl -n hw.logicalcpu` - echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV - - name: Install arrow source package - env: - # Test source build so be sure not to download a binary - LIBARROW_BINARY: "FALSE" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - - name: Retry with verbosity if that failed - if: failure() - env: - LIBARROW_BINARY: "FALSE" - ARROW_R_DEV: "TRUE" - CMAKE_FIND_DEBUG_MODE: "ON" - {{ macros.github_set_sccache_envvars()|indent(8) }} - shell: Rscript {0} - run: | - {{ macros.github_test_r_src_pkg()|indent(8) }} - - upload-binaries: - # Only upload binaries if all tests pass. - needs: [r-packages, test-source, test-linux-binary] - name: Upload artifacts - runs-on: ubuntu-latest - steps: - {{ macros.github_checkout_arrow()|indent }} - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - - name: Install R - uses: r-lib/actions/setup-r@v2 - with: - install-r: true - - name: Move libarrow artifacts - run: | - mkdir -p binaries/ - mv artifacts/r-libarrow-*/* binaries/ - - name: Rename artifacts - shell: Rscript {0} - run: | - file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) - new_names <- file.path("binaries", sub("/", "__", file_paths)) - dir.create("binaries", showWarnings = FALSE) - file.copy(file.path("artifacts", file_paths), new_names) - - {{ macros.github_upload_releases("binaries/r-*")|indent }} +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{% set is_fork = macros.is_fork %} + +{{ macros.github_header() }} + +jobs: + source: + # This job will change the version to either the custom_version param or YMD format. + # The output allows other steps to use the exact version to prevent issues (e.g. date changes during run) + name: Source Package + runs-on: ubuntu-latest + outputs: + pkg_version: {{ '${{ steps.save-version.outputs.pkg_version }}' }} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, arrow.no_rc_r_version)|indent }} + - name: Save Version + id: save-version + shell: bash + run: | + echo "pkg_version=$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)" >> $GITHUB_OUTPUT + + - uses: r-lib/actions/setup-r@v2 + with: + install-r: true + + - name: Build R source package + shell: bash + run: | + cd arrow/r + # Copy in the Arrow C++ source + make sync-cpp + R CMD build --no-build-vignettes . + + - name: Upload package artifact + uses: actions/upload-artifact@v4 + with: + name: r-pkg__src__contrib + path: arrow/r/arrow_*.tar.gz + + macos-cpp: + name: C++ Binary macOS {{ '${{ matrix.platform.arch }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + needs: source + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: macos-15-intel, arch: "x86_64" } + - { runs_on: macos-14, arch: "arm64" } + env: + PKG_ID: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }} + PKG_FILE: r-libarrow-darwin-{{ '${{ matrix.platform.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + {{ macros.github_checkout_arrow(action_v="3")|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + - name: Install Deps + run: | + brew install sccache ninja + brew install openssl@3.0 + brew install libxml2 + - name: Build libarrow + shell: bash + env: + {{ macros.github_set_sccache_envvars()|indent(8) }} + MACOSX_DEPLOYMENT_TARGET: "11.6" + ARROW_GCS: ON + ARROW_AZURE: ON + ARROW_DEPENDENCY_SOURCE: BUNDLED + CMAKE_GENERATOR: Ninja + LIBARROW_MINIMAL: false + run: | + sccache --start-server + export EXTRA_CMAKE_FLAGS="-DOPENSSL_ROOT_DIR=$(brew --prefix openssl@3.0)" + cd arrow + r/inst/build_arrow_static.sh + - name: Bundle libarrow + shell: bash + run: | + cd arrow/r/libarrow/dist + zip -r $PKG_FILE lib/ include/ + - name: Create Checksum + shell: bash + run: | + cd arrow/r/libarrow/dist + shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../../../../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + linux-cpp: + name: C++ Binary Linux {{ '${{ matrix.arch }}' }} + runs-on: {{ '${{ matrix.runs-on }}' }} + needs: source + strategy: + fail-fast: false + matrix: + include: + - arch: x86_64 + runs-on: ubuntu-latest + ubuntu: "22.04" + - arch: arm64 + runs-on: ubuntu-24.04-arm + ubuntu: "22.04" + env: + PKG_ID: r-libarrow-linux-{{ '${{ matrix.arch }}' }} + PKG_FILE: r-libarrow-linux-{{ '${{ matrix.arch }}' }}-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + {{ macros.github_install_archery()|indent }} + - name: Build libarrow + shell: bash + env: + ARCH: {{ "${{ matrix.arch == 'x86_64' && 'amd64' || 'arm64/v8' }}" }} + UBUNTU: {{ '${{ matrix.ubuntu }}' }} + {{ macros.github_set_sccache_envvars()|indent(8) }} + run: | + source arrow/ci/scripts/util_enable_core_dumps.sh + archery docker run ubuntu-cpp-static + - name: Bundle libarrow + shell: bash + run: | + # These files were created by the docker user so we have to chown them + sudo chown -R $USER:$USER arrow/r/libarrow + + cd arrow/r/libarrow/dist + zip -r $PKG_FILE lib/ include/ + - name: Create Checksum + shell: bash + run: | + cd arrow/r/libarrow/dist + shasum -a 512 ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../../../../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + windows-cpp: + name: C++ Binary Windows RTools (40 only) + needs: source + runs-on: windows-latest + env: + PKG_ID: r-libarrow-windows-x86_64 + PKG_FILE: r-libarrow-windows-x86_64-{{ '${{ needs.source.outputs.pkg_version }}' }}.zip + steps: + - run: git config --global core.autocrlf false + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_change_r_pkg_version(is_fork, '${{ needs.source.outputs.pkg_version }}')|indent }} + - uses: r-lib/actions/setup-r@v2 + with: + rtools-version: 40 + r-version: "4.0" + Ncpus: 2 + - name: Install sccache + shell: bash + run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache + - name: Build Arrow C++ with rtools40 + shell: bash + env: + ARROW_HOME: "arrow" + MINGW_ARCH: "ucrt64" + RTOOLS_VERSION: 40 + {{ macros.github_set_sccache_envvars()|indent(8) }} + run: arrow/ci/scripts/r_windows_build.sh + - name: Create Checksum + shell: bash + run: | + cd build + sha512sum ${PKG_FILE} > ${PKG_FILE}.sha512 + mv ${PKG_FILE}{,.sha512} ../ + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: {{ '${{ env.PKG_ID }}' }} + path: | + {{ '${{ env.PKG_FILE }}' }} + {{ '${{ env.PKG_FILE }}' }}.sha512 + + r-packages: + needs: [source, windows-cpp, macos-cpp] + name: {{ '${{ matrix.platform.name }} ${{ matrix.r_version }}' }} + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + strategy: + fail-fast: false + matrix: + platform: + - { runs_on: 'windows-latest', name: "Windows"} + - { runs_on: macos-15-intel, name: "macOS x86_64"} + - { runs_on: macos-14, name: "macOS arm64" } + r_version: [oldrel, release] + steps: + - uses: r-lib/actions/setup-r@v2 + with: + r-version: {{ '${{ matrix.r_version }}' }} + {{ macros.github_setup_local_r_repo(false, true, true)|indent }} + - name: Prepare Dependency Installation + shell: bash + run: | + tar -xzf repo/src/contrib/arrow_*.tar.gz arrow/DESCRIPTION + - name: Install dependencies + uses: r-lib/actions/setup-r-dependencies@v2 + with: + working-directory: 'arrow' + extra-packages: cpp11 + - name: Set CRAN like openssl + if: contains(matrix.platform.name, 'macOS') + # The -E forwards the GITHUB_* envvars + shell: sudo -E Rscript {0} + run: | + # get the mac-recipes version of openssl from CRAN + source("https://mac.R-project.org/bin/install.R") + install.libs("openssl") + + # override our cmakes default setting of the brew --prefix as root dir to avoid version conflicts. + if (Sys.info()[["machine"]] == "arm64"){ + cat("OPENSSL_ROOT_DIR=/opt/R/arm64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) + } else { + cat("OPENSSL_ROOT_DIR=/opt/R/x86_64\n", file=Sys.getenv("GITHUB_ENV"), append = TRUE) + } + - name: Build Binary + id: build + shell: Rscript {0} + env: + NOT_CRAN: "false" # actions/setup-r sets this implicitly + ARROW_R_DEV: "true" + LIBARROW_BINARY: "true" # has to be set as long as allowlist not updated + LIBARROW_BUILD: "false" + ARROW_R_ENFORCE_CHECKSUM: "true" + ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" + run: | + on_windows <- tolower(Sys.info()[["sysname"]]) == "windows" + + # Build + Sys.setenv(MAKEFLAGS = paste0("-j", parallel::detectCores())) + INSTALL_opts <- "--build" + if (!on_windows) { + # Windows doesn't support the --strip arg + INSTALL_opts <- c(INSTALL_opts, "--strip") + } + + # always remove arrow (mainly for self-hosted runners) + try(remove.packages("arrow"), silent = TRUE) + + cat("Install arrow from dev repo.\n") + install.packages( + "arrow", + type = "source", + # The sub is necessary to prevent an error on windows. + repos = sub("file://", "file:", getOption("arrow.dev_repo")),, + INSTALL_opts = INSTALL_opts + ) + + + # Test + library(arrow) + arrow_info() + read_parquet(system.file("v0.7.1.parquet", package = "arrow")) + + # encode contrib.url for artifact name + cmd <- paste0( + "path=", + gsub( + "/", "__", + contrib.url("", type = "binary") + ), + "\n" + ) + cat(cmd, file = Sys.getenv("GITHUB_OUTPUT"), append = TRUE) + + - name: Upload binary artifact + uses: actions/upload-artifact@v4 + with: + name: r-pkg{{ '${{ steps.build.outputs.path }}' }} + path: arrow_* + test-linux-binary: + needs: [source, linux-cpp] + name: Test binary {{ '${{ matrix.config.image }}' }} {{ '${{ matrix.config.runner }}' }} + runs-on: {{ '${{ matrix.config.runner }}' }} + container: {{ '${{ matrix.config.image }}' }} + strategy: + fail-fast: false + matrix: + config: + # If libarrow_binary is unset, we're testing that we're automatically + # choosing a binary on this OS. If libarrow_binary is TRUE, we're on + # an OS that is not in the allowlist, so we have to opt-in to use the + # binary. Other env vars used in r_docker_configure.sh can be added + # here and wired up in the later steps. + # x86_64 tests + - {image: "rhub/ubuntu-clang", libarrow_binary: "TRUE", runner: "ubuntu-latest"} + # fedora-clang-devel cannot use binaries bc of libc++ (uncomment to see the error) + # - {image: "rhub/fedora-clang-devel", libarrow_binary: "TRUE", runner: "ubuntu-latest"} + - {image: "rhub/ubuntu-release", runner: "ubuntu-latest"} # currently ubuntu-24.04 + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.4-noble", runner: "ubuntu-latest"} + - {image: "posit/r-base:4.5-noble", runner: "ubuntu-latest"} + # ARM64 tests + - {image: "posit/r-base:4.3-noble", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.4-noble", runner: "ubuntu-24.04-arm"} + - {image: "posit/r-base:4.5-noble", runner: "ubuntu-24.04-arm"} + steps: + # Get the arrow checkout just for the docker config scripts + # Don't need submodules for this (hence false arg to macro): they fail on + # actions/checkout for some reason in this context + {{ macros.github_checkout_arrow(1, false, '3')|indent }} + + - name: Install system requirements + env: + ARROW_R_DEV: "TRUE" # To install curl/openssl in r_docker_configure.sh + shell: bash + run: | + # Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) + echo "${RPREFIX}/bin" >> $GITHUB_PATH + export PATH="${RPREFIX}/bin:${PATH}" + cd arrow && ARROW_SOURCE_HOME=$(pwd) ./ci/scripts/r_docker_configure.sh + {{ macros.github_setup_local_r_repo(true, false)|indent }} + - name: Install arrow from our repo + env: + ARROW_R_DEV: "TRUE" + LIBARROW_BUILD: "FALSE" + LIBARROW_BINARY: {{ '${{ matrix.config.libarrow_binary }}' }} + ARROW_R_ENFORCE_CHECKSUM: "true" + ARROW_R_CHECKSUM_PATH: "{{ '${{ github.workspace }}' }}/repo/libarrow" + shell: bash + run: | + Rscript -e ' + {{ macros.github_test_r_src_pkg()|indent(8) }} + ' + test-source: + needs: source + name: Test {{ '${{ matrix.platform.name }}' }} source build + runs-on: {{ '${{ matrix.platform.runs_on }}' }} + strategy: + fail-fast: false + matrix: + platform: + - {runs_on: "ubuntu-latest", name: "Linux"} + - {runs_on: "macos-15-intel" , name: "macOS"} + steps: + - name: Install R + uses: r-lib/actions/setup-r@v2 + {{ macros.github_setup_local_r_repo(false, false)|indent }} + {{ macros.github_checkout_arrow(action_v="3")|indent }} + - name: Install sccache + if: matrix.platform.name == 'Linux' + shell: bash + run: | + arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin + - name: Install R package system dependencies (Linux) + if: matrix.platform.name == 'Linux' + run: sudo apt-get install -y libcurl4-openssl-dev libssl-dev libuv1-dev + - name: Install R package system dependencies (macOS) + if: matrix.platform.name == 'macOS' + run: brew install sccache openssl curl libuv + - name: Remove arrow/ + run: | + rm -rf arrow/ + - name: Enable parallel build + run: | + cores=`nproc || sysctl -n hw.logicalcpu` + echo "MAKEFLAGS=-j$cores" >> $GITHUB_ENV + - name: Install arrow source package + env: + # Test source build so be sure not to download a binary + LIBARROW_BINARY: "FALSE" + {{ macros.github_set_sccache_envvars()|indent(8) }} + shell: Rscript {0} + run: | + {{ macros.github_test_r_src_pkg()|indent(8) }} + + - name: Retry with verbosity if that failed + if: failure() + env: + LIBARROW_BINARY: "FALSE" + ARROW_R_DEV: "TRUE" + CMAKE_FIND_DEBUG_MODE: "ON" + {{ macros.github_set_sccache_envvars()|indent(8) }} + shell: Rscript {0} + run: | + {{ macros.github_test_r_src_pkg()|indent(8) }} + + upload-binaries: + # Only upload binaries if all tests pass. + needs: [r-packages, test-source, test-linux-binary] + name: Upload artifacts + runs-on: ubuntu-latest + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + - name: Install R + uses: r-lib/actions/setup-r@v2 + with: + install-r: true + - name: Move libarrow artifacts + run: | + mkdir -p binaries/ + mv artifacts/r-libarrow-*/* binaries/ + - name: Rename artifacts + shell: Rscript {0} + run: | + file_paths <- list.files("artifacts", include.dirs = FALSE, recursive = TRUE) + new_names <- file.path("binaries", sub("/", "__", file_paths)) + dir.create("binaries", showWarnings = FALSE) + file.copy(file.path("artifacts", file_paths), new_names) + + {{ macros.github_upload_releases("binaries/r-*")|indent }} From 84f3b741ba5c803a3a4ad50fdf713f1bf13ef59c Mon Sep 17 00:00:00 2001 From: Steve Martin Date: Sun, 28 Jun 2026 21:46:27 -0400 Subject: [PATCH 88/89] libxml2 warning in with_cloud_support() is now conditional on azure being requested --- r/tools/nixlibs.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R index a92e8eb1610b..3c184f6fe232 100644 --- a/r/tools/nixlibs.R +++ b/r/tools/nixlibs.R @@ -911,6 +911,7 @@ with_wasm_support <- function(env_var_list) { ARROW_DEPENDENCY_USE_SHARED = "OFF", ARROW_ENABLE_THREADING = "OFF", ARROW_GCS = "OFF", + ARROW_AZURE = "OFF", ARROW_JEMALLOC = "OFF", ARROW_MIMALLOC = "OFF", ARROW_S3 = "OFF", @@ -960,7 +961,7 @@ with_cloud_support <- function(env_var_list) { arrow_s3 <- FALSE arrow_gcs <- FALSE arrow_azure <- FALSE - } else if (!cmake_find_package("libxml2", NULL, env_var_list)) { + } else if (arrow_azure && !cmake_find_package("libxml2", NULL, env_var_list)) { print_warning("requires libxml2-devel (rpm), or libxml2-dev (deb), libxml2 (brew)", "AZURE") arrow_azure <- FALSE } From 9dd6793d3749da8aa38335c6e73d7518aedaf691 Mon Sep 17 00:00:00 2001 From: Jonathan Keane Date: Fri, 10 Jul 2026 14:37:24 -0500 Subject: [PATCH 89/89] Add libxml2 --- ci/scripts/r_install_system_dependencies.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/scripts/r_install_system_dependencies.sh b/ci/scripts/r_install_system_dependencies.sh index 955d1d6d58d3..265569b11ecf 100755 --- a/ci/scripts/r_install_system_dependencies.sh +++ b/ci/scripts/r_install_system_dependencies.sh @@ -35,20 +35,22 @@ else apt-get update fi -# Install curl, OpenSSL, and libuv +# Install curl, OpenSSL, libuv, and libxml2 # - curl/OpenSSL: technically only needed for S3/GCS support, but # installing the R curl package fails without it # - libpng: required by the png R package # - libuv: required by the fs R package (no longer bundles libuv by default) +# - libxml2: the bundled Azure C++ SDK links against libxml2, so the final +# link of arrow.so needs it (even when installing a prebuilt libarrow binary) case "$PACKAGE_MANAGER" in apt-get) - apt-get install -y libcurl4-openssl-dev libpng-dev libssl-dev libuv1-dev + apt-get install -y libcurl4-openssl-dev libpng-dev libssl-dev libuv1-dev libxml2-dev ;; apk) - $PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev + $PACKAGE_MANAGER add curl-dev openssl-dev libuv-dev libxml2-dev ;; *) - $PACKAGE_MANAGER install -y libcurl-devel openssl-devel libuv-devel + $PACKAGE_MANAGER install -y libcurl-devel openssl-devel libuv-devel libxml2-devel ;; esac