diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff3989c..a4ecd3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,6 @@ name: Check and test -on: +on: workflow_dispatch: pull_request: push: @@ -12,8 +12,8 @@ jobs: name: build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 - name: "build all" # We keep these separate since sometimes the derive fails when # independently built. @@ -26,9 +26,9 @@ jobs: name: Clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 + with: components: clippy - name: "clippy --all" run: cargo clippy --all --tests -- -D warnings @@ -37,9 +37,9 @@ jobs: name: fmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 + with: components: rustfmt - name: Run run: cargo fmt --all -- --check @@ -48,9 +48,9 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - - uses: Swatinem/rust-cache@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 + - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@nextest - name: Run run: | @@ -60,17 +60,17 @@ jobs: coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 - with: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@1.85.0 + with: components: llvm-tools-preview - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - - uses: Swatinem/rust-cache@v1 + - uses: Swatinem/rust-cache@v2 - name: Generate code coverage run: cargo llvm-cov --lcov --output-path lcov.info - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos files: lcov.info diff --git a/Cargo.toml b/Cargo.toml index 30b3a1e..2a68fcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,9 +18,9 @@ repository = "https://github.com/Swoorup/arrow-convert" arrow_convert = { path = "arrow_convert", version = "0.11.4" } arrow_convert_derive = { path = "arrow_convert_derive", version = "0.11.4" } -arrow = { version = "57", default-features = false } arrow-array = { version = "57" } arrow-buffer = { version = "57" } +arrow-cast = { version = "57" } arrow-data = { version = "57" } arrow-schema = { version = "57" } chrono = { version = "0.4", default-features = false } diff --git a/README.md b/README.md index 05de141..9dc7026 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The example below performs a round trip conversion of a struct with a single fie Please see the [complex_example.rs](https://github.com/Swoorup/arrow-convert/blob/main/arrow_convert/tests/complex_example.rs) for usage of the full functionality. ```rust -use arrow::array::{Array, ArrayRef}; +use arrow_array::{Array, ArrayRef, StructArray}; use arrow_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField, ArrowSerialize, ArrowDeserialize}; #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)] @@ -30,8 +30,7 @@ let original_array = [ let arrow_array: ArrayRef = original_array.try_into_arrow().unwrap(); // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. -// supported by `arrow` -let struct_array= arrow_array.as_any().downcast_ref::().unwrap(); +let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 3); // deserialize back to our original vector via TryIntoCollection trait. @@ -154,7 +153,8 @@ struct S { A `vec` can be converted. to/from arrow by using the `arrow_serialize_to_mutable_array` and `arrow_array_deserialize_iterator_as_type` methods. ```rust -use arrow::array::{Array, ArrayBuilder, ArrayRef}; +use arrow_array::{Array, ArrayRef}; +use arrow_array::builder::ArrayBuilder; use arrow_convert::serialize::arrow_serialize_to_mutable_array; use arrow_convert::deserialize::arrow_array_deserialize_iterator_as_type; use arrow_convert::field::I128; diff --git a/arrow_convert/Cargo.toml b/arrow_convert/Cargo.toml index 924aa68..aa30e00 100644 --- a/arrow_convert/Cargo.toml +++ b/arrow_convert/Cargo.toml @@ -20,6 +20,7 @@ uuid = ["dep:uuid", "arrow-schema/canonical_extension_types"] [dependencies] arrow-array = { workspace = true } arrow-buffer = { workspace = true } +arrow-cast = { workspace = true } arrow-data = { workspace = true } arrow-schema = { workspace = true } arrow_convert_derive = { workspace = true, optional = true } @@ -34,7 +35,9 @@ tinystr = { workspace = true, optional = true } uuid = { workspace = true, optional = true } [dev-dependencies] -arrow = { workspace = true } +arrow-array = { workspace = true } +arrow-buffer = { workspace = true } +arrow-schema = { workspace = true } arrow_convert_derive = { workspace = true } criterion = { workspace = true } glam = { workspace = true } diff --git a/arrow_convert/benches/bench.rs b/arrow_convert/benches/bench.rs index 3aa6841..40c967a 100644 --- a/arrow_convert/benches/bench.rs +++ b/arrow_convert/benches/bench.rs @@ -1,11 +1,9 @@ -use arrow::{ - array::ArrayRef, - buffer::{Buffer, ScalarBuffer}, -}; +use arrow_array::ArrayRef; +use arrow_buffer::{Buffer, ScalarBuffer}; use arrow_convert::{ - deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize, + ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow, }; -use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; // Arrow stores U8 arrays as `arrow::array::BinaryArray` #[derive(ArrowField, ArrowSerialize, ArrowDeserialize)] diff --git a/arrow_convert/src/deserialize/iterable.rs b/arrow_convert/src/deserialize/iterable.rs index fd2b684..b815559 100644 --- a/arrow_convert/src/deserialize/iterable.rs +++ b/arrow_convert/src/deserialize/iterable.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use super::{BufferBinaryArray, BufferBinaryArrayIter}; -use arrow_array::{iterator::*, ArrowPrimitiveType, *}; +use arrow_array::{ArrowPrimitiveType, iterator::*, *}; /// A trait for Arrow arrays that can be transformed into an iterator. pub trait ArrowArrayIterable { diff --git a/arrow_convert/src/deserialize/mod.rs b/arrow_convert/src/deserialize/mod.rs index b1178e9..4772c3c 100644 --- a/arrow_convert/src/deserialize/mod.rs +++ b/arrow_convert/src/deserialize/mod.rs @@ -2,7 +2,7 @@ mod iterable; pub use iterable::*; -use arrow_array::{types, ArrowPrimitiveType, *}; +use arrow_array::{ArrowPrimitiveType, types, *}; use arrow_buffer::{ArrowNativeType, Buffer, ScalarBuffer}; use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc}; diff --git a/arrow_convert/src/features/glam.rs b/arrow_convert/src/features/glam.rs index afd68f6..cb91f35 100644 --- a/arrow_convert/src/features/glam.rs +++ b/arrow_convert/src/features/glam.rs @@ -7,9 +7,9 @@ use crate::serialize::ArrowSerialize; use arrow_schema::Field; use crate::deserialize::arrow_deserialize_vec_helper; -use arrow_array::builder::{BooleanBuilder, Float32Builder, Float64Builder}; use arrow_array::ArrayRef; -use arrow_array::{builder::FixedSizeListBuilder, FixedSizeListArray}; +use arrow_array::builder::{BooleanBuilder, Float32Builder, Float64Builder}; +use arrow_array::{FixedSizeListArray, builder::FixedSizeListBuilder}; use std::sync::Arc; /// This macro implements the `ArrowSerialize` and `ArrowDeserialize` traits for a given `glam` vector or matrix type. diff --git a/arrow_convert/src/features/rust_decimal.rs b/arrow_convert/src/features/rust_decimal.rs index 878e85d..1f64a73 100644 --- a/arrow_convert/src/features/rust_decimal.rs +++ b/arrow_convert/src/features/rust_decimal.rs @@ -3,10 +3,10 @@ use crate::deserialize::ArrowDeserialize; use crate::field::ArrowField; use crate::serialize::ArrowSerialize; -use arrow_schema::{DataType, DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE}; +use arrow_schema::{DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION, DataType}; use rust_decimal::Decimal; -use arrow_array::{builder::Decimal128Builder, Decimal128Array}; +use arrow_array::{Decimal128Array, builder::Decimal128Builder}; impl ArrowField for Decimal { type Type = Decimal; diff --git a/arrow_convert/src/features/tinystr.rs b/arrow_convert/src/features/tinystr.rs index 7d643ef..2790fcb 100644 --- a/arrow_convert/src/features/tinystr.rs +++ b/arrow_convert/src/features/tinystr.rs @@ -5,7 +5,7 @@ use crate::deserialize::ArrowDeserialize; use crate::field::ArrowField; use crate::serialize::ArrowSerialize; -use arrow_array::{builder::FixedSizeBinaryBuilder, FixedSizeBinaryArray}; +use arrow_array::{FixedSizeBinaryArray, builder::FixedSizeBinaryBuilder}; impl ArrowField for TinyAsciiStr { type Type = Self; diff --git a/arrow_convert/src/features/uuid.rs b/arrow_convert/src/features/uuid.rs index 92b0d09..679f6ba 100644 --- a/arrow_convert/src/features/uuid.rs +++ b/arrow_convert/src/features/uuid.rs @@ -8,9 +8,9 @@ use crate::deserialize::ArrowDeserialize; use crate::field::ArrowField; use crate::serialize::ArrowSerialize; use crate::serialize::PushNull; -use arrow_array::builder::FixedSizeBinaryBuilder; use arrow_array::ArrayRef; use arrow_array::FixedSizeBinaryArray; +use arrow_array::builder::FixedSizeBinaryBuilder; use arrow_schema::DataType; use uuid::Uuid; diff --git a/arrow_convert/src/field.rs b/arrow_convert/src/field.rs index 7f58881..1c07b91 100644 --- a/arrow_convert/src/field.rs +++ b/arrow_convert/src/field.rs @@ -93,7 +93,7 @@ pub trait ArrowField { #[inline] #[doc(hidden)] /// For internal use and not meant to be reimplemented. - /// returns the [`arrow::datatypes::Field`] for this field + /// returns the [`arrow_schema::Field`] for this field fn field(name: &str) -> Field { Field::new(name, Self::data_type(), Self::is_nullable()) } diff --git a/arrow_convert/src/lib.rs b/arrow_convert/src/lib.rs index b21405a..850df20 100644 --- a/arrow_convert/src/lib.rs +++ b/arrow_convert/src/lib.rs @@ -6,6 +6,14 @@ pub mod deserialize; pub mod field; pub mod serialize; +#[doc(hidden)] +pub mod _private { + pub use arrow_array; + pub use arrow_buffer; + pub use arrow_cast; + pub use arrow_schema; +} + // The proc macro is implemented in derive_internal, and re-exported by this // crate. This is because a single crate can not define both a proc macro and a // macro_rules macro. diff --git a/arrow_convert/tests/complex_example.rs b/arrow_convert/tests/complex_example.rs index 33642f3..7bffe1d 100644 --- a/arrow_convert/tests/complex_example.rs +++ b/arrow_convert/tests/complex_example.rs @@ -1,6 +1,5 @@ -use arrow::array::*; -use arrow::datatypes::DataType; -use arrow_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoCollection}; +use arrow_array::*; +use arrow_convert::deserialize::{TryIntoCollection, arrow_array_deserialize_iterator}; use arrow_convert::field::ArrowField; use arrow_convert::serialize::TryIntoArrow; /// Complex example that uses the following features: @@ -8,6 +7,7 @@ use arrow_convert::serialize::TryIntoArrow; /// - Deeply Nested structs and lists /// - Custom types use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; +use arrow_schema::DataType; use chrono::DateTime; use std::borrow::Borrow; @@ -88,13 +88,13 @@ impl arrow_convert::field::ArrowField for CustomType { type Type = Self; #[inline] - fn data_type() -> arrow::datatypes::DataType { - arrow::datatypes::DataType::UInt64 + fn data_type() -> arrow_schema::DataType { + arrow_schema::DataType::UInt64 } } impl arrow_convert::serialize::ArrowSerialize for CustomType { - type ArrayBuilderType = arrow::array::UInt64Builder; + type ArrayBuilderType = arrow_array::builder::UInt64Builder; #[inline] fn new_array() -> Self::ArrayBuilderType { @@ -102,14 +102,14 @@ impl arrow_convert::serialize::ArrowSerialize for CustomType { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> arrow::error::Result<()> { + fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> Result<(), arrow_schema::ArrowError> { array.append_option(Some(v.0)); Ok(()) } } impl arrow_convert::deserialize::ArrowDeserialize for CustomType { - type ArrayType = arrow::array::UInt64Array; + type ArrayType = arrow_array::UInt64Array; #[inline] fn arrow_deserialize(v: Option) -> Option { @@ -217,13 +217,13 @@ fn item2() -> Root { } #[test] -fn test_round_trip() -> arrow::error::Result<()> { +fn test_round_trip() -> Result<(), arrow_schema::ArrowError> { // serialize to an arrow array let original_array = [item1(), item2()]; let array: ArrayRef = original_array.try_into_arrow()?; - let struct_array = array.as_any().downcast_ref::().unwrap(); + let struct_array = array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 2); let values = struct_array.columns(); diff --git a/arrow_convert/tests/simple_example.rs b/arrow_convert/tests/simple_example.rs index 0a5210e..ee2b744 100644 --- a/arrow_convert/tests/simple_example.rs +++ b/arrow_convert/tests/simple_example.rs @@ -1,9 +1,9 @@ use std::sync::Arc; /// Simple example -use arrow::array::Array; +use arrow_array::Array; use arrow_convert::{ - deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize, + ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow, }; #[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)] @@ -30,10 +30,7 @@ fn test_simple_roundtrip() { // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. // supported by `arrow` - let struct_array = arrow_array - .as_any() - .downcast_ref::() - .unwrap(); + let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 3); // deserialize back to our original vector via TryIntoCollection trait. diff --git a/arrow_convert/tests/test_array.rs b/arrow_convert/tests/test_array.rs index daa6e46..6942c35 100644 --- a/arrow_convert/tests/test_array.rs +++ b/arrow_convert/tests/test_array.rs @@ -1,5 +1,4 @@ -use arrow::array::Array; -use arrow::array::ArrayRef; +use arrow_array::{Array, ArrayRef}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::TryIntoArrow; /// Simple example @@ -56,10 +55,7 @@ fn test_simple_roundtrip() { // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. // supported by `arrow` - let struct_array = arrow_array - .as_any() - .downcast_ref::() - .unwrap(); + let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 3); // deserialize back to our original vector via TryIntoCollection trait. diff --git a/arrow_convert/tests/test_datetime.rs b/arrow_convert/tests/test_datetime.rs index c93666a..9b0dc9b 100644 --- a/arrow_convert/tests/test_datetime.rs +++ b/arrow_convert/tests/test_datetime.rs @@ -1,13 +1,12 @@ //! Tests for `chrono::DateTime` support. use std::sync::Arc; -use arrow::array::ArrayRef; -use arrow::datatypes::*; -use arrow::error::Result; +use arrow_array::ArrayRef; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::field::{ArrowField, DEFAULT_FIELD_NAME}; use arrow_convert::serialize::TryIntoArrow; use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; +use arrow_schema::*; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use pretty_assertions::assert_eq; @@ -98,8 +97,8 @@ fn test_naive_vs_utc_schema_differs() { ); } -fn data_mismatch_error() -> arrow::error::ArrowError { - arrow::error::ArrowError::InvalidArgumentError(format!( +fn data_mismatch_error() -> ArrowError { + ArrowError::InvalidArgumentError(format!( "Data type mismatch. Expected type={:#?} is_nullable={}, but was type={:#?} is_nullable={}", Expected::data_type(), Expected::is_nullable(), @@ -116,7 +115,7 @@ fn test_datetime_utc_to_naive_mismatch_error() { let arrow_array: ArrayRef = original_array.try_into_arrow().unwrap(); // Attempting to deserialize as NaiveDateTime should fail - let result: Result> = arrow_array.try_into_collection(); + let result: Result, ArrowError> = arrow_array.try_into_collection(); assert_eq!( result.unwrap_err().to_string(), @@ -135,7 +134,7 @@ fn test_naive_to_datetime_utc_mismatch_error() { let arrow_array: ArrayRef = original_array.try_into_arrow().unwrap(); // Attempting to deserialize as DateTime should fail - let result: Result>> = arrow_array.try_into_collection(); + let result: Result>, ArrowError> = arrow_array.try_into_collection(); assert_eq!( result.unwrap_err().to_string(), @@ -288,7 +287,7 @@ fn test_struct_datetime_mismatch() { let arrow_array: ArrayRef = original.try_into_arrow().unwrap(); // Attempting to deserialize as WithNaive should fail - let result: Result> = arrow_array.try_into_collection(); + let result: Result, ArrowError> = arrow_array.try_into_collection(); assert_eq!( result.unwrap_err().to_string(), diff --git a/arrow_convert/tests/test_deserialize.rs b/arrow_convert/tests/test_deserialize.rs index 4942f53..0c7acf7 100644 --- a/arrow_convert/tests/test_deserialize.rs +++ b/arrow_convert/tests/test_deserialize.rs @@ -1,12 +1,12 @@ -use arrow::buffer::ScalarBuffer; -use arrow::error::Result; -use arrow::{array::*, buffer::Buffer}; +use arrow_array::*; +use arrow_buffer::{Buffer, ScalarBuffer}; use arrow_convert::field::ArrowField; -use arrow_convert::{deserialize::*, serialize::*, ArrowDeserialize, ArrowField, ArrowSerialize}; +use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::*, serialize::*}; +use arrow_schema::ArrowError; #[test] fn test_deserialize_iterator() { - use arrow::array::*; + use arrow_array::*; use arrow_convert::deserialize::*; use arrow_convert::serialize::*; use std::borrow::Borrow; @@ -32,8 +32,8 @@ fn test_deserialize_iterator() { } } -fn data_mismatch_error() -> arrow::error::ArrowError { - arrow::error::ArrowError::InvalidArgumentError(format!( +fn data_mismatch_error() -> ArrowError { + ArrowError::InvalidArgumentError(format!( "Data type mismatch. Expected type={:#?} is_nullable={}, but was type={:#?} is_nullable={}", Expected::data_type(), Expected::is_nullable(), @@ -55,7 +55,7 @@ fn test_deserialize_schema_mismatch_error() { let arr1 = vec![S1 { a: 1 }, S1 { a: 2 }]; let arr1: ArrayRef = arr1.try_into_arrow().unwrap(); - let result: Result> = arr1.try_into_collection(); + let result: Result, ArrowError> = arr1.try_into_collection(); assert_eq!( result.unwrap_err().to_string(), data_mismatch_error::().to_string() @@ -63,7 +63,7 @@ fn test_deserialize_schema_mismatch_error() { let arr1 = vec![S1 { a: 1 }, S1 { a: 2 }]; let arr1: ArrayRef = arr1.try_into_arrow().unwrap(); - let result: Result> = arr1.try_into_collection_as_type::(); + let result: Result, ArrowError> = arr1.try_into_collection_as_type::(); assert_eq!( result.unwrap_err().to_string(), data_mismatch_error::().to_string() @@ -85,7 +85,7 @@ fn test_deserialize_large_types_schema_mismatch_error() { let arr1 = vec![S1 { a: "123".to_string() }, S1 { a: "333".to_string() }]; let arr1: ArrayRef = arr1.try_into_arrow().unwrap(); - let result: Result> = arr1.try_into_collection(); + let result: Result, ArrowError> = arr1.try_into_collection(); assert_eq!( result.unwrap_err().to_string(), data_mismatch_error::().to_string() diff --git a/arrow_convert/tests/test_enum.rs b/arrow_convert/tests/test_enum.rs index 92846e2..bf218af 100644 --- a/arrow_convert/tests/test_enum.rs +++ b/arrow_convert/tests/test_enum.rs @@ -1,7 +1,8 @@ -use arrow::{array::*, datatypes::*}; +use arrow_array::*; use arrow_convert::{ - deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize, + ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow, }; +use arrow_schema::*; use pretty_assertions::assert_eq; #[test] diff --git a/arrow_convert/tests/test_flatten_record_batch.rs b/arrow_convert/tests/test_flatten_record_batch.rs index ce71864..6415de3 100644 --- a/arrow_convert/tests/test_flatten_record_batch.rs +++ b/arrow_convert/tests/test_flatten_record_batch.rs @@ -1,7 +1,7 @@ -use arrow::datatypes::{Field, Schema}; -use arrow::record_batch::RecordBatch; -use arrow::{array::*, datatypes::DataType}; -use arrow_convert::{serialize::*, ArrowField, ArrowSerialize}; +use arrow_array::RecordBatch; +use arrow_array::*; +use arrow_convert::{ArrowField, ArrowSerialize, serialize::*}; +use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; #[test] diff --git a/arrow_convert/tests/test_glam.rs b/arrow_convert/tests/test_glam.rs index 7c75af9..cf12a5d 100644 --- a/arrow_convert/tests/test_glam.rs +++ b/arrow_convert/tests/test_glam.rs @@ -3,7 +3,7 @@ fn test_vec3_roundtrip() { use glam::*; - use arrow::array::{Array, ArrayRef}; + use arrow_array::{Array, ArrayRef}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::TryIntoArrow; use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; @@ -76,11 +76,8 @@ fn test_vec3_roundtrip() { // Serialize to Arrow let arrow_array: ArrayRef = original.try_into_arrow().unwrap(); - assert!(arrow_array.as_any().is::()); - let struct_array = arrow_array - .as_any() - .downcast_ref::() - .unwrap(); + assert!(arrow_array.as_any().is::()); + let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); // Verify the number of fields in the struct assert_eq!(struct_array.num_columns(), 15); @@ -95,7 +92,7 @@ fn test_vec3_roundtrip() { #[cfg(feature = "glam")] #[test] fn test_glam_edge_values() { - use arrow::array::{Array, ArrayRef}; + use arrow_array::{Array, ArrayRef}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::TryIntoArrow; use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize}; @@ -174,11 +171,8 @@ fn test_glam_edge_values() { }]; let arrow_array: ArrayRef = original.try_into_arrow().expect("Failed to convert to Arrow array"); - assert!(arrow_array.as_any().is::()); - let struct_array = arrow_array - .as_any() - .downcast_ref::() - .unwrap(); + assert!(arrow_array.as_any().is::()); + let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.num_columns(), 15); diff --git a/arrow_convert/tests/test_round_trip.rs b/arrow_convert/tests/test_round_trip.rs index e7f3fa1..116a8cb 100644 --- a/arrow_convert/tests/test_round_trip.rs +++ b/arrow_convert/tests/test_round_trip.rs @@ -1,14 +1,14 @@ -use arrow::array::*; -use arrow::datatypes::*; +use arrow_array::*; use arrow_convert::deserialize::arrow_array_deserialize_iterator_as_type; use arrow_convert::deserialize::*; use arrow_convert::field::DEFAULT_FIELD_NAME; -use arrow_convert::field::{LargeBinary, I128}; +use arrow_convert::field::{I128, LargeBinary}; use arrow_convert::serialize::*; use arrow_convert::{ - field::{FixedSizeBinary, FixedSizeVec, LargeString, LargeVec}, ArrowDeserialize, ArrowField, ArrowSerialize, + field::{FixedSizeBinary, FixedSizeVec, LargeString, LargeVec}, }; +use arrow_schema::*; use half::f16; use std::sync::Arc; diff --git a/arrow_convert/tests/test_rust_decimal.rs b/arrow_convert/tests/test_rust_decimal.rs index 50e515e..65342fc 100644 --- a/arrow_convert/tests/test_rust_decimal.rs +++ b/arrow_convert/tests/test_rust_decimal.rs @@ -1,11 +1,10 @@ #[cfg(feature = "rust_decimal")] #[test] fn test_decimal_roundtrip() { - use arrow::array::{Array, ArrayRef}; - use arrow::datatypes::DECIMAL_DEFAULT_SCALE; - use arrow::{array::Decimal128Array, datatypes::DECIMAL128_MAX_PRECISION}; + use arrow_array::{Array, ArrayRef, Decimal128Array}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::*; + use arrow_schema::{DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION}; use pretty_assertions::assert_eq; use rust_decimal::Decimal; @@ -41,11 +40,10 @@ fn test_decimal_roundtrip() { #[cfg(feature = "rust_decimal")] #[test] fn test_decimal_edge_values() { - use arrow::array::{Array, ArrayRef}; - use arrow::datatypes::DECIMAL_DEFAULT_SCALE; - use arrow::{array::Decimal128Array, datatypes::DECIMAL128_MAX_PRECISION}; + use arrow_array::{Array, ArrayRef, Decimal128Array}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::*; + use arrow_schema::{DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION}; use rust_decimal::Decimal; let original: Vec = vec![ diff --git a/arrow_convert/tests/test_schema.rs b/arrow_convert/tests/test_schema.rs index bffb9bb..3c75903 100644 --- a/arrow_convert/tests/test_schema.rs +++ b/arrow_convert/tests/test_schema.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use arrow::datatypes::*; use arrow_convert::{ - field::{with_list_element_metadata, with_list_element_name, DEFAULT_FIELD_NAME}, - serialize::TryIntoArrow, ArrowField, ArrowSerialize, + field::{DEFAULT_FIELD_NAME, with_list_element_metadata, with_list_element_name}, + serialize::TryIntoArrow, }; +use arrow_schema::*; use pretty_assertions::assert_eq; #[test] @@ -460,10 +460,10 @@ fn test_serialize_respects_list_element_name_and_metadata() { } let rows = vec![Root { bids: vec![1, 2, 3] }]; - let array: arrow::array::ArrayRef = rows.try_into_arrow().expect("serialization should succeed"); + let array: arrow_array::ArrayRef = rows.try_into_arrow().expect("serialization should succeed"); let struct_array = array .as_any() - .downcast_ref::() + .downcast_ref::() .expect("expected StructArray"); let fields = struct_array.fields(); diff --git a/arrow_convert/tests/test_serialize.rs b/arrow_convert/tests/test_serialize.rs index 4edf57b..4ed5bd0 100644 --- a/arrow_convert/tests/test_serialize.rs +++ b/arrow_convert/tests/test_serialize.rs @@ -1,13 +1,12 @@ -use arrow::array::{Array, ArrayRef}; -use arrow::buffer::{Buffer, ScalarBuffer}; -use arrow::record_batch::RecordBatch; +use arrow_array::{Array, ArrayRef, RecordBatch}; +use arrow_buffer::{Buffer, ScalarBuffer}; use arrow_convert::field::{ArrowField, FixedSizeBinary}; use arrow_convert::serialize::*; #[test] fn test_error_exceed_fixed_size_binary() { let strs = [b"abc".to_vec()]; - let r: arrow::error::Result = strs.try_into_arrow_as_type::>(); + let r: Result = strs.try_into_arrow_as_type::>(); assert!(r.is_err()) } diff --git a/arrow_convert/tests/test_struct.rs b/arrow_convert/tests/test_struct.rs index cf9b1fc..5b9c935 100644 --- a/arrow_convert/tests/test_struct.rs +++ b/arrow_convert/tests/test_struct.rs @@ -1,5 +1,5 @@ -use arrow::array::*; -use arrow_convert::{deserialize::*, serialize::*, ArrowDeserialize, ArrowField, ArrowSerialize}; +use arrow_array::*; +use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::*, serialize::*}; #[test] fn test_nested_optional_struct_array() { diff --git a/arrow_convert/tests/test_tinystr.rs b/arrow_convert/tests/test_tinystr.rs index c8cdcac..11d1bfe 100644 --- a/arrow_convert/tests/test_tinystr.rs +++ b/arrow_convert/tests/test_tinystr.rs @@ -1,7 +1,7 @@ #[cfg(feature = "tinystr")] #[test] fn test_tinyasciistr_roundtrip() { - use arrow::array::{Array, ArrayRef, FixedSizeBinaryArray}; + use arrow_array::{Array, ArrayRef, FixedSizeBinaryArray}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::TryIntoArrow; use tinystr::TinyAsciiStr; @@ -30,7 +30,7 @@ fn test_tinyasciistr_roundtrip() { #[cfg(feature = "tinystr")] #[test] fn test_tinyasciistr_max_length() { - use arrow::array::{Array, ArrayRef, FixedSizeBinaryArray}; + use arrow_array::{Array, ArrayRef, FixedSizeBinaryArray}; use arrow_convert::deserialize::TryIntoCollection; use arrow_convert::serialize::TryIntoArrow; use tinystr::TinyAsciiStr; diff --git a/arrow_convert/tests/test_uuid.rs b/arrow_convert/tests/test_uuid.rs index 9629a20..9fa2a4e 100644 --- a/arrow_convert/tests/test_uuid.rs +++ b/arrow_convert/tests/test_uuid.rs @@ -1,7 +1,6 @@ #[cfg(feature = "uuid")] mod uuid { - use arrow::array::{Array, ArrayRef}; - use arrow::record_batch::RecordBatch; + use arrow_array::{Array, ArrayRef, RecordBatch}; use arrow_convert::deserialize::*; use arrow_convert::field::ArrowField; use arrow_convert::serialize::*; @@ -16,10 +15,7 @@ mod uuid { let array: ArrayRef = uuids.iter().collect::>().try_into_arrow().unwrap(); assert_eq!(array.len(), 2); assert_eq!(array.data_type(), &::data_type()); - assert_eq!( - array.data_type(), - &arrow::datatypes::DataType::FixedSizeBinary(16) - ); + assert_eq!(array.data_type(), &arrow_schema::DataType::FixedSizeBinary(16)); let rb: RecordBatch = uuids.iter().collect::>().try_into_arrow().unwrap(); assert_eq!(rb.num_rows(), 2); diff --git a/arrow_convert_derive/src/derive_enum.rs b/arrow_convert_derive/src/derive_enum.rs index 65490f3..3d50f3e 100644 --- a/arrow_convert_derive/src/derive_enum.rs +++ b/arrow_convert_derive/src/derive_enum.rs @@ -1,5 +1,5 @@ -use proc_macro2::TokenStream; use proc_macro_error2::abort; +use proc_macro2::TokenStream; use quote::{quote, quote_spanned}; use syn::spanned::Spanned; @@ -26,9 +26,9 @@ impl<'a> From<&'a DeriveEnum> for Common<'a> { let variants = &input.variants; let union_type = if is_dense { - quote!(arrow::datatypes::UnionMode::Dense) + quote!(arrow_convert::_private::arrow_schema::UnionMode::Dense) } else { - quote!(arrow::datatypes::UnionMode::Sparse) + quote!(arrow_convert::_private::arrow_schema::UnionMode::Sparse) }; let variant_names = variants.iter().map(|v| v.syn.ident.clone()).collect::>(); @@ -93,9 +93,9 @@ pub fn expand_field(input: DeriveEnum) -> TokenStream { impl arrow_convert::field::ArrowField for #original_name { type Type = Self; - fn data_type() -> arrow::datatypes::DataType { - arrow::datatypes::DataType::Union( - arrow::datatypes::UnionFields::new( + fn data_type() -> arrow_convert::_private::arrow_schema::DataType { + arrow_convert::_private::arrow_schema::DataType::Union( + arrow_convert::_private::arrow_schema::UnionFields::new( 0..#num_variants, // basically union tag id or here called type_id vec![ #( @@ -137,8 +137,8 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { quote! { offsets: Vec, }, quote! { offsets: vec![], }, quote! { self.offsets.reserve(additional); }, - quote! { Some(arrow::buffer::ScalarBuffer::from_iter(std::mem::take(&mut self.offsets))) }, - quote! { Some(self.offsets.iter().cloned().collect::>()) }, + quote! { Some(arrow_convert::_private::arrow_buffer::ScalarBuffer::from_iter(std::mem::take(&mut self.offsets))) }, + quote! { Some(self.offsets.iter().cloned().collect::>()) }, quote! { self.offsets.shrink_to_fit(); }, ) } else { @@ -240,7 +240,7 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { #( #variant_names: #mutable_variant_array_types, )* - data_type: arrow::datatypes::DataType, + data_type: arrow_convert::_private::arrow_schema::DataType, type_ids: Vec, #offsets_decl } @@ -257,7 +257,7 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { } } - fn data_type(&self) -> &arrow::datatypes::DataType { + fn data_type(&self) -> &arrow_convert::_private::arrow_schema::DataType { &self.data_type } @@ -266,7 +266,7 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { self.try_push(None::<#original_name>).unwrap(); } - fn validity(&self) -> Option<&arrow::array::BooleanBufferBuilder> { + fn validity(&self) -> Option<&arrow_convert::_private::arrow_array::builder::BooleanBufferBuilder> { None } @@ -280,7 +280,8 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { #offsets_reserve } - fn try_push(&mut self, item: Option>) -> arrow::error::Result<()> { + fn try_push(&mut self, item: Option>) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { + use arrow_convert::_private::arrow_array::builder::ArrayBuilder; use arrow_convert::serialize::{ArrowSerialize, PushNull}; match item { Some(i) => { @@ -297,8 +298,8 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { Ok(()) } - // fn try_extend>>(&mut self, iter: impl arrow_convert::deserialize::ArrowArrayIterable>>) -> arrow::error::Result<()> { - fn try_extend(&mut self, iter: impl IntoIterator>>) -> arrow::error::Result<()> { + // fn try_extend>>(&mut self, iter: impl arrow_convert::deserialize::ArrowArrayIterable>>) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { + fn try_extend(&mut self, iter: impl IntoIterator>>) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { use arrow_convert::serialize::PushNull; for i in iter { self.try_push(i)?; @@ -328,6 +329,7 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { let array_push_null_impl = quote! { impl arrow_convert::serialize::PushNull for #mutable_array_name { fn push_null(&mut self) { + use arrow_convert::_private::arrow_array::builder::ArrayBuilder; use arrow_convert::serialize::PushNull; #push_null_impl } @@ -343,25 +345,25 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { }; let array_mutable_array_impl = quote! { - impl arrow::array::ArrayBuilder for #mutable_array_name { + impl arrow_convert::_private::arrow_array::builder::ArrayBuilder for #mutable_array_name { fn len(&self) -> usize { self.type_ids.len() } - fn finish(&mut self) -> arrow::array::ArrayRef { - let arrow::datatypes::DataType::Union(union_fields, _) = + fn finish(&mut self) -> arrow_convert::_private::arrow_array::ArrayRef { + let arrow_convert::_private::arrow_schema::DataType::Union(union_fields, _) = <#original_name as arrow_convert::field::ArrowField>::data_type() .clone() else { panic!("datatype is not a union") }; let children = vec![#( - <#mutable_variant_array_types as arrow::array::ArrayBuilder>::finish(&mut self.#variant_names), + <#mutable_variant_array_types as arrow_convert::_private::arrow_array::builder::ArrayBuilder>::finish(&mut self.#variant_names), )*]; - let type_ids = arrow::buffer::ScalarBuffer::from_iter(std::mem::take(&mut self.type_ids)); + let type_ids = arrow_convert::_private::arrow_buffer::ScalarBuffer::from_iter(std::mem::take(&mut self.type_ids)); - std::sync::Arc::new(arrow::array::UnionArray::try_new( + std::sync::Arc::new(arrow_convert::_private::arrow_array::UnionArray::try_new( union_fields, type_ids, #offsets_take, @@ -369,20 +371,20 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { ).unwrap()) } - fn finish_cloned(&self) -> arrow::array::ArrayRef { - let arrow::datatypes::DataType::Union(union_fields, _) = + fn finish_cloned(&self) -> arrow_convert::_private::arrow_array::ArrayRef { + let arrow_convert::_private::arrow_schema::DataType::Union(union_fields, _) = <#original_name as arrow_convert::field::ArrowField>::data_type() .clone() else { panic!("datatype is not a union") }; let children = vec![#( - <#mutable_variant_array_types as arrow::array::ArrayBuilder>::finish_cloned(&self.#variant_names), + <#mutable_variant_array_types as arrow_convert::_private::arrow_array::builder::ArrayBuilder>::finish_cloned(&self.#variant_names), )*]; - let type_ids = self.type_ids.iter().cloned().collect::>(); + let type_ids = self.type_ids.iter().cloned().collect::>(); - std::sync::Arc::new(arrow::array::UnionArray::try_new( + std::sync::Arc::new(arrow_convert::_private::arrow_array::UnionArray::try_new( union_fields, type_ids, #offsets_clone, @@ -415,7 +417,7 @@ pub fn expand_serialize(input: DeriveEnum) -> TokenStream { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> arrow::error::Result<()> { + fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { array.try_push(Some(v)) } } @@ -484,16 +486,16 @@ pub fn expand_deserialize(input: DeriveEnum) -> TokenStream { let array_impl = quote! { impl arrow_convert::deserialize::ArrowArray for #array_name { - type BaseArrayType = arrow::array::UnionArray; + type BaseArrayType = arrow_convert::_private::arrow_array::UnionArray; #[inline] - fn iter_from_array_ref<'a>(b: &'a dyn arrow::array::Array) -> ::Iter<'a> + fn iter_from_array_ref<'a>(b: &'a dyn arrow_convert::_private::arrow_array::Array) -> ::Iter<'a> { - let arr = b.as_any().downcast_ref::().unwrap(); + let arr = b.as_any().downcast_ref::().unwrap(); #iterator_name { arr, - index_iter: 0..arrow::array::Array::len(&arr), + index_iter: 0..arrow_convert::_private::arrow_array::Array::len(&arr), } } } @@ -514,7 +516,7 @@ pub fn expand_deserialize(input: DeriveEnum) -> TokenStream { let array_iterator_decl = quote! { #[allow(non_snake_case)] #visibility struct #iterator_name<'a> { - arr: &'a arrow::array::UnionArray, + arr: &'a arrow_convert::_private::arrow_array::UnionArray, index_iter: std::ops::Range, } }; diff --git a/arrow_convert_derive/src/derive_struct.rs b/arrow_convert_derive/src/derive_struct.rs index c97fa27..0dfcf82 100644 --- a/arrow_convert_derive/src/derive_struct.rs +++ b/arrow_convert_derive/src/derive_struct.rs @@ -1,5 +1,5 @@ -use proc_macro2::TokenStream; use proc_macro_error2::abort; +use proc_macro2::TokenStream; use quote::{format_ident, quote, quote_spanned}; use syn::spanned::Spanned; @@ -157,8 +157,8 @@ pub fn expand_field(input: DeriveStruct) -> TokenStream { } else { quote! { impl #original_name { - pub fn arrow_schema() -> arrow::datatypes::Schema { - arrow::datatypes::Schema::new(vec![ + pub fn arrow_schema() -> arrow_convert::_private::arrow_schema::Schema { + arrow_convert::_private::arrow_schema::Schema::new(vec![ #( { let field = <#field_types as arrow_convert::field::ArrowField>::field(#field_names); @@ -191,7 +191,9 @@ pub fn expand_field(input: DeriveStruct) -> TokenStream { <#ty as arrow_convert::field::ArrowField>::data_type() ) } else { - quote!(arrow::datatypes::DataType::Struct(Self::arrow_schema().fields)) + quote!(arrow_convert::_private::arrow_schema::DataType::Struct( + Self::arrow_schema().fields + )) } }; @@ -201,7 +203,7 @@ pub fn expand_field(input: DeriveStruct) -> TokenStream { impl arrow_convert::field::ArrowField for #original_name { type Type = Self; - fn data_type() -> arrow::datatypes::DataType { + fn data_type() -> arrow_convert::_private::arrow_schema::DataType { #data_type_impl } } @@ -232,8 +234,8 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { #( #field_idents: #mutable_field_array_types, )* - data_type: arrow::datatypes::DataType, - validity: Option, + data_type: arrow_convert::_private::arrow_schema::DataType, + validity: Option, } }; @@ -248,14 +250,14 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { } fn init_validity(&mut self) { - let length = ::len(self); - let mut validity = arrow::array::BooleanBufferBuilder::new(length); + let length = ::len(self); + let mut validity = arrow_convert::_private::arrow_array::builder::BooleanBufferBuilder::new(length); validity.append_n(length - 1, true); validity.append(false); self.validity = Some(validity); } - fn data_type(&self) -> &arrow::datatypes::DataType { + fn data_type(&self) -> &arrow_convert::_private::arrow_schema::DataType { &self.data_type } @@ -263,12 +265,12 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { self.try_push(None::<&#original_name>).unwrap(); } - fn validity(&self) -> Option<&arrow::array::BooleanBufferBuilder> { + fn validity(&self) -> Option<&arrow_convert::_private::arrow_array::builder::BooleanBufferBuilder> { self.validity.as_ref() } - fn try_push(&mut self, item: Option>) -> arrow::error::Result<()> { - use arrow::array::ArrayBuilder; + fn try_push(&mut self, item: Option>) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { + use arrow_convert::_private::arrow_array::builder::ArrayBuilder; use std::borrow::Borrow; match item { @@ -289,7 +291,7 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { Ok(()) } - fn try_extend<'a, I: IntoIterator>>(&mut self, iter: I) -> arrow::error::Result<()> { + fn try_extend<'a, I: IntoIterator>>(&mut self, iter: I) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { for i in iter { self.try_push(i)?; } @@ -297,9 +299,9 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { } fn align_struct_values( - values: Vec, - fields: &arrow::datatypes::Fields, - ) -> Vec { + values: Vec, + fields: &arrow_convert::_private::arrow_schema::Fields, + ) -> Vec { values .into_iter() .zip(fields.iter()) @@ -307,7 +309,7 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { if value.data_type() == field.data_type() { value } else { - arrow::compute::cast(value.as_ref(), field.data_type()).unwrap_or_else(|e| { + arrow_convert::_private::arrow_cast::cast(value.as_ref(), field.data_type()).unwrap_or_else(|e| { panic!( "failed to cast field '{}' from {:?} to {:?}: {}", field.name(), @@ -334,7 +336,7 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { let array_push_null_impl = quote! { impl arrow_convert::serialize::PushNull for #mutable_array_name { fn push_null(&mut self) { - use arrow::array::ArrayBuilder; + use arrow_convert::_private::arrow_array::builder::ArrayBuilder; use arrow_convert::serialize::{ArrowSerialize, PushNull}; use std::borrow::Borrow; @@ -356,43 +358,43 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { let first_ident = &field_idents[0]; let array_mutable_array_impl = quote! { - impl arrow::array::ArrayBuilder for #mutable_array_name { + impl arrow_convert::_private::arrow_array::builder::ArrayBuilder for #mutable_array_name { fn len(&self) -> usize { self.#first_ident.len() } - fn finish(&mut self) -> arrow::array::ArrayRef { + fn finish(&mut self) -> arrow_convert::_private::arrow_array::ArrayRef { let values = vec![#( - <#mutable_field_array_types as arrow::array::ArrayBuilder>::finish(&mut self.#field_idents), + <#mutable_field_array_types as arrow_convert::_private::arrow_array::builder::ArrayBuilder>::finish(&mut self.#field_idents), )*]; - let arrow::datatypes::DataType::Struct(fields) = + let arrow_convert::_private::arrow_schema::DataType::Struct(fields) = <#original_name as arrow_convert::field::ArrowField>::data_type() .clone() else { panic!("datatype is not struct") }; let values = #mutable_array_name::align_struct_values(values, &fields); - std::sync::Arc::new(arrow::array::StructArray::new( + std::sync::Arc::new(arrow_convert::_private::arrow_array::StructArray::new( fields, values, std::mem::take(&mut self.validity).map(|mut x| x.finish().into()), )) } - fn finish_cloned(&self) -> arrow::array::ArrayRef { + fn finish_cloned(&self) -> arrow_convert::_private::arrow_array::ArrayRef { let values = vec![#( - <#mutable_field_array_types as arrow::array::ArrayBuilder>::finish_cloned(&self.#field_idents), + <#mutable_field_array_types as arrow_convert::_private::arrow_array::builder::ArrayBuilder>::finish_cloned(&self.#field_idents), )*]; - let arrow::datatypes::DataType::Struct(fields) = + let arrow_convert::_private::arrow_schema::DataType::Struct(fields) = <#original_name as arrow_convert::field::ArrowField>::data_type() .clone() else { panic!("datatype is not struct") }; let values = #mutable_array_name::align_struct_values(values, &fields); - std::sync::Arc::new(arrow::array::StructArray::new( + std::sync::Arc::new(arrow_convert::_private::arrow_array::StructArray::new( fields, values, self.validity.as_ref().map(|x| x.finish_cloned().into()) @@ -428,7 +430,7 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> arrow::error::Result<()> { + fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { <#first_type as arrow_convert::serialize::ArrowSerialize>::arrow_serialize(&v.#first_field, array) } } @@ -444,7 +446,7 @@ pub fn expand_serialize(input: DeriveStruct) -> TokenStream { } #[inline] - fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> arrow::error::Result<()> { + fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> std::result::Result<(), arrow_convert::_private::arrow_schema::ArrowError> { array.try_push(Some(v)) } } @@ -484,15 +486,15 @@ pub fn expand_deserialize(input: DeriveStruct) -> TokenStream { let array_impl = quote! { impl arrow_convert::deserialize::ArrowArray for #array_name { - type BaseArrayType = arrow::array::StructArray; + type BaseArrayType = arrow_convert::_private::arrow_array::StructArray; #[inline] - fn iter_from_array_ref<'a>(b: &'a dyn arrow::array::Array) -> ::Iter<'a> + fn iter_from_array_ref<'a>(b: &'a dyn arrow_convert::_private::arrow_array::Array) -> ::Iter<'a> { use core::ops::Deref; - use arrow::array::Array; + use arrow_convert::_private::arrow_array::Array; - let arr = b.as_any().downcast_ref::().unwrap(); + let arr = b.as_any().downcast_ref::().unwrap(); let values = arr.columns(); let validity = arr.nulls(); // for now do a straight comp @@ -501,7 +503,7 @@ pub fn expand_deserialize(input: DeriveStruct) -> TokenStream { #field_idents: <<#field_types as arrow_convert::deserialize::ArrowDeserialize>::ArrayType as arrow_convert::deserialize::ArrowArray>::iter_from_array_ref(values[#field_indices].deref()), )* has_validity: validity.as_ref().is_some(), - validity_iter: validity.as_ref().map(|x| x.iter()).unwrap_or_else(|| arrow::util::bit_iterator::BitIterator::new(&[], 0, 0)) + validity_iter: validity.as_ref().map(|x| x.iter()).unwrap_or_else(|| arrow_convert::_private::arrow_buffer::bit_iterator::BitIterator::new(&[], 0, 0)) } } } @@ -524,7 +526,7 @@ pub fn expand_deserialize(input: DeriveStruct) -> TokenStream { #( #field_idents: <<#field_types as arrow_convert::deserialize::ArrowDeserialize>::ArrayType as arrow_convert::deserialize::ArrowArrayIterable>::Iter<'a>, )* - validity_iter: arrow::util::bit_iterator::BitIterator<'a>, + validity_iter: arrow_convert::_private::arrow_buffer::bit_iterator::BitIterator<'a>, has_validity: bool } }; diff --git a/examples/simple/Cargo.toml b/examples/simple/Cargo.toml index df3a874..9f71290 100644 --- a/examples/simple/Cargo.toml +++ b/examples/simple/Cargo.toml @@ -7,5 +7,5 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { workspace = true } +arrow-array = { workspace = true } arrow_convert = { workspace = true } diff --git a/examples/simple/src/main.rs b/examples/simple/src/main.rs index a5018f5..ef92bb3 100644 --- a/examples/simple/src/main.rs +++ b/examples/simple/src/main.rs @@ -1,7 +1,7 @@ /// Simple example -use arrow::array::{Array, ArrayRef}; +use arrow_array::{Array, ArrayRef}; use arrow_convert::{ - deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize, + ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow, }; #[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)] @@ -28,10 +28,7 @@ fn main() { // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc. // supported by `arrow` - let struct_array = arrow_array - .as_any() - .downcast_ref::() - .unwrap(); + let struct_array = arrow_array.as_any().downcast_ref::().unwrap(); assert_eq!(struct_array.len(), 3); // deserialize back to our original vector via TryIntoCollection trait.