Arrow restructure

.github/workflows/test.yml

@@ -1,6 +1,6 @@
 name: Check and test
 
-on: 
+on:
   workflow_dispatch:
   pull_request:
   push:
@@ -12,8 +12,8 @@ jobs:
     name: build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.85.0
       - name: "build all"
         # We keep these separate since sometimes the derive fails when
         # independently built.
@@ -26,9 +26,9 @@ jobs:
     name: Clippy
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with: 
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.85.0
+        with:
           components: clippy
       - name: "clippy --all"
         run: cargo clippy --all --tests -- -D warnings
@@ -37,9 +37,9 @@ jobs:
     name: fmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-        with: 
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.85.0
+        with:
           components: rustfmt
       - name: Run
         run: cargo fmt --all -- --check
@@ -48,9 +48,9 @@ jobs:
     name: Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions-rs/toolchain@v1
-      - uses: Swatinem/rust-cache@v1
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.85.0
+      - uses: Swatinem/rust-cache@v2
       - uses: taiki-e/install-action@nextest
       - name: Run
         run: |
@@ -60,17 +60,17 @@ jobs:
   coverage:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with: 
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@1.85.0
+        with:
           components: llvm-tools-preview
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
-      - uses: Swatinem/rust-cache@v1
+      - uses: Swatinem/rust-cache@v2
       - name: Generate code coverage
         run: cargo llvm-cov --lcov --output-path lcov.info
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
           files: lcov.info

Cargo.toml

@@ -18,9 +18,9 @@ repository = "https://github.com/Swoorup/arrow-convert"
 arrow_convert = { path = "arrow_convert", version = "0.11.4" }
 arrow_convert_derive = { path = "arrow_convert_derive", version = "0.11.4" }
 
-arrow = { version = "57", default-features = false }
 arrow-array = { version = "57" }
 arrow-buffer = { version = "57" }
+arrow-cast = { version = "57" }
 arrow-data = { version = "57" }
 arrow-schema = { version = "57" }
 chrono = { version = "0.4", default-features = false }

README.md

@@ -11,7 +11,7 @@ The example below performs a round trip conversion of a struct with a single fie
 Please see the [complex_example.rs](https://github.com/Swoorup/arrow-convert/blob/main/arrow_convert/tests/complex_example.rs) for usage of the full functionality.
 
 ```rust
-use arrow::array::{Array, ArrayRef};
+use arrow_array::{Array, ArrayRef, StructArray};
 use arrow_convert::{deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowField, ArrowSerialize, ArrowDeserialize};
 
 #[derive(Debug, Clone, PartialEq, ArrowField, ArrowSerialize, ArrowDeserialize)]
@@ -30,8 +30,7 @@ let original_array = [
 let arrow_array: ArrayRef = original_array.try_into_arrow().unwrap();
 
 // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc.
-// supported by `arrow`
-let struct_array= arrow_array.as_any().downcast_ref::<arrow::array::StructArray>().unwrap();
+let struct_array = arrow_array.as_any().downcast_ref::<StructArray>().unwrap();
 assert_eq!(struct_array.len(), 3);
 
 // deserialize back to our original vector via TryIntoCollection trait.
@@ -154,7 +153,8 @@ struct S {
 A `vec<i128>` can be converted. to/from arrow by using the `arrow_serialize_to_mutable_array` and `arrow_array_deserialize_iterator_as_type` methods. 
 
 ```rust
-use arrow::array::{Array, ArrayBuilder, ArrayRef};
+use arrow_array::{Array, ArrayRef};
+use arrow_array::builder::ArrayBuilder;
 use arrow_convert::serialize::arrow_serialize_to_mutable_array;
 use arrow_convert::deserialize::arrow_array_deserialize_iterator_as_type;
 use arrow_convert::field::I128;

arrow_convert/Cargo.toml

@@ -20,6 +20,7 @@ uuid = ["dep:uuid", "arrow-schema/canonical_extension_types"]
 [dependencies]
 arrow-array = { workspace = true }
 arrow-buffer = { workspace = true }
+arrow-cast = { workspace = true }
 arrow-data = { workspace = true }
 arrow-schema = { workspace = true }
 arrow_convert_derive = { workspace = true, optional = true }
@@ -34,7 +35,9 @@ tinystr = { workspace = true, optional = true }
 uuid = { workspace = true, optional = true }
 
 [dev-dependencies]
-arrow = { workspace = true }
+arrow-array = { workspace = true }
+arrow-buffer = { workspace = true }
+arrow-schema = { workspace = true }
 arrow_convert_derive = { workspace = true }
 criterion = { workspace = true }
 glam = { workspace = true }

arrow_convert/benches/bench.rs

@@ -1,11 +1,9 @@
-use arrow::{
-    array::ArrayRef,
-    buffer::{Buffer, ScalarBuffer},
-};
+use arrow_array::ArrayRef;
+use arrow_buffer::{Buffer, ScalarBuffer};
 use arrow_convert::{
-    deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize,
+    ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow,
 };
-use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
 
 // Arrow stores U8 arrays as `arrow::array::BinaryArray`
 #[derive(ArrowField, ArrowSerialize, ArrowDeserialize)]

arrow_convert/src/deserialize/iterable.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use super::{BufferBinaryArray, BufferBinaryArrayIter};
-use arrow_array::{iterator::*, ArrowPrimitiveType, *};
+use arrow_array::{ArrowPrimitiveType, iterator::*, *};
 
 /// A trait for Arrow arrays that can be transformed into an iterator.
 pub trait ArrowArrayIterable {

arrow_convert/src/deserialize/mod.rs

@@ -2,7 +2,7 @@
 mod iterable;
 pub use iterable::*;
 
-use arrow_array::{types, ArrowPrimitiveType, *};
+use arrow_array::{ArrowPrimitiveType, types, *};
 use arrow_buffer::{ArrowNativeType, Buffer, ScalarBuffer};
 use chrono::{DateTime, NaiveDate, NaiveDateTime, TimeZone, Utc};
 

arrow_convert/src/features/glam.rs

@@ -7,9 +7,9 @@ use crate::serialize::ArrowSerialize;
 use arrow_schema::Field;
 
 use crate::deserialize::arrow_deserialize_vec_helper;
-use arrow_array::builder::{BooleanBuilder, Float32Builder, Float64Builder};
 use arrow_array::ArrayRef;
-use arrow_array::{builder::FixedSizeListBuilder, FixedSizeListArray};
+use arrow_array::builder::{BooleanBuilder, Float32Builder, Float64Builder};
+use arrow_array::{FixedSizeListArray, builder::FixedSizeListBuilder};
 use std::sync::Arc;
 
 /// This macro implements the `ArrowSerialize` and `ArrowDeserialize` traits for a given `glam` vector or matrix type.

arrow_convert/src/features/rust_decimal.rs

@@ -3,10 +3,10 @@ use crate::deserialize::ArrowDeserialize;
 use crate::field::ArrowField;
 use crate::serialize::ArrowSerialize;
 
-use arrow_schema::{DataType, DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE};
+use arrow_schema::{DECIMAL_DEFAULT_SCALE, DECIMAL128_MAX_PRECISION, DataType};
 use rust_decimal::Decimal;
 
-use arrow_array::{builder::Decimal128Builder, Decimal128Array};
+use arrow_array::{Decimal128Array, builder::Decimal128Builder};
 
 impl ArrowField for Decimal {
     type Type = Decimal;

arrow_convert/src/features/tinystr.rs

@@ -5,7 +5,7 @@ use crate::deserialize::ArrowDeserialize;
 use crate::field::ArrowField;
 use crate::serialize::ArrowSerialize;
 
-use arrow_array::{builder::FixedSizeBinaryBuilder, FixedSizeBinaryArray};
+use arrow_array::{FixedSizeBinaryArray, builder::FixedSizeBinaryBuilder};
 
 impl<const N: usize> ArrowField for TinyAsciiStr<N> {
     type Type = Self;

arrow_convert/src/features/uuid.rs

@@ -8,9 +8,9 @@ use crate::deserialize::ArrowDeserialize;
 use crate::field::ArrowField;
 use crate::serialize::ArrowSerialize;
 use crate::serialize::PushNull;
-use arrow_array::builder::FixedSizeBinaryBuilder;
 use arrow_array::ArrayRef;
 use arrow_array::FixedSizeBinaryArray;
+use arrow_array::builder::FixedSizeBinaryBuilder;
 use arrow_schema::DataType;
 use uuid::Uuid;
 

arrow_convert/src/field.rs

@@ -93,7 +93,7 @@ pub trait ArrowField {
     #[inline]
     #[doc(hidden)]
     /// For internal use and not meant to be reimplemented.
-    /// returns the [`arrow::datatypes::Field`] for this field
+    /// returns the [`arrow_schema::Field`] for this field
     fn field(name: &str) -> Field {
         Field::new(name, Self::data_type(), Self::is_nullable())
     }

arrow_convert/src/lib.rs

@@ -6,6 +6,14 @@ pub mod deserialize;
 pub mod field;
 pub mod serialize;
 
+#[doc(hidden)]
+pub mod _private {
+    pub use arrow_array;
+    pub use arrow_buffer;
+    pub use arrow_cast;
+    pub use arrow_schema;
+}
+
 // The proc macro is implemented in derive_internal, and re-exported by this
 // crate. This is because a single crate can not define both a proc macro and a
 // macro_rules macro.

arrow_convert/tests/complex_example.rs

@@ -1,13 +1,13 @@
-use arrow::array::*;
-use arrow::datatypes::DataType;
-use arrow_convert::deserialize::{arrow_array_deserialize_iterator, TryIntoCollection};
+use arrow_array::*;
+use arrow_convert::deserialize::{TryIntoCollection, arrow_array_deserialize_iterator};
 use arrow_convert::field::ArrowField;
 use arrow_convert::serialize::TryIntoArrow;
 /// Complex example that uses the following features:
 ///
 /// - Deeply Nested structs and lists
 /// - Custom types
 use arrow_convert::{ArrowDeserialize, ArrowField, ArrowSerialize};
+use arrow_schema::DataType;
 use chrono::DateTime;
 use std::borrow::Borrow;
 
@@ -88,28 +88,28 @@ impl arrow_convert::field::ArrowField for CustomType {
     type Type = Self;
 
     #[inline]
-    fn data_type() -> arrow::datatypes::DataType {
-        arrow::datatypes::DataType::UInt64
+    fn data_type() -> arrow_schema::DataType {
+        arrow_schema::DataType::UInt64
     }
 }
 
 impl arrow_convert::serialize::ArrowSerialize for CustomType {
-    type ArrayBuilderType = arrow::array::UInt64Builder;
+    type ArrayBuilderType = arrow_array::builder::UInt64Builder;
 
     #[inline]
     fn new_array() -> Self::ArrayBuilderType {
         Self::ArrayBuilderType::default()
     }
 
     #[inline]
-    fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> arrow::error::Result<()> {
+    fn arrow_serialize(v: &Self, array: &mut Self::ArrayBuilderType) -> Result<(), arrow_schema::ArrowError> {
         array.append_option(Some(v.0));
         Ok(())
     }
 }
 
 impl arrow_convert::deserialize::ArrowDeserialize for CustomType {
-    type ArrayType = arrow::array::UInt64Array;
+    type ArrayType = arrow_array::UInt64Array;
 
     #[inline]
     fn arrow_deserialize(v: Option<u64>) -> Option<Self> {
@@ -217,13 +217,13 @@ fn item2() -> Root {
 }
 
 #[test]
-fn test_round_trip() -> arrow::error::Result<()> {
+fn test_round_trip() -> Result<(), arrow_schema::ArrowError> {
     // serialize to an arrow array
     let original_array = [item1(), item2()];
 
     let array: ArrayRef = original_array.try_into_arrow()?;
 
-    let struct_array = array.as_any().downcast_ref::<arrow::array::StructArray>().unwrap();
+    let struct_array = array.as_any().downcast_ref::<arrow_array::StructArray>().unwrap();
     assert_eq!(struct_array.len(), 2);
 
     let values = struct_array.columns();

arrow_convert/tests/simple_example.rs

@@ -1,9 +1,9 @@
 use std::sync::Arc;
 
 /// Simple example
-use arrow::array::Array;
+use arrow_array::Array;
 use arrow_convert::{
-    deserialize::TryIntoCollection, serialize::TryIntoArrow, ArrowDeserialize, ArrowField, ArrowSerialize,
+    ArrowDeserialize, ArrowField, ArrowSerialize, deserialize::TryIntoCollection, serialize::TryIntoArrow,
 };
 
 #[derive(Debug, Clone, PartialEq, Eq, ArrowField, ArrowSerialize, ArrowDeserialize)]
@@ -30,10 +30,7 @@ fn test_simple_roundtrip() {
 
     // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc.
     // supported by `arrow`
-    let struct_array = arrow_array
-        .as_any()
-        .downcast_ref::<arrow::array::StructArray>()
-        .unwrap();
+    let struct_array = arrow_array.as_any().downcast_ref::<arrow_array::StructArray>().unwrap();
     assert_eq!(struct_array.len(), 3);
 
     // deserialize back to our original vector via TryIntoCollection trait.

arrow_convert/tests/test_array.rs

@@ -1,5 +1,4 @@
-use arrow::array::Array;
-use arrow::array::ArrayRef;
+use arrow_array::{Array, ArrayRef};
 use arrow_convert::deserialize::TryIntoCollection;
 use arrow_convert::serialize::TryIntoArrow;
 /// Simple example
@@ -56,10 +55,7 @@ fn test_simple_roundtrip() {
 
     // which can be cast to an Arrow StructArray and be used for all kinds of IPC, FFI, etc.
     // supported by `arrow`
-    let struct_array = arrow_array
-        .as_any()
-        .downcast_ref::<arrow::array::StructArray>()
-        .unwrap();
+    let struct_array = arrow_array.as_any().downcast_ref::<arrow_array::StructArray>().unwrap();
     assert_eq!(struct_array.len(), 3);
 
     // deserialize back to our original vector via TryIntoCollection trait.