From e74aac3783c161d375598723b0b90b38ba9035ed Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 12 Jun 2026 16:25:04 +0100 Subject: [PATCH 001/162] fix: update doc links (#8375) fixup links for vortex from spiral and the `docs.` subdomain Signed-off-by: Joe Isaacs --- CONTRIBUTING.md | 2 +- docs/developer-guide/embedding/index.md | 4 ++-- docs/developer-guide/extending/index.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aef2e0c33ac..6d32374293a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ We ask that you read the guidelines below in order to make the process as stream > > For the full AI policy — including disclosure requirements, review standards for > AI-generated PRs, and rules for autonomous agents — see the -> [contributing guide](https://vortex.dev/project/contributing.html#ai-assistance). +> [contributing guide](https://docs.vortex.dev/project/contributing#ai-assistance). ## Code Contributions diff --git a/docs/developer-guide/embedding/index.md b/docs/developer-guide/embedding/index.md index f09d1841762..e204939431b 100644 --- a/docs/developer-guide/embedding/index.md +++ b/docs/developer-guide/embedding/index.md @@ -2,8 +2,8 @@ :::{warning} This section is under construction. For guidance on embedding Vortex, please join the -[Vortex Slack channel](https://join.slack.com/t/vortex-array/shared_invite/zt-2ycp2w24h-sRdrGbMGPmQwCuPQT40Jig) -or start a [GitHub Discussion](https://github.com/spiraldb/vortex/discussions). +[Vortex Slack channel](https://vortex.dev/slack) +or start a [GitHub Discussion](https://github.com/vortex-data/vortex/discussions). ::: Vortex can be embedded into applications and services via its C FFI, C++ wrapper, or the Scan API. diff --git a/docs/developer-guide/extending/index.md b/docs/developer-guide/extending/index.md index 78b7a1829e3..ff004a69bc5 100644 --- a/docs/developer-guide/extending/index.md +++ b/docs/developer-guide/extending/index.md @@ -3,7 +3,7 @@ :::{warning} This section is under construction. For guidance on extending Vortex, please join the [Vortex Slack channel](https://vortex.dev/slack) -or start a [GitHub Discussion](https://github.com/spiraldb/vortex/discussions). +or start a [GitHub Discussion](https://github.com/vortex-data/vortex/discussions). ::: Vortex is designed to be extended with custom types, encodings, layouts, and compute functions. From 46e725313588dd839fdc46e5691175e439c33f2f Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Fri, 12 Jun 2026 16:49:56 +0100 Subject: [PATCH 002/162] perf: non-blocking Arrow Device array release (#8390) Signed-off-by: Alexander Droste --- vortex-cuda/src/arrow/canonical.rs | 19 ++++++++----------- vortex-cuda/src/arrow/mod.rs | 22 +++++++++------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 6c0f53ef249..33cde27520b 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -11,7 +11,6 @@ use cudarc::driver::CudaSlice; use cudarc::driver::DeviceRepr; use cudarc::driver::LaunchConfig; use cudarc::driver::PushKernelArg; -use cudarc::driver::result as cuda_driver; use futures::future::BoxFuture; use futures::future::join; use vortex::array::ArrayRef; @@ -1147,9 +1146,9 @@ fn export_fixed_size( } unsafe extern "C" fn release_array(array: *mut ArrowArray) { - // SAFETY: this is only safe if we're dropping an ArrowArray that was created from Rust - // code. This is necessary to ensure that the fields inside the CudaPrivateData - // get dropped to free native/GPU memory. + // SAFETY: this is only safe if we're dropping an ArrowArray that was + // created from Rust code. This is necessary to ensure that the fields + // inside the CudaPrivateData get dropped to free native/GPU memory. unsafe { if array.is_null() || (*array).release.is_none() { return; @@ -1159,13 +1158,11 @@ unsafe extern "C" fn release_array(array: *mut ArrowArray) { if !private_data_ptr.is_null() { let mut private_data = Box::from_raw(private_data_ptr.cast::()); - // Release may run on a foreign thread; bind this array's context before synchronizing - // so async frees cannot race consumer-side reads. - let cuda_context = Arc::clone(private_data.cuda_stream.context()); - match cuda_context.bind_to_thread() { - Ok(()) => cuda_context.record_err(cuda_driver::ctx::synchronize()), - Err(err) => cuda_context.record_err(Err::<(), _>(err)), - } + // Queue a non-blocking wait so CudaSlice::Drop frees after export work. + let wait_result = private_data.cuda_stream.wait(&private_data.export_event); + // Arrow release callbacks cannot return errors, so record wait-enqueue + // failures for later CUDA operations to observe on this context. + private_data.cuda_stream.context().record_err(wait_result); release_children(&mut private_data); release_dictionary(&mut private_data); } diff --git a/vortex-cuda/src/arrow/mod.rs b/vortex-cuda/src/arrow/mod.rs index 7dac0c8631d..22cb871d8f5 100644 --- a/vortex-cuda/src/arrow/mod.rs +++ b/vortex-cuda/src/arrow/mod.rs @@ -110,21 +110,18 @@ impl ArrowArray { unsafe impl Send for ArrowArray {} unsafe impl Sync for ArrowArray {} -#[expect( - unused, - reason = "cuda_stream and cuda_buffers need to have deferred drop" -)] pub(crate) struct PrivateData { /// Hold a reference to the CudaStream so that it stays alive even after CudaExecutionCtx /// has been dropped. pub(crate) cuda_stream: Arc, /// The single boxed slice which owns all buffers that the Rust code allocated on the device. + #[allow(dead_code, reason = "buffers are retained for deferred drop")] pub(crate) buffers: Box<[Option]>, /// Boxed slice of buffer pointers. We return a pointer to the start of this allocation over /// the interface, so we hold it here so the Box contents are not freed. pub(crate) buffer_ptrs: Box<[*const c_void]>, - pub(crate) cuda_event: CudaEvent, - pub(crate) cuda_event_ptr: cudaEvent_t, + pub(crate) export_event: CudaEvent, + pub(crate) export_event_ptr: cudaEvent_t, pub(crate) children: Box<[*mut ArrowArray]>, pub(crate) dictionary: *mut ArrowArray, } @@ -168,11 +165,10 @@ impl PrivateData { .map(|array| Box::into_raw(Box::new(array))) .collect::>(); - // generate the synchronization event - let cuda_event = ctx + let export_event = ctx .stream() .record_event(None) - .map_err(|_| vortex_err!("failed to create cudaEvent_t"))?; + .map_err(|_| vortex_err!("failed to create CUDA export event"))?; let dictionary = dictionary .map(|array| Box::into_raw(Box::new(array))) @@ -184,14 +180,14 @@ impl PrivateData { cuda_stream: Arc::clone(ctx.stream()), children, dictionary, - cuda_event_ptr: cuda_event.cu_event().cast(), - cuda_event, + export_event_ptr: export_event.cu_event().cast(), + export_event, })) } - /// Return a stable pointer to the recorded CUDA event handle. + /// Return a stable pointer to the recorded CUDA export event handle. pub(crate) fn sync_event(&mut self) -> SyncEvent { - (&raw mut self.cuda_event_ptr).cast() + (&raw mut self.export_event_ptr).cast() } } From fb489a1cad4f60968c6961fdbbf3b7c12cca71a2 Mon Sep 17 00:00:00 2001 From: Nemo Yu <83347615+HarukiMoriarty@users.noreply.github.com> Date: Fri, 12 Jun 2026 12:11:09 -0400 Subject: [PATCH 003/162] feat(vortex-geo): Arrow import/export for the native Point type (#8374) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR adds support for import/export to Arrow for the `Point` extension type, as the `geoarrow.point` Arrow extension with separated (struct) coordinates. Stacked on #8372. ## Testing Unit tests are added to exercise both code paths, plus a Vortex → Arrow → Vortex round-trip. Signed-off-by: Nemo Yu --- Cargo.lock | 1 + vortex-geo/Cargo.toml | 1 + vortex-geo/src/extension/coordinate.rs | 88 ++++++- vortex-geo/src/extension/mod.rs | 29 +++ vortex-geo/src/extension/point.rs | 214 +++++++++++++---- vortex-geo/src/extension/wkb.rs | 34 +-- vortex-geo/src/lib.rs | 311 +------------------------ vortex-geo/src/scalar_fn/distance.rs | 21 +- vortex-geo/src/test_harness.rs | 29 +++ vortex-geo/src/tests/mod.rs | 20 ++ vortex-geo/src/tests/point.rs | 191 +++++++++++++++ vortex-geo/src/tests/wkb.rs | 255 ++++++++++++++++++++ 12 files changed, 790 insertions(+), 404 deletions(-) create mode 100644 vortex-geo/src/test_harness.rs create mode 100644 vortex-geo/src/tests/mod.rs create mode 100644 vortex-geo/src/tests/point.rs create mode 100644 vortex-geo/src/tests/wkb.rs diff --git a/Cargo.lock b/Cargo.lock index fd66ecc5670..57be369d7c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9867,6 +9867,7 @@ dependencies = [ "geo-types", "geoarrow", "prost 0.14.4", + "rstest", "vortex-array", "vortex-error", "vortex-session", diff --git a/vortex-geo/Cargo.toml b/vortex-geo/Cargo.toml index 4e76e2eefcb..3b09cb77cf8 100644 --- a/vortex-geo/Cargo.toml +++ b/vortex-geo/Cargo.toml @@ -26,6 +26,7 @@ wkb = { workspace = true } [dev-dependencies] geo-traits = { workspace = true } geo-types = { workspace = true } +rstest = { workspace = true } [lints] workspace = true diff --git a/vortex-geo/src/extension/coordinate.rs b/vortex-geo/src/extension/coordinate.rs index 5624886dbb5..3e560bc4734 100644 --- a/vortex-geo/src/extension/coordinate.rs +++ b/vortex-geo/src/extension/coordinate.rs @@ -14,6 +14,7 @@ use std::fmt::Display; use std::fmt::Formatter; +use geoarrow::datatypes::Dimension as GeoArrowDimension; use vortex_array::ArrayRef; use vortex_array::ExecutionCtx; use vortex_array::arrays::ExtensionArray; @@ -25,6 +26,7 @@ use vortex_array::dtype::DType; use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; +use vortex_array::dtype::StructFields; use vortex_array::scalar::Scalar; use vortex_error::VortexResult; use vortex_error::vortex_bail; @@ -63,6 +65,38 @@ impl Dimension { _ => vortex_bail!("not a valid GeoArrow coordinate dimension: {names:?}"), }) } + + /// The coordinate field names of this dimension, in GeoArrow order. + pub(crate) fn field_names(self) -> &'static [&'static str] { + match self { + Dimension::Xy => &["x", "y"], + Dimension::Xyz => &["x", "y", "z"], + Dimension::Xym => &["x", "y", "m"], + Dimension::Xyzm => &["x", "y", "z", "m"], + } + } +} + +impl From for Dimension { + fn from(dim: GeoArrowDimension) -> Self { + match dim { + GeoArrowDimension::XY => Dimension::Xy, + GeoArrowDimension::XYZ => Dimension::Xyz, + GeoArrowDimension::XYM => Dimension::Xym, + GeoArrowDimension::XYZM => Dimension::Xyzm, + } + } +} + +impl From for GeoArrowDimension { + fn from(dim: Dimension) -> Self { + match dim { + Dimension::Xy => GeoArrowDimension::XY, + Dimension::Xyz => GeoArrowDimension::XYZ, + Dimension::Xym => GeoArrowDimension::XYM, + Dimension::Xyzm => GeoArrowDimension::XYZM, + } + } } /// A decoded coordinate. `z`/`m` are `Some` iff the storage dimension includes them. @@ -122,6 +156,21 @@ pub(crate) fn coordinate_dimension(dtype: &DType) -> VortexResult { Dimension::from_field_names(fields.names()) } +/// The canonical storage dtype for `dim`: a `Struct` of non-nullable `f64` coordinate fields, +/// with `nullability` at the struct (per-point) level. Inverse of [`coordinate_dimension`]. +pub(crate) fn coordinate_storage_dtype(dim: Dimension, nullability: Nullability) -> DType { + let names = dim.field_names(); + let fields = std::iter::repeat_n( + DType::Primitive(PType::F64, Nullability::NonNullable), + names.len(), + ) + .collect::>(); + DType::Struct( + StructFields::new(FieldNames::from(names), fields), + nullability, + ) +} + /// Decode a [`Coordinate`] from a coordinate `Struct` scalar (`z`/`m` read iff /// present, so the same decoder serves every dimension). pub(crate) fn coordinate_from_struct(scalar: &Scalar) -> VortexResult { @@ -196,12 +245,14 @@ pub(crate) fn parse_storage( #[cfg(test)] mod tests { + use rstest::rstest; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::FieldNames; + use vortex_array::dtype::Nullability; use vortex_array::dtype::extension::ExtDType; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; @@ -209,26 +260,43 @@ mod tests { use vortex_session::VortexSession; use super::Coordinate; + use super::Dimension; + use super::coordinate_dimension; + use super::coordinate_storage_dtype; use super::parse_storage; use crate::extension::GeoMetadata; use crate::extension::Point; + /// Each dimension round-trips through its field names and canonical storage dtype. + #[rstest] + #[case::xy(Dimension::Xy, &["x", "y"])] + #[case::xyz(Dimension::Xyz, &["x", "y", "z"])] + #[case::xym(Dimension::Xym, &["x", "y", "m"])] + #[case::xyzm(Dimension::Xyzm, &["x", "y", "z", "m"])] + fn storage_dtype_roundtrips_dimension( + #[case] dim: Dimension, + #[case] names: &[&str], + ) -> VortexResult<()> { + assert_eq!(dim.field_names(), names); + let dtype = coordinate_storage_dtype(dim, Nullability::NonNullable); + assert_eq!(coordinate_dimension(&dtype)?, dim); + Ok(()) + } + /// Display emits WKT, including `z`/`m` when present. - #[test] - fn display_is_wkt() { - let coordinate = |z, m| Coordinate { + #[rstest] + #[case::xy(None, None, "POINT(1 2)")] + #[case::xyz(Some(3.0), None, "POINT Z (1 2 3)")] + #[case::xym(None, Some(4.0), "POINT M (1 2 4)")] + #[case::xyzm(Some(3.0), Some(4.0), "POINT ZM (1 2 3 4)")] + fn display_is_wkt(#[case] z: Option, #[case] m: Option, #[case] expected: &str) { + let coordinate = Coordinate { x: 1.0, y: 2.0, z, m, }; - assert_eq!(coordinate(None, None).to_string(), "POINT(1 2)"); - assert_eq!(coordinate(Some(3.0), None).to_string(), "POINT Z (1 2 3)"); - assert_eq!(coordinate(None, Some(4.0)).to_string(), "POINT M (1 2 4)"); - assert_eq!( - coordinate(Some(3.0), Some(4.0)).to_string(), - "POINT ZM (1 2 3 4)" - ); + assert_eq!(coordinate.to_string(), expected); } /// [`parse_storage`] reads the coordinate fields unmasked, so a nullable point column must diff --git a/vortex-geo/src/extension/mod.rs b/vortex-geo/src/extension/mod.rs index d69dd239d14..cc3a1f6e532 100644 --- a/vortex-geo/src/extension/mod.rs +++ b/vortex-geo/src/extension/mod.rs @@ -6,7 +6,10 @@ mod point; mod wkb; use std::fmt::Display; +use std::sync::Arc; +use geoarrow::datatypes::Crs; +use geoarrow::datatypes::Metadata; pub use point::*; pub use wkb::*; @@ -30,6 +33,32 @@ impl Display for GeoMetadata { } } +/// The GeoArrow [`Metadata`] equivalent of `geo_metadata`. +pub(crate) fn geoarrow_metadata(geo_metadata: &GeoMetadata) -> Arc { + Arc::new(Metadata::new( + geo_metadata + .crs + .as_ref() + .map(|crs| Crs::from_unknown_crs_type(crs.to_string())) + .unwrap_or_default(), + None, + )) +} + +/// Recover [`GeoMetadata`] from GeoArrow metadata. +pub(crate) fn geo_metadata_from_arrow(metadata: &Metadata) -> GeoMetadata { + let crs = metadata.crs().crs_value().map(|value| { + // `Crs::from_unknown_crs_type` stores the user's string verbatim as a JSON string + // value, so prefer the raw string when available to round-trip cleanly. For other + // CRS encodings (PROJJSON object, etc.), fall back to the JSON-encoded form. + value + .as_str() + .map(str::to_string) + .unwrap_or_else(|| value.to_string()) + }); + GeoMetadata { crs } +} + #[cfg(test)] mod tests { use prost::Message; diff --git a/vortex-geo/src/extension/point.rs b/vortex-geo/src/extension/point.rs index 119de3a7e45..ecd544ca51e 100644 --- a/vortex-geo/src/extension/point.rs +++ b/vortex-geo/src/extension/point.rs @@ -6,18 +6,47 @@ //! XYZM — tagged with [`GeoMetadata`] (CRS). `z` is an optional elevation and `m` an optional //! measure: an arbitrary per-point value such as distance along a route or a timestamp. +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_schema::DataType; +use arrow_schema::Field; +use arrow_schema::extension::ExtensionType; +use geoarrow::array::IntoArrow; +use geoarrow::array::PointArray; +use geoarrow::datatypes::CoordType; +use geoarrow::datatypes::PointType; use prost::Message; +use vortex_array::ArrayRef; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::extension::ExtensionArrayExt; +use vortex_array::arrow::ArrowExport; +use vortex_array::arrow::ArrowExportVTable; +use vortex_array::arrow::ArrowImport; +use vortex_array::arrow::ArrowImportVTable; +use vortex_array::arrow::ArrowSession; +use vortex_array::arrow::ArrowSessionExt; +use vortex_array::arrow::FromArrowArray; +use vortex_array::dtype::DType; use vortex_array::dtype::extension::ExtDType; use vortex_array::dtype::extension::ExtId; use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::Scalar; use vortex_array::scalar::ScalarValue; use vortex_error::VortexResult; +use vortex_error::vortex_ensure; +use vortex_error::vortex_err; +use vortex_session::registry::CachedId; +use vortex_session::registry::Id; use super::GeoMetadata; use super::coordinate::Coordinate; +use super::coordinate::Dimension; use super::coordinate::coordinate_dimension; use super::coordinate::coordinate_from_struct; +use super::coordinate::coordinate_storage_dtype; +use super::geo_metadata_from_arrow; +use super::geoarrow_metadata; /// A single location: `geoarrow.point`, stored as `Struct` of non-nullable `f64`. #[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] @@ -55,18 +84,143 @@ impl ExtVTable for Point { } } +static ARROW_POINT: CachedId = CachedId::new(PointType::NAME); + +/// The `geoarrow.point` extension type for `dimension`, with separated (struct) coordinates +/// matching `Point` storage. +fn point_type(geo_metadata: &GeoMetadata, dimension: Dimension) -> PointType { + PointType::new(dimension.into(), geoarrow_metadata(geo_metadata)) +} + +impl ArrowExportVTable for Point { + fn arrow_ext_id(&self) -> Id { + *ARROW_POINT + } + + fn vortex_id(&self) -> Id { + self.id() + } + + fn to_arrow_field( + &self, + name: &str, + dtype: &DType, + session: &ArrowSession, + ) -> VortexResult> { + let ext_type = dtype.as_extension(); + let geo_metadata = ext_type.metadata::(); + let dimension = coordinate_dimension(ext_type.storage_dtype())?; + + let mut field = session.to_arrow_field(name, ext_type.storage_dtype())?; + field.try_with_extension_type(point_type(geo_metadata, dimension))?; + + Ok(Some(field)) + } + + fn execute_arrow( + &self, + array: ArrayRef, + target: &Field, + ctx: &mut ExecutionCtx, + ) -> VortexResult { + let is_point = array + .dtype() + .as_extension_opt() + .map(|ext| ext.is::()) + .unwrap_or(false); + if !is_point { + return Ok(ArrowExport::Unsupported(array)); + } + + let Ok(point_meta) = target.try_extension_type::() else { + return Ok(ArrowExport::Unsupported(array)); + }; + if point_meta.coord_type() != CoordType::Separated { + return Ok(ArrowExport::Unsupported(array)); + } + + let executed = array.execute::(ctx)?; + let storage = executed.storage_array().clone(); + + let storage_field = Field::new( + String::new(), + target.data_type().clone(), + target.is_nullable(), + ); + let session = ctx.session().clone(); + let arrow_storage = session + .arrow() + .execute_arrow(storage, Some(&storage_field), ctx)?; + + // Round-trip through the GeoArrow point array type: this validates that the storage is + // the separated-coordinate struct layout expected for a `PointType` extension field. + let points = PointArray::try_from((arrow_storage.as_ref(), point_meta)) + .map_err(|e| vortex_err!("failed to construct PointArray: {e}"))?; + + Ok(ArrowExport::Exported(points.into_arrow())) + } +} + +impl ArrowImportVTable for Point { + fn arrow_ext_id(&self) -> Id { + *ARROW_POINT + } + + fn from_arrow_field(&self, field: &Field) -> VortexResult> { + let Ok(point_meta) = field.try_extension_type::() else { + return Ok(None); + }; + vortex_ensure!( + point_meta.coord_type() == CoordType::Separated, + "geoarrow.point with interleaved coordinates is not supported; \ + re-encode with separated (struct) coordinates" + ); + + let storage_dtype = + coordinate_storage_dtype(point_meta.dimension().into(), field.is_nullable().into()); + Ok(Some(DType::Extension( + ExtDType::try_with_vtable( + Point, + geo_metadata_from_arrow(point_meta.metadata()), + storage_dtype, + )? + .erased(), + ))) + } + + fn from_arrow_array( + &self, + array: ArrowArrayRef, + field: &Field, + dtype: &DType, + ) -> VortexResult { + let Some(ext_dtype) = dtype.as_extension_opt() else { + return Ok(ArrowImport::Unsupported(array)); + }; + if !ext_dtype.is::() + || field.try_extension_type::().is_err() + || !matches!(array.data_type(), DataType::Struct(_)) + { + return Ok(ArrowImport::Unsupported(array)); + } + + let storage = ArrayRef::from_arrow(array.as_ref(), field.is_nullable())?; + Ok(ArrowImport::Imported( + ExtensionArray::try_new(ext_dtype.clone(), storage)?.into_array(), + )) + } +} + #[cfg(test)] mod tests { + use rstest::rstest; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; - use vortex_array::arrays::ExtensionArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::dtype::DType; - use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::dtype::StructFields; use vortex_array::dtype::extension::ExtDType; use vortex_array::session::ArraySession; use vortex_error::VortexResult; @@ -76,8 +230,9 @@ mod tests { use crate::extension::GeoMetadata; use crate::extension::coordinate::Coordinate; use crate::extension::coordinate::Dimension; - use crate::extension::coordinate::coordinate_dimension; use crate::extension::coordinate::coordinate_from_scalar; + use crate::extension::coordinate::coordinate_storage_dtype; + use crate::test_harness::point_column; fn geo_meta() -> GeoMetadata { GeoMetadata { @@ -85,34 +240,15 @@ mod tests { } } - /// A coordinate storage dtype with the given field names, non-nullable `f64` per field. - fn coordinate_dtype(names: &[&'static str]) -> DType { - let fields = std::iter::repeat_n( - DType::Primitive(PType::F64, Nullability::NonNullable), - names.len(), - ) - .collect::>(); - DType::Struct( - StructFields::new(FieldNames::from(names), fields), - Nullability::NonNullable, - ) - } - - /// `Point` accepts every GeoArrow dimension; the canonical field names round-trip to their - /// dimension, so a `z`/`m` swap or a mislabel would be caught. - #[test] - fn point_validates_every_dimension() -> VortexResult<()> { - let cases = [ - (Dimension::Xy, ["x", "y"].as_slice()), - (Dimension::Xyz, ["x", "y", "z"].as_slice()), - (Dimension::Xym, ["x", "y", "m"].as_slice()), - (Dimension::Xyzm, ["x", "y", "z", "m"].as_slice()), - ]; - for (dim, names) in cases { - let storage = coordinate_dtype(names); - assert_eq!(coordinate_dimension(&storage)?, dim); - ExtDType::::try_new(geo_meta(), storage)?; - } + /// `Point` accepts the canonical coordinate storage of every GeoArrow dimension. + #[rstest] + #[case::xy(Dimension::Xy)] + #[case::xyz(Dimension::Xyz)] + #[case::xym(Dimension::Xym)] + #[case::xyzm(Dimension::Xyzm)] + fn point_validates_every_dimension(#[case] dim: Dimension) -> VortexResult<()> { + let storage = coordinate_storage_dtype(dim, Nullability::NonNullable); + ExtDType::::try_new(geo_meta(), storage)?; Ok(()) } @@ -138,19 +274,7 @@ mod tests { let session = VortexSession::empty().with::(); let mut ctx = session.create_execution_ctx(); - let storage = StructArray::from_fields(&[ - ( - "x", - PrimitiveArray::from_iter(vec![1.0f64, -111.7610]).into_array(), - ), - ( - "y", - PrimitiveArray::from_iter(vec![2.0f64, 34.8697]).into_array(), - ), - ])? - .into_array(); - let dtype = ExtDType::::try_new(geo_meta(), storage.dtype().clone())?; - let points = ExtensionArray::new(dtype.erased(), storage).into_array(); + let points = point_column(vec![1.0, -111.7610], vec![2.0, 34.8697])?; assert_eq!( coordinate_from_scalar(&points.execute_scalar(0, &mut ctx)?)?, diff --git a/vortex-geo/src/extension/wkb.rs b/vortex-geo/src/extension/wkb.rs index 172da41b93b..5491d8673d7 100644 --- a/vortex-geo/src/extension/wkb.rs +++ b/vortex-geo/src/extension/wkb.rs @@ -12,8 +12,6 @@ use arrow_schema::extension::ExtensionType; use geoarrow::array::GenericWkbArray; use geoarrow::array::IntoArrow; use geoarrow::array::WkbViewArray; -use geoarrow::datatypes::Crs; -use geoarrow::datatypes::Metadata; use geoarrow::datatypes::WkbType; use prost::Message; use vortex_array::ArrayRef; @@ -35,7 +33,6 @@ use vortex_array::dtype::extension::ExtVTable; use vortex_array::scalar::ScalarValue; use vortex_error::VortexError; use vortex_error::VortexResult; -use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_err; use vortex_session::registry::CachedId; @@ -43,6 +40,8 @@ use vortex_session::registry::Id; use wkb::reader::GeometryType; use crate::extension::GeoMetadata; +use crate::extension::geo_metadata_from_arrow; +use crate::extension::geoarrow_metadata; /// A typed handle to an [`ExtensionArray`] that contains WKB-encoded data. /// @@ -71,9 +70,11 @@ impl TryFrom for WellKnownBinaryData { type Error = VortexError; fn try_from(ext: ExtensionArray) -> Result { - if !ext.ext_dtype().is::() { - vortex_bail!("array extension dtype {} is not a WKB", ext.ext_dtype()); - } + vortex_ensure!( + ext.ext_dtype().is::(), + "array extension dtype {} is not a WKB", + ext.ext_dtype() + ); Ok(Self { ext }) } @@ -292,26 +293,9 @@ impl ArrowImportVTable for WellKnownBinary { } fn wkb_type(geo_metadata: &GeoMetadata) -> WkbType { - let metadata = Metadata::new( - geo_metadata - .crs - .as_ref() - .map(|crs| Crs::from_unknown_crs_type(crs.to_string())) - .unwrap_or_default(), - None, - ); - WkbType::new(Arc::new(metadata)) + WkbType::new(geoarrow_metadata(geo_metadata)) } fn geo_metadata(wkb_type: &WkbType) -> GeoMetadata { - let crs = wkb_type.metadata().crs().crs_value().map(|value| { - // `Crs::from_unknown_crs_type` stores the user's string verbatim as a JSON string - // value, so prefer the raw string when available to round-trip cleanly. For other - // CRS encodings (PROJJSON object, etc.), fall back to the JSON-encoded form. - value - .as_str() - .map(str::to_string) - .unwrap_or_else(|| value.to_string()) - }); - GeoMetadata { crs } + geo_metadata_from_arrow(wkb_type.metadata()) } diff --git a/vortex-geo/src/lib.rs b/vortex-geo/src/lib.rs index 9d0cde26f9e..d47b0976bc6 100644 --- a/vortex-geo/src/lib.rs +++ b/vortex-geo/src/lib.rs @@ -14,6 +14,11 @@ use crate::scalar_fn::distance::GeoDistance; pub mod extension; pub mod scalar_fn; +#[cfg(test)] +mod test_harness; +#[cfg(test)] +mod tests; + /// Set up a session with support for geospatial extension types, encodings and layouts. pub fn initialize(session: &VortexSession) { // Register the geospatial extension types. @@ -21,311 +26,9 @@ pub fn initialize(session: &VortexSession) { session.arrow().register_exporter(Arc::new(WellKnownBinary)); session.arrow().register_importer(Arc::new(WellKnownBinary)); session.dtypes().register(Point); + session.arrow().register_exporter(Arc::new(Point)); + session.arrow().register_importer(Arc::new(Point)); // Register the geometry scalar functions. session.scalar_fns().register(GeoDistance); } - -#[cfg(test)] -mod tests { - use std::sync::Arc; - use std::sync::LazyLock; - - use arrow_array::Array as _; - use arrow_array::ArrayRef as ArrowArrayRef; - use arrow_array::BinaryArray; - use arrow_array::BinaryViewArray; - use arrow_array::LargeBinaryArray; - use arrow_array::cast::AsArray; - use arrow_schema::DataType; - use arrow_schema::Field; - use arrow_schema::extension::ExtensionType as _; - use geo_traits::to_geo::ToGeoGeometry; - use geo_types::Coord; - use geo_types::Geometry; - use geo_types::LineString; - use geo_types::Polygon; - use geoarrow::datatypes::Crs; - use geoarrow::datatypes::Metadata; - use geoarrow::datatypes::WkbType; - use vortex_array::IntoArray; - use vortex_array::VortexSessionExecute; - use vortex_array::arrays::ExtensionArray; - use vortex_array::arrays::varbin::builder::VarBinBuilder; - use vortex_array::arrow::ArrowSessionExt; - use vortex_array::dtype::DType; - use vortex_array::dtype::Nullability; - use vortex_array::dtype::extension::ExtDType; - use vortex_array::dtype::extension::ExtVTable; - use vortex_array::session::ArraySession; - use vortex_error::VortexResult; - use vortex_error::vortex_err; - use vortex_session::VortexSession; - use wkb::writer::WriteOptions; - - use crate::extension::GeoMetadata; - use crate::extension::WellKnownBinary; - - static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); - crate::initialize(&session); - session - }); - - #[test] - fn test_array() -> VortexResult<()> { - let mut execution_ctx = SESSION.create_execution_ctx(); - - let mut buf = Vec::new(); - - let mut builder = VarBinBuilder::::with_capacity(3); - - let polygon = Geometry::Polygon(Polygon::new( - LineString::new(vec![ - Coord::zero(), - Coord { x: 100.0, y: 0.0 }, - Coord { x: 100.0, y: 100.0 }, - Coord { x: 0.0, y: 100.0 }, - Coord::zero(), - ]), - vec![], - )); - - // We should always prefer to write little-endian, which is the default option. - wkb::writer::write_geometry(&mut buf, &polygon, &WriteOptions::default()) - .map_err(|e| vortex_err!("writing WKB failed: {e}"))?; - - // Push same polygon 3 times - builder.append_value(&buf); - builder.append_value(&buf); - builder.append_value(&buf); - - let dtype = ExtDType::::try_new( - GeoMetadata { - crs: Some("EPSG:4326".to_string()), - }, - DType::Binary(Nullability::NonNullable), - )?; - - let array = builder.finish(DType::Binary(Nullability::NonNullable)); - let array = ExtensionArray::new(dtype.clone().erased(), array.into_array()).into_array(); - - for idx in 0..3 { - let geom = array.execute_scalar(idx, &mut execution_ctx)?; - let wkb = WellKnownBinary::unpack_native(&dtype, geom.value().unwrap())?; - - assert_eq!(wkb.to_geometry(), polygon); - } - - Ok(()) - } - - fn wkb_extension_array() -> VortexResult<(Vec, vortex_array::ArrayRef)> { - let polygon = Geometry::Polygon(Polygon::new( - LineString::new(vec![ - Coord::zero(), - Coord { x: 100.0, y: 0.0 }, - Coord { x: 100.0, y: 100.0 }, - Coord { x: 0.0, y: 100.0 }, - Coord::zero(), - ]), - vec![], - )); - let mut buf = Vec::new(); - wkb::writer::write_geometry(&mut buf, &polygon, &WriteOptions::default()) - .map_err(|e| vortex_err!("writing WKB failed: {e}"))?; - - let mut builder = VarBinBuilder::::with_capacity(3); - builder.append_value(&buf); - builder.append_value(&buf); - builder.append_value(&buf); - - let dtype = ExtDType::::try_new( - GeoMetadata { - crs: Some("EPSG:4326".to_string()), - }, - DType::Binary(Nullability::NonNullable), - )?; - let storage = builder.finish(DType::Binary(Nullability::NonNullable)); - let array = ExtensionArray::new(dtype.erased(), storage.into_array()).into_array(); - Ok((buf, array)) - } - - #[test] - fn export_arrow_field_carries_wkb_extension() -> VortexResult<()> { - let (_, array) = wkb_extension_array()?; - let field = SESSION.arrow().to_arrow_field("geom", array.dtype())?; - assert_eq!(field.extension_type_name(), Some(WkbType::NAME)); - // Vortex's canonical mapping of `DType::Binary` is Arrow's `BinaryView`. - assert_eq!(field.data_type(), &DataType::BinaryView); - Ok(()) - } - - #[test] - fn execute_arrow_exports_wkb_to_binary() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let (wkb_bytes, array) = wkb_extension_array()?; - - let field = Field::new("geom", DataType::Binary, false) - .with_extension_type(WkbType::new(Default::default())); - let exported = SESSION - .arrow() - .execute_arrow(array, Some(&field), &mut ctx)?; - - assert_eq!(exported.data_type(), &DataType::Binary); - let binary = exported.as_binary::(); - assert_eq!(binary.len(), 3); - for idx in 0..3 { - assert_eq!(binary.value(idx), wkb_bytes.as_slice()); - } - Ok(()) - } - - #[test] - fn execute_arrow_exports_wkb_to_large_binary() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let (wkb_bytes, array) = wkb_extension_array()?; - - let field = Field::new("geom", DataType::LargeBinary, false) - .with_extension_type(WkbType::new(Default::default())); - let exported = SESSION - .arrow() - .execute_arrow(array, Some(&field), &mut ctx)?; - - assert_eq!(exported.data_type(), &DataType::LargeBinary); - let binary = exported.as_binary::(); - assert_eq!(binary.len(), 3); - for idx in 0..3 { - assert_eq!(binary.value(idx), wkb_bytes.as_slice()); - } - Ok(()) - } - - #[test] - fn execute_arrow_exports_wkb_to_binary_view() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let (wkb_bytes, array) = wkb_extension_array()?; - - let field = Field::new("geom", DataType::BinaryView, false) - .with_extension_type(WkbType::new(Default::default())); - let exported = SESSION - .arrow() - .execute_arrow(array, Some(&field), &mut ctx)?; - - assert_eq!(exported.data_type(), &DataType::BinaryView); - let binary = exported.as_binary_view(); - assert_eq!(binary.len(), 3); - for idx in 0..3 { - assert_eq!(binary.value(idx), wkb_bytes.as_slice()); - } - Ok(()) - } - - fn wkb_field(name: &str, data_type: DataType, nullable: bool, crs: Option<&str>) -> Field { - let crs = crs - .map(|crs| Crs::from_unknown_crs_type(crs.to_string())) - .unwrap_or_default(); - let wkb_type = WkbType::new(Arc::new(Metadata::new(crs, None))); - Field::new(name, data_type, nullable).with_extension_type(wkb_type) - } - - fn assert_imported_wkb_dtype(dtype: &DType, expected_crs: Option<&str>, nullable: bool) { - let DType::Extension(ext) = dtype else { - panic!("expected Extension dtype, got {dtype}"); - }; - assert!(ext.is::()); - assert_eq!(ext.storage_dtype(), &DType::Binary(nullable.into())); - let geo = ext.metadata::(); - assert_eq!(geo.crs.as_deref(), expected_crs); - } - - #[test] - fn import_arrow_field_recovers_wkb_extension() -> VortexResult<()> { - let field = wkb_field("geom", DataType::Binary, false, Some("EPSG:4326")); - let dtype = SESSION.arrow().from_arrow_field(&field)?; - assert_imported_wkb_dtype(&dtype, Some("EPSG:4326"), false); - Ok(()) - } - - #[test] - fn import_arrow_field_without_crs() -> VortexResult<()> { - let field = wkb_field("geom", DataType::BinaryView, true, None); - let dtype = SESSION.arrow().from_arrow_field(&field)?; - assert_imported_wkb_dtype(&dtype, None, true); - Ok(()) - } - - #[test] - fn import_arrow_array_from_binary() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let (wkb_bytes, _) = wkb_extension_array()?; - let arrow: ArrowArrayRef = Arc::new(BinaryArray::from_iter_values([ - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - ])); - let field = wkb_field("geom", DataType::Binary, false, Some("EPSG:4326")); - - let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; - assert_imported_wkb_dtype(imported.dtype(), Some("EPSG:4326"), false); - - for idx in 0..3 { - let geom = imported.execute_scalar(idx, &mut ctx)?; - assert_eq!(geom.value().unwrap().as_binary().as_slice(), wkb_bytes); - } - Ok(()) - } - - #[test] - fn import_arrow_array_from_large_binary() -> VortexResult<()> { - let (wkb_bytes, _) = wkb_extension_array()?; - let arrow: ArrowArrayRef = Arc::new(LargeBinaryArray::from_iter_values([ - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - ])); - let field = wkb_field("geom", DataType::LargeBinary, false, Some("EPSG:4326")); - - let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; - assert_imported_wkb_dtype(imported.dtype(), Some("EPSG:4326"), false); - assert_eq!(imported.len(), 2); - Ok(()) - } - - #[test] - fn import_arrow_array_from_binary_view() -> VortexResult<()> { - let (wkb_bytes, _) = wkb_extension_array()?; - let arrow: ArrowArrayRef = Arc::new(BinaryViewArray::from_iter_values([ - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - wkb_bytes.as_slice(), - ])); - let field = wkb_field("geom", DataType::BinaryView, false, None); - - let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; - assert_imported_wkb_dtype(imported.dtype(), None, false); - assert_eq!(imported.len(), 4); - Ok(()) - } - - #[test] - fn import_arrow_array_roundtrips_through_export() -> VortexResult<()> { - let mut ctx = SESSION.create_execution_ctx(); - let (wkb_bytes, original) = wkb_extension_array()?; - - let field = Field::new("geom", DataType::BinaryView, false) - .with_extension_type(WkbType::new(Default::default())); - let exported = SESSION - .arrow() - .execute_arrow(original, Some(&field), &mut ctx)?; - let arrow_field = wkb_field("geom", DataType::BinaryView, false, Some("EPSG:4326")); - let reimported = SESSION.arrow().from_arrow_array(exported, &arrow_field)?; - - assert_imported_wkb_dtype(reimported.dtype(), Some("EPSG:4326"), false); - for idx in 0..3 { - let geom = reimported.execute_scalar(idx, &mut ctx)?; - assert_eq!(geom.value().unwrap().as_binary().as_slice(), wkb_bytes); - } - Ok(()) - } -} diff --git a/vortex-geo/src/scalar_fn/distance.rs b/vortex-geo/src/scalar_fn/distance.rs index e621c4bbad1..c65f1ab78dc 100644 --- a/vortex-geo/src/scalar_fn/distance.rs +++ b/vortex-geo/src/scalar_fn/distance.rs @@ -153,32 +153,13 @@ mod tests { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ConstantArray; - use vortex_array::arrays::ExtensionArray; - use vortex_array::arrays::PrimitiveArray; - use vortex_array::arrays::StructArray; - use vortex_array::dtype::extension::ExtDType; use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; use super::GeoDistance; use super::euclidean_distance; - use crate::extension::GeoMetadata; - use crate::extension::Point; - - /// A `Point` column (CRS `EPSG:4326`) over the given x/y coordinates. - fn point_column(xs: Vec, ys: Vec) -> VortexResult { - let storage = StructArray::from_fields(&[ - ("x", PrimitiveArray::from_iter(xs).into_array()), - ("y", PrimitiveArray::from_iter(ys).into_array()), - ])? - .into_array(); - let metadata = GeoMetadata { - crs: Some("EPSG:4326".to_string()), - }; - let dtype = ExtDType::::try_new(metadata, storage.dtype().clone())?; - Ok(ExtensionArray::new(dtype.erased(), storage).into_array()) - } + use crate::test_harness::point_column; /// A constant `Point` column of length `len`, every row at `(x, y)`. fn point_constant( diff --git a/vortex-geo/src/test_harness.rs b/vortex-geo/src/test_harness.rs new file mode 100644 index 00000000000..9f066fb4d14 --- /dev/null +++ b/vortex-geo/src/test_harness.rs @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Shared test helpers for the geospatial extension types. + +use vortex_array::ArrayRef; +use vortex_array::IntoArray; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::arrays::StructArray; +use vortex_array::dtype::extension::ExtDType; +use vortex_error::VortexResult; + +use crate::extension::GeoMetadata; +use crate::extension::Point; + +/// A `Point` column (CRS `EPSG:4326`) over the given x/y coordinates. +pub(crate) fn point_column(xs: Vec, ys: Vec) -> VortexResult { + let storage = StructArray::from_fields(&[ + ("x", PrimitiveArray::from_iter(xs).into_array()), + ("y", PrimitiveArray::from_iter(ys).into_array()), + ])? + .into_array(); + let metadata = GeoMetadata { + crs: Some("EPSG:4326".to_string()), + }; + let dtype = ExtDType::::try_new(metadata, storage.dtype().clone())?; + Ok(ExtensionArray::new(dtype.erased(), storage).into_array()) +} diff --git a/vortex-geo/src/tests/mod.rs b/vortex-geo/src/tests/mod.rs new file mode 100644 index 00000000000..86a89a60b5b --- /dev/null +++ b/vortex-geo/src/tests/mod.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Arrow interop tests for the geospatial extension types, exercising the session wiring set up +//! by [`crate::initialize`]. + +mod point; +mod wkb; + +use std::sync::LazyLock; + +use vortex_array::session::ArraySession; +use vortex_session::VortexSession; + +/// A session with the geospatial types and functions registered. +static SESSION: LazyLock = LazyLock::new(|| { + let session = VortexSession::empty().with::(); + crate::initialize(&session); + session +}); diff --git a/vortex-geo/src/tests/point.rs b/vortex-geo/src/tests/point.rs new file mode 100644 index 00000000000..074ef97f0d3 --- /dev/null +++ b/vortex-geo/src/tests/point.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Arrow interop for the `vortex.geo.point` extension type (`geoarrow.point`). + +use std::sync::Arc; + +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_array::Float64Array; +use arrow_array::StructArray as ArrowStructArray; +use arrow_array::cast::AsArray; +use arrow_array::types::Float64Type; +use arrow_schema::DataType; +use arrow_schema::Field; +use arrow_schema::Fields; +use arrow_schema::extension::ExtensionType as _; +use geoarrow::datatypes::CoordType; +use geoarrow::datatypes::Crs; +use geoarrow::datatypes::Dimension as GeoArrowDimension; +use geoarrow::datatypes::Metadata; +use geoarrow::datatypes::PointType; +use vortex_array::VortexSessionExecute; +use vortex_array::arrow::ArrowSessionExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_error::VortexResult; +use vortex_error::vortex_err; + +use super::SESSION; +use crate::extension::Point; +use crate::extension::coordinate::Coordinate; +use crate::extension::coordinate::coordinate_from_scalar; +use crate::test_harness::point_column; + +/// A `geoarrow.point` Arrow field with separated (struct) XY coordinates. +fn point_field(name: &str, nullable: bool, crs: Option<&str>) -> Field { + let crs = crs + .map(|crs| Crs::from_unknown_crs_type(crs.to_string())) + .unwrap_or_default(); + let metadata = Arc::new(Metadata::new(crs, None)); + PointType::new(GeoArrowDimension::XY, metadata).to_field(name, nullable) +} + +/// An Arrow `Struct` point array with non-nullable `Float64` children. +fn arrow_point_struct(xs: Vec, ys: Vec) -> ArrowStructArray { + let fields: Fields = vec![ + Field::new("x", DataType::Float64, false), + Field::new("y", DataType::Float64, false), + ] + .into(); + ArrowStructArray::new( + fields, + vec![ + Arc::new(Float64Array::from(xs)) as ArrowArrayRef, + Arc::new(Float64Array::from(ys)), + ], + None, + ) +} + +/// The exported Arrow field carries the `geoarrow.point` extension over the separated +/// `Struct` coordinate layout. +#[test] +fn export_field_carries_extension() -> VortexResult<()> { + let array = point_column(vec![1.0], vec![2.0])?; + let field = SESSION.arrow().to_arrow_field("loc", array.dtype())?; + + assert_eq!(field.extension_type_name(), Some(PointType::NAME)); + let DataType::Struct(fields) = field.data_type() else { + panic!("expected Struct, got {}", field.data_type()); + }; + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].name(), "x"); + assert_eq!(fields[0].data_type(), &DataType::Float64); + assert_eq!(fields[1].name(), "y"); + assert_eq!(fields[1].data_type(), &DataType::Float64); + Ok(()) +} + +/// Export materializes the point column as an Arrow struct with the original ordinates. +#[test] +fn exports_to_struct() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let array = point_column(vec![1.0, 3.0], vec![2.0, 4.0])?; + + let target = point_field("loc", false, Some("EPSG:4326")); + let exported = SESSION + .arrow() + .execute_arrow(array, Some(&target), &mut ctx)?; + + let points = exported.as_struct(); + let ordinates = |name: &str| -> VortexResult> { + Ok(points + .column_by_name(name) + .ok_or_else(|| vortex_err!("missing {name} column"))? + .as_primitive::() + .values() + .to_vec()) + }; + assert_eq!(ordinates("x")?, vec![1.0, 3.0]); + assert_eq!(ordinates("y")?, vec![2.0, 4.0]); + Ok(()) +} + +/// An imported `geoarrow.point` field maps to the Point extension dtype, recovering the +/// CRS, coordinate field names, and nullability. +#[test] +fn import_field_recovers_extension() -> VortexResult<()> { + let field = point_field("loc", true, Some("EPSG:4326")); + let dtype = SESSION.arrow().from_arrow_field(&field)?; + + let DType::Extension(ext) = &dtype else { + panic!("expected Extension dtype, got {dtype}"); + }; + assert!(ext.is::()); + assert_eq!(ext.metadata::().crs.as_deref(), Some("EPSG:4326")); + + let DType::Struct(fields, nullability) = ext.storage_dtype() else { + panic!("expected Struct storage, got {}", ext.storage_dtype()); + }; + assert_eq!(*nullability, Nullability::Nullable); + let names: Vec<&str> = fields.names().iter().map(|n| n.as_ref()).collect(); + assert_eq!(names, vec!["x", "y"]); + Ok(()) +} + +/// A field with interleaved (`FixedSizeList`) coordinates fails to import. +#[test] +fn import_interleaved_field_fails() { + let point_type = PointType::new(GeoArrowDimension::XY, Default::default()) + .with_coord_type(CoordType::Interleaved); + let field = point_type.to_field("loc", false); + assert!(SESSION.arrow().from_arrow_field(&field).is_err()); +} + +/// Import wraps the Arrow struct's coordinate buffers into a Point column. +#[test] +fn imports_from_struct() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let arrow: ArrowArrayRef = + Arc::new(arrow_point_struct(vec![1.0, -111.7610], vec![2.0, 34.8697])); + let field = point_field("loc", false, Some("EPSG:4326")); + + let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; + assert!( + imported + .dtype() + .as_extension_opt() + .map(|ext| ext.is::()) + .unwrap_or(false) + ); + + assert_eq!( + coordinate_from_scalar(&imported.execute_scalar(0, &mut ctx)?)?, + Coordinate::xy(1.0, 2.0) + ); + assert_eq!( + coordinate_from_scalar(&imported.execute_scalar(1, &mut ctx)?)?, + Coordinate::xy(-111.7610, 34.8697) + ); + Ok(()) +} + +/// A point column exported to Arrow and imported back is unchanged, including the CRS. +#[test] +fn roundtrips_through_arrow() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let original = point_column(vec![0.0, 3.0], vec![4.0, 0.0])?; + + let target = point_field("loc", false, Some("EPSG:4326")); + let exported = SESSION + .arrow() + .execute_arrow(original, Some(&target), &mut ctx)?; + let reimported = SESSION.arrow().from_arrow_array(exported, &target)?; + + let ext = reimported + .dtype() + .as_extension_opt() + .ok_or_else(|| vortex_err!("expected Extension dtype"))?; + assert_eq!(ext.metadata::().crs.as_deref(), Some("EPSG:4326")); + + assert_eq!( + coordinate_from_scalar(&reimported.execute_scalar(0, &mut ctx)?)?, + Coordinate::xy(0.0, 4.0) + ); + assert_eq!( + coordinate_from_scalar(&reimported.execute_scalar(1, &mut ctx)?)?, + Coordinate::xy(3.0, 0.0) + ); + Ok(()) +} diff --git a/vortex-geo/src/tests/wkb.rs b/vortex-geo/src/tests/wkb.rs new file mode 100644 index 00000000000..4ef3d6e2ba8 --- /dev/null +++ b/vortex-geo/src/tests/wkb.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Arrow interop for the `vortex.geo.wkb` extension type (`geoarrow.wkb`). + +use std::sync::Arc; + +use arrow_array::Array as _; +use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_array::BinaryArray; +use arrow_array::BinaryViewArray; +use arrow_array::LargeBinaryArray; +use arrow_array::cast::AsArray; +use arrow_schema::DataType; +use arrow_schema::Field; +use arrow_schema::extension::ExtensionType as _; +use geo_traits::to_geo::ToGeoGeometry; +use geo_types::Coord; +use geo_types::Geometry; +use geo_types::LineString; +use geo_types::Polygon; +use geoarrow::datatypes::Crs; +use geoarrow::datatypes::Metadata; +use geoarrow::datatypes::WkbType; +use rstest::rstest; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::ExtensionArray; +use vortex_array::arrays::varbin::builder::VarBinBuilder; +use vortex_array::arrow::ArrowSessionExt; +use vortex_array::dtype::DType; +use vortex_array::dtype::Nullability; +use vortex_array::dtype::extension::ExtDType; +use vortex_array::dtype::extension::ExtVTable; +use vortex_error::VortexResult; +use vortex_error::vortex_err; +use wkb::writer::WriteOptions; + +use super::SESSION; +use crate::extension::GeoMetadata; +use crate::extension::WellKnownBinary; + +/// The polygon geometry encoded by these tests. +fn test_polygon() -> Geometry { + Geometry::Polygon(Polygon::new( + LineString::new(vec![ + Coord::zero(), + Coord { x: 100.0, y: 0.0 }, + Coord { x: 100.0, y: 100.0 }, + Coord { x: 0.0, y: 100.0 }, + Coord::zero(), + ]), + vec![], + )) +} + +/// A WKB column (CRS `EPSG:4326`) holding [`test_polygon`] three times, along with the +/// polygon's WKB bytes. +fn wkb_extension_array() -> VortexResult<(Vec, vortex_array::ArrayRef)> { + let mut buf = Vec::new(); + // We should always prefer to write little-endian, which is the default option. + wkb::writer::write_geometry(&mut buf, &test_polygon(), &WriteOptions::default()) + .map_err(|e| vortex_err!("writing WKB failed: {e}"))?; + + let mut builder = VarBinBuilder::::with_capacity(3); + builder.append_value(&buf); + builder.append_value(&buf); + builder.append_value(&buf); + + let dtype = ExtDType::::try_new( + GeoMetadata { + crs: Some("EPSG:4326".to_string()), + }, + DType::Binary(Nullability::NonNullable), + )?; + let storage = builder.finish(DType::Binary(Nullability::NonNullable)); + let array = ExtensionArray::new(dtype.erased(), storage.into_array()).into_array(); + Ok((buf, array)) +} + +/// A `geoarrow.wkb` Arrow field over the given binary data type. +fn wkb_field(name: &str, data_type: DataType, nullable: bool, crs: Option<&str>) -> Field { + let crs = crs + .map(|crs| Crs::from_unknown_crs_type(crs.to_string())) + .unwrap_or_default(); + let wkb_type = WkbType::new(Arc::new(Metadata::new(crs, None))); + Field::new(name, data_type, nullable).with_extension_type(wkb_type) +} + +fn assert_imported_wkb_dtype(dtype: &DType, expected_crs: Option<&str>, nullable: bool) { + let DType::Extension(ext) = dtype else { + panic!("expected Extension dtype, got {dtype}"); + }; + assert!(ext.is::()); + assert_eq!(ext.storage_dtype(), &DType::Binary(nullable.into())); + let geo = ext.metadata::(); + assert_eq!(geo.crs.as_deref(), expected_crs); +} + +/// WKB scalars unpack back to the geometry they encode. +#[test] +fn scalar_unpacks_to_geometry() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let (_, array) = wkb_extension_array()?; + + let dtype = ExtDType::::try_new( + GeoMetadata { + crs: Some("EPSG:4326".to_string()), + }, + DType::Binary(Nullability::NonNullable), + )?; + + for idx in 0..3 { + let geom = array.execute_scalar(idx, &mut ctx)?; + let wkb = WellKnownBinary::unpack_native(&dtype, geom.value().unwrap())?; + assert_eq!(wkb.to_geometry(), test_polygon()); + } + Ok(()) +} + +/// The exported Arrow field carries the `geoarrow.wkb` extension over Vortex's canonical +/// binary mapping (`BinaryView`). +#[test] +fn export_field_carries_extension() -> VortexResult<()> { + let (_, array) = wkb_extension_array()?; + let field = SESSION.arrow().to_arrow_field("geom", array.dtype())?; + assert_eq!(field.extension_type_name(), Some(WkbType::NAME)); + assert_eq!(field.data_type(), &DataType::BinaryView); + Ok(()) +} + +/// Export materializes the WKB bytes into the requested binary-family Arrow array. +#[rstest] +#[case::binary(DataType::Binary)] +#[case::large_binary(DataType::LargeBinary)] +#[case::binary_view(DataType::BinaryView)] +fn exports_to_binary_family(#[case] data_type: DataType) -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let (wkb_bytes, array) = wkb_extension_array()?; + + let field = Field::new("geom", data_type.clone(), false) + .with_extension_type(WkbType::new(Default::default())); + let exported = SESSION + .arrow() + .execute_arrow(array, Some(&field), &mut ctx)?; + + assert_eq!(exported.data_type(), &data_type); + let values: Vec<&[u8]> = match &data_type { + DataType::Binary => exported.as_binary::().iter().flatten().collect(), + DataType::LargeBinary => exported.as_binary::().iter().flatten().collect(), + DataType::BinaryView => exported.as_binary_view().iter().flatten().collect(), + _ => unreachable!("cases cover only the binary family"), + }; + assert_eq!(values.len(), 3); + for value in values { + assert_eq!(value, wkb_bytes.as_slice()); + } + Ok(()) +} + +/// An imported `geoarrow.wkb` field maps to the WKB extension dtype, recovering the CRS. +#[test] +fn import_field_recovers_extension() -> VortexResult<()> { + let field = wkb_field("geom", DataType::Binary, false, Some("EPSG:4326")); + let dtype = SESSION.arrow().from_arrow_field(&field)?; + assert_imported_wkb_dtype(&dtype, Some("EPSG:4326"), false); + Ok(()) +} + +/// A `geoarrow.wkb` field without a CRS imports as an unreferenced geometry. +#[test] +fn import_field_without_crs() -> VortexResult<()> { + let field = wkb_field("geom", DataType::BinaryView, true, None); + let dtype = SESSION.arrow().from_arrow_field(&field)?; + assert_imported_wkb_dtype(&dtype, None, true); + Ok(()) +} + +/// Import wraps the binary-family Arrow array's WKB values unchanged. +#[test] +fn imports_from_binary() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let (wkb_bytes, _) = wkb_extension_array()?; + let arrow: ArrowArrayRef = Arc::new(BinaryArray::from_iter_values([ + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + ])); + let field = wkb_field("geom", DataType::Binary, false, Some("EPSG:4326")); + + let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; + assert_imported_wkb_dtype(imported.dtype(), Some("EPSG:4326"), false); + + for idx in 0..3 { + let geom = imported.execute_scalar(idx, &mut ctx)?; + assert_eq!(geom.value().unwrap().as_binary().as_slice(), wkb_bytes); + } + Ok(()) +} + +/// Import wraps the binary-family Arrow array's WKB values unchanged. +#[test] +fn imports_from_large_binary() -> VortexResult<()> { + let (wkb_bytes, _) = wkb_extension_array()?; + let arrow: ArrowArrayRef = Arc::new(LargeBinaryArray::from_iter_values([ + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + ])); + let field = wkb_field("geom", DataType::LargeBinary, false, Some("EPSG:4326")); + + let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; + assert_imported_wkb_dtype(imported.dtype(), Some("EPSG:4326"), false); + assert_eq!(imported.len(), 2); + Ok(()) +} + +/// Import wraps the binary-family Arrow array's WKB values unchanged. +#[test] +fn imports_from_binary_view() -> VortexResult<()> { + let (wkb_bytes, _) = wkb_extension_array()?; + let arrow: ArrowArrayRef = Arc::new(BinaryViewArray::from_iter_values([ + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + wkb_bytes.as_slice(), + ])); + let field = wkb_field("geom", DataType::BinaryView, false, None); + + let imported = SESSION.arrow().from_arrow_array(arrow, &field)?; + assert_imported_wkb_dtype(imported.dtype(), None, false); + assert_eq!(imported.len(), 4); + Ok(()) +} + +/// A WKB column exported to Arrow and imported back is unchanged, byte for byte. +#[test] +fn roundtrips_through_arrow() -> VortexResult<()> { + let mut ctx = SESSION.create_execution_ctx(); + let (wkb_bytes, original) = wkb_extension_array()?; + + let field = Field::new("geom", DataType::BinaryView, false) + .with_extension_type(WkbType::new(Default::default())); + let exported = SESSION + .arrow() + .execute_arrow(original, Some(&field), &mut ctx)?; + let arrow_field = wkb_field("geom", DataType::BinaryView, false, Some("EPSG:4326")); + let reimported = SESSION.arrow().from_arrow_array(exported, &arrow_field)?; + + assert_imported_wkb_dtype(reimported.dtype(), Some("EPSG:4326"), false); + for idx in 0..3 { + let geom = reimported.execute_scalar(idx, &mut ctx)?; + assert_eq!(geom.value().unwrap().as_binary().as_slice(), wkb_bytes); + } + Ok(()) +} From 37ae2d73a4c4727bf4572bee9ea6419a04acb30c Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Fri, 12 Jun 2026 17:49:29 +0100 Subject: [PATCH 004/162] REGEX support for sqllogictests (#8392) Support `:` and `:` in sqllogictests Signed-off-by: Mikhail Kot --- Cargo.lock | 1 + vortex-sqllogictest/Cargo.toml | 1 + vortex-sqllogictest/src/duckdb.rs | 115 +++++++++++++++++++++++++++--- 3 files changed, 108 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57be369d7c3..00f5345aa76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10227,6 +10227,7 @@ dependencies = [ "datafusion-sqllogictest", "futures", "indicatif", + "regex", "rstest", "sqllogictest", "thiserror 2.0.18", diff --git a/vortex-sqllogictest/Cargo.toml b/vortex-sqllogictest/Cargo.toml index 757abb49689..499702454f3 100644 --- a/vortex-sqllogictest/Cargo.toml +++ b/vortex-sqllogictest/Cargo.toml @@ -22,6 +22,7 @@ datafusion = { workspace = true } datafusion-sqllogictest = { workspace = true } futures.workspace = true indicatif.workspace = true +regex = { workspace = true } rstest = { workspace = true } sqllogictest = "0.29.1" thiserror = { workspace = true } diff --git a/vortex-sqllogictest/src/duckdb.rs b/vortex-sqllogictest/src/duckdb.rs index 182110e1f45..821eaeeed6b 100644 --- a/vortex-sqllogictest/src/duckdb.rs +++ b/vortex-sqllogictest/src/duckdb.rs @@ -10,11 +10,13 @@ use async_trait::async_trait; use bigdecimal::BigDecimal; use datafusion_sqllogictest::DFColumnType; use indicatif::ProgressBar; +use regex::RegexBuilder; use sqllogictest::DBOutput; use sqllogictest::Normalizer; use sqllogictest::runner::AsyncDB; use vortex::error::VortexError; use vortex::error::VortexExpect; +use vortex::error::vortex_err; use vortex_duckdb::duckdb::Connection; use vortex_duckdb::duckdb::Database; use vortex_duckdb::duckdb::ExtractedValue; @@ -99,20 +101,48 @@ impl DuckDB { } } +// Regex matching for output inspired by Duckdb's .test files. Rows joined +// by newline are matched against positive or negative match. +const REGEX_MARKER: &str = ":"; +const NEG_REGEX_MARKER: &str = ":"; + +fn regex_matches(pattern: &str, text: &str) -> bool { + RegexBuilder::new(pattern) + .dot_matches_new_line(true) + .build() + .map_err(|e| vortex_err!("invalid regex pattern: {e}")) + .vortex_expect("invalid pattern") + .is_match(text) +} + pub fn duckdb_validator( normalizer: Normalizer, actual: &[Vec], expected: &[String], ) -> bool { - let actual = actual.iter().flat_map(|strings| { - strings - .join(" ") - .trim_end() - .split('\n') - .map(|line| line.trim_end().to_string()) - .collect::>() - }); - Iterator::eq(actual, expected.iter().map(normalizer)) + let actual = actual + .iter() + .flat_map(|strings| { + strings + .join(" ") + .trim_end() + .split('\n') + .map(|line| line.trim_end().to_string()) + .collect::>() + }) + .collect::>(); + let expected = expected.iter().map(normalizer).collect::>(); + + if let [line] = expected.as_slice() { + if let Some(pattern) = line.strip_prefix(NEG_REGEX_MARKER) { + return !regex_matches(pattern, &actual.join("\n")); + } + if let Some(pattern) = line.strip_prefix(REGEX_MARKER) { + return regex_matches(pattern, &actual.join("\n")); + } + } + + Iterator::eq(actual.iter(), expected.iter()) } #[async_trait] @@ -274,11 +304,78 @@ mod tests { use vortex_duckdb::duckdb::Value; use super::ValueDisplayAdapter; + use super::duckdb_validator; + use super::regex_matches; fn display(value: Value) -> String { ValueDisplayAdapter(value).to_string() } + #[rstest] + #[case("foo", "foo", true)] + #[case("foo", "xfooy", true)] + #[case("foo", "bar", false)] + #[case("^foo$", "foo", true)] + #[case("^foo$", "xfoo", false)] + #[case("a.*b", "axxxb", true)] + #[case("a.*b", "a\nxx\nb", true)] + #[case("needle", "line1\nneedle here\nline3", true)] + #[case("missing", "line1\nline2\nline3", false)] + fn test_regex_matches(#[case] pattern: &str, #[case] text: &str, #[case] expected: bool) { + assert_eq!(regex_matches(pattern, text), expected); + } + + // clippy: Signature must match sqllogictest::Normalizer + #[expect(clippy::ptr_arg)] + fn identity(s: &String) -> String { + s.clone() + } + + #[test] + fn validator_regex_directive_spans_multiple_lines() { + let actual = vec![vec![ + "physical_plan".to_string(), + "[\n {\n \"SELECT projections\": \"str\"\n }\n]".to_string(), + ]]; + + assert!(duckdb_validator( + identity, + &actual, + &[":SELECT projections".to_string()] + )); + assert!(!duckdb_validator( + identity, + &actual, + &[":NOT PRESENT".to_string()] + )); + + assert!(duckdb_validator( + identity, + &actual, + &[":NOT PRESENT".to_string()] + )); + assert!(!duckdb_validator( + identity, + &actual, + &[":SELECT projections".to_string()] + )); + } + + #[test] + fn validator_without_marker_requires_exact_match() { + let actual = vec![vec!["5".to_string()], vec!["3".to_string()]]; + assert!(duckdb_validator( + identity, + &actual, + &["5".to_string(), "3".to_string()] + )); + assert!(!duckdb_validator( + identity, + &actual, + &["5".to_string(), "2".to_string()] + )); + } + #[test] fn test_null() { assert_eq!(display(Value::sql_null()), "NULL"); From 31167ca37f6818d498d2e78e77834059538f2c5e Mon Sep 17 00:00:00 2001 From: Andrew Duffy Date: Fri, 12 Jun 2026 14:05:21 -0400 Subject: [PATCH 005/162] remove unused lockfile (#8394) We are no longer using the lockfiles, but I accidentally added this one as part of #7722 Signed-off-by: Andrew Duffy --- vortex-geo/public-api.lock | 127 ------------------------------------- 1 file changed, 127 deletions(-) delete mode 100644 vortex-geo/public-api.lock diff --git a/vortex-geo/public-api.lock b/vortex-geo/public-api.lock deleted file mode 100644 index 3d399aace98..00000000000 --- a/vortex-geo/public-api.lock +++ /dev/null @@ -1,127 +0,0 @@ -pub mod vortex_geo - -pub mod vortex_geo::extension - -pub struct vortex_geo::extension::GeoMetadata - -pub vortex_geo::extension::GeoMetadata::crs: core::option::Option - -impl vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::crs(&self) -> &str - -impl core::clone::Clone for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::clone(&self) -> vortex_geo::extension::GeoMetadata - -impl core::cmp::Eq for vortex_geo::extension::GeoMetadata - -impl core::cmp::PartialEq for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::eq(&self, &vortex_geo::extension::GeoMetadata) -> bool - -impl core::default::Default for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::default() -> Self - -impl core::fmt::Debug for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::fmt::Display for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::hash::Hash for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::hash<__H: core::hash::Hasher>(&self, &mut __H) - -impl core::marker::StructuralPartialEq for vortex_geo::extension::GeoMetadata - -impl prost::message::Message for vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::GeoMetadata::clear(&mut self) - -pub fn vortex_geo::extension::GeoMetadata::encoded_len(&self) -> usize - -pub struct vortex_geo::extension::WellKnownBinary - -impl core::clone::Clone for vortex_geo::extension::WellKnownBinary - -pub fn vortex_geo::extension::WellKnownBinary::clone(&self) -> vortex_geo::extension::WellKnownBinary - -impl core::cmp::Eq for vortex_geo::extension::WellKnownBinary - -impl core::cmp::PartialEq for vortex_geo::extension::WellKnownBinary - -pub fn vortex_geo::extension::WellKnownBinary::eq(&self, &vortex_geo::extension::WellKnownBinary) -> bool - -impl core::default::Default for vortex_geo::extension::WellKnownBinary - -pub fn vortex_geo::extension::WellKnownBinary::default() -> vortex_geo::extension::WellKnownBinary - -impl core::fmt::Debug for vortex_geo::extension::WellKnownBinary - -pub fn vortex_geo::extension::WellKnownBinary::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl core::hash::Hash for vortex_geo::extension::WellKnownBinary - -pub fn vortex_geo::extension::WellKnownBinary::hash<__H: core::hash::Hasher>(&self, &mut __H) - -impl core::marker::StructuralPartialEq for vortex_geo::extension::WellKnownBinary - -impl vortex_array::dtype::extension::vtable::ExtVTable for vortex_geo::extension::WellKnownBinary - -pub type vortex_geo::extension::WellKnownBinary::Metadata = vortex_geo::extension::GeoMetadata - -pub type vortex_geo::extension::WellKnownBinary::NativeValue<'a> = vortex_geo::extension::Wkb<'a> - -pub fn vortex_geo::extension::WellKnownBinary::deserialize_metadata(&self, &[u8]) -> vortex_error::VortexResult - -pub fn vortex_geo::extension::WellKnownBinary::id(&self) -> vortex_array::dtype::extension::ExtId - -pub fn vortex_geo::extension::WellKnownBinary::serialize_metadata(&self, &Self::Metadata) -> vortex_error::VortexResult> - -pub fn vortex_geo::extension::WellKnownBinary::unpack_native<'a>(&'a vortex_array::dtype::extension::typed::ExtDType, &'a vortex_array::scalar::scalar_value::ScalarValue) -> vortex_error::VortexResult - -pub fn vortex_geo::extension::WellKnownBinary::validate_dtype(&vortex_array::dtype::extension::typed::ExtDType) -> vortex_error::VortexResult<()> - -pub struct vortex_geo::extension::WellKnownBinaryData - -impl vortex_geo::extension::WellKnownBinaryData - -pub fn vortex_geo::extension::WellKnownBinaryData::geo_metadata(&self) -> &vortex_geo::extension::GeoMetadata - -pub fn vortex_geo::extension::WellKnownBinaryData::wkb_values(&self) -> &vortex_array::array::erased::ArrayRef - -impl core::clone::Clone for vortex_geo::extension::WellKnownBinaryData - -pub fn vortex_geo::extension::WellKnownBinaryData::clone(&self) -> vortex_geo::extension::WellKnownBinaryData - -impl core::convert::TryFrom> for vortex_geo::extension::WellKnownBinaryData - -pub type vortex_geo::extension::WellKnownBinaryData::Error = vortex_error::VortexError - -pub fn vortex_geo::extension::WellKnownBinaryData::try_from(vortex_array::arrays::extension::vtable::ExtensionArray) -> core::result::Result - -impl core::fmt::Debug for vortex_geo::extension::WellKnownBinaryData - -pub fn vortex_geo::extension::WellKnownBinaryData::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -pub struct vortex_geo::extension::Wkb<'a>(_) - -impl<'a> vortex_geo::extension::Wkb<'a> - -pub fn vortex_geo::extension::Wkb<'a>::try_from_bytes(&'a [u8]) -> vortex_error::VortexResult - -impl<'a> core::fmt::Display for vortex_geo::extension::Wkb<'a> - -pub fn vortex_geo::extension::Wkb<'a>::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result - -impl<'a> core::ops::deref::Deref for vortex_geo::extension::Wkb<'a> - -pub type vortex_geo::extension::Wkb<'a>::Target = wkb::reader::geometry::Wkb<'a> - -pub fn vortex_geo::extension::Wkb<'a>::deref(&self) -> &wkb::reader::geometry::Wkb<'a> - -pub fn vortex_geo::initialize(&vortex_session::VortexSession) From 5580295eed269900b562eb8249d355aeec239733 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 12 Jun 2026 20:13:55 +0100 Subject: [PATCH 006/162] Fix weird malformed license notice in test file (#8395) ## Summary I was trying [hawkeye](https://github.com/korandoru/hawkeye) which has some nice features `reuse` doesn't have, and it found this weird issue. Signed-off-by: Adam Gutglick --- vortex-array/src/arrays/datetime/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vortex-array/src/arrays/datetime/test.rs b/vortex-array/src/arrays/datetime/test.rs index 511b8196a79..ed2adc7cfc8 100644 --- a/vortex-array/src/arrays/datetime/test.rs +++ b/vortex-array/src/arrays/datetime/test.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributorsuse crate::dtype::Nullability; +// SPDX-FileCopyrightText: Copyright the Vortex contributors use rstest::rstest; use vortex_buffer::buffer; From d0013ff6a2b898c9722e73c55e509298e6f811eb Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 12 Jun 2026 16:22:20 -0400 Subject: [PATCH 007/162] Add binary operator benchmarks (#8396) Adds some benchmarks for binary operators as I have some plans to improve Signed-off-by: Nicholas Gates --- vortex-array/Cargo.toml | 4 + vortex-array/benches/binary_ops.rs | 154 +++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 vortex-array/benches/binary_ops.rs diff --git a/vortex-array/Cargo.toml b/vortex-array/Cargo.toml index 4ec8a83575e..de3bb416985 100644 --- a/vortex-array/Cargo.toml +++ b/vortex-array/Cargo.toml @@ -125,6 +125,10 @@ harness = false name = "compare" harness = false +[[bench]] +name = "binary_ops" +harness = false + [[bench]] name = "interleave" harness = false diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs new file mode 100644 index 00000000000..a5e3f9a5594 --- /dev/null +++ b/vortex-array/benches/binary_ops.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#![expect(clippy::unwrap_used)] +#![expect( + clippy::cast_possible_truncation, + reason = "benchmark fixtures use indices that fit in the chosen widths" +)] + +use std::sync::LazyLock; + +use divan::Bencher; +use divan::counter::ItemsCount; +use vortex_array::ArrayRef; +use vortex_array::Executable; +use vortex_array::IntoArray; +use vortex_array::VortexSessionExecute; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::scalar_fn::fns::operators::Operator; +use vortex_array::session::ArraySession; +use vortex_session::VortexSession; + +fn main() { + divan::main(); +} + +static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + +const LEN: usize = 65_536; + +#[divan::bench] +fn add_i64_nonnull(bencher: Bencher) { + let lhs = primitive_nonnull(0).into_array(); + let rhs = primitive_nonnull(1_000_000).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Add); +} + +#[divan::bench] +fn add_i64_nullable(bencher: Bencher) { + let lhs = primitive_nullable(0, 7).into_array(); + let rhs = primitive_nullable(1_000_000, 5).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Add); +} + +#[divan::bench] +fn mul_i64_nonnull(bencher: Bencher) { + let lhs = primitive_small_nonnull(1).into_array(); + let rhs = primitive_small_nonnull(17).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn div_i64_nonnull(bencher: Bencher) { + let lhs = primitive_nonnull(1_000_000).into_array(); + let rhs = primitive_nonzero().into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Div); +} + +#[divan::bench] +fn sub_i64_constant(bencher: Bencher) { + let lhs = primitive_nonnull(0).into_array(); + let rhs = ConstantArray::new(37i64, LEN).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Sub); +} + +#[divan::bench] +fn eq_i64_constant(bencher: Bencher) { + let lhs = primitive_nonnull(0).into_array(); + let rhs = ConstantArray::new(1024i64, LEN).into_array(); + + bench_bool(bencher, lhs, rhs, Operator::Eq); +} + +#[divan::bench] +fn lt_i64_nullable(bencher: Bencher) { + let lhs = primitive_nullable(0, 7).into_array(); + let rhs = primitive_nullable(1_000_000, 5).into_array(); + + bench_bool(bencher, lhs, rhs, Operator::Lt); +} + +#[divan::bench] +fn and_bool_nullable(bencher: Bencher) { + let lhs = bool_nullable(2, 7).into_array(); + let rhs = bool_nullable(3, 5).into_array(); + + bench_bool(bencher, lhs, rhs, Operator::And); +} + +#[divan::bench] +fn or_bool_constant(bencher: Bencher) { + let lhs = bool_nullable(2, 7).into_array(); + let rhs = ConstantArray::new(true, LEN).into_array(); + + bench_bool(bencher, lhs, rhs, Operator::Or); +} + +fn bench_primitive(bencher: Bencher, lhs: ArrayRef, rhs: ArrayRef, operator: Operator) { + bench_binary::(bencher, lhs, rhs, operator); +} + +fn bench_bool(bencher: Bencher, lhs: ArrayRef, rhs: ArrayRef, operator: Operator) { + bench_binary::(bencher, lhs, rhs, operator); +} + +fn bench_binary( + bencher: Bencher, + lhs: ArrayRef, + rhs: ArrayRef, + operator: Operator, +) { + let mut ctx = SESSION.create_execution_ctx(); + + bencher.counter(ItemsCount::new(LEN)).bench_local(|| { + lhs.clone() + .binary(rhs.clone(), operator) + .unwrap() + .execute::(&mut ctx) + .unwrap() + }); +} + +fn primitive_nonnull(base: i64) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN as i64).map(|i| base + i)) +} + +fn primitive_small_nonnull(offset: i64) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN as i64).map(|i| ((i + offset) % 1024) + 1)) +} + +fn primitive_nonzero() -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN as i64).map(|i| (i % 255) + 1)) +} + +fn primitive_nullable(base: i64, null_every: usize) -> PrimitiveArray { + PrimitiveArray::from_option_iter( + (0..LEN as i64).map(|i| (!(i as usize).is_multiple_of(null_every)).then_some(base + i)), + ) +} + +fn bool_nullable(true_every: usize, null_every: usize) -> BoolArray { + BoolArray::from_iter( + (0..LEN).map(|i| (!i.is_multiple_of(null_every)).then_some(i.is_multiple_of(true_every))), + ) +} From 9444d20aed2149022778ee93a5fb255c45d12783 Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Fri, 12 Jun 2026 19:54:49 -0400 Subject: [PATCH 008/162] More arithmetic benchmarks (#8402) Signed-off-by: Nicholas Gates --- vortex-array/benches/binary_ops.rs | 95 ++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index a5e3f9a5594..31cc5d2ec42 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -56,6 +56,70 @@ fn mul_i64_nonnull(bencher: Bencher) { bench_primitive(bencher, lhs, rhs, Operator::Mul); } +#[divan::bench] +fn mul_i8_nonnull(bencher: Bencher) { + let lhs = primitive_i8_small_nonnull(1).into_array(); + let rhs = primitive_i8_small_nonnull(7).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_u8_nonnull(bencher: Bencher) { + let lhs = primitive_u8_small_nonnull(1).into_array(); + let rhs = primitive_u8_small_nonnull(7).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_i16_nonnull(bencher: Bencher) { + let lhs = primitive_i16_small_nonnull(1).into_array(); + let rhs = primitive_i16_small_nonnull(17).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_u16_nonnull(bencher: Bencher) { + let lhs = primitive_u16_small_nonnull(1).into_array(); + let rhs = primitive_u16_small_nonnull(17).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_i32_nonnull(bencher: Bencher) { + let lhs = primitive_i32_small_nonnull(1).into_array(); + let rhs = primitive_i32_small_nonnull(17).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_u32_nonnull(bencher: Bencher) { + let lhs = primitive_u32_small_nonnull(1).into_array(); + let rhs = primitive_u32_small_nonnull(17).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_i32_nullable(bencher: Bencher) { + let lhs = primitive_i32_small_nullable(1, 7).into_array(); + let rhs = primitive_i32_small_nullable(17, 5).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + +#[divan::bench] +fn mul_i32_constant(bencher: Bencher) { + let lhs = primitive_i32_small_nonnull(1).into_array(); + let rhs = ConstantArray::new(31i32, LEN).into_array(); + + bench_primitive(bencher, lhs, rhs, Operator::Mul); +} + #[divan::bench] fn div_i64_nonnull(bencher: Bencher) { let lhs = primitive_nonnull(1_000_000).into_array(); @@ -137,6 +201,30 @@ fn primitive_small_nonnull(offset: i64) -> PrimitiveArray { PrimitiveArray::from_iter((0..LEN as i64).map(|i| ((i + offset) % 1024) + 1)) } +fn primitive_i8_small_nonnull(offset: i8) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| (((i as i32 + offset as i32) % 21) - 10) as i8)) +} + +fn primitive_u8_small_nonnull(offset: u8) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| ((i + offset as usize) % 15 + 1) as u8)) +} + +fn primitive_i16_small_nonnull(offset: i16) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| (((i as i32 + offset as i32) % 255) - 127) as i16)) +} + +fn primitive_u16_small_nonnull(offset: u16) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| ((i + offset as usize) % 251 + 1) as u16)) +} + +fn primitive_i32_small_nonnull(offset: i32) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| (((i as i64 + offset as i64) % 4096) - 2048) as i32)) +} + +fn primitive_u32_small_nonnull(offset: u32) -> PrimitiveArray { + PrimitiveArray::from_iter((0..LEN).map(|i| ((i + offset as usize) % 4096 + 1) as u32)) +} + fn primitive_nonzero() -> PrimitiveArray { PrimitiveArray::from_iter((0..LEN as i64).map(|i| (i % 255) + 1)) } @@ -147,6 +235,13 @@ fn primitive_nullable(base: i64, null_every: usize) -> PrimitiveArray { ) } +fn primitive_i32_small_nullable(offset: i32, null_every: usize) -> PrimitiveArray { + PrimitiveArray::from_option_iter((0..LEN).map(|i| { + (!i.is_multiple_of(null_every)) + .then_some((((i as i64 + offset as i64) % 4096) - 2048) as i32) + })) +} + fn bool_nullable(true_every: usize, null_every: usize) -> BoolArray { BoolArray::from_iter( (0..LEN).map(|i| (!i.is_multiple_of(null_every)).then_some(i.is_multiple_of(true_every))), From 4ea4f546aa35cea796c19ff6b7f9e6ad846fbe01 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 15 Jun 2026 12:06:05 +0100 Subject: [PATCH 009/162] Constant comparison and byte_length OnPair kernels (#8371) This PR adds 2 kernels for OnPair: - byte_length which uses uncompressed_lengths only - constant comparison to avoid decompressing for queries like URL != '' This PR also adds validity() for FSST's kernel for byte_length so that evaluating validity wouldn't mean decompressing whole FSST array. Signed-off-by: Mikhail Kot --- .../onpair/src/compute/byte_length.rs | 36 +++++ .../onpair/src/compute/compare.rs | 143 ++++++++++++++++++ .../experimental/onpair/src/compute/mod.rs | 2 + encodings/experimental/onpair/src/kernel.rs | 9 +- vortex-array/src/scalar_fn/fns/byte_length.rs | 9 ++ 5 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 encodings/experimental/onpair/src/compute/byte_length.rs create mode 100644 encodings/experimental/onpair/src/compute/compare.rs diff --git a/encodings/experimental/onpair/src/compute/byte_length.rs b/encodings/experimental/onpair/src/compute/byte_length.rs new file mode 100644 index 00000000000..933e8bccd85 --- /dev/null +++ b/encodings/experimental/onpair/src/compute/byte_length.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::IntoArray; +use vortex_array::ValidityVTable; +use vortex_array::arrays::ConstantArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::dtype::PType; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::fns::byte_length::ByteLengthKernel; +use vortex_array::validity::Validity; + +use crate::OnPair; +use crate::OnPairArraySlotsExt; + +// TODO(myrrc): this and FSST comparison should be in the same trait. +// https://github.com/vortex-data/vortex/tree/myrrc/onpair-compare-trait +impl ByteLengthKernel for OnPair { + fn byte_length( + array: vortex_array::ArrayView<'_, Self>, + _ctx: &mut vortex_array::ExecutionCtx, + ) -> vortex_error::VortexResult> { + let nullable = array.dtype().nullability(); + let dtype = DType::Primitive(PType::U64, nullable); + // Uncompressed lengths are non-nullable and may be less than u64 each + let lengths = array.uncompressed_lengths().cast(dtype.clone())?; + Ok(Some(match OnPair::validity(array)? { + Validity::NonNullable | Validity::AllValid => lengths, + Validity::Array(v) => lengths.mask(v)?, + Validity::AllInvalid => { + ConstantArray::new(Scalar::null(dtype), lengths.len()).into_array() + } + })) + } +} diff --git a/encodings/experimental/onpair/src/compute/compare.rs b/encodings/experimental/onpair/src/compute/compare.rs new file mode 100644 index 00000000000..c33f1f10a08 --- /dev/null +++ b/encodings/experimental/onpair/src/compute/compare.rs @@ -0,0 +1,143 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_array::ArrayRef; +use vortex_array::ArrayView; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::BoolArray; +use vortex_array::arrays::ConstantArray; +use vortex_array::builtins::ArrayBuiltins; +use vortex_array::dtype::DType; +use vortex_array::scalar::Scalar; +use vortex_array::scalar_fn::fns::binary::CompareKernel; +use vortex_array::scalar_fn::fns::operators::CompareOperator; +use vortex_buffer::BitBuffer; +use vortex_error::VortexResult; + +use crate::OnPair; +use crate::OnPairArraySlotsExt; + +impl CompareKernel for OnPair { + fn compare( + lhs: ArrayView<'_, Self>, + rhs: &ArrayRef, + operator: CompareOperator, + ctx: &mut ExecutionCtx, + ) -> VortexResult> { + let Some(constant) = rhs.as_constant() else { + return Ok(None); + }; + let is_empty = match constant.dtype() { + DType::Utf8(_) => constant.as_utf8().is_empty(), + DType::Binary(_) => constant.as_binary().is_empty(), + _ => return Ok(None), + }; + if is_empty != Some(true) { + return Ok(None); + } + + let lengths = lhs.uncompressed_lengths(); + let buffer = match operator { + // every value is greater than an empty string + CompareOperator::Gte => BitBuffer::new_set(lhs.len()), + // no value is less than an empty string + CompareOperator::Lt => BitBuffer::new_unset(lhs.len()), + _ => lengths + .binary( + ConstantArray::new(Scalar::zero_value(lengths.dtype()), lengths.len()) + .into_array(), + operator.into(), + )? + .execute(ctx)?, + }; + Ok(Some( + BoolArray::new( + buffer, + lhs.validity()? + .union_nullability(constant.dtype().nullability()), + ) + .into_array(), + )) + } +} + +#[cfg(test)] +mod tests { + use std::sync::LazyLock; + + use rstest::rstest; + use vortex_array::IntoArray; + use vortex_array::VortexSessionExecute; + use vortex_array::arrays::BoolArray; + use vortex_array::arrays::ConstantArray; + use vortex_array::arrays::VarBinArray; + use vortex_array::assert_arrays_eq; + use vortex_array::builtins::ArrayBuiltins; + use vortex_array::dtype::DType; + use vortex_array::dtype::Nullability; + use vortex_array::scalar::Scalar; + use vortex_array::scalar_fn::fns::operators::Operator; + use vortex_array::session::ArraySession; + use vortex_error::VortexResult; + use vortex_session::VortexSession; + + use crate::compress::DEFAULT_DICT12_CONFIG; + use crate::compress::onpair_compress; + + static SESSION: LazyLock = + LazyLock::new(|| VortexSession::empty().with::()); + + #[cfg_attr(miri, ignore)] + #[rstest] + #[case(Operator::Eq, [true, false, true, false])] + #[case(Operator::NotEq, [false, true, false, true])] + #[case(Operator::Gt, [false, true, false, true])] + #[case(Operator::Gte, [true, true, true, true])] + #[case(Operator::Lt, [false, false, false, false])] + #[case(Operator::Lte, [true, false, true, false])] + fn compare_empty_string(#[case] op: Operator, #[case] expected: [bool; 4]) -> VortexResult<()> { + let input = VarBinArray::from_iter( + [Some(""), Some("a"), Some(""), Some("bbb")], + DType::Utf8(Nullability::NonNullable), + ); + let arr = onpair_compress(&input, input.len(), input.dtype(), DEFAULT_DICT12_CONFIG)? + .into_array(); + + let mut ctx = SESSION.create_execution_ctx(); + let result = arr + .binary(ConstantArray::new("", input.len()).into_array(), op)? + .execute::(&mut ctx)?; + assert_arrays_eq!(&result, &BoolArray::from_iter(expected)); + Ok(()) + } + + #[cfg_attr(miri, ignore)] + #[test] + fn compare_empty_string_nullable() -> VortexResult<()> { + let input = VarBinArray::from_iter( + [Some(""), None, Some("x")], + DType::Utf8(Nullability::Nullable), + ); + let arr = onpair_compress(&input, input.len(), input.dtype(), DEFAULT_DICT12_CONFIG)? + .into_array(); + let mut ctx = SESSION.create_execution_ctx(); + + let eq_empty = arr + .clone() + .binary(ConstantArray::new("", arr.len()).into_array(), Operator::Eq)? + .execute::(&mut ctx)?; + assert_arrays_eq!( + &eq_empty, + &BoolArray::from_iter([Some(true), None, Some(false)]) + ); + + let null_rhs = + ConstantArray::new(Scalar::null(DType::Utf8(Nullability::Nullable)), arr.len()); + let eq_null = arr + .binary(null_rhs.into_array(), Operator::Eq)? + .execute::(&mut ctx)?; + assert_arrays_eq!(&eq_null, &BoolArray::from_iter([None::, None, None])); + Ok(()) + } +} diff --git a/encodings/experimental/onpair/src/compute/mod.rs b/encodings/experimental/onpair/src/compute/mod.rs index 4cb15868625..4ad5f48f578 100644 --- a/encodings/experimental/onpair/src/compute/mod.rs +++ b/encodings/experimental/onpair/src/compute/mod.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod byte_length; mod cast; +mod compare; mod filter; mod slice; diff --git a/encodings/experimental/onpair/src/kernel.rs b/encodings/experimental/onpair/src/kernel.rs index fdd521e887e..7eb7b761b4f 100644 --- a/encodings/experimental/onpair/src/kernel.rs +++ b/encodings/experimental/onpair/src/kernel.rs @@ -3,9 +3,14 @@ use vortex_array::arrays::filter::FilterExecuteAdaptor; use vortex_array::kernel::ParentKernelSet; +use vortex_array::scalar_fn::fns::binary::CompareExecuteAdaptor; +use vortex_array::scalar_fn::fns::byte_length::ByteLengthExecuteAdaptor; use crate::OnPair; // TODO: implement ListExecute & TakeExecute for OnPair -pub(super) const PARENT_KERNELS: ParentKernelSet = - ParentKernelSet::new(&[ParentKernelSet::lift(&FilterExecuteAdaptor(OnPair))]); +pub(super) const PARENT_KERNELS: ParentKernelSet = ParentKernelSet::new(&[ + ParentKernelSet::lift(&FilterExecuteAdaptor(OnPair)), + ParentKernelSet::lift(&CompareExecuteAdaptor(OnPair)), + ParentKernelSet::lift(&ByteLengthExecuteAdaptor(OnPair)), +]); diff --git a/vortex-array/src/scalar_fn/fns/byte_length.rs b/vortex-array/src/scalar_fn/fns/byte_length.rs index 13a4f3158b5..aa9c508ea89 100644 --- a/vortex-array/src/scalar_fn/fns/byte_length.rs +++ b/vortex-array/src/scalar_fn/fns/byte_length.rs @@ -24,6 +24,7 @@ use crate::arrays::varbinview::VarBinViewArrayExt; use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; +use crate::expr::Expression; use crate::kernel::ExecuteParentKernel; use crate::scalar::Scalar; use crate::scalar_fn::Arity; @@ -122,6 +123,14 @@ impl ScalarFnVTable for ByteLength { } } + fn validity( + &self, + _: &Self::Options, + expression: &Expression, + ) -> VortexResult> { + Ok(Some(expression.child(0).validity()?)) + } + fn is_null_sensitive(&self, _options: &Self::Options) -> bool { false } From d66b9739542c8e76819d5634113475389f3b556d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 11:37:11 +0000 Subject: [PATCH 010/162] Update plugin com.palantir.java-format to v2.93.0 (#8416) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | com.palantir.java-format | `2.91.0` → `2.93.0` | ![age](https://developer.mend.io/api/mc/badges/age/maven/com.palantir.java-format:com.palantir.java-format.gradle.plugin/2.93.0?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/maven/com.palantir.java-format:com.palantir.java-format.gradle.plugin/2.91.0/2.93.0?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, only on Monday (`* 0-3 * * 1`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Enabled. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- java/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/build.gradle.kts b/java/build.gradle.kts index 43cc24b48ed..8afda976316 100644 --- a/java/build.gradle.kts +++ b/java/build.gradle.kts @@ -6,7 +6,7 @@ import net.ltgt.gradle.errorprone.errorprone plugins { id("com.diffplug.spotless") version "8.6.0" id("com.palantir.git-version") version "5.0.0" - id("com.palantir.java-format") version "2.91.0" + id("com.palantir.java-format") version "2.93.0" id("net.ltgt.errorprone") version "5.1.0" apply false id("com.vanniktech.maven.publish") version "0.36.0" apply false } From d42380602c7723c294649c042a9cdf40de2b363f Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 15 Jun 2026 13:51:26 +0100 Subject: [PATCH 011/162] Align data buffers in fastlanes benchmarks (#8393) ## Summary An experiment to align the packed buffers in the benchmarks to try and reduce some of the noise around the `compare` benchmark. If this works maybe its a path towards more consistent benchmarks? Signed-off-by: Adam Gutglick --- .../fastlanes/benches/bitpack_compare.rs | 37 ++++++++++++++++--- .../benches/bitpack_compare_sweep.rs | 37 ++++++++++++++++--- 2 files changed, 64 insertions(+), 10 deletions(-) diff --git a/encodings/fastlanes/benches/bitpack_compare.rs b/encodings/fastlanes/benches/bitpack_compare.rs index c704f2b35ed..fc6e67bb11d 100644 --- a/encodings/fastlanes/benches/bitpack_compare.rs +++ b/encodings/fastlanes/benches/bitpack_compare.rs @@ -25,7 +25,10 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; +use vortex_buffer::Alignment; use vortex_buffer::BufferMut; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::BitPackedArray; use vortex_fastlanes::BitPackedData; use vortex_session::VortexSession; @@ -39,17 +42,41 @@ static SESSION: LazyLock = const LENS: &[usize] = &[1024, 64 * 1024]; const BIT_WIDTHS: &[u8] = &[4, 16]; +/// Rebuild the array with its packed buffer copied to a page-aligned allocation. +/// +/// `bitpack_encode` aligns the packed buffer only to the element type, so its cache-line +/// placement depends on allocator state, which shifts with any change to the bench binary. +/// CodSpeed's simulated cache misses are deterministic in those addresses, which made these +/// benches flip ~30% between two layout modes across unrelated commits. Pinning the buffer to a +/// page boundary makes the layout, and therefore the measurement, reproducible. +fn page_aligned(array: BitPackedArray) -> BitPackedArray { + let ptype = array.dtype().as_ptype(); + let parts = BitPacked::into_parts(array); + BitPacked::try_new( + parts.packed.ensure_aligned(Alignment::new(4096)).unwrap(), + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + ) + .unwrap() +} + /// Build a packed array of varied in-range values, plus an out-of-range constant RHS for /// the fast-path benches. fn build_inputs(len: usize) -> (ArrayRef, ArrayRef, ExecutionCtx) { let mut ctx = SESSION.create_execution_ctx(); let buf: BufferMut = (0..len).map(|i| (i as u32) % (1 << BW)).collect(); - let array = BitPackedData::encode( - &PrimitiveArray::new(buf.freeze(), Validity::NonNullable).into_array(), - BW, - &mut ctx, + let array = page_aligned( + BitPackedData::encode( + &PrimitiveArray::new(buf.freeze(), Validity::NonNullable).into_array(), + BW, + &mut ctx, + ) + .unwrap(), ) - .unwrap() .into_array(); // 1 << BW is just past the packable range, so the out-of-range fast path fires. let constant = 1u32 << BW; diff --git a/encodings/fastlanes/benches/bitpack_compare_sweep.rs b/encodings/fastlanes/benches/bitpack_compare_sweep.rs index 0be525566e7..570fa9ceec6 100644 --- a/encodings/fastlanes/benches/bitpack_compare_sweep.rs +++ b/encodings/fastlanes/benches/bitpack_compare_sweep.rs @@ -29,7 +29,10 @@ use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_array::session::ArraySession; use vortex_array::validity::Validity; +use vortex_buffer::Alignment; use vortex_buffer::BufferMut; +use vortex_fastlanes::BitPacked; +use vortex_fastlanes::BitPackedArray; use vortex_fastlanes::BitPackedData; use vortex_session::VortexSession; @@ -65,6 +68,28 @@ macro_rules! impl_bench_int { impl_bench_int!(u8, u16, u32, u64, i8, i16, i32, i64); +/// Rebuild the array with its packed buffer copied to a page-aligned allocation. +/// +/// `bitpack_encode` aligns the packed buffer only to the element type, so its cache-line +/// placement depends on allocator state, which shifts with any change to the bench binary. +/// CodSpeed's simulated cache misses are deterministic in those addresses, which made the whole +/// sweep flip ~40% between two layout modes across unrelated commits. Pinning the buffer to a +/// page boundary makes the layout, and therefore the measurement, reproducible. +fn page_aligned(array: BitPackedArray) -> BitPackedArray { + let ptype = array.dtype().as_ptype(); + let parts = BitPacked::into_parts(array); + BitPacked::try_new( + parts.packed.ensure_aligned(Alignment::new(4096)).unwrap(), + ptype, + parts.validity, + parts.patches, + parts.bit_width, + parts.len, + parts.offset, + ) + .unwrap() +} + /// Encode `LEN` in-range values of type `T` at the given bit width, returning the packed array, a /// mid-range constant to compare against, and an execution context. fn setup(width: usize) -> (ArrayRef, ArrayRef, ExecutionCtx) { @@ -73,12 +98,14 @@ fn setup(width: usize) -> (ArrayRef, ArrayRef, ExecutionCtx) { let buf: BufferMut = (0..LEN) .map(|i| T::from_counter((i as u64) % cap)) .collect(); - let array = BitPackedData::encode( - &PrimitiveArray::new(buf.freeze(), Validity::NonNullable).into_array(), - width as u8, - &mut ctx, + let array = page_aligned( + BitPackedData::encode( + &PrimitiveArray::new(buf.freeze(), Validity::NonNullable).into_array(), + width as u8, + &mut ctx, + ) + .unwrap(), ) - .unwrap() .into_array(); let rhs = ConstantArray::new(T::from_counter(cap / 2), LEN).into_array(); (array, rhs, ctx) From 916a1f7914b017db8e21b9ab90cccb34fcc1cc38 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 15 Jun 2026 14:03:51 +0100 Subject: [PATCH 012/162] Yet another renovate configuration (#8427) ## Summary Yet another iteration of our renovate config, including the following changes: 1. Removes some already-default base configs or moved things there. 2. Lock files updated weekly, everything else every second Monday 3. JS/NPM - only update the lockfile and security updates 4. The lance benchmarks have their own group, so they shouldn't impact other dependencies Signed-off-by: Adam Gutglick --- renovate.json | 61 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/renovate.json b/renovate.json index 80fb4c20238..460cd9ba807 100644 --- a/renovate.json +++ b/renovate.json @@ -2,34 +2,30 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": [ "config:recommended", - ":automergeStableNonMajor", - ":automergePr", ":automergeRequireAllStatusChecks", - ":combinePatchMinorReleases", - ":dependencyDashboard", ":separateMultipleMajorReleases", ":configMigration", "group:rust-futuresMonorepo", "helpers:pinGitHubActionDigests", - "schedule:earlyMondays", - "docker:disable" + "docker:disable", + ":maintainLockFilesWeekly" ], - "lockFileMaintenance": { - "enabled": true - }, + "schedule": "* 0-3 * * 1#2,1#4", "autoApprove": true, "automergeStrategy": "squash", "prHourlyLimit": 0, "prConcurrentLimit": 20, "rebaseWhen": "conflicted", - "cloneSubmodules": true, - "platformAutomerge": true, - "patch": { - "groupName": "all patch updates" - }, "labels": [ "changelog/chore" ], + "minor": { "enabled": false }, + "patch": { "enabled": false }, + "osvVulnerabilityAlerts": true, + "vulnerabilityAlerts": { + "enabled": true, + "vulnerabilityFixStrategy": "lowest" + }, "packageRules": [ { "matchPackageNames": [ @@ -40,15 +36,29 @@ }, { "groupName": "arrow-rs", + "matchManagers": ["cargo"], "matchUpdateTypes": [ - "major", - "minor", - "patch" + "major" ], "matchSourceUrls": [ "https://github.com/apache/arrow-rs" ] }, + { + "description": "Keep lance-bench dependency majors together because they are tested as one benchmark stack", + "groupName": "lance benchmark dependencies", + "groupSlug": "lance-bench", + "matchManagers": [ + "cargo" + ], + "matchFileNames": [ + "benchmarks/lance-bench/Cargo.toml" + ], + "matchUpdateTypes": [ + "major" + ], + "separateMultipleMajor": false + }, { "groupName": "flatbuffers", "matchSourceUrls": [ @@ -61,6 +71,21 @@ "https://github.com/hyperium/tonic" ] }, + { + "description": "Disable routine npm dependency updates; keep security alerts and lock file maintenance", + "matchManagers": [ + "npm" + ], + "matchUpdateTypes": [ + "digest", + "major", + "minor", + "patch", + "pin", + "replacement" + ], + "enabled": false + }, { "groupName": "Rust lock file maintenance", "groupSlug": "rust-lock-file-maintenance", @@ -100,7 +125,7 @@ "/vortex-duckdb/build\\.rs/" ], "matchStrings": [ - "const DUCKDB_VERSION: &str = \"(?[0-9]+\\.[0-9]+\\.[0-9]+)\";" + "const DEFAULT_DUCKDB_VERSION: &str = \"(?[0-9]+\\.[0-9]+\\.[0-9]+)\";" ], "depNameTemplate": "duckdb/duckdb", "datasourceTemplate": "github-releases", From 01ea0fd950317de775a42f8ec106ffb2dbbfe12d Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Mon, 15 Jun 2026 14:20:55 +0100 Subject: [PATCH 013/162] Fix renovate config (#8430) ## Summary Closes: #8429 Split the lance-specific rule into two. Signed-off-by: Adam Gutglick --- renovate.json | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/renovate.json b/renovate.json index 460cd9ba807..68a14baa7ca 100644 --- a/renovate.json +++ b/renovate.json @@ -44,6 +44,16 @@ "https://github.com/apache/arrow-rs" ] }, + { + "description": "Do not split lance-bench grouped majors by target major", + "matchManagers": [ + "cargo" + ], + "matchFileNames": [ + "benchmarks/lance-bench/Cargo.toml" + ], + "separateMultipleMajor": false + }, { "description": "Keep lance-bench dependency majors together because they are tested as one benchmark stack", "groupName": "lance benchmark dependencies", @@ -56,8 +66,7 @@ ], "matchUpdateTypes": [ "major" - ], - "separateMultipleMajor": false + ] }, { "groupName": "flatbuffers", From 8cd57c8463af66b24952d6c67ce881599c52ddca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:35:18 +0000 Subject: [PATCH 014/162] Update dependency pip to v26.1.2 [SECURITY] (#8432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [pip](https://redirect.github.com/pypa/pip) ([changelog](https://pip.pypa.io/en/stable/news/)) | `26.1.1` → `26.1.2` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/pip/26.1.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/pip/26.1.1/26.1.2?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### [CVE-2026-8643](https://nvd.nist.gov/vuln/detail/CVE-2026-8643) / PYSEC-2026-196
More information #### Details pip would treat console_scripts and gui_scripts as paths instead of file names without sanitizing the resolved absolute path to the installation directory, leading to entry points being installed outside the installation directory. #### Severity - CVSS Score: 5.5 / 10 (Medium) - Vector String: `CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:N` #### References - [http://www.openwall.com/lists/oss-security/2026/06/01/5](http://www.openwall.com/lists/oss-security/2026/06/01/5) - [https://mail.python.org/archives/list/security-announce@python.org/thread/YV63UET5D3OOJY7O4M5XCVYO2YM4NBYJ/](https://mail.python.org/archives/list/security-announce@python.org/thread/YV63UET5D3OOJY7O4M5XCVYO2YM4NBYJ/) - [https://github.com/pypa/pip/pull/14000](https://redirect.github.com/pypa/pip/pull/14000) This data is provided by [OSV](https://osv.dev/vulnerability/PYSEC-2026-196) and the [PyPI Advisory Database](https://redirect.github.com/pypa/advisory-database) ([CC-BY 4.0](https://redirect.github.com/pypa/advisory-database/blob/main/LICENSE)).
--- ### Release Notes
pypa/pip (pip) ### [`v26.1.2`](https://redirect.github.com/pypa/pip/compare/26.1.1...26.1.2) [Compare Source](https://redirect.github.com/pypa/pip/compare/26.1.1...26.1.2)
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - At any time (no schedule defined) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index ef6e5670aaf..62ce5f50ccd 100644 --- a/uv.lock +++ b/uv.lock @@ -942,11 +942,11 @@ wheels = [ [[package]] name = "pip" -version = "26.1.1" +version = "26.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b6/48/cb9b7a682f6fe01a4221e1728941dd4ac3cd9090a17db3779d6ff490b602/pip-26.1.1.tar.gz", hash = "sha256:d36762751d156a4ee895de8af39aa0abeeeb577f93a2eca6ab62467bbf0f8a78", size = 1840400, upload-time = "2026-05-04T19:02:21.248Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/91/47e7d486260f618783899587af63ccf7980fb60245c3e63dd4571c6b57ad/pip-26.1.2.tar.gz", hash = "sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605", size = 1840799, upload-time = "2026-05-31T17:33:58.56Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/eb/fea4d1d51c49832120f7f285d07306db3960f423a2612c6057caf3e8196f/pip-26.1.1-py3-none-any.whl", hash = "sha256:99cb1c2899893b075ff56e4ed0af55669a955b49ad7fb8d8603ecdaf4ed653fb", size = 1812777, upload-time = "2026-05-04T19:02:18.9Z" }, + { url = "https://files.pythonhosted.org/packages/5d/95/6b5cb3461ea5673ba0995989746db58eb18b91b54dbf331e72f569540946/pip-26.1.2-py3-none-any.whl", hash = "sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab", size = 1813144, upload-time = "2026-05-31T17:33:56.772Z" }, ] [[package]] From 051183280a30af40d9afe5367a5efeb00e1bf172 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:36:19 +0000 Subject: [PATCH 015/162] Update dependency duckdb to v1.4.2 [SECURITY] (#8431) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [duckdb](https://redirect.github.com/duckdb/duckdb-python) ([changelog](https://redirect.github.com/duckdb/duckdb-python/releases)) | `1.4.1` → `1.4.2` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/duckdb/1.4.2?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/duckdb/1.4.1/1.4.2?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### [CVE-2025-64429](https://nvd.nist.gov/vuln/detail/CVE-2025-64429) / [GHSA-vmp8-hg63-v2hp](https://redirect.github.com/advisories/GHSA-vmp8-hg63-v2hp) / PYSEC-2025-112
More information #### Details DuckDB is a SQL database management system. DuckDB implemented block-based encryption of DB on the filesystem starting with DuckDB 1.4.0. There are a few issues related to this implementation. The DuckDB can fall back to an insecure random number generator (pcg32) to generate cryptographic keys or IVs. When clearing keys from memory, the compiler may remove the memset() and leave sensitive data on the heap. By modifying the database header, an attacker could downgrade the encryption mode from GCM to CTR to bypass integrity checks. There may be a failure to check return value on call to OpenSSL `rand_bytes()`. An attacker could use public IVs to compromise the internal state of RNG and determine the randomly generated key used to encrypt temporary files, get access to cryptographic keys if they have access to process memory (e.g. through memory leak),circumvent GCM integrity checks, and/or influence the OpenSSL random number generator and DuckDB would not be able to detect a failure of the generator. Version 1.4.2 has disabled the insecure random number generator by no longer using the fallback to write to or create databases. Instead, DuckDB will now attempt to install and load the OpenSSL implementation in the `httpfs` extension. DuckDB now uses secure MbedTLS primitive to clear memory as recommended and requires explicit specification of ciphers without integrity checks like CTR on `ATTACH`. Additionally, DuckDB now checks the return code. #### Severity - CVSS Score: 6.5 / 10 (Medium) - Vector String: `CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:L/I:L/A:N` #### References - [https://github.com/duckdb/duckdb/blob/029a5b87ff5b1cd22f7f9717d48cd8830d00807c/src/common/random_engine.cpp#L20](https://redirect.github.com/duckdb/duckdb/blob/029a5b87ff5b1cd22f7f9717d48cd8830d00807c/src/common/random_engine.cpp#L20) - [https://duckdb.org/2025/09/16/announcing-duckdb-140.html](https://duckdb.org/2025/09/16/announcing-duckdb-140.html) - [https://github.com/duckdb/duckdb/security/advisories/GHSA-vmp8-hg63-v2hp](https://redirect.github.com/duckdb/duckdb/security/advisories/GHSA-vmp8-hg63-v2hp) - [https://github.com/duckdb/duckdb/pull/17275](https://redirect.github.com/duckdb/duckdb/pull/17275) This data is provided by [OSV](https://osv.dev/vulnerability/PYSEC-2025-112) and the [PyPI Advisory Database](https://redirect.github.com/pypa/advisory-database) ([CC-BY 4.0](https://redirect.github.com/pypa/advisory-database/blob/main/LICENSE)).
--- ### Release Notes
duckdb/duckdb-python (duckdb) ### [`v1.4.2`](https://redirect.github.com/duckdb/duckdb-python/releases/tag/v1.4.2): Python DuckDB v1.4.2 [Compare Source](https://redirect.github.com/duckdb/duckdb-python/compare/v1.4.1...v1.4.2) This is a bug fix release for various issues discovered after we released v1.4.1. Also see the [DuckDB v1.4.2 Changelog](https://redirect.github.com/duckdb/duckdb/releases/tag/v1.4.2). #### What's Changed - Add Python 3.14 support by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​116](https://redirect.github.com/duckdb/duckdb-python/pull/116) - Fix ADBC driver path resolution when `importlib.util` was not implicitly loaded by [@​henryharbeck](https://redirect.github.com/henryharbeck) in [#​135](https://redirect.github.com/duckdb/duckdb-python/pull/135) - add targeted test workflow by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​145](https://redirect.github.com/duckdb/duckdb-python/pull/145) - Remove xfail annotations on adbc tests by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​147](https://redirect.github.com/duckdb/duckdb-python/pull/147) - fix config dict value typehint by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​151](https://redirect.github.com/duckdb/duckdb-python/pull/151) - Add df data and tz type columns back into the same loc after type con… by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​150](https://redirect.github.com/duckdb/duckdb-python/pull/150) - Enable pyarrow with python 3.14 by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​152](https://redirect.github.com/duckdb/duckdb-python/pull/152) - spark imports by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​157](https://redirect.github.com/duckdb/duckdb-python/pull/157) - Fix failing test due to changed error msg by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​158](https://redirect.github.com/duckdb/duckdb-python/pull/158) - Add explicit .pl(lazy=True) overload by [@​J-Meyers](https://redirect.github.com/J-Meyers) in [#​172](https://redirect.github.com/duckdb/duckdb-python/pull/172) - Fix InsertRelation on attached database by [@​evertlammerts](https://redirect.github.com/evertlammerts) in [#​155](https://redirect.github.com/duckdb/duckdb-python/pull/155) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - At any time (no schedule defined) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about these updates again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- uv.lock | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/uv.lock b/uv.lock index 62ce5f50ccd..0d325a5287e 100644 --- a/uv.lock +++ b/uv.lock @@ -286,28 +286,38 @@ wheels = [ [[package]] name = "duckdb" -version = "1.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ea/e7/21cf50a3d52ffceee1f0bcc3997fa96a5062e6bab705baee4f6c4e33cce5/duckdb-1.4.1.tar.gz", hash = "sha256:f903882f045d057ebccad12ac69975952832edfe133697694854bb784b8d6c76", size = 18461687, upload-time = "2025-10-07T10:37:28.605Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/52/606f13fa9669a24166d2fe523e28982d8ef9039874b4de774255c7806d1f/duckdb-1.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:605d563c1d5203ca992497cd33fb386ac3d533deca970f9dcf539f62a34e22a9", size = 29065894, upload-time = "2025-10-07T10:36:29.837Z" }, - { url = "https://files.pythonhosted.org/packages/84/57/138241952ece868b9577e607858466315bed1739e1fbb47205df4dfdfd88/duckdb-1.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d3305c7c4b70336171de7adfdb50431f23671c000f11839b580c4201d9ce6ef5", size = 16163720, upload-time = "2025-10-07T10:36:32.241Z" }, - { url = "https://files.pythonhosted.org/packages/a3/81/afa3a0a78498a6f4acfea75c48a70c5082032d9ac87822713d7c2d164af1/duckdb-1.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a063d6febbe34b32f1ad2e68822db4d0e4b1102036f49aaeeb22b844427a75df", size = 13756223, upload-time = "2025-10-07T10:36:34.673Z" }, - { url = "https://files.pythonhosted.org/packages/47/dd/5f6064fbd9248e37a3e806a244f81e0390ab8f989d231b584fb954f257fc/duckdb-1.4.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1ffcaaf74f7d1df3684b54685cbf8d3ce732781c541def8e1ced304859733ae", size = 18487022, upload-time = "2025-10-07T10:36:36.759Z" }, - { url = "https://files.pythonhosted.org/packages/a1/10/b54969a1c42fd9344ad39228d671faceb8aa9f144b67cd9531a63551757f/duckdb-1.4.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:685d3d1599dc08160e0fa0cf09e93ac4ff8b8ed399cb69f8b5391cd46b5b207c", size = 20491004, upload-time = "2025-10-07T10:36:39.318Z" }, - { url = "https://files.pythonhosted.org/packages/ed/d5/7332ae8f804869a4e895937821b776199a283f8d9fc775fd3ae5a0558099/duckdb-1.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:78f1d28a15ae73bd449c43f80233732adffa49be1840a32de8f1a6bb5b286764", size = 12327619, upload-time = "2025-10-07T10:36:41.509Z" }, - { url = "https://files.pythonhosted.org/packages/0e/6c/906a3fe41cd247b5638866fc1245226b528de196588802d4df4df1e6e819/duckdb-1.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cd1765a7d180b7482874586859fc23bc9969d7d6c96ced83b245e6c6f49cde7f", size = 29076820, upload-time = "2025-10-07T10:36:43.782Z" }, - { url = "https://files.pythonhosted.org/packages/66/c7/01dd33083f01f618c2a29f6dd068baf16945b8cbdb132929d3766610bbbb/duckdb-1.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8ed7a86725185470953410823762956606693c0813bb64e09c7d44dbd9253a64", size = 16167558, upload-time = "2025-10-07T10:36:46.003Z" }, - { url = "https://files.pythonhosted.org/packages/81/e2/f983b4b7ae1dfbdd2792dd31dee9a0d35f88554452cbfc6c9d65e22fdfa9/duckdb-1.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a189bdfc64cfb9cc1adfbe4f2dcfde0a4992ec08505ad8ce33c886e4813f0bf", size = 13762226, upload-time = "2025-10-07T10:36:48.55Z" }, - { url = "https://files.pythonhosted.org/packages/ed/34/fb69a7be19b90f573b3cc890961be7b11870b77514769655657514f10a98/duckdb-1.4.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a9090089b6486f7319c92acdeed8acda022d4374032d78a465956f50fc52fabf", size = 18500901, upload-time = "2025-10-07T10:36:52.445Z" }, - { url = "https://files.pythonhosted.org/packages/e4/a5/1395d7b49d5589e85da9a9d7ffd8b50364c9d159c2807bef72d547f0ad1e/duckdb-1.4.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:142552ea3e768048e0e8c832077a545ca07792631c59edaee925e3e67401c2a0", size = 20514177, upload-time = "2025-10-07T10:36:55.358Z" }, - { url = "https://files.pythonhosted.org/packages/c0/21/08f10706d30252753349ec545833fc0cea67c11abd0b5223acf2827f1056/duckdb-1.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:567f3b3a785a9e8650612461893c49ca799661d2345a6024dda48324ece89ded", size = 12336422, upload-time = "2025-10-07T10:36:57.521Z" }, - { url = "https://files.pythonhosted.org/packages/d7/08/705988c33e38665c969f7876b3ca4328be578554aa7e3dc0f34158da3e64/duckdb-1.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:46496a2518752ae0c6c5d75d4cdecf56ea23dd098746391176dd8e42cf157791", size = 29077070, upload-time = "2025-10-07T10:36:59.83Z" }, - { url = "https://files.pythonhosted.org/packages/99/c5/7c9165f1e6b9069441bcda4da1e19382d4a2357783d37ff9ae238c5c41ac/duckdb-1.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1c65ae7e9b541cea07d8075343bcfebdecc29a3c0481aa6078ee63d51951cfcd", size = 16167506, upload-time = "2025-10-07T10:37:02.24Z" }, - { url = "https://files.pythonhosted.org/packages/38/46/267f4a570a0ee3ae6871ddc03435f9942884284e22a7ba9b7cb252ee69b6/duckdb-1.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:598d1a314e34b65d9399ddd066ccce1eeab6a60a2ef5885a84ce5ed62dbaf729", size = 13762330, upload-time = "2025-10-07T10:37:04.581Z" }, - { url = "https://files.pythonhosted.org/packages/15/7b/c4f272a40c36d82df20937d93a1780eb39ab0107fe42b62cba889151eab9/duckdb-1.4.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2f16b8def782d484a9f035fc422bb6f06941ed0054b4511ddcdc514a7fb6a75", size = 18504687, upload-time = "2025-10-07T10:37:06.991Z" }, - { url = "https://files.pythonhosted.org/packages/17/fc/9b958751f0116d7b0406406b07fa6f5a10c22d699be27826d0b896f9bf51/duckdb-1.4.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a5a7d0aed068a5c33622a8848857947cab5cfb3f2a315b1251849bac2c74c492", size = 20513823, upload-time = "2025-10-07T10:37:09.349Z" }, - { url = "https://files.pythonhosted.org/packages/30/79/4f544d73fcc0513b71296cb3ebb28a227d22e80dec27204977039b9fa875/duckdb-1.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:280fd663dacdd12bb3c3bf41f3e5b2e5b95e00b88120afabb8b8befa5f335c6f", size = 12336460, upload-time = "2025-10-07T10:37:12.154Z" }, +version = "1.5.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/00/d579dcb2a536b6ea3a2563cdad6844f77d81a9b2d4b22a858097f2468acf/duckdb-1.5.3.tar.gz", hash = "sha256:df39428eb130faa35ae96fd35245bdeae6ecf43936250b116b5fead568eb9f16", size = 18026640, upload-time = "2026-05-20T11:55:31.901Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/fc/a8a89c6c73f31c2b58c6abbc2f543e0b736042dd5ef7cc1784c24ec31428/duckdb-1.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:341a2672e2551ba51c95c1898f0ade983e76675e79038ccb16342c3d6cfb82d7", size = 32583465, upload-time = "2026-05-20T11:54:13.132Z" }, + { url = "https://files.pythonhosted.org/packages/63/f1/3423a2f523dd034e505d4a5dd8e210ae577212e152598dc13b6a5e736e1b/duckdb-1.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c9e8fa408705081160ede7ead238d16e73a36b8561b700f2bf2d650ae48e7b92", size = 17278520, upload-time = "2026-05-20T11:54:16.368Z" }, + { url = "https://files.pythonhosted.org/packages/e1/1a/7bf5ba1b7ea520557e6b2dbee1c85abab016bdac0c1779d9d0ef76c87300/duckdb-1.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:70a18f932cf6d87bd0e554613657a515c1443a1724aacfc7ec5137dd28698b03", size = 15424794, upload-time = "2026-05-20T11:54:19.891Z" }, + { url = "https://files.pythonhosted.org/packages/ad/16/ce4b1e386e45fab0268edbf1b85bace20e9437589e9edb2bd5f9a226fa44/duckdb-1.5.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e80eb4d0fb59869cb2c7d7ef494c07fb92014fe8e77d96c170cd1ebc1488a708", size = 19306666, upload-time = "2026-05-20T11:54:22.77Z" }, + { url = "https://files.pythonhosted.org/packages/99/1f/651f8453f26931e8061b7e27b3090f868868185814ecb9216d0bd71ec8ef/duckdb-1.5.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3248b49cd835ea322574bc6aac0ae7a83be85547f49d4f5f5777cb380ee6627f", size = 21418306, upload-time = "2026-05-20T11:54:25.616Z" }, + { url = "https://files.pythonhosted.org/packages/bc/64/e1ffebf010b1631a6fef8d1508f46d4eab3e97c18729af986bb796fa8452/duckdb-1.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:f4eff89c12c3a362efa012262e57b7b4ab904a7f79bad9178fe365510077abe8", size = 13101423, upload-time = "2026-05-20T11:54:28.107Z" }, + { url = "https://files.pythonhosted.org/packages/e7/42/b1d4e34f9658cc0e13d7aae581ab82643f50a548d5aee8767f0c587cc3a4/duckdb-1.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:75d13308c9da3ee431d1e72b8ab720aa74a1b3e9159d4124cb62435924496334", size = 13951740, upload-time = "2026-05-20T11:54:30.886Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c4/2e34929b16c8d544ef664fad8f7f3a2a9db05746aae1e7c8c4ee3a8b23e4/duckdb-1.5.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ff11a457258148337ef9a392148a8cdbd1069b6c27c21958816c7b67fe6c542d", size = 32626494, upload-time = "2026-05-20T11:54:33.738Z" }, + { url = "https://files.pythonhosted.org/packages/3a/53/3af681793d03771365ae3e2215331151c196a3ac8193f613344840694671/duckdb-1.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5fd25f533cb1b6b2c84cc767a9a9bab7769bb1aa44571a2a0bfc91ac3e4a38ac", size = 17301121, upload-time = "2026-05-20T11:54:36.928Z" }, + { url = "https://files.pythonhosted.org/packages/15/e2/c80af1eac2ab5d35fc2c372ef0a84668842e549fbbf7799277b3fccf3e39/duckdb-1.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:10960400ed60cdf0fe05bab2086fa8eb733889cb0ceca18d07ff9a00c0e0be7b", size = 15449283, upload-time = "2026-05-20T11:54:39.777Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/c63af233c9f761bf5178a5210437e1bc6bcb30fa8a9073de6398cfb12c03/duckdb-1.5.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5f18e7561403054433706c187589e86629a7af09a7efc23a06a8b308e6acc68", size = 19332762, upload-time = "2026-05-20T11:54:42.51Z" }, + { url = "https://files.pythonhosted.org/packages/21/cc/2d77af4fff86012f334ef82e6d54a995a86c8745e58074f1218ed7d25171/duckdb-1.5.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fb7516255a8764545e30f7efacea408cc847764a3027b3b0b3e7d1a7bebbc5c", size = 21453290, upload-time = "2026-05-20T11:54:45.272Z" }, + { url = "https://files.pythonhosted.org/packages/8d/5e/9bc4817a98feb4dab83e56f2245cd3a30d00ee646d4dec7926464e2b3f28/duckdb-1.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:8001eccbc28be244dfd04d708526f34ddd6460b47a8aeb5d0e39d6f7f9e3fe15", size = 13118308, upload-time = "2026-05-20T11:54:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/81/35/e3f32e4e53e2450ddb1db8312a17d1ce455d60cc4941b6ad2cfc908794b0/duckdb-1.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:6d2835e39bb6af73891f73c0f8d4324f98afe00d0b00c6d34b2a582c2256cbb0", size = 13927187, upload-time = "2026-05-20T11:54:50.584Z" }, + { url = "https://files.pythonhosted.org/packages/cc/9c/a528eb09d8be51954c485864bd06753e616939a080cbc3dd4417e8c94a57/duckdb-1.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e75a6122c12579a99848517f6f00a4e342aebda3590c30fe9b5cc5f39d5e6afc", size = 32626254, upload-time = "2026-05-20T11:54:53.65Z" }, + { url = "https://files.pythonhosted.org/packages/ec/3c/1534c0a6db347c05eb7d0f6ecfb7aefbe74cbff398e4892a8fd1903a20e8/duckdb-1.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fd3963c1cb9d9567777f4a898a9dbe388a2fe9724681801b1e7d6d93eecf1b76", size = 17300917, upload-time = "2026-05-20T11:54:56.628Z" }, + { url = "https://files.pythonhosted.org/packages/23/fa/beafb91e6e152d2161c4a9cbc472334c87607eb61ad7104b5a7fa8d8d7b1/duckdb-1.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3d5db8c0b55e072cf437948ebb5d7e23d7b9d03d905fa5f9145583e65aa447f7", size = 15449411, upload-time = "2026-05-20T11:54:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/50/0a/49b6fe04e2fcd63729eb607dadd44818dde77342a4f5ce086c6c92f1dd4d/duckdb-1.5.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ce80aed7a538422129a57eaca9141e3afb51f8bf562b1908b1576c9725b5b22", size = 19333120, upload-time = "2026-05-20T11:55:01.727Z" }, + { url = "https://files.pythonhosted.org/packages/63/4c/0907c3f76adb9dd90e67610b31e0304a35814e65c4c41a354a262c09b885/duckdb-1.5.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:787df63824f07bf18022dbc3b8ca4b2bfab0ebe616464f55c6e8cd0f59ea762e", size = 21453266, upload-time = "2026-05-20T11:55:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/6d/9c/d2f23a7803ddbbd9413f7572ecf66a15120ed5ced7ce5c73e698c1406b76/duckdb-1.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:bb5bb5dcdd09d62ee60f0ddbbef918e71cce304ffe28428b1131949d39ffaabf", size = 13118640, upload-time = "2026-05-20T11:55:07.389Z" }, + { url = "https://files.pythonhosted.org/packages/27/d5/7ba2316415bcdab6edd765bbbe35c2ca8a3800f2fe695cd70e3cdb997f09/duckdb-1.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2fa17ecdd5d3db122836cb71bb93601c2106a3be883c17dffddc02fbf3fa7888", size = 13926409, upload-time = "2026-05-20T11:55:10.166Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c2/d4b6f8a5e4d3bc25773be6da76a99d9661ebbf3552c007c460d2dd59dbf8/duckdb-1.5.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:4bfa9a4dadf71e83e2c4eaca2f9421c82a54defecc1b0b4c0be95e2389dec4fe", size = 32636685, upload-time = "2026-05-20T11:55:13.158Z" }, + { url = "https://files.pythonhosted.org/packages/42/58/e835c8298979d29db7a62cb5acc29e9b57aeaca7cdde2fcd3ac980f5cb18/duckdb-1.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:aea7baf67ad7e1829ac76f67d7dcbd7fb1f57c3eb179d55ac30952df4709ae30", size = 17308134, upload-time = "2026-05-20T11:55:16.194Z" }, + { url = "https://files.pythonhosted.org/packages/c9/46/617b51363f5613418c8b224b3cce16b58e6dde80904566bec232579c1d4e/duckdb-1.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b0b4f088a65d77e1217ce5d7eff889e63fedc44281200d899ff47c84d8ff836", size = 15449891, upload-time = "2026-05-20T11:55:18.687Z" }, + { url = "https://files.pythonhosted.org/packages/b3/72/354146656e8d9ba3853d3a5ee80a481b8c5f70edfc3d5ae80a8c4479c967/duckdb-1.5.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe8d0c1f6a120aa03fa6e0d03897c71a1842e6cf7afd31d181348391f7108fe1", size = 19338499, upload-time = "2026-05-20T11:55:21.34Z" }, + { url = "https://files.pythonhosted.org/packages/56/8f/65fc623b51448f2bfba1a9ec6ab3debb4664c0876c0113a5e782600b53ac/duckdb-1.5.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d0405eae18ec6e8210a471c97dbfe87a7e4d605274b7fe572a1f276e92158f13", size = 21455828, upload-time = "2026-05-20T11:55:23.847Z" }, + { url = "https://files.pythonhosted.org/packages/2b/db/d0274cbe9f5fe219f77c0bdf900ac77103569e83c102a4225ce04cbc607d/duckdb-1.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:33ae08b3e818d7613d8936744b67718c2062c2f530376895bfd89efb51b81538", size = 13640011, upload-time = "2026-05-20T11:55:26.276Z" }, + { url = "https://files.pythonhosted.org/packages/07/5d/8f1899b8bef291caf953992fcd6c24df9f29387a35645e58c2504a5ca473/duckdb-1.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:746433e49bbc667b4df283153415fbe37e9083e0eff6c3cd6e54de7536869cd4", size = 14411554, upload-time = "2026-05-20T11:55:29.037Z" }, ] [[package]] From 727e6946b945aa6b7588d2ecb7259b6eebe2452f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:37:29 +0000 Subject: [PATCH 016/162] Update anthropics/claude-code-action digest to d5726de (#8407) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [anthropics/claude-code-action](https://redirect.github.com/anthropics/claude-code-action) ([changelog](https://redirect.github.com/anthropics/claude-code-action/compare/fbda2eb1bdc90d319b8d853f5deb53bca199a7c1..d5726de019ec4498aa667642bc3a80fca83aa102)) | action | digest | `fbda2eb` → `d5726de` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, on the second Monday and fourth Monday of the month (`* 0-3 * * 1#2,1#4`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/fuzzer-fix-automation.yml | 2 +- .github/workflows/report-fuzz-crash.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fuzzer-fix-automation.yml b/.github/workflows/fuzzer-fix-automation.yml index 01df7162e78..35b423c0506 100644 --- a/.github/workflows/fuzzer-fix-automation.yml +++ b/.github/workflows/fuzzer-fix-automation.yml @@ -273,7 +273,7 @@ jobs: CRASH_FILE: ${{ steps.extract.outputs.crash_file }} CRASH_FILE_PATH: ${{ steps.download.outputs.crash_file_path }} ARTIFACT_URL: ${{ steps.extract.outputs.artifact_url }} - uses: anthropics/claude-code-action@fbda2eb1bdc90d319b8d853f5deb53bca199a7c1 # v1 + uses: anthropics/claude-code-action@d5726de019ec4498aa667642bc3a80fca83aa102 # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # Use the App token (not GITHUB_TOKEN) so the committed fix branch and diff --git a/.github/workflows/report-fuzz-crash.yml b/.github/workflows/report-fuzz-crash.yml index ea68e9a832a..69d5a461010 100644 --- a/.github/workflows/report-fuzz-crash.yml +++ b/.github/workflows/report-fuzz-crash.yml @@ -118,7 +118,7 @@ jobs: steps.dedup.outputs.duplicate != 'true' || steps.dedup.outputs.confidence != 'exact' continue-on-error: true - uses: anthropics/claude-code-action@fbda2eb1bdc90d319b8d853f5deb53bca199a7c1 # v1 + uses: anthropics/claude-code-action@d5726de019ec4498aa667642bc3a80fca83aa102 # v1 with: claude_code_oauth_token: ${{ secrets.claude_code_oauth_token }} github_token: ${{ secrets.gh_token }} From 68698eff52bab18bc5e934aaab4c43c54e279257 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:37:45 +0000 Subject: [PATCH 017/162] Update gradle/actions digest to 3f131e8 (#8408) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [gradle/actions](https://redirect.github.com/gradle/actions) ([changelog](https://redirect.github.com/gradle/actions/compare/50e97c2cd7a37755bbfafc9c5b7cafaece252f6e..3f131e8634966bd73d06cc69884922b02e6faf92)) | action | digest | `50e97c2` → `3f131e8` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, on the second Monday and fourth Monday of the month (`* 0-3 * * 1#2,1#4`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6398c54c20e..6b428aeaca5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -93,7 +93,7 @@ jobs: with: distribution: "corretto" java-version: "17" - - uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6 + - uses: gradle/actions/setup-gradle@3f131e8634966bd73d06cc69884922b02e6faf92 # v6 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: libvortex_jni_*.zip From 2528dca1f9a25b3e97596044f6cbe857857d5766 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 15 Jun 2026 13:37:57 +0000 Subject: [PATCH 018/162] Update taiki-e/install-action digest to 7a79fe8 (#8409) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [taiki-e/install-action](https://redirect.github.com/taiki-e/install-action) ([changelog](https://redirect.github.com/taiki-e/install-action/compare/0631aa6515c7d545823c67cfae7ef4fc7f490154..7a79fe8c3a13344501c80d99cae481c1c9085912)) | action | digest | `0631aa6` → `7a79fe8` | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - Between 12:00 AM and 03:59 AM, on the second Monday and fourth Monday of the month (`* 0-3 * * 1#2,1#4`) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/cuda.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 00faca7fd43..37ec5545686 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -70,7 +70,7 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install nextest - uses: taiki-e/install-action@0631aa6515c7d545823c67cfae7ef4fc7f490154 # v2 + uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # v2 with: tool: nextest - name: Rust Tests From d737639760896e1421e8dadf123e108d7f71ac2a Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Mon, 15 Jun 2026 15:00:53 +0100 Subject: [PATCH 019/162] ffi: create data source from memory buffer (#8426) Match rust API: buffer is borrowed under the condition it won't be modified until data source is freed Signed-off-by: Mikhail Kot --- Cargo.lock | 1 + vortex-ffi/Cargo.toml | 1 + vortex-ffi/cinclude/vortex.h | 15 +++++++ vortex-ffi/src/data_source.rs | 76 +++++++++++++++++++++++++++++++++++ vortex-ffi/test/scan.cpp | 68 +++++++++++++++++++++++-------- 5 files changed, 144 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00f5345aa76..2f0965f5a8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9744,6 +9744,7 @@ dependencies = [ "arrow-array", "arrow-schema", "async-fs", + "bytes", "cbindgen", "futures", "itertools 0.14.0", diff --git a/vortex-ffi/Cargo.toml b/vortex-ffi/Cargo.toml index 6407f97fe46..0fddf153d2b 100644 --- a/vortex-ffi/Cargo.toml +++ b/vortex-ffi/Cargo.toml @@ -23,6 +23,7 @@ all-features = true arrow-array = { workspace = true, features = ["ffi"] } arrow-schema = { workspace = true } async-fs = { workspace = true } +bytes = { workspace = true } futures = { workspace = true } itertools = { workspace = true } mimalloc = { workspace = true, optional = true } diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index 26c7b409cd9..b67202a4dc8 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -884,6 +884,21 @@ void vx_data_source_free(const vx_data_source *ptr); const vx_data_source * vx_data_source_new(const vx_session *session, const vx_data_source_options *options, vx_error **err); +/** + * Create a data source from a single in-memory Vortex file. + * + * "buffer_len" is the length of "buffer" in bytes. + * The bytes are borrowed, not copied: the caller must keep "buffer" alive and + * unmodified until the data source is freed. + * + * The returned pointer is owned by the caller and must be freed with + * vx_data_source_free. + * + * On error, returns NULL and sets "err". + */ +const vx_data_source * +vx_data_source_new_buffer(const vx_session *session, const void *buffer, size_t buffer_len, vx_error **err); + /** * Return the schema of the data source as a non-owned dtype. * The returned pointer is valid as long as "ds" is alive. Do not free it. diff --git a/vortex-ffi/src/data_source.rs b/vortex-ffi/src/data_source.rs index b1a25c30ddf..376a61e3096 100644 --- a/vortex-ffi/src/data_source.rs +++ b/vortex-ffi/src/data_source.rs @@ -5,16 +5,22 @@ use std::ffi::CStr; use std::ffi::c_char; +use std::ffi::c_void; use std::ptr; +use std::slice; use std::sync::Arc; +use bytes::Bytes; +use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; use vortex::error::vortex_ensure; use vortex::expr::stats::Precision::Absent; use vortex::expr::stats::Precision::Exact; use vortex::expr::stats::Precision::Inexact; +use vortex::file::OpenOptionsSessionExt; use vortex::file::multi::MultiFileDataSource; use vortex::io::runtime::BlockingRuntime; +use vortex::layout::scan::multi::MultiLayoutDataSource; use vortex::scan::DataSource; use vortex::scan::DataSourceRef; @@ -104,6 +110,38 @@ pub unsafe extern "C-unwind" fn vx_data_source_new( }) } +/// Create a data source from a single in-memory Vortex file. +/// +/// "buffer_len" is the length of "buffer" in bytes. +/// The bytes are borrowed, not copied: the caller must keep "buffer" alive and +/// unmodified until the data source is freed. +/// +/// The returned pointer is owned by the caller and must be freed with +/// vx_data_source_free. +/// +/// On error, returns NULL and sets "err". +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_data_source_new_buffer( + session: *const vx_session, + buffer: *const c_void, + buffer_len: usize, + err: *mut *mut vx_error, +) -> *const vx_data_source { + try_or(err, ptr::null(), || { + vortex_ensure!(!session.is_null()); + vortex_ensure!(!buffer.is_null()); + + let session = vx_session::as_ref(session); + let bytes: &'static [u8] = + unsafe { slice::from_raw_parts(buffer.cast::(), buffer_len) }; + let buffer = ByteBuffer::from(Bytes::from_static(bytes)); + let file = session.open_options().open_buffer(buffer)?; + let ds = MultiLayoutDataSource::new_with_first(file.layout_reader()?, Vec::new(), session); + + Ok(vx_data_source::new(Arc::new(ds) as DataSourceRef)) + }) +} + /// Return the schema of the data source as a non-owned dtype. /// The returned pointer is valid as long as "ds" is alive. Do not free it. #[unsafe(no_mangle)] @@ -140,12 +178,15 @@ pub unsafe extern "C-unwind" fn vx_data_source_get_row_count( #[cfg(test)] mod tests { use std::ffi::CString; + use std::ffi::c_void; + use std::fs::read; use std::ptr; use crate::data_source::vx_data_source_dtype; use crate::data_source::vx_data_source_free; use crate::data_source::vx_data_source_get_row_count; use crate::data_source::vx_data_source_new; + use crate::data_source::vx_data_source_new_buffer; use crate::data_source::vx_data_source_options; use crate::dtype::vx_dtype; use crate::scan::vx_estimate; @@ -220,4 +261,39 @@ mod tests { vx_session_free(session); } } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_buffer() { + unsafe { + let session = vx_session_new(); + let (sample, struct_array) = write_sample(session); + + let mut error = ptr::null_mut(); + let ds = vx_data_source_new_buffer(session, ptr::null(), 0, &raw mut error); + assert_error(error); + assert!(ds.is_null()); + + let file = read(sample).unwrap(); + let ds = vx_data_source_new_buffer( + session, + file.as_ptr() as *const c_void, + file.len(), + &raw mut error, + ); + assert_no_error(error); + assert!(!ds.is_null()); + + let dtype = vx_dtype::as_ref(vx_data_source_dtype(ds)); + assert_eq!(dtype, struct_array.dtype()); + + let mut row_count = vx_estimate::default(); + vx_data_source_get_row_count(ds, &raw mut row_count); + assert_eq!(row_count.r#type, vx_estimate_type::VX_ESTIMATE_EXACT); + assert_eq!(row_count.estimate, SAMPLE_ROWS as u64); + + vx_data_source_free(ds); + vx_session_free(session); + } + } } diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp index 2e138a12913..6857ea82bc1 100644 --- a/vortex-ffi/test/scan.cpp +++ b/vortex-ffi/test/scan.cpp @@ -9,6 +9,7 @@ #include #include #include +#include using FFI_ArrowArrayStream = ArrowArrayStream; using FFI_ArrowArray = ArrowArray; @@ -218,6 +219,11 @@ TEST_CASE("Creating datasources", "[datasource]") { REQUIRE(error != nullptr); vx_error_free(error); + ds = vx_data_source_new_buffer(session, nullptr, 0, &error); + REQUIRE(ds == nullptr); + REQUIRE(error != nullptr); + vx_error_free(error); + TempPath file = write_sample(session); opts.paths = file.c_str(); ds = vx_data_source_new(session, &opts, &error); @@ -383,24 +389,8 @@ TEST_CASE("Requesting scans", "[datasource]") { } } -TEST_CASE("Basic scan", "[datasource]") { - vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; - TempPath path = write_sample(session); +void basic_scan(const vx_data_source *ds) { vx_error *error = nullptr; - - vx_data_source_options ds_options = {}; - ds_options.paths = path.c_str(); - - const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); - require_no_error(error); - REQUIRE(ds != nullptr); - defer { - vx_data_source_free(ds); - }; - vx_estimate estimate = {}; vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); require_no_error(error); @@ -439,6 +429,50 @@ TEST_CASE("Basic scan", "[datasource]") { verify_sample_array(array); } +TEST_CASE("Basic scan", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + vx_error *error = nullptr; + + vx_data_source_options ds_options = {}; + ds_options.paths = path.c_str(); + const vx_data_source *ds = vx_data_source_new(session, &ds_options, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + basic_scan(ds); +} + +TEST_CASE("Basic scan from memory", "[datasource]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + TempPath path = write_sample(session); + + std::ifstream file(path, std::ios::binary | std::ios::ate); + const std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(size); + REQUIRE(file.read(buffer.data(), size)); + + vx_error *error = nullptr; + const vx_data_source *ds = vx_data_source_new_buffer(session, buffer.data(), size, &error); + require_no_error(error); + REQUIRE(ds != nullptr); + defer { + vx_data_source_free(ds); + }; + + basic_scan(ds); +} + TEST_CASE("Multithreaded scan", "[datasource]") { vx_session *session = vx_session_new(); defer { From 49b0088fe14043133c451901aa3869461349d5dc Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Mon, 15 Jun 2026 22:59:22 +0100 Subject: [PATCH 020/162] Support reporting statistics in spark datasource (#8057) Spark mostly focuses on sizeInBytes which we populate from file sizes with scaling. We also report numRows since that exists in our datasource. Signed-off-by: Robert Kruszewski --- .../main/java/dev/vortex/api/DataSource.java | 51 ++++ .../java/dev/vortex/jni/NativeDataSource.java | 6 + .../test/java/dev/vortex/api/TestMinimal.java | 2 + .../vortex/spark/read/VortexBatchExec.java | 20 +- .../dev/vortex/spark/read/VortexScan.java | 98 ++++++- .../vortex/spark/read/VortexScanBuilder.java | 24 +- .../spark/VortexDataSourceStatsTest.java | 241 ++++++++++++++++++ vortex-duckdb/src/table_function.rs | 2 +- vortex-file/src/multi/mod.rs | 63 +++-- vortex-jni/src/data_source.rs | 21 ++ vortex-layout/src/scan/multi.rs | 151 ++++++++++- 11 files changed, 627 insertions(+), 52 deletions(-) create mode 100644 java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceStatsTest.java diff --git a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java index 24780785274..93c5df2c61a 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java +++ b/java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java @@ -128,6 +128,57 @@ public OptionalLong asOptional() { } } + /** + * Sum of the on-storage byte sizes of all files included in this data source along with the precision of that + * estimate. Mirrors the Rust {@code Option>} returned by {@code DataSource::byte_size}: + * {@link ByteSize.Unknown} when no estimate is available (for example when the filesystem listing did not return + * sizes), {@link ByteSize.Estimate} for an inexact hint (some files contribute extrapolated sizes), and + * {@link ByteSize.Exact} when every file has a known size. + */ + public ByteSize byteSize() { + long[] out = new long[2]; + NativeDataSource.byteSize(pointer, out); + return switch ((int) out[1]) { + case 1 -> new ByteSize.Estimate(out[0]); + case 2 -> new ByteSize.Exact(out[0]); + default -> ByteSize.Unknown.INSTANCE; + }; + } + + /** Precision-aware byte size. See {@link #byteSize()}. */ + public sealed interface ByteSize { + /** Returns the byte size as a long, or {@code OptionalLong.empty()} when unknown. */ + OptionalLong asOptional(); + + /** Byte size is not known. */ + final class Unknown implements ByteSize { + public static final Unknown INSTANCE = new Unknown(); + + private Unknown() {} + + @Override + public OptionalLong asOptional() { + return OptionalLong.empty(); + } + } + + /** Estimated byte size; the actual value may differ. */ + record Estimate(long value) implements ByteSize { + @Override + public OptionalLong asOptional() { + return OptionalLong.of(value); + } + } + + /** Exact byte size. */ + record Exact(long value) implements ByteSize { + @Override + public OptionalLong asOptional() { + return OptionalLong.of(value); + } + } + } + /** Submit a scan. */ public Scan scan(ScanOptions options) { Objects.requireNonNull(options, "options"); diff --git a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java index b7e58d2dc21..cc2aa163cee 100644 --- a/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java +++ b/java/vortex-jni/src/main/java/dev/vortex/jni/NativeDataSource.java @@ -33,4 +33,10 @@ private NativeDataSource() {} * {@code 1=estimate}, {@code 2=exact}. */ public static native void rowCount(long pointer, long[] out); + + /** + * Populate {@code out} with {@code [bytes, precision]}, the sum of on-storage file sizes for the data source. + * Precision is one of {@code 0=unknown}, {@code 1=estimate}, {@code 2=exact}. + */ + public static native void byteSize(long pointer, long[] out); } diff --git a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java index 322dc5522c3..aeee8febcf4 100644 --- a/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java +++ b/java/vortex-jni/src/test/java/dev/vortex/api/TestMinimal.java @@ -11,6 +11,7 @@ import dev.vortex.jni.NativeLoader; import java.io.IOException; import java.math.BigDecimal; +import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; @@ -132,6 +133,7 @@ public void testFullScan() throws Exception { DataSource ds = DataSource.open(session, writePath); assertEquals(new DataSource.RowCount.Exact(10L), ds.rowCount()); + assertEquals(new DataSource.ByteSize.Exact(Files.size(tempDir.resolve("minimal.vortex"))), ds.byteSize()); var schema = ds.arrowSchema(allocator); assertEquals( diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java index 8df7ce8e1db..198dfd6c77c 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexBatchExec.java @@ -4,6 +4,7 @@ package dev.vortex.spark.read; import com.google.common.collect.ImmutableMap; +import dev.vortex.api.Session; import dev.vortex.jni.NativeFiles; import dev.vortex.spark.VortexFilePartition; import dev.vortex.spark.VortexSparkSession; @@ -76,14 +77,19 @@ public PartitionReaderFactory createReaderFactory() { } private List resolvePaths() { - var session = VortexSparkSession.get(formatOptions); + return resolveVortexPaths(VortexSparkSession.get(formatOptions), paths, formatOptions); + } + + /** + * Expands directory-like entries to concrete {@code .vortex} files; entries that already name a {@code .vortex} + * file are kept as-is. Shared with {@link VortexScan#estimateStatistics()} so planning and execution resolve paths + * identically. + */ + static List resolveVortexPaths(Session session, List paths, Map formatOptions) { return paths.stream() - .flatMap(path -> { - if (path.endsWith(".vortex")) { - return Stream.of(path); - } - return NativeFiles.listFiles(session, path, formatOptions).stream(); - }) + .flatMap(path -> path.endsWith(".vortex") + ? Stream.of(path) + : NativeFiles.listFiles(session, path, formatOptions).stream()) .collect(Collectors.toList()); } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java index d5949b57a4d..02a7563f925 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScan.java @@ -3,38 +3,62 @@ package dev.vortex.spark.read; +import dev.vortex.api.DataSource; +import dev.vortex.api.Session; +import dev.vortex.spark.VortexSparkSession; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.OptionalLong; import org.apache.spark.sql.connector.catalog.CatalogV2Util; import org.apache.spark.sql.connector.catalog.Column; +import org.apache.spark.sql.connector.expressions.NamedReference; import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.read.Batch; import org.apache.spark.sql.connector.read.Scan; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.connector.read.SupportsReportStatistics; +import org.apache.spark.sql.connector.read.colstats.ColumnStatistics; +import org.apache.spark.sql.internal.SQLConf; import org.apache.spark.sql.types.StructType; -/** Spark V2 {@link Scan} over a table of Vortex files. */ -public final class VortexScan implements Scan { +/** + * Spark V2 {@link Scan} over a table of Vortex files. + * + *

Implements {@link SupportsReportStatistics} to surface both the row count Vortex records in each file footer and a + * Spark scan-size estimate. The byte estimate starts from the on-storage file sizes collected by + * {@code MultiFileDataSource}, then follows Spark's file scan convention by applying the SQL file-compression factor + * and scaling by the pushed read schema's default size relative to the full table schema's default size. When the + * listing did not return a size for one or more files the file-byte total is extrapolated before Spark scaling is + * applied. + */ +public final class VortexScan implements Scan, SupportsReportStatistics { private final List paths; + private final List tableColumns; private final List readColumns; private final Map formatOptions; private final Predicate[] pushedPredicates; + private volatile Statistics cachedStatistics; + /** * Creates a new VortexScan for the specified file paths and columns. The caller is responsible for passing * immutable collections; the constructor does not copy. * * @param paths the list of Vortex file paths to scan + * @param tableColumns the full table columns before projection pushdown * @param readColumns the list of columns to read from the files * @param pushedPredicates predicates pushed down by Spark; {@code null} or empty means no pushdown */ public VortexScan( List paths, + List tableColumns, List readColumns, - Map formatOptions, - Predicate[] pushedPredicates) { + Predicate[] pushedPredicates, + Map formatOptions) { this.paths = paths; + this.tableColumns = tableColumns; this.readColumns = readColumns; this.formatOptions = formatOptions; this.pushedPredicates = pushedPredicates == null ? new Predicate[0] : pushedPredicates.clone(); @@ -83,4 +107,70 @@ public Batch toBatch() { public ColumnarSupportMode columnarSupportMode() { return ColumnarSupportMode.SUPPORTED; } + + /** + * Returns statistics for this scan. + * + *

Opens the Vortex {@link DataSource} on first invocation and caches the result. The row count is taken from the + * data source (sum of file-footer row counts; extrapolated from the first opened file when other files are + * deferred). {@link Statistics#sizeInBytes()} is derived from the per-file sizes reported by the filesystem + * listing, then adjusted by Spark's compression factor and the ratio between the pushed read schema and the full + * table schema. When a listing did not return a size for some file the file-byte total is extrapolated. When no + * file size is known at all the value is left empty so Spark falls back to its default heuristic. + * + * @return statistics with row-count and Spark scan-size estimates + */ + @Override + public Statistics estimateStatistics() { + Statistics local = cachedStatistics; + if (local != null) { + return local; + } + synchronized (this) { + if (cachedStatistics == null) { + cachedStatistics = computeStatistics(); + } + return cachedStatistics; + } + } + + private Statistics computeStatistics() { + Session session = VortexSparkSession.get(formatOptions); + List resolvedPaths = VortexBatchExec.resolveVortexPaths(session, paths, formatOptions); + if (resolvedPaths.isEmpty()) { + return new VortexStatistics(OptionalLong.empty(), OptionalLong.empty()); + } + + DataSource source = DataSource.open(session, resolvedPaths, formatOptions); + return new VortexStatistics( + source.rowCount().asOptional(), + scaleSizeInBytes(source.byteSize().asOptional())); + } + + private OptionalLong scaleSizeInBytes(OptionalLong fileBytes) { + if (fileBytes.isEmpty()) { + return OptionalLong.empty(); + } + + StructType tableSchema = CatalogV2Util.v2ColumnsToStructType(tableColumns.toArray(new Column[0])); + StructType readSchema = readSchema(); + int tableDefaultSize = tableSchema.defaultSize(); + if (tableDefaultSize <= 0) { + return fileBytes; + } + + double scaled = SQLConf.get().fileCompressionFactor() + * fileBytes.getAsLong() + / tableDefaultSize + * readSchema.defaultSize(); + return OptionalLong.of((long) scaled); + } + + private record VortexStatistics(OptionalLong numRows, OptionalLong sizeInBytes) implements Statistics { + + @Override + public Map columnStats() { + return Map.of(); + } + } } diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java index 94990432b45..107d22b30f6 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexScanBuilder.java @@ -31,7 +31,8 @@ public final class VortexScanBuilder implements ScanBuilder, SupportsPushDownRequiredColumns, SupportsPushDownV2Filters { private final ImmutableList.Builder paths; - private final List columns; + private final List tableColumns; + private final List readColumns; private final Map formatOptions; private final Set partitionColumnNames; private Predicate[] pushedPredicates = new Predicate[0]; @@ -48,10 +49,11 @@ public VortexScanBuilder(Map formatOptions) { */ public VortexScanBuilder(Map formatOptions, Transform[] partitionTransforms) { this.paths = ImmutableList.builder(); - this.columns = new ArrayList<>(); Map options = Maps.newHashMap(); options.put("vortex.workerThreads", "4"); options.putAll(formatOptions); + this.tableColumns = new ArrayList<>(); + this.readColumns = new ArrayList<>(); this.formatOptions = options; this.partitionColumnNames = collectPartitionColumnNames(partitionTransforms); } @@ -74,7 +76,8 @@ public VortexScanBuilder addPath(String path) { * @return this builder for method chaining */ public VortexScanBuilder addColumn(Column column) { - this.columns.add(column); + this.tableColumns.add(column); + this.readColumns.add(column); return this; } @@ -97,7 +100,7 @@ public VortexScanBuilder addAllPaths(Iterable paths) { */ public VortexScanBuilder addAllColumns(Iterable columns) { for (Column column : columns) { - this.columns.add(column); + addColumn(column); } return this; } @@ -116,7 +119,12 @@ public Scan build() { // Allow empty columns for operations like count() that don't need actual column data // If no columns are specified, we'll read the minimal schema needed - return new VortexScan(paths, List.copyOf(this.columns), this.formatOptions, pushedPredicates); + return new VortexScan( + paths, + List.copyOf(this.tableColumns), + List.copyOf(this.readColumns), + pushedPredicates, + this.formatOptions); } /** @@ -129,8 +137,8 @@ public Scan build() { */ @Override public void pruneColumns(StructType requiredSchema) { - columns.clear(); - columns.addAll(Arrays.asList(CatalogV2Util.structTypeToV2Columns(requiredSchema))); + readColumns.clear(); + readColumns.addAll(Arrays.asList(CatalogV2Util.structTypeToV2Columns(requiredSchema))); } /** @@ -145,7 +153,7 @@ public void pruneColumns(StructType requiredSchema) { @Override public Predicate[] pushPredicates(Predicate[] predicates) { Map dataColumnTypes = new HashMap<>(); - for (Column column : columns) { + for (Column column : readColumns) { if (!partitionColumnNames.contains(column.name())) { dataColumnTypes.put(column.name(), column.dataType()); } diff --git a/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceStatsTest.java b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceStatsTest.java new file mode 100644 index 00000000000..0595349a49a --- /dev/null +++ b/java/vortex-spark/src/test/java/dev/vortex/spark/VortexDataSourceStatsTest.java @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +package dev.vortex.spark; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.vortex.spark.read.VortexScan; +import dev.vortex.spark.read.VortexScanBuilder; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Map; +import java.util.stream.Stream; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; +import org.apache.spark.sql.connector.catalog.Column; +import org.apache.spark.sql.connector.read.Statistics; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; + +/** + * Integration tests for {@link VortexScan#estimateStatistics()}. + * + *

Verifies that the Spark V2 scan surfaces both the row count Vortex stores in each file footer and the sum of the + * on-storage file sizes reported by the filesystem listing. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public final class VortexDataSourceStatsTest { + private static final String FILE_COMPRESSION_FACTOR_KEY = "spark.sql.sources.fileCompressionFactor"; + + private SparkSession spark; + + @TempDir + Path tempDir; + + @BeforeAll + public void setUp() { + spark = SparkSession.builder() + .appName("VortexStatsTest") + .master("local[2]") + .config("spark.driver.host", "127.0.0.1") + .config("spark.sql.shuffle.partitions", "2") + .config("spark.sql.adaptive.enabled", "false") + .config("spark.ui.enabled", "false") + .getOrCreate(); + } + + @AfterAll + public void tearDown() { + if (spark != null) { + spark.stop(); + } + } + + @Test + @DisplayName("VortexScan reports exact row count for single-file scans") + public void testEstimateStatisticsReportsRowCount() throws IOException { + int numRows = 250; + Path outputPath = writeRows(numRows, "single_file"); + + VortexScan scan = buildScan(outputPath); + Statistics stats = scan.estimateStatistics(); + + assertTrue( + stats.numRows().isPresent(), + "VortexScan should report a row count for a Vortex dataset with a populated footer"); + assertEquals(numRows, stats.numRows().getAsLong(), "Row count should match the rows we wrote"); + } + + @Test + @DisplayName("VortexScan reports aggregate row count across multi-file scans") + public void testEstimateStatisticsAcrossMultipleFiles() throws IOException { + int numRows = 400; + Path outputPath = writeRows(numRows, "multi_file", 4); + + VortexScan scan = buildScan(outputPath); + Statistics stats = scan.estimateStatistics(); + + assertTrue(stats.numRows().isPresent(), "Row count should be reported for multi-file Vortex datasets"); + assertEquals(numRows, stats.numRows().getAsLong(), "Row count should sum across all files"); + } + + @Test + @DisplayName("VortexScan reports sizeInBytes equal to the sum of on-storage file sizes") + public void testEstimateStatisticsReportsSizeInBytes() throws IOException { + Path outputPath = writeRows(120, "with_size", 3); + + long fileBytes = totalVortexFileBytes(outputPath); + assertTrue(fileBytes > 0, "Test setup should produce at least one non-empty .vortex file"); + + VortexScan scan = buildScan(outputPath); + Statistics stats = scan.estimateStatistics(); + + assertTrue( + stats.sizeInBytes().isPresent(), + "VortexScan should surface a sizeInBytes when the filesystem listing reports file sizes"); + // Mirror the scan's Spark-convention scaling (factor 1.0, unpruned schema), which divides and + // re-multiplies by the schema default size in double arithmetic before truncating; asserting + // against the raw byte sum would be sensitive to the floating-point round trip. + StructType schema = spark.read() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .load() + .schema(); + long expectedSize = (long) (1.0 * fileBytes / schema.defaultSize() * schema.defaultSize()); + assertEquals( + expectedSize, + stats.sizeInBytes().getAsLong(), + "sizeInBytes should equal the sum of on-storage .vortex file sizes"); + } + + @Test + @DisplayName("VortexScan scales sizeInBytes by the pushed read schema") + public void testEstimateStatisticsScalesSizeInBytesForProjection() throws IOException { + Path outputPath = writeRows(120, "projected_size", 3); + long fileBytes = totalVortexFileBytes(outputPath); + + StructType fullSchema = spark.read() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .load() + .schema(); + StructType idOnlySchema = new StructType(new StructField[] {fullSchema.fields()[0]}); + + String previousCompressionFactor = spark.conf().get(FILE_COMPRESSION_FACTOR_KEY); + spark.conf().set(FILE_COMPRESSION_FACTOR_KEY, "0.5"); + try { + VortexScan scan = buildScan(outputPath, idOnlySchema); + Statistics stats = scan.estimateStatistics(); + + long expectedSize = (long) (0.5 * fileBytes / fullSchema.defaultSize() * idOnlySchema.defaultSize()); + assertTrue(stats.sizeInBytes().isPresent(), "Projected scans should still surface sizeInBytes"); + assertEquals( + expectedSize, + stats.sizeInBytes().getAsLong(), + "sizeInBytes should follow Spark FileScan's compression and schema-width scaling"); + assertTrue( + stats.sizeInBytes().getAsLong() < fileBytes, + "Projected scan stats should be smaller than full file bytes"); + } finally { + spark.conf().set(FILE_COMPRESSION_FACTOR_KEY, previousCompressionFactor); + } + } + + @Test + @DisplayName("VortexScan caches statistics across repeated calls") + public void testEstimateStatisticsIsCached() throws IOException { + Path outputPath = writeRows(50, "cached", 1); + + VortexScan scan = buildScan(outputPath); + Statistics first = scan.estimateStatistics(); + Statistics second = scan.estimateStatistics(); + + // Same instance returned -- the second call hits the cached value. + assertEquals(first, second, "estimateStatistics() should return the same Statistics object on repeat calls"); + assertInstanceOf(Statistics.class, first); + } + + private VortexScan buildScan(Path outputPath) { + return buildScan(outputPath, null); + } + + private VortexScan buildScan(Path outputPath, StructType requiredSchema) { + Dataset readDf = spark.read() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .load(); + StructType readSchema = readDf.schema(); + + VortexScanBuilder builder = new VortexScanBuilder(Map.of()); + builder.addPath(outputPath.toUri().toString()); + for (StructField field : readSchema.fields()) { + builder.addColumn(Column.create(field.name(), field.dataType())); + } + if (requiredSchema != null) { + builder.pruneColumns(requiredSchema); + } + return (VortexScan) builder.build(); + } + + private Path writeRows(int numRows, String name) throws IOException { + return writeRows(numRows, name, 1); + } + + private long totalVortexFileBytes(Path outputPath) throws IOException { + try (Stream paths = Files.walk(outputPath)) { + return paths.filter(Files::isRegularFile) + .filter(path -> path.getFileName().toString().endsWith(".vortex")) + .mapToLong(path -> { + try { + return Files.size(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .sum(); + } + } + + private Path writeRows(int numRows, String name, int partitions) throws IOException { + Path outputPath = tempDir.resolve(name); + Dataset df = spark.range(0, numRows) + .selectExpr("cast(id as int) as id", "concat('value_', cast(id as string)) as value"); + + df.repartition(partitions) + .write() + .format("vortex") + .option("path", outputPath.toUri().toString()) + .mode(SaveMode.Overwrite) + .save(); + return outputPath; + } + + @AfterEach + public void cleanupTempFiles() throws IOException { + if (tempDir != null && Files.exists(tempDir)) { + try (Stream paths = Files.walk(tempDir)) { + paths.sorted(Comparator.reverseOrder()).forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + System.err.println("Failed to delete: " + path); + } + }); + } + } + } +} diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index 11c5851af27..7b34725b123 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -439,7 +439,7 @@ pub fn statistics(bind_data: &TableFunctionBind, column_index: usize) -> Option< if children.len() != 1 { return None; } - let MultiLayoutChild::Opened(reader) = &children[0] else { + let MultiLayoutChild::Opened { reader, .. } = &children[0] else { return None; }; let stats_sets = match reader.as_any().downcast_ref::() { diff --git a/vortex-file/src/multi/mod.rs b/vortex-file/src/multi/mod.rs index 3abb4ebea2a..2baf53ff845 100644 --- a/vortex-file/src/multi/mod.rs +++ b/vortex-file/src/multi/mod.rs @@ -8,9 +8,12 @@ mod session; use std::sync::Arc; use async_trait::async_trait; +use futures::StreamExt; use futures::TryStreamExt; +use futures::stream; use session::MultiFileSessionExt; use tracing::debug; +use vortex_error::VortexError; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_io::filesystem::FileListing; @@ -63,6 +66,11 @@ pub struct MultiFileDataSource { open_options_fn: Arc VortexOpenOptions + Send + Sync>, } +/// In-flight glob resolutions in [`MultiFileDataSource::build`]. Callers like the JNI data +/// source add one exact path per glob source, where each resolution is a single remote +/// metadata lookup; resolving them concurrently avoids one round trip of latency per file. +const GLOB_RESOLUTION_CONCURRENCY: usize = 16; + impl MultiFileDataSource { /// Create a new [`MultiFileDataSource`] builder. pub fn new(session: VortexSession) -> Self { @@ -122,31 +130,39 @@ impl MultiFileDataSource { .then(|| create_local_filesystem(&self.session)) .transpose()?; - // Collect files from all glob sources. - let mut all_files: Vec<(FileListing, FileSystemRef)> = Vec::new(); - for (glob, maybe_fs) in &self.glob_sources { - // Use the provided filesystem, or fall back to the local filesystem. - // We know local_fs is Some when maybe_fs is None (by construction above). - let fs = maybe_fs - .as_ref() - .or(local_fs.as_ref()) - .map(Arc::clone) - .unwrap_or_else(|| { - unreachable!("local_fs is set when any glob lacks a filesystem") - }); - let files: Vec = fs.glob(glob)?.try_collect().await?; - for file in files { - all_files.push((file, Arc::clone(&fs))); - } - } + let globs: Vec = self.glob_sources.iter().map(|(g, _)| g.clone()).collect(); + + // Resolve glob sources concurrently while preserving their order, since the order + // determines partition indices and which file is opened eagerly for the schema. + let resolved: Vec> = + stream::iter(self.glob_sources.into_iter().map(|(glob, maybe_fs)| { + // Use the provided filesystem, or fall back to the local filesystem. + // We know local_fs is Some when maybe_fs is None (by construction above). + let fs = maybe_fs + .or_else(|| local_fs.as_ref().map(Arc::clone)) + .unwrap_or_else(|| { + unreachable!("local_fs is set when any glob lacks a filesystem") + }); + async move { + let files: Vec = fs.glob(&glob)?.try_collect().await?; + Ok::<_, VortexError>( + files + .into_iter() + .map(|file| (file, Arc::clone(&fs))) + .collect(), + ) + } + })) + .buffered(GLOB_RESOLUTION_CONCURRENCY) + .try_collect() + .await?; + let all_files: Vec<(FileListing, FileSystemRef)> = resolved.into_iter().flatten().collect(); if all_files.is_empty() { - let globs: Vec<_> = self.glob_sources.iter().map(|(g, _)| g.as_str()).collect(); vortex_bail!("No files matched the glob pattern(s): {:?}", globs); } let file_count = all_files.len(); - let globs: Vec<_> = self.glob_sources.iter().map(|(g, _)| g.as_str()).collect(); debug!(file_count, glob = ?globs, "discovered files"); // Open first file eagerly for dtype. @@ -155,6 +171,8 @@ impl MultiFileDataSource { let first_file = open_file(first_fs, first_file_listing, &self.session, open_fn).await?; let first_reader = first_file.layout_reader()?; + let byte_sizes: Vec> = all_files.iter().map(|(file, _)| file.size).collect(); + let factories: Vec> = all_files[1..] .iter() .map(|(file, fs)| { @@ -167,7 +185,12 @@ impl MultiFileDataSource { }) .collect(); - let inner = MultiLayoutDataSource::new_with_first(first_reader, factories, &self.session); + let inner = MultiLayoutDataSource::new_with_first( + first_reader, + factories, + byte_sizes, + &self.session, + ); debug!(file_count, dtype = %inner.dtype(), "built MultiFileDataSource"); diff --git a/vortex-jni/src/data_source.rs b/vortex-jni/src/data_source.rs index d733c2db48f..5f244998f67 100644 --- a/vortex-jni/src/data_source.rs +++ b/vortex-jni/src/data_source.rs @@ -211,6 +211,27 @@ pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_rowCount( }); } +/// Write the byte size into the two-slot jlong pair `out`: +/// `out[0]` receives the size in bytes (0 when unknown), `out[1]` the precision (0=unknown, 1=estimate, 2=exact). +#[unsafe(no_mangle)] +pub extern "system" fn Java_dev_vortex_jni_NativeDataSource_byteSize( + mut env: EnvUnowned, + _class: JClass, + pointer: jlong, + out: JLongArray, +) { + try_or_throw(&mut env, |env| { + let ds = unsafe { NativeDataSource::from_ptr(pointer) }; + let (bytes, precision) = match ds.inner.byte_size() { + Precision::Exact(b) => (b as jlong, 2), + Precision::Inexact(b) => (b as jlong, 1), + Precision::Absent => (0, 0), + }; + out.set_region(env, 0, &[bytes, precision])?; + Ok(()) + }); +} + #[cfg(test)] mod tests { use super::*; diff --git a/vortex-layout/src/scan/multi.rs b/vortex-layout/src/scan/multi.rs index 8b8376b3b0b..cb516d2500d 100644 --- a/vortex-layout/src/scan/multi.rs +++ b/vortex-layout/src/scan/multi.rs @@ -32,6 +32,7 @@ use async_trait::async_trait; use futures::FutureExt; use futures::StreamExt; use futures::stream; +use itertools::Itertools; use tracing::Instrument; use vortex_array::dtype::DType; use vortex_array::dtype::FieldPath; @@ -87,26 +88,68 @@ pub struct MultiLayoutDataSource { } pub enum MultiLayoutChild { - Opened(LayoutReaderRef), - Deferred(Arc), + Opened { + reader: LayoutReaderRef, + /// On-storage file size in bytes, if known from the listing metadata. + byte_size: Option, + }, + Deferred { + factory: Arc, + /// On-storage file size in bytes, if known from the listing metadata. + byte_size: Option, + }, +} + +impl MultiLayoutChild { + /// On-storage file size in bytes for this child, if known. + pub fn byte_size(&self) -> Option { + match self { + MultiLayoutChild::Opened { byte_size, .. } => *byte_size, + MultiLayoutChild::Deferred { byte_size, .. } => *byte_size, + } + } } impl MultiLayoutDataSource { /// Creates a multi-layout data source with the first reader pre-opened. /// /// The first reader determines the dtype. Remaining readers are opened lazily during - /// scanning via their factories. + /// scanning via their factories. `byte_sizes` carries the on-storage file size in bytes for + /// each child (first followed by remaining); pass `None` for entries where the size is + /// unknown. Must be empty or have length `1 + remaining.len()`. pub fn new_with_first( first: LayoutReaderRef, remaining: Vec>, + byte_sizes: Vec>, session: &VortexSession, ) -> Self { let dtype = first.dtype().clone(); let concurrency = get_available_parallelism().unwrap_or(DEFAULT_CONCURRENCY); - let mut children = Vec::with_capacity(1 + remaining.len()); - children.push(MultiLayoutChild::Opened(first)); - children.extend(remaining.into_iter().map(MultiLayoutChild::Deferred)); + let total = 1 + remaining.len(); + let mut sizes = byte_sizes; + if sizes.is_empty() { + sizes = vec![None; total]; + } + debug_assert_eq!( + sizes.len(), + total, + "byte_sizes length must match the number of children" + ); + + let mut children = Vec::with_capacity(total); + let mut sizes_iter = sizes.into_iter(); + let first_size = sizes_iter.next().unwrap_or(None); + children.push(MultiLayoutChild::Opened { + reader: first, + byte_size: first_size, + }); + children.extend( + remaining + .into_iter() + .zip_eq(sizes_iter) + .map(|(factory, byte_size)| MultiLayoutChild::Deferred { factory, byte_size }), + ); Self { dtype, @@ -120,20 +163,34 @@ impl MultiLayoutDataSource { /// /// The dtype must be provided externally since there is no pre-opened reader to infer it /// from. This avoids eagerly opening any file when the schema is already known (e.g. from - /// a catalog or a prior scan). + /// a catalog or a prior scan). `byte_sizes` carries the on-storage file size in bytes for + /// each factory; pass `None` for entries where the size is unknown. Must be empty or have + /// the same length as `factories`. pub fn new_deferred( dtype: DType, factories: Vec>, + byte_sizes: Vec>, session: &VortexSession, ) -> Self { let concurrency = get_available_parallelism().unwrap_or(DEFAULT_CONCURRENCY); + let mut sizes = byte_sizes; + if sizes.is_empty() { + sizes = vec![None; factories.len()]; + } + debug_assert_eq!( + sizes.len(), + factories.len(), + "byte_sizes length must match the number of factories" + ); + Self { dtype, session: session.clone(), children: factories .into_iter() - .map(MultiLayoutChild::Deferred) + .zip_eq(sizes) + .map(|(factory, byte_size)| MultiLayoutChild::Deferred { factory, byte_size }) .collect(), concurrency, } @@ -166,11 +223,11 @@ impl DataSource for MultiLayoutDataSource { for child in &self.children { match child { - MultiLayoutChild::Opened(reader) => { + MultiLayoutChild::Opened { reader, .. } => { opened_count += 1; sum = sum.saturating_add(reader.row_count()); } - MultiLayoutChild::Deferred(_) => { + MultiLayoutChild::Deferred { .. } => { deferred_count += 1; } } @@ -192,6 +249,34 @@ impl DataSource for MultiLayoutDataSource { } } + fn byte_size(&self) -> Precision { + let total_count = self.children.len() as u64; + if total_count == 0 { + return Precision::exact(0u64); + } + + let mut sum: u64 = 0; + let mut known_count: u64 = 0; + for child in &self.children { + if let Some(size) = child.byte_size() { + sum = sum.saturating_add(size); + known_count += 1; + } + } + + if known_count == 0 { + return Precision::Absent; + } + + if known_count == total_count { + Precision::exact(sum) + } else { + let avg = sum / known_count; + let extrapolated = avg.saturating_mul(total_count); + Precision::inexact(extrapolated) + } + } + fn deserialize_partition( &self, _data: &[u8], @@ -206,8 +291,10 @@ impl DataSource for MultiLayoutDataSource { for child in &self.children { match child { - MultiLayoutChild::Opened(reader) => ready.push_back(Arc::clone(reader)), - MultiLayoutChild::Deferred(factory) => deferred.push_back(Arc::clone(factory)), + MultiLayoutChild::Opened { reader, .. } => ready.push_back(Arc::clone(reader)), + MultiLayoutChild::Deferred { factory, .. } => { + deferred.push_back(Arc::clone(factory)) + } } } @@ -443,3 +530,43 @@ impl Partition for MultiLayoutPartition { ))) } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use vortex_array::dtype::Nullability; + + use super::*; + use crate::scan::test::new_session; + + struct NeverOpened; + + #[async_trait] + impl LayoutReaderFactory for NeverOpened { + async fn open(&self) -> VortexResult> { + unreachable!("byte_size must not open readers") + } + } + + fn deferred_source(byte_sizes: Vec>) -> MultiLayoutDataSource { + let factories: Vec> = byte_sizes + .iter() + .map(|_| Arc::new(NeverOpened) as _) + .collect(); + MultiLayoutDataSource::new_deferred( + DType::Bool(Nullability::NonNullable), + factories, + byte_sizes, + &new_session(), + ) + } + + #[rstest] + #[case::all_known(vec![Some(10), Some(20), Some(30)], Precision::exact(60u64))] + #[case::some_known_extrapolates(vec![Some(10), None, Some(30)], Precision::inexact(60u64))] + #[case::none_known(vec![None, None], Precision::Absent)] + #[case::no_children(vec![], Precision::exact(0u64))] + fn byte_size_precision(#[case] sizes: Vec>, #[case] expected: Precision) { + assert_eq!(deferred_source(sizes).byte_size(), expected); + } +} From e935986bf3a52e928da40c776520bdeaf4cc6170 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 10:30:01 +0100 Subject: [PATCH 021/162] fix: `vx_data_source_new_buffer` sem-merge conflict (#8437) Signed-off-by: Joe Isaacs --- vortex-ffi/src/data_source.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vortex-ffi/src/data_source.rs b/vortex-ffi/src/data_source.rs index 376a61e3096..bdbb9b53972 100644 --- a/vortex-ffi/src/data_source.rs +++ b/vortex-ffi/src/data_source.rs @@ -136,7 +136,12 @@ pub unsafe extern "C-unwind" fn vx_data_source_new_buffer( unsafe { slice::from_raw_parts(buffer.cast::(), buffer_len) }; let buffer = ByteBuffer::from(Bytes::from_static(bytes)); let file = session.open_options().open_buffer(buffer)?; - let ds = MultiLayoutDataSource::new_with_first(file.layout_reader()?, Vec::new(), session); + let ds = MultiLayoutDataSource::new_with_first( + file.layout_reader()?, + Vec::new(), + vec![Some(buffer_len as u64)], + session, + ); Ok(vx_data_source::new(Arc::new(ds) as DataSourceRef)) }) From a17de8c8ce1958ca271ed1b4640e2dcb08dbaadc Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Tue, 16 Jun 2026 05:48:51 -0400 Subject: [PATCH 022/162] Ignore rustsec (#8381) ## Summary Closes: #000 ## Testing --------- Signed-off-by: Connor Tsui --- deny.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index b9f9c79a796..a2a6935ba5b 100644 --- a/deny.toml +++ b/deny.toml @@ -16,7 +16,12 @@ ignore = [ # Paste is no longer maintained because its essentially "finished". "RUSTSEC-2024-0436", # proc-macro-error-2 is unmaintained, only used by the `test_with` test dependency - "RUSTSEC-2026-0173" + "RUSTSEC-2026-0173", + # Out-of-bounds read in `nth`/`nth_back` on pyo3 list/tuple iterators, fixed only in pyo3 + # 0.29.0. We cannot bump until pyo3-bytes, pyo3-log, and pyo3-object_store support 0.29 (all + # pin pyo3 to <0.29, and pyo3-ffi `links = "python"` forbids two pyo3 versions in the graph). + # Not exploitable here: `vortex-python` never calls `nth`/`nth_back` on these iterators. + "RUSTSEC-2026-0176" ] [licenses] From 739b7451b2ffad6d3cceb7bf6bd9c16ede4ed2a8 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 11:01:43 +0100 Subject: [PATCH 023/162] ci: temporarily disable rust-ffi-test-sanitizer (#8441) See https://github.com/vortex-data/vortex/actions/runs/27608329337/job/81625965956?pr=8121, This job hangs Signed-off-by: Joe Isaacs --- .github/workflows/rust-instrumented.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-instrumented.yml b/.github/workflows/rust-instrumented.yml index d58463d53d2..e4dc48ee8b7 100644 --- a/.github/workflows/rust-instrumented.yml +++ b/.github/workflows/rust-instrumented.yml @@ -221,7 +221,8 @@ jobs: - name: Run tests run: | set -o pipefail - ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt +# re-enable once we can fix this hang +# ./vortex-ffi/build/test/vortex_ffi_test 2>&1 | rustfilt - name: Run examples run: | set -o pipefail From 9c4e0e47e936005e1198b3bd9a99035a35fc188b Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 11:15:38 +0100 Subject: [PATCH 024/162] Make `Mask` execution strict about nullable booleans (#8121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Previously `Mask::execute` accepted **nullable** boolean arrays and silently coerced null elements to `false` (folding validity into the value bits). That behaviour was flagged with a `TODO` questioning whether the implicit coercion was correct. This PR makes `Mask` execution **strict**: `Mask::execute` now requires a non-nullable boolean array (`Bool(NonNullable)`) and errors on nullable input. The null-as-`false` coercion is no longer implicit — callers that want SQL-style `NULL`-as-not-matching semantics opt in explicitly via `array.fill_null(false)` before executing into a `Mask`. --------- Signed-off-by: Joe Isaacs Co-authored-by: Claude --- .../src/aggregate_fn/fns/is_constant/mod.rs | 2 +- vortex-array/src/mask.rs | 83 ++++++++++++++----- vortex-cuda/src/layout.rs | 5 +- vortex-layout/src/layouts/dict/reader.rs | 6 +- vortex-layout/src/layouts/flat/reader.rs | 5 +- vortex-layout/src/layouts/partitioned.rs | 6 +- vortex-layout/src/layouts/row_idx/mod.rs | 6 +- vortex-layout/src/layouts/zoned/zone_map.rs | 5 +- 8 files changed, 89 insertions(+), 29 deletions(-) diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index b14b2a050f8..d1dd6bef39c 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -74,7 +74,7 @@ fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> Vor // Compare values element-wise. Result is null where both inputs are null, // true/false where both are valid. let eq_result = a.binary(b.clone(), Operator::Eq)?; - let eq_result = eq_result.execute::(ctx)?; + let eq_result = eq_result.fill_null(false)?.execute::(ctx)?; Ok(eq_result.true_count() == valid_count) } diff --git a/vortex-array/src/mask.rs b/vortex-array/src/mask.rs index ab035670e24..09032aabbd0 100644 --- a/vortex-array/src/mask.rs +++ b/vortex-array/src/mask.rs @@ -1,8 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::ops::BitAnd; - use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_mask::Mask; @@ -12,19 +10,22 @@ use crate::Executable; use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::BoolArray; -use crate::arrays::Constant; use crate::columnar::Columnar; use crate::dtype::DType; +use crate::dtype::Nullability; impl Executable for Mask { + /// Executes a boolean array into a [`Mask`]. + /// + /// The array must have a non-nullable boolean dtype. To execute a nullable boolean array, + /// coercing null elements to `false`, first call + /// [`ArrayRef::fill_null(false)`](crate::builtins::ArrayBuiltins::fill_null). fn execute(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { - if !matches!(array.dtype(), DType::Bool(_)) { - vortex_bail!("Mask array must have boolean dtype, not {}", array.dtype()); - } - - if let Some(constant) = array.as_opt::() { - let mask_value = constant.scalar().as_bool().value().unwrap_or(false); - return Ok(Mask::new(array.len(), mask_value)); + if !matches!(array.dtype(), DType::Bool(Nullability::NonNullable)) { + vortex_bail!( + "Mask array must have boolean(NonNullable) dtype, not {}", + array.dtype() + ); } let array_len = array.len(); @@ -34,16 +35,60 @@ impl Executable for Mask { } Columnar::Canonical(a) => { let bool = a.into_array().execute::(ctx)?; - let mask = bool - .as_ref() - .validity()? - .execute_mask(bool.as_ref().len(), ctx)?; - let bits = bool.into_bit_buffer(); - // To handle nullable boolean arrays, we treat nulls as false in the mask. - // TODO(ngates): is this correct? Feels like we should just force the caller to - // pass non-nullable boolean arrays. - mask.bitand(&Mask::from(bits)) + Mask::from(bool.into_bit_buffer()) } }) } } + +#[cfg(test)] +mod tests { + use vortex_error::VortexResult; + use vortex_mask::Mask; + + use crate::ExecutionCtx; + use crate::IntoArray; + use crate::LEGACY_SESSION; + use crate::VortexSessionExecute; + use crate::arrays::BoolArray; + use crate::arrays::ConstantArray; + use crate::builtins::ArrayBuiltins; + use crate::dtype::DType; + use crate::dtype::Nullability; + use crate::scalar::Scalar; + + fn ctx() -> ExecutionCtx { + LEGACY_SESSION.create_execution_ctx() + } + + #[test] + fn mask_non_nullable() -> VortexResult<()> { + let array = BoolArray::from_iter([true, false, true]).into_array(); + let mask = array.execute::(&mut ctx())?; + assert_eq!(mask, Mask::from_iter([true, false, true])); + Ok(()) + } + + #[test] + fn mask_rejects_nullable() { + let array = BoolArray::from_iter([Some(true), None]).into_array(); + assert!(array.execute::(&mut ctx()).is_err()); + } + + #[test] + fn fill_null_then_mask_coerces_nulls() -> VortexResult<()> { + let array = BoolArray::from_iter([Some(true), None, Some(false), None]).into_array(); + let mask = array.fill_null(false)?.execute::(&mut ctx())?; + assert_eq!(mask, Mask::from_iter([true, false, false, false])); + Ok(()) + } + + #[test] + fn fill_null_then_mask_null_constant() -> VortexResult<()> { + let array = + ConstantArray::new(Scalar::null(DType::Bool(Nullability::Nullable)), 4).into_array(); + let mask = array.fill_null(false)?.execute::(&mut ctx())?; + assert_eq!(mask, Mask::new_false(4)); + Ok(()) + } +} diff --git a/vortex-cuda/src/layout.rs b/vortex-cuda/src/layout.rs index 20aa26e0b3b..9e6111fc371 100644 --- a/vortex-cuda/src/layout.rs +++ b/vortex-cuda/src/layout.rs @@ -22,6 +22,7 @@ use vortex::array::MaskFuture; use vortex::array::ProstMetadata; use vortex::array::VortexSessionExecute; use vortex::array::arrays::Constant; +use vortex::array::builtins::ArrayBuiltins; use vortex::array::expr::Expression; use vortex::array::expr::stats::Precision; use vortex::array::expr::stats::Stat; @@ -331,12 +332,12 @@ impl LayoutReader for CudaFlatReader { let array = array.apply(&expr)?; let array = array.filter(mask.clone())?; let mut ctx = session.create_execution_ctx(); - let array_mask = array.execute::(&mut ctx)?; + let array_mask = array.fill_null(false)?.execute::(&mut ctx)?; mask.intersect_by_rank(&array_mask) } else { let array = array.apply(&expr)?; let mut ctx = session.create_execution_ctx(); - let array_mask = array.execute::(&mut ctx)?; + let array_mask = array.fill_null(false)?.execute::(&mut ctx)?; mask.bitand(&array_mask) }; diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 002b4b1e902..6c8d91706db 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -16,6 +16,7 @@ use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::SharedArray; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; @@ -216,7 +217,10 @@ impl LayoutReader for DictReader { let mask = mask.await?; let mut ctx = session.create_execution_ctx(); - let dict_mask = values.take(codes)?.execute::(&mut ctx)?; + let dict_mask = values + .take(codes)? + .fill_null(false)? + .execute::(&mut ctx)?; Ok(mask.bitand(&dict_mask)) })) diff --git a/vortex-layout/src/layouts/flat/reader.rs b/vortex-layout/src/layouts/flat/reader.rs index 3f3527aaa6a..e03ffc20e1a 100644 --- a/vortex-layout/src/layouts/flat/reader.rs +++ b/vortex-layout/src/layouts/flat/reader.rs @@ -11,6 +11,7 @@ use tracing::trace; use vortex_array::ArrayRef; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; @@ -156,14 +157,14 @@ impl LayoutReader for FlatReader { let array = array.apply(&expr)?; let array = array.filter(mask.clone())?; let mut ctx = session.create_execution_ctx(); - let array_mask = array.execute::(&mut ctx)?; + let array_mask = array.fill_null(false)?.execute::(&mut ctx)?; mask.intersect_by_rank(&array_mask) } else { // Run over the full array, with a simpler bitand at the end. let array = array.apply(&expr)?; let mut ctx = session.create_execution_ctx(); - let array_mask = array.execute::(&mut ctx)?; + let array_mask = array.fill_null(false)?.execute::(&mut ctx)?; mask.bitand(&array_mask) }; diff --git a/vortex-layout/src/layouts/partitioned.rs b/vortex-layout/src/layouts/partitioned.rs index d5ac3d44c55..cd927bbc074 100644 --- a/vortex-layout/src/layouts/partitioned.rs +++ b/vortex-layout/src/layouts/partitioned.rs @@ -11,6 +11,7 @@ use vortex_array::IntoArray; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::expr::Expression; @@ -90,7 +91,10 @@ impl PartitionedExprEval

for PartitionedExpr

{ .into_array(); let mut ctx = session.create_execution_ctx(); - let root_mask = root_scope.apply(&self.root)?.execute::(&mut ctx)?; + let root_mask = root_scope + .apply(&self.root)? + .fill_null(false)? + .execute::(&mut ctx)?; let mask = mask.bitand(&root_mask); diff --git a/vortex-layout/src/layouts/row_idx/mod.rs b/vortex-layout/src/layouts/row_idx/mod.rs index 3814c2f8093..2347d05b0bd 100644 --- a/vortex-layout/src/layouts/row_idx/mod.rs +++ b/vortex-layout/src/layouts/row_idx/mod.rs @@ -19,6 +19,7 @@ use vortex_array::Canonical; use vortex_array::IntoArray; use vortex_array::MaskFuture; use vortex_array::VortexSessionExecute; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::dtype::FieldName; @@ -295,7 +296,10 @@ fn row_idx_mask_future( let array = idx_array(row_offset, &row_range).into_array(); let mut ctx = session.create_execution_ctx(); - let result_mask = array.apply(&expr)?.execute::(&mut ctx)?; + let result_mask = array + .apply(&expr)? + .fill_null(false)? + .execute::(&mut ctx)?; Ok(result_mask.bitand(&mask.await?)) }) diff --git a/vortex-layout/src/layouts/zoned/zone_map.rs b/vortex-layout/src/layouts/zoned/zone_map.rs index 96154e69571..d7a5b7cd769 100644 --- a/vortex-layout/src/layouts/zoned/zone_map.rs +++ b/vortex-layout/src/layouts/zoned/zone_map.rs @@ -17,6 +17,7 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::StructArray; use vortex_array::arrays::struct_::StructArrayExt; +use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::expr::Expression; @@ -123,12 +124,12 @@ impl ZoneMap { let applied = self.array.clone().into_array().apply(&predicate)?; if !contains_row_count(&applied) { - return applied.execute::(&mut ctx); + return applied.fill_null(false)?.execute::(&mut ctx); } let row_count_array = row_count_array(self.zone_len, self.row_count, num_zones)?; let substituted = substitute_row_count(applied, &row_count_array)?; - substituted.execute::(&mut ctx) + substituted.fill_null(false)?.execute::(&mut ctx) } fn lower_stats(&self, predicate: Expression) -> VortexResult { From fb5cbbea07555b376b5a25852e40af5fce4c9a6f Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 11:28:32 +0100 Subject: [PATCH 025/162] ci: gate cuda san on cuda changes (#8438) CUDA san is slow gate it on CUDA changes. This is will post commit with san --------- Signed-off-by: Joe Isaacs --- .github/workflows/cuda.yaml | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 37ec5545686..6fc19e9c7b1 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -21,6 +21,26 @@ env: RUST_BACKTRACE: 1 jobs: + changes: + name: "Detect CUDA changes" + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + pull-requests: read + outputs: + run-cuda-san: ${{ github.event_name != 'pull_request' || steps.filter.outputs.cuda == 'true' }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4 + id: filter + if: github.event_name == 'pull_request' + with: + filters: | + cuda: + - "vortex-cuda/**" + - "vortex-test/**" + - ".github/workflows/**" + cuda-build-lint: if: github.repository == 'vortex-data/vortex' name: "CUDA build & lint" @@ -92,7 +112,10 @@ jobs: --verbose cuda-test-sanitizer: - if: github.repository == 'vortex-data/vortex' + needs: [changes] + if: >- + always() && github.repository == 'vortex-data/vortex' && + needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests (${{ matrix.sanitizer }})" timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-sanitizer From 9b5447a91b1d1dc2799ce043e5b2e8d515e3e92f Mon Sep 17 00:00:00 2001 From: Han Damin Date: Tue, 16 Jun 2026 19:30:03 +0900 Subject: [PATCH 026/162] perf(buffer): read byte-aligned bit operands directly as words (#8436) ## Summary Split out of #8425 so each perf change can be looked at on its own, as suggested by @joseph-isaacs. This is the "bit collect" half, no count fusion. `bitwise_binary_op` only took the direct word path when both operands were 64-bit aligned, otherwise falling back to `iter_padded`. It now reads the backing bytes as `u64` via `as_chunks::<8>` for any byte-aligned offset, leaving `iter_padded` for sub-byte offsets only. This also drops the `from_trusted_len_iter` unsafe for a safe `BufferMut` push. --------- Signed-off-by: Han Damin Co-authored-by: Joe Isaacs --- vortex-buffer/src/bit/ops.rs | 106 ++++++++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 28 deletions(-) diff --git a/vortex-buffer/src/bit/ops.rs b/vortex-buffer/src/bit/ops.rs index be1401d912a..5006fbcd4cc 100644 --- a/vortex-buffer/src/bit/ops.rs +++ b/vortex-buffer/src/bit/ops.rs @@ -5,9 +5,8 @@ use std::mem::MaybeUninit; use crate::BitBuffer; use crate::BitBufferMut; -use crate::Buffer; +use crate::BufferMut; use crate::ByteBufferMut; -use crate::trusted_len::TrustedLenExt; /// Read up to 8 bytes as a little-endian `u64`, zero-padding the high bytes when fewer than 8 are /// supplied. Using [`u64::from_le_bytes`] keeps the bit-numbering identical on little- and @@ -188,36 +187,52 @@ pub(super) fn bitwise_binary_op u64>( mut op: F, ) -> BitBuffer { assert_eq!(left.len(), right.len()); - - // If the buffers are aligned, we can use the fast path. - if left.offset().is_multiple_of(8) && right.offset().is_multiple_of(8) { - let left_chunks = left.unaligned_chunks(); - let right_chunks = right.unaligned_chunks(); - if left_chunks.lead_padding() == 0 - && left_chunks.trailing_padding() == 0 - && right_chunks.lead_padding() == 0 - && right_chunks.trailing_padding() == 0 - { - let iter = left_chunks - .iter() - .zip(right_chunks.iter()) - .map(|(l, r)| op(l, r)); - let iter = unsafe { iter.trusted_len() }; - let result = Buffer::::from_trusted_len_iter(iter).into_byte_buffer(); - return BitBuffer::new(result, left.len()); - } + let len = left.len(); + if len == 0 { + return BitBuffer::empty(); } - let iter = left - .chunks() - .iter_padded() - .zip(right.chunks().iter_padded()) - .map(|(l, r)| op(l, r)); - let iter = unsafe { iter.trusted_len() }; + let n_bytes = len.div_ceil(8); + let out = if left.offset().is_multiple_of(8) && right.offset().is_multiple_of(8) { + // Byte-aligned operands: logical bits map onto physical `u64` words, so read the backing + // bytes straight as words and build the result from a `TrustedLen` iterator. + let l_start = left.offset() / 8; + let r_start = right.offset() / 8; + let lhs = &left.inner().as_slice()[l_start..l_start + n_bytes]; + let rhs = &right.inner().as_slice()[r_start..r_start + n_bytes]; - let result = Buffer::::from_trusted_len_iter(iter).into_byte_buffer(); + let (lhs_words, lhs_tail) = lhs.as_chunks::<8>(); + let (rhs_words, rhs_tail) = rhs.as_chunks::<8>(); + + let mut out = BufferMut::::from_trusted_len_iter( + lhs_words + .iter() + .zip(rhs_words) + .map(|(l, r)| op(u64::from_le_bytes(*l), u64::from_le_bytes(*r))), + ); + if !lhs_tail.is_empty() { + out.push(op(read_u64_le(lhs_tail), read_u64_le(rhs_tail))); + } + out + } else { + // Sub-byte offset: `iter_padded` realigns the bits and appends one pad word, so take + // exactly `ceil(len / 64)` words. + let n_words = len.div_ceil(64); + let mut out = BufferMut::::with_capacity(n_words); + for (l, r) in left + .chunks() + .iter_padded() + .zip(right.chunks().iter_padded()) + .take(n_words) + { + out.push(op(l, r)); + } + out + }; - BitBuffer::new(result, left.len()) + let mut bytes = out.into_byte_buffer(); + bytes.truncate(n_bytes); + BitBuffer::new(bytes.freeze(), len) } #[cfg(test)] @@ -317,6 +332,41 @@ mod tests { assert_eq!(result, bitbuffer![false, true, true, false]); } + /// `bitwise_binary_op` must match a naive per-bit reference for every op, offset and length, + /// independent of the chunked kernels. + #[rstest] + #[case::aligned(0, 0)] + #[case::byte_aligned(8, 16)] + #[case::byte_aligned_mismatch(16, 0)] + #[case::sub_byte(3, 3)] + #[case::sub_byte_mismatch(0, 5)] + fn binary_op_matches_naive(#[case] left_offset: usize, #[case] right_offset: usize) { + #[allow(clippy::cast_possible_truncation)] + let make = |offset: usize, len: usize, salt: u8| -> BitBuffer { + let bytes: ByteBufferMut = (0..(offset + len).div_ceil(8).max(1)) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(salt)) + .collect(); + BitBufferMut::from_buffer(bytes, offset, len).freeze() + }; + let ops: [fn(u64, u64) -> u64; 4] = + [|a, b| a & b, |a, b| a | b, |a, b| a ^ b, |a, b| a & !b]; + + for len in [1usize, 5, 8, 63, 64, 65, 127, 128, 200, 256] { + let left = make(left_offset, len, 0xC3); + let right = make(right_offset, len, 0x5A); + for op in ops { + let got = bitwise_binary_op(&left, &right, op); + let expected: BitBuffer = (0..len) + .map(|i| op(u64::from(left.value(i)), u64::from(right.value(i))) & 1 == 1) + .collect(); + assert_eq!( + got, expected, + "loff={left_offset} roff={right_offset} len={len}" + ); + } + } + } + /// Regression test for a bug where [`bitwise_unary_op`] produced corrupt results when /// the [`BitBuffer`]'s underlying byte pointer was not u64-aligned. Slicing a buffer by /// a non-multiple-of-8 number of bytes can cause this misalignment. The bug only From e1c6ef523ac6d816f16076cb7dc8ea46fea13ba2 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 13:49:01 +0100 Subject: [PATCH 027/162] break: add `Validity::definitely_all_null()` method (#8447) ## Summary This PR introduces a new `Validity::definitely_all_null()` method that provides a fast path to check if a validity is definitively all-null (i.e., `Validity::AllInvalid`). This is the counterpart to the existing `definitely_no_nulls()` method. This is a rename, but not a change in semantics --------- Signed-off-by: Joe Isaacs Co-authored-by: Claude --- vortex-array/src/arrays/dict/vtable/mod.rs | 3 +-- vortex-array/src/arrays/masked/vtable/mod.rs | 2 +- .../src/arrays/primitive/array/top_value.rs | 3 +-- vortex-array/src/arrays/varbin/array.rs | 2 +- vortex-array/src/scalar_fn/fns/fill_null/kernel.rs | 2 +- vortex-array/src/scalar_fn/fns/list_contains/mod.rs | 2 +- vortex-array/src/validity.rs | 13 +++++++++++++ vortex-cuda/src/dynamic_dispatch/mod.rs | 3 +-- vortex-cuda/src/kernel/encodings/date_time_parts.rs | 2 +- vortex-cuda/src/kernel/encodings/fsst.rs | 3 +-- vortex-cuda/src/kernel/encodings/runend.rs | 2 +- vortex-duckdb/src/exporter/bool.rs | 3 +-- vortex-duckdb/src/exporter/decimal.rs | 3 +-- vortex-duckdb/src/exporter/fixed_size_list.rs | 3 +-- vortex-duckdb/src/exporter/list.rs | 3 +-- vortex-duckdb/src/exporter/list_view.rs | 3 +-- vortex-duckdb/src/exporter/primitive.rs | 3 +-- vortex-duckdb/src/exporter/struct_.rs | 3 +-- vortex-duckdb/src/exporter/varbinview.rs | 3 +-- 19 files changed, 31 insertions(+), 30 deletions(-) diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index fa8515dd986..eb742182a10 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -46,7 +46,6 @@ use crate::executor::ExecutionResult; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::validity::Validity; mod kernel; mod operations; @@ -179,7 +178,7 @@ impl VTable for Dict { let array = require_child!(array, array.codes(), DictSlots::CODES => Primitive); - if matches!(array.codes().validity()?, Validity::AllInvalid) { + if array.codes().validity()?.definitely_all_null() { return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 4409c8ff3c6..ed5ea67cd62 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -165,7 +165,7 @@ impl VTable for Masked { let validity = array.masked_validity(); // Fast path: all masked means result is all nulls. - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(ExecutionResult::done( ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index d3ee5eb5a65..7ae45c89405 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -17,7 +17,6 @@ use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; use crate::match_each_native_ptype; use crate::scalar::PValue; -use crate::validity::Validity; impl PrimitiveArray { /// Compute most common present value of this array @@ -26,7 +25,7 @@ impl PrimitiveArray { return Ok(None); } - if matches!(self.validity()?, Validity::AllInvalid) { + if self.validity()?.definitely_all_null() { return Ok(None); } diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 4d435471ce0..220fe948a73 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -255,7 +255,7 @@ impl VarBinData { } _ => None, }; - let all_invalid = matches!(validity, Validity::AllInvalid); + let all_invalid = validity.definitely_all_null(); match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| { let offsets_slice = primitive_offsets.as_slice::(); diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index eea3dd6ef7b..147658ddfb2 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -79,7 +79,7 @@ pub(super) fn precondition( } // If all values are null, replace the entire array with the fill value. - if matches!(array.validity()?, Validity::AllInvalid) { + if array.validity()?.definitely_all_null() { return Ok(Some( ConstantArray::new(fill_value.clone(), array.len()).into_array(), )); diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 978a1da1caf..e16991763ed 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -415,7 +415,7 @@ fn list_is_not_empty( ctx: &mut ExecutionCtx, ) -> VortexResult { // Short-circuit for all invalid. - if matches!(list_array.validity()?, Validity::AllInvalid) { + if list_array.validity()?.definitely_all_null() { return Ok(ConstantArray::new( Scalar::null(DType::Bool(Nullability::Nullable)), list_array.len(), diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index 8d36c07405e..c45c4cf3094 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -123,6 +123,19 @@ impl Validity { matches!(self, Self::NonNullable | Self::AllValid) } + /// Returns `true` if this validity is *definitely* all-null (every value is null), i.e. it + /// is [`Validity::AllInvalid`]. + /// + /// Returning `false` does not prove that any value is valid: a [`Validity::Array`] may still + /// resolve to all-null once executed. Callers must treat `false` as "unknown without + /// compute". For a definitive answer, execute the validity with [`Self::execute_mask`] and + /// check whether the resulting [`Mask`] is all-false (`Mask::all_false`). This is the + /// all-null counterpart to [`Self::definitely_no_nulls`]. + #[inline] + pub fn definitely_all_null(&self) -> bool { + matches!(self, Self::AllInvalid) + } + /// Returns whether this validity contains no null values, executing the validity array if /// necessary. /// diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 186f2297db1..557ec7468f5 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -32,7 +32,6 @@ use vortex::array::buffer::BufferHandle; use vortex::array::buffer::DeviceBufferExt; use vortex::array::match_each_unsigned_integer_ptype; use vortex::array::scalar::Scalar; -use vortex::array::validity::Validity; use vortex::buffer::Alignment; use vortex::buffer::ByteBuffer; use vortex::buffer::ByteBufferMut; @@ -434,7 +433,7 @@ impl MaterializedPlan { let output_ptype = self.dispatch_plan.output_ptype(); // All values are null — no need to touch the GPU. - if matches!(self.validity, Validity::AllInvalid) { + if self.validity.definitely_all_null() { let dtype = DType::Primitive(output_ptype, Nullability::Nullable); return ConstantArray::new(Scalar::null(dtype), len) .into_array() diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index d8c655c2558..7c4f4580c30 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -72,7 +72,7 @@ impl CudaExecute for DateTimePartsExecutor { return Ok(Canonical::empty(array.dtype())); } - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { let storage_ptype = ext.storage_dtype().as_ptype(); return Ok(Canonical::Extension( TemporalArray::new_timestamp( diff --git a/vortex-cuda/src/kernel/encodings/fsst.rs b/vortex-cuda/src/kernel/encodings/fsst.rs index 5d3d66eaf04..20a18b4e538 100644 --- a/vortex-cuda/src/kernel/encodings/fsst.rs +++ b/vortex-cuda/src/kernel/encodings/fsst.rs @@ -23,7 +23,6 @@ use vortex::array::arrays::varbinview::build_views::build_views; use vortex::array::buffer::DeviceBuffer; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_unsigned_integer_ptype; -use vortex::array::validity::Validity; use vortex::buffer::Alignment; use vortex::buffer::Buffer; use vortex::dtype::NativePType; @@ -62,7 +61,7 @@ impl CudaExecute for FSSTExecutor { let dtype = fsst.dtype().clone(); let validity = fsst.codes().validity()?; - if fsst.is_empty() || matches!(validity, Validity::AllInvalid) { + if fsst.is_empty() || validity.definitely_all_null() { let empty = unsafe { VarBinViewArray::new_unchecked( Buffer::::zeroed(fsst.len()), diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index fca435478d8..01fc57abb86 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -75,7 +75,7 @@ impl CudaExecute for RunEndExecutor { ))); } - if matches!(values.validity()?, Validity::AllInvalid) { + if values.validity()?.definitely_all_null() { return ConstantArray::new(Scalar::null(values.dtype().clone()), output_len) .into_array() .execute::(ctx.execution_ctx()); diff --git a/vortex-duckdb/src/exporter/bool.rs b/vortex-duckdb/src/exporter/bool.rs index 84fd17f0789..4d0a2b29883 100644 --- a/vortex-duckdb/src/exporter/bool.rs +++ b/vortex-duckdb/src/exporter/bool.rs @@ -4,7 +4,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::BoolArray; use vortex::array::arrays::bool::BoolArrayExt; -use vortex::array::validity::Validity; use vortex::buffer::BitBuffer; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -26,7 +25,7 @@ pub(crate) fn new_exporter( let bits = array.to_bit_buffer(); let validity = array.validity()?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index f6b8d9607e7..b0683190b3a 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -8,7 +8,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::decimal::DecimalDataParts; use vortex::array::match_each_decimal_value_type; -use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::BigCast; use vortex::dtype::DecimalDType; @@ -49,7 +48,7 @@ pub(crate) fn new_exporter( } = array.into_data_parts(); let dest_values_type = precision_to_duckdb_storage_size(&decimal_dtype)?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/fixed_size_list.rs b/vortex-duckdb/src/exporter/fixed_size_list.rs index 11b6e29a605..8fdb91df336 100644 --- a/vortex-duckdb/src/exporter/fixed_size_list.rs +++ b/vortex-duckdb/src/exporter/fixed_size_list.rs @@ -11,7 +11,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::FixedSizeListArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex::array::validity::Validity; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -43,7 +42,7 @@ pub(crate) fn new_exporter( let elements = parts.elements; let validity = parts.validity; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index a0eb65ffd64..8569150a607 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -11,7 +11,6 @@ use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::list::ListDataParts; use vortex::array::match_each_integer_ptype; -use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -55,7 +54,7 @@ pub(crate) fn new_exporter( } = array.into_data_parts(); let num_elements = elements.len(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(array_len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index a4cb61895f3..baf27949e5d 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -16,7 +16,6 @@ use vortex::array::arrays::listview::ListViewArrayExt; use vortex::array::arrays::listview::ListViewDataParts; use vortex::array::arrays::listview::ListViewRebuildMode; use vortex::array::match_each_integer_ptype; -use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -92,7 +91,7 @@ pub(crate) fn new_exporter( // Cache an `elements` vector up front so that future exports can reference it. let num_elements = elements.len(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs index a0b08c80e4b..4bda3fb3f0b 100644 --- a/vortex-duckdb/src/exporter/primitive.rs +++ b/vortex-duckdb/src/exporter/primitive.rs @@ -6,7 +6,6 @@ use std::marker::PhantomData; use vortex::array::ExecutionCtx; use vortex::array::arrays::PrimitiveArray; use vortex::array::match_each_native_ptype; -use vortex::array::validity::Validity; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -29,7 +28,7 @@ pub fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let validity = array.validity()?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); }; let validity = validity.to_array(array.len()).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 76c07d672a3..e9aca03a3bb 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -8,7 +8,6 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; -use vortex::array::validity::Validity; use vortex::error::VortexResult; use crate::duckdb::VectorRef; @@ -35,7 +34,7 @@ pub(crate) fn new_exporter( .. } = array.into_data_parts(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); }; let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index 557795f45f3..3e00384cd93 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -9,7 +9,6 @@ use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; use vortex::array::arrays::varbinview::VarBinViewDataParts; -use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; @@ -39,7 +38,7 @@ pub(crate) fn new_exporter( buffers, } = array.into_data_parts(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; From 67a2b2277a64e4e954dd3e1a59d089bae348c659 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Tue, 16 Jun 2026 15:21:24 +0100 Subject: [PATCH 028/162] fix: cuda san always run (#8449) Signed-off-by: Joe Isaacs --- .github/workflows/cuda.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cuda.yaml b/.github/workflows/cuda.yaml index 6fc19e9c7b1..a8b702188c3 100644 --- a/.github/workflows/cuda.yaml +++ b/.github/workflows/cuda.yaml @@ -42,7 +42,10 @@ jobs: - ".github/workflows/**" cuda-build-lint: - if: github.repository == 'vortex-data/vortex' + needs: [changes] + if: >- + always() && github.repository == 'vortex-data/vortex' && + needs.changes.outputs.run-cuda-san == 'true' name: "CUDA build & lint" timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-build @@ -72,7 +75,10 @@ jobs: -- -D warnings cuda-test: - if: github.repository == 'vortex-data/vortex' + needs: [changes] + if: >- + always() && github.repository == 'vortex-data/vortex' && + needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests" timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-tests @@ -153,7 +159,10 @@ jobs: run: cargo test --profile ci --locked -p vortex-cuda --all-features --target x86_64-unknown-linux-gnu cuda-test-cudf: - if: github.repository == 'vortex-data/vortex' + needs: [changes] + if: >- + always() && github.repository == 'vortex-data/vortex' && + needs.changes.outputs.run-cuda-san == 'true' name: "CUDA tests (cudf)" timeout-minutes: 30 runs-on: runs-on=${{ github.run_id }}/runner=gpu/tag=cuda-test-cudf From 7dbe5acf1f363298467b335b62b3fd0c481782ca Mon Sep 17 00:00:00 2001 From: Nicholas Gates Date: Tue, 16 Jun 2026 18:01:37 -0700 Subject: [PATCH 029/162] Arithmetic Operations (#8398) The only remaining calls to Arrow are for kleene logic, arithmetic, and LIKE expressions. If we make these native Vortex, we can pull out Arrow into a vortex-arrow engine/integration crate. --------- Signed-off-by: Nicholas Gates Signed-off-by: "Nicholas Gates" --- .../src/scalar_fn/fns/binary/numeric.rs | 1046 ++++++++++++++++- 1 file changed, 1008 insertions(+), 38 deletions(-) diff --git a/vortex-array/src/scalar_fn/fns/binary/numeric.rs b/vortex-array/src/scalar_fn/fns/binary/numeric.rs index 2ef58fb5209..cb85b727503 100644 --- a/vortex-array/src/scalar_fn/fns/binary/numeric.rs +++ b/vortex-array/src/scalar_fn/fns/binary/numeric.rs @@ -1,77 +1,924 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +use vortex_buffer::BitBuffer; +use vortex_buffer::Buffer; +use vortex_buffer::BufferMut; use vortex_error::VortexResult; +use vortex_error::vortex_bail; +use vortex_error::vortex_err; +use vortex_mask::AllOr; +use vortex_mask::Mask; use crate::ArrayRef; +use crate::ExecutionCtx; use crate::IntoArray; use crate::arrays::Constant; use crate::arrays::ConstantArray; -use crate::arrow::Datum; -use crate::arrow::from_arrow_columnar; -use crate::executor::ExecutionCtx; +use crate::arrays::PrimitiveArray; +use crate::builtins::ArrayBuiltins; +use crate::dtype::DType; +use crate::dtype::NativePType; +use crate::dtype::PType; +use crate::dtype::half::f16; +use crate::match_each_native_ptype; use crate::scalar::NumericOperator; +use crate::scalar::Scalar; +use crate::validity::Validity; + +struct CheckedAdd; + +struct CheckedSub; + +struct CheckedMul; + +struct CheckedDiv; + +trait CheckedPrimitiveOp: Sized { + const ERROR: &'static str; + const CHECKED_VALUE_LOOP: bool = false; + + // The vectorizable kernels need an overflowing-style result, not + // `Option`: every lane can write a value unconditionally while reducing + // the error flag separately. `checked` is still available for scalar and + // one-pass paths where early exit is the right shape. + fn apply(lhs: T, rhs: T) -> (T, bool); + + #[inline(always)] + fn checked(lhs: T, rhs: T) -> Option { + let (value, failed) = Self::apply(lhs, rhs); + if failed { None } else { Some(value) } + } +} + +impl CheckedPrimitiveOp for CheckedAdd { + const ERROR: &'static str = "integer overflow in checked add"; + + #[inline(always)] + fn apply(lhs: T, rhs: T) -> (T, bool) { + (lhs.add_value(rhs), lhs.add_error(rhs)) + } +} + +impl CheckedPrimitiveOp for CheckedSub { + const ERROR: &'static str = "integer overflow in checked sub"; + + #[inline(always)] + fn apply(lhs: T, rhs: T) -> (T, bool) { + (lhs.sub_value(rhs), lhs.sub_error(rhs)) + } +} + +impl CheckedPrimitiveOp for CheckedMul { + const ERROR: &'static str = "integer overflow in checked mul"; + + #[inline(always)] + fn apply(lhs: T, rhs: T) -> (T, bool) { + (lhs.mul_value(rhs), lhs.mul_error(rhs)) + } +} + +impl CheckedPrimitiveOp for CheckedDiv { + const ERROR: &'static str = "integer division by zero or overflow in checked div"; + // Integer division still lowers to scalar divides, so the split + // value/error-scan loop used to auto-vectorize add/sub/mul only adds a + // second full scan. Use the generic branchy checked value loop for integer + // division, matching Arrow/Velox. Float division has no checked errors and + // stays on the split/vectorizable default path. + const CHECKED_VALUE_LOOP: bool = T::DIV_CHECKS_IN_VALUE_LOOP; + + #[inline(always)] + fn apply(lhs: T, rhs: T) -> (T, bool) { + let failed = lhs.div_error(rhs); + let value = if failed { + T::default() + } else { + lhs.div_value(rhs) + }; + (value, failed) + } + + #[inline(always)] + fn checked(lhs: T, rhs: T) -> Option { + lhs.div_checked(rhs) + } +} /// Execute a numeric operation between two arrays. -/// -/// This is the entry point for numeric operations from the binary expression. -/// Handles constant-constant directly, otherwise falls back to Arrow. pub(crate) fn execute_numeric( lhs: &ArrayRef, rhs: &ArrayRef, op: NumericOperator, ctx: &mut ExecutionCtx, ) -> VortexResult { - if let Some(result) = constant_numeric(lhs, rhs, op)? { - return Ok(result); + let ptype = PType::try_from(lhs.dtype())?; + if !lhs.dtype().eq_ignore_nullability(rhs.dtype()) { + vortex_bail!( + "numeric operator requires matching primitive types, got {} and {}", + lhs.dtype(), + rhs.dtype() + ); } - arrow_numeric(lhs, rhs, op, ctx) + + match_each_native_ptype!(ptype, |T| { + match op { + NumericOperator::Add => execute_checked_typed::(lhs, rhs, ctx), + NumericOperator::Sub => execute_checked_typed::(lhs, rhs, ctx), + NumericOperator::Mul => execute_checked_typed::(lhs, rhs, ctx), + NumericOperator::Div => execute_checked_typed::(lhs, rhs, ctx), + } + }) } -/// Implementation of numeric operations using the Arrow crate. -pub(crate) fn arrow_numeric( +fn execute_checked_typed( lhs: &ArrayRef, rhs: &ArrayRef, - operator: NumericOperator, ctx: &mut ExecutionCtx, -) -> VortexResult { - let nullable = lhs.dtype().is_nullable() || rhs.dtype().is_nullable(); +) -> VortexResult +where + T: NativePType, + Op: CheckedPrimitiveOp, + Scalar: From, + Scalar: From>, +{ + let result_dtype = lhs + .dtype() + .with_nullability(lhs.dtype().nullability() | rhs.dtype().nullability()); + let lhs = PrimitiveOperand::::try_new(lhs, ctx)?; + let rhs = PrimitiveOperand::::try_new(rhs, ctx)?; let len = lhs.len(); + if len != rhs.len() { + vortex_bail!( + "numeric operator requires equal lengths, got {} and {}", + len, + rhs.len() + ); + } - let left = Datum::try_new(lhs, ctx)?; - let right = Datum::try_new_with_target_datatype(rhs, left.data_type(), ctx)?; + let validity = lhs.validity().and(rhs.validity())?; + let valid_rows = validity.execute_mask(len, ctx)?; - let array = match operator { - NumericOperator::Add => arrow_arith::numeric::add(&left, &right)?, - NumericOperator::Sub => arrow_arith::numeric::sub(&left, &right)?, - NumericOperator::Mul => arrow_arith::numeric::mul(&left, &right)?, - NumericOperator::Div => arrow_arith::numeric::div(&left, &right)?, + let checked = match (&lhs, &rhs) { + ( + PrimitiveOperand::Array { values: lhs, .. }, + PrimitiveOperand::Array { values: rhs, .. }, + ) => checked_array_array::(lhs, rhs, &valid_rows), + ( + PrimitiveOperand::Array { values: lhs, .. }, + PrimitiveOperand::Constant { value: rhs, .. }, + ) => checked_array_constant::(lhs, *rhs, &valid_rows), + ( + PrimitiveOperand::Constant { value: lhs, .. }, + PrimitiveOperand::Array { values: rhs, .. }, + ) => checked_constant_array::(*lhs, rhs, &valid_rows), + ( + PrimitiveOperand::Constant { value: lhs, .. }, + PrimitiveOperand::Constant { value: rhs, .. }, + ) => { + let value = Op::checked(*lhs, *rhs) + .ok_or_else(|| vortex_err!(InvalidArgument: "{}", Op::ERROR))?; + return Ok(constant_result_array(value, len, &result_dtype)); + } + (PrimitiveOperand::Null(_), _) | (_, PrimitiveOperand::Null(_)) => { + CheckedValues::zeroed(len) + } }; + check_numeric_errors(checked.failed, Op::ERROR)?; - from_arrow_columnar(array.as_ref(), len, nullable, ctx) + primitive_result_array::(checked.values, validity, &result_dtype) } -fn constant_numeric( - lhs: &ArrayRef, - rhs: &ArrayRef, - op: NumericOperator, -) -> VortexResult> { - let (Some(lhs), Some(rhs)) = (lhs.as_opt::(), rhs.as_opt::()) else { - return Ok(None); +fn primitive_result_array( + values: Buffer, + validity: Validity, + dtype: &DType, +) -> VortexResult { + let array = PrimitiveArray::new(values, validity).into_array(); + if array.dtype() == dtype { + return Ok(array); + } + array.cast(dtype.clone()) +} + +fn constant_result_array(value: T, len: usize, dtype: &DType) -> ArrayRef +where + T: NativePType, + Scalar: From + From>, +{ + if dtype.is_nullable() { + ConstantArray::new(Some(value), len).into_array() + } else { + ConstantArray::new(value, len).into_array() + } +} + +enum PrimitiveOperand { + Array { + values: Buffer, + validity: Validity, + }, + Constant { + value: T, + len: usize, + validity: Validity, + }, + Null(usize), +} + +impl PrimitiveOperand { + fn try_new(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { + if let Some(constant) = array.as_opt::() { + return Ok( + match constant.scalar().as_primitive().try_typed_value::()? { + Some(value) => Self::Constant { + value, + len: array.len(), + validity: if constant.scalar().dtype().is_nullable() { + Validity::AllValid + } else { + Validity::NonNullable + }, + }, + None => Self::Null(array.len()), + }, + ); + } + + let array = array.clone().execute::(ctx)?; + let validity = array.validity()?; + let values = array.into_buffer::(); + Ok(Self::Array { values, validity }) + } + + fn len(&self) -> usize { + match self { + Self::Array { values, .. } => values.len(), + Self::Constant { len, .. } | Self::Null(len) => *len, + } + } + + fn validity(&self) -> Validity { + match self { + Self::Array { validity, .. } => validity.clone(), + Self::Constant { validity, .. } => validity.clone(), + Self::Null(_) => Validity::AllInvalid, + } + } +} + +struct CheckedValues { + values: Buffer, + failed: bool, +} + +impl CheckedValues { + fn zeroed(len: usize) -> Self { + Self { + values: Buffer::::zeroed(len), + failed: false, + } + } + + fn failed(len: usize) -> Self { + Self { + values: Buffer::::zeroed(len), + failed: true, + } + } +} + +fn checked_array_array(lhs: &[T], rhs: &[T], valid_rows: &Mask) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + debug_assert_eq!(lhs.len(), rhs.len()); + + match valid_rows.bit_buffer() { + AllOr::All if Op::CHECKED_VALUE_LOOP => checked_array_array_one_pass::(lhs, rhs), + AllOr::All => checked_array_array_all_lanes::(lhs, rhs), + AllOr::None => CheckedValues::zeroed(lhs.len()), + AllOr::Some(valid_bits) if Op::CHECKED_VALUE_LOOP => { + checked_array_array_valid_lanes_one_pass::(lhs, rhs, valid_bits) + } + AllOr::Some(valid_bits) => checked_array_array_valid_lanes::(lhs, rhs, valid_bits), + } +} + +fn checked_array_constant(lhs: &[T], rhs: T, valid_rows: &Mask) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + match valid_rows.bit_buffer() { + AllOr::All if Op::CHECKED_VALUE_LOOP => checked_array_constant_one_pass::(lhs, rhs), + AllOr::All => checked_array_constant_all_lanes::(lhs, rhs), + AllOr::None => CheckedValues::zeroed(lhs.len()), + AllOr::Some(valid_bits) if Op::CHECKED_VALUE_LOOP => { + checked_array_constant_valid_lanes_one_pass::(lhs, rhs, valid_bits) + } + AllOr::Some(valid_bits) => { + checked_array_constant_valid_lanes::(lhs, rhs, valid_bits) + } + } +} + +fn checked_constant_array(lhs: T, rhs: &[T], valid_rows: &Mask) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + match valid_rows.bit_buffer() { + AllOr::All if Op::CHECKED_VALUE_LOOP => checked_constant_array_one_pass::(lhs, rhs), + AllOr::All => checked_constant_array_all_lanes::(lhs, rhs), + AllOr::None => CheckedValues::zeroed(rhs.len()), + AllOr::Some(valid_bits) if Op::CHECKED_VALUE_LOOP => { + checked_constant_array_valid_lanes_one_pass::(lhs, rhs, valid_bits) + } + AllOr::Some(valid_bits) => { + checked_constant_array_valid_lanes::(lhs, rhs, valid_bits) + } + } +} + +fn checked_array_array_all_lanes(lhs: &[T], rhs: &[T]) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + collect_all_lanes(lhs.len(), |idx| Op::apply(lhs[idx], rhs[idx])) +} + +fn checked_array_array_valid_lanes( + lhs: &[T], + rhs: &[T], + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + let mut checked = collect_all_lanes(lhs.len(), |idx| Op::apply(lhs[idx], rhs[idx])); + + checked.failed = checked.failed + && any_valid_error(lhs.len(), valid_bits, |idx| Op::apply(lhs[idx], rhs[idx]).1); + checked +} + +fn checked_array_constant_all_lanes(lhs: &[T], rhs: T) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + collect_all_lanes(lhs.len(), |idx| Op::apply(lhs[idx], rhs)) +} + +fn checked_array_constant_valid_lanes( + lhs: &[T], + rhs: T, + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + let mut checked = collect_all_lanes(lhs.len(), |idx| Op::apply(lhs[idx], rhs)); + + checked.failed = + checked.failed && any_valid_error(lhs.len(), valid_bits, |idx| Op::apply(lhs[idx], rhs).1); + checked +} + +fn checked_constant_array_all_lanes(lhs: T, rhs: &[T]) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + collect_all_lanes(rhs.len(), |idx| Op::apply(lhs, rhs[idx])) +} + +fn checked_constant_array_valid_lanes( + lhs: T, + rhs: &[T], + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + let mut checked = collect_all_lanes(rhs.len(), |idx| Op::apply(lhs, rhs[idx])); + + checked.failed = + checked.failed && any_valid_error(rhs.len(), valid_bits, |idx| Op::apply(lhs, rhs[idx]).1); + checked +} + +fn collect_all_lanes(len: usize, mut value_and_error_at: F) -> CheckedValues +where + T: NativePType, + F: FnMut(usize) -> (T, bool), +{ + let mut values = BufferMut::::with_capacity(len); + let slots = values.spare_capacity_mut().as_mut_ptr(); + let mut failed = false; + for idx in 0..len { + let (value, is_error) = value_and_error_at(idx); + failed |= is_error; + // SAFETY: `idx < len <= capacity`, and each slot is written once. + unsafe { slots.add(idx).write(std::mem::MaybeUninit::new(value)) }; + } + // SAFETY: every slot in `0..len` was initialized above. + unsafe { values.set_len(len) }; + CheckedValues { + values: values.freeze(), + failed, + } +} + +fn checked_array_array_one_pass(lhs: &[T], rhs: &[T]) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_all_lanes(lhs.len(), |idx| Op::checked(lhs[idx], rhs[idx])) +} + +fn checked_array_array_valid_lanes_one_pass( + lhs: &[T], + rhs: &[T], + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_valid_lanes(lhs.len(), valid_bits, |idx| Op::checked(lhs[idx], rhs[idx])) +} + +fn checked_array_constant_one_pass(lhs: &[T], rhs: T) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_all_lanes(lhs.len(), |idx| Op::checked(lhs[idx], rhs)) +} + +fn checked_array_constant_valid_lanes_one_pass( + lhs: &[T], + rhs: T, + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_valid_lanes(lhs.len(), valid_bits, |idx| Op::checked(lhs[idx], rhs)) +} + +fn checked_constant_array_one_pass(lhs: T, rhs: &[T]) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_all_lanes(rhs.len(), |idx| Op::checked(lhs, rhs[idx])) +} + +fn checked_constant_array_valid_lanes_one_pass( + lhs: T, + rhs: &[T], + valid_bits: &BitBuffer, +) -> CheckedValues +where + T: NativePType, + Op: CheckedPrimitiveOp, +{ + checked_valid_lanes(rhs.len(), valid_bits, |idx| Op::checked(lhs, rhs[idx])) +} + +// Checked one-pass ops delay early exit until the end of a small block. This +// keeps the loop generic while avoiding a branch-driven exit decision on every +// lane; it is deliberately independent of mask density or input length. +const CHECKED_BLOCK_LANES: usize = 16; + +fn checked_all_lanes(len: usize, mut checked_at: F) -> CheckedValues +where + T: NativePType, + F: FnMut(usize) -> Option, +{ + let mut values = BufferMut::::with_capacity(len); + let mut base = 0; + + while base + CHECKED_BLOCK_LANES <= len { + let mut block_failed = false; + for idx in base..base + CHECKED_BLOCK_LANES { + match checked_at(idx) { + Some(value) => { + // SAFETY: the buffer is allocated with capacity `len`, and + // this loop pushes at most one value for each `idx`. + unsafe { values.push_unchecked(value) }; + } + None => { + block_failed = true; + // SAFETY: the buffer is allocated with capacity `len`, and + // this loop pushes at most one value for each `idx`. + unsafe { values.push_unchecked(T::default()) }; + } + } + } + + if block_failed { + return CheckedValues::failed(len); + } + base += CHECKED_BLOCK_LANES; + } + + for idx in base..len { + let Some(value) = checked_at(idx) else { + return CheckedValues::failed(len); + }; + // SAFETY: the buffer is allocated with capacity `len`, and this loop + // pushes at most one value for each `idx`. + unsafe { values.push_unchecked(value) }; + } + + CheckedValues { + values: values.freeze(), + failed: false, + } +} + +fn checked_valid_lanes( + len: usize, + valid_bits: &BitBuffer, + mut checked_at: F, +) -> CheckedValues +where + T: NativePType, + F: FnMut(usize) -> Option, +{ + let mut values = BufferMut::::zeroed(len); + let mut failed = false; + { + let values = values.as_mut_slice(); + for_each_valid_idx(len, valid_bits, |idx| { + let Some(value) = checked_at(idx) else { + failed = true; + return false; + }; + values[idx] = value; + true + }); + } + + CheckedValues { + values: values.freeze(), + failed, + } +} + +fn any_valid_error(len: usize, valid_bits: &BitBuffer, is_error: F) -> bool +where + F: Fn(usize) -> bool, +{ + !for_each_valid_idx(len, valid_bits, |idx| !is_error(idx)) +} + +fn for_each_valid_idx(len: usize, valid_bits: &BitBuffer, mut f: F) -> bool +where + F: FnMut(usize) -> bool, +{ + debug_assert_eq!(len, valid_bits.len()); + + for (word_idx, valid_word) in valid_bits.chunks().iter_padded().enumerate() { + if valid_word == 0 { + continue; + } + + let offset = word_idx * 64; + let lanes = len.saturating_sub(offset).min(64); + if lanes == 64 && valid_word == u64::MAX { + for bit_idx in 0..64 { + if !f(offset + bit_idx) { + return false; + } + } + continue; + } + + let mut valid_word = if lanes == 64 { + valid_word + } else { + valid_word & ((1u64 << lanes) - 1) + }; + while valid_word != 0 { + let bit_idx = valid_word.trailing_zeros() as usize; + if !f(offset + bit_idx) { + return false; + } + valid_word &= valid_word - 1; + } + } + + true +} + +trait CheckedArithmetic: NativePType { + const DIV_CHECKS_IN_VALUE_LOOP: bool; + + fn add_value(self, rhs: Self) -> Self; + fn add_error(self, rhs: Self) -> bool; + fn sub_value(self, rhs: Self) -> Self; + fn sub_error(self, rhs: Self) -> bool; + fn mul_value(self, rhs: Self) -> Self; + fn mul_error(self, rhs: Self) -> bool; + fn div_value(self, rhs: Self) -> Self; + fn div_error(self, rhs: Self) -> bool; + fn div_checked(self, rhs: Self) -> Option; +} + +macro_rules! impl_checked_unsigned { + ($ty:ty,widening_mul: $wide:ty) => { + impl CheckedArithmetic for $ty { + const DIV_CHECKS_IN_VALUE_LOOP: bool = true; + + #[inline(always)] + fn add_value(self, rhs: Self) -> Self { + self.wrapping_add(rhs) + } + + #[inline(always)] + fn add_error(self, rhs: Self) -> bool { + self > <$ty>::MAX - rhs + } + + #[inline(always)] + fn sub_value(self, rhs: Self) -> Self { + self.wrapping_sub(rhs) + } + + #[inline(always)] + fn sub_error(self, rhs: Self) -> bool { + self < rhs + } + + #[inline(always)] + fn mul_value(self, rhs: Self) -> Self { + self.wrapping_mul(rhs) + } + + #[inline(always)] + fn mul_error(self, rhs: Self) -> bool { + (self as $wide) * (rhs as $wide) > <$ty>::MAX as $wide + } + + #[inline(always)] + fn div_value(self, rhs: Self) -> Self { + self / rhs + } + + #[inline(always)] + fn div_error(self, rhs: Self) -> bool { + rhs == 0 + } + + #[inline(always)] + fn div_checked(self, rhs: Self) -> Option { + self.checked_div(rhs) + } + } + }; + ($ty:ty,overflowing_mul) => { + impl CheckedArithmetic for $ty { + const DIV_CHECKS_IN_VALUE_LOOP: bool = true; + + #[inline(always)] + fn add_value(self, rhs: Self) -> Self { + self.wrapping_add(rhs) + } + + #[inline(always)] + fn add_error(self, rhs: Self) -> bool { + self > <$ty>::MAX - rhs + } + + #[inline(always)] + fn sub_value(self, rhs: Self) -> Self { + self.wrapping_sub(rhs) + } + + #[inline(always)] + fn sub_error(self, rhs: Self) -> bool { + self < rhs + } + + #[inline(always)] + fn mul_value(self, rhs: Self) -> Self { + self.wrapping_mul(rhs) + } + + #[inline(always)] + fn mul_error(self, rhs: Self) -> bool { + self.overflowing_mul(rhs).1 + } + + #[inline(always)] + fn div_value(self, rhs: Self) -> Self { + self / rhs + } + + #[inline(always)] + fn div_error(self, rhs: Self) -> bool { + rhs == 0 + } + + #[inline(always)] + fn div_checked(self, rhs: Self) -> Option { + self.checked_div(rhs) + } + } + }; +} + +macro_rules! impl_checked_signed { + ($ty:ty,widening_mul: $wide:ty) => { + impl CheckedArithmetic for $ty { + const DIV_CHECKS_IN_VALUE_LOOP: bool = true; + + #[inline(always)] + fn add_value(self, rhs: Self) -> Self { + self.wrapping_add(rhs) + } + + #[inline(always)] + fn add_error(self, rhs: Self) -> bool { + let value = self.wrapping_add(rhs); + ((self ^ value) & (rhs ^ value)) < 0 + } + + #[inline(always)] + fn sub_value(self, rhs: Self) -> Self { + self.wrapping_sub(rhs) + } + + #[inline(always)] + fn sub_error(self, rhs: Self) -> bool { + let value = self.wrapping_sub(rhs); + ((self ^ rhs) & (self ^ value)) < 0 + } + + #[inline(always)] + fn mul_value(self, rhs: Self) -> Self { + self.wrapping_mul(rhs) + } + + #[inline(always)] + fn mul_error(self, rhs: Self) -> bool { + let product = (self as $wide) * (rhs as $wide); + product < <$ty>::MIN as $wide || product > <$ty>::MAX as $wide + } + + #[inline(always)] + fn div_value(self, rhs: Self) -> Self { + self / rhs + } + + #[inline(always)] + fn div_error(self, rhs: Self) -> bool { + rhs == 0 || (self == <$ty>::MIN && rhs == -1) + } + + #[inline(always)] + fn div_checked(self, rhs: Self) -> Option { + self.checked_div(rhs) + } + } }; + ($ty:ty,overflowing_mul) => { + impl CheckedArithmetic for $ty { + const DIV_CHECKS_IN_VALUE_LOOP: bool = true; + + #[inline(always)] + fn add_value(self, rhs: Self) -> Self { + self.wrapping_add(rhs) + } + + #[inline(always)] + fn add_error(self, rhs: Self) -> bool { + let value = self.wrapping_add(rhs); + ((self ^ value) & (rhs ^ value)) < 0 + } + + #[inline(always)] + fn sub_value(self, rhs: Self) -> Self { + self.wrapping_sub(rhs) + } + + #[inline(always)] + fn sub_error(self, rhs: Self) -> bool { + let value = self.wrapping_sub(rhs); + ((self ^ rhs) & (self ^ value)) < 0 + } + + #[inline(always)] + fn mul_value(self, rhs: Self) -> Self { + self.wrapping_mul(rhs) + } + + #[inline(always)] + fn mul_error(self, rhs: Self) -> bool { + self.overflowing_mul(rhs).1 + } + + #[inline(always)] + fn div_value(self, rhs: Self) -> Self { + self / rhs + } - let Some(result) = lhs - .scalar() - .as_primitive() - .checked_binary_numeric(&rhs.scalar().as_primitive(), op) - else { - // Overflow detected — fall through to arrow_numeric which uses wrapping arithmetic. - return Ok(None); + #[inline(always)] + fn div_error(self, rhs: Self) -> bool { + rhs == 0 || (self == <$ty>::MIN && rhs == -1) + } + + #[inline(always)] + fn div_checked(self, rhs: Self) -> Option { + self.checked_div(rhs) + } + } }; +} + +macro_rules! impl_checked_float { + ($($ty:ty),+ $(,)?) => { + $( + impl CheckedArithmetic for $ty { + const DIV_CHECKS_IN_VALUE_LOOP: bool = false; + + #[inline(always)] + fn add_value(self, rhs: Self) -> Self { + self + rhs + } + + #[inline(always)] + fn add_error(self, _rhs: Self) -> bool { + false + } + + #[inline(always)] + fn sub_value(self, rhs: Self) -> Self { + self - rhs + } + + #[inline(always)] + fn sub_error(self, _rhs: Self) -> bool { + false + } + + #[inline(always)] + fn mul_value(self, rhs: Self) -> Self { + self * rhs + } - Ok(Some(ConstantArray::new(result, lhs.len()).into_array())) + #[inline(always)] + fn mul_error(self, _rhs: Self) -> bool { + false + } + + #[inline(always)] + fn div_value(self, rhs: Self) -> Self { + self / rhs + } + + #[inline(always)] + fn div_error(self, _rhs: Self) -> bool { + false + } + + #[inline(always)] + fn div_checked(self, rhs: Self) -> Option { + Some(self / rhs) + } + } + )+ + }; } +impl_checked_unsigned!(u8, widening_mul: u16); +impl_checked_unsigned!(u16, widening_mul: u32); +impl_checked_unsigned!(u32, widening_mul: u64); +impl_checked_unsigned!(u64, overflowing_mul); +impl_checked_signed!(i8, widening_mul: i16); +impl_checked_signed!(i16, widening_mul: i32); +impl_checked_signed!(i32, widening_mul: i64); +impl_checked_signed!(i64, overflowing_mul); +impl_checked_float!(f16, f32, f64); + +fn check_numeric_errors(failed: bool, error: &'static str) -> VortexResult<()> { + if failed { + return Err(vortex_err!(InvalidArgument: "{}", error)); + } + + Ok(()) +} #[cfg(test)] mod test { use vortex_buffer::buffer; @@ -82,12 +929,13 @@ mod test { use crate::LEGACY_SESSION; use crate::RecursiveCanonical; use crate::VortexSessionExecute; + use crate::arrays::ConstantArray; use crate::arrays::PrimitiveArray; use crate::assert_arrays_eq; use crate::builtins::ArrayBuiltins; use crate::scalar::Scalar; - use crate::scalar_fn::fns::binary::numeric::ConstantArray; use crate::scalar_fn::fns::operators::Operator; + use crate::validity::Validity; fn sub_scalar(array: &ArrayRef, scalar: impl Into) -> VortexResult { array @@ -138,4 +986,126 @@ mod test { let _results = sub_scalar(&values, 1.0f32).unwrap(); let _results = sub_scalar(&values, f32::MAX).unwrap(); } + + #[test] + fn test_float_divide_by_zero_is_ok() { + let values = buffer![1.0f64, -1.0].into_array(); + let result = values + .binary( + ConstantArray::new(0.0f64, values.len()).into_array(), + Operator::Div, + ) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())) + .unwrap(); + + assert_arrays_eq!( + result, + PrimitiveArray::from_iter([f64::INFINITY, f64::NEG_INFINITY]) + ); + } + + #[test] + fn test_integer_overflow_errors() { + let values = buffer![u8::MAX].into_array(); + let result = values + .binary( + ConstantArray::new(1u8, values.len()).into_array(), + Operator::Add, + ) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())); + + assert!(result.is_err()); + } + + #[test] + fn test_integer_divide_by_zero_errors() { + let values = buffer![1i32].into_array(); + let result = values + .binary( + ConstantArray::new(0i32, values.len()).into_array(), + Operator::Div, + ) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())); + + assert!(result.is_err()); + } + + #[test] + fn test_integer_divide_overflow_errors() { + let values = buffer![i64::MIN].into_array(); + let result = values + .binary( + ConstantArray::new(-1i64, values.len()).into_array(), + Operator::Div, + ) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())); + + assert!(result.is_err()); + } + + #[test] + fn test_integer_divide_errors_ignore_null_lanes() { + let lhs = PrimitiveArray::new(buffer![10i32, 10], Validity::from_iter([false, true])) + .into_array(); + let rhs = buffer![0i32, 2].into_array(); + let result = lhs + .binary(rhs, Operator::Div) + .and_then(|a| { + a.execute::(&mut LEGACY_SESSION.create_execution_ctx()) + }) + .map(|a| a.0.into_array()) + .unwrap(); + + assert_arrays_eq!(result, PrimitiveArray::from_option_iter([None, Some(5i32)])); + } + + #[test] + fn test_integer_errors_ignore_null_lanes() { + let values = PrimitiveArray::new(buffer![u8::MAX, 1], Validity::from_iter([false, true])) + .into_array(); + let result = values + .binary( + ConstantArray::new(1u8, values.len()).into_array(), + Operator::Add, + ) + .and_then(|a| { + a.execute::(&mut LEGACY_SESSION.create_execution_ctx()) + }) + .map(|a| a.0.into_array()) + .unwrap(); + + assert_arrays_eq!(result, PrimitiveArray::from_option_iter([None, Some(2u8)])); + } + + #[test] + fn test_integer_array_array_errors_on_valid_lanes() { + let lhs = PrimitiveArray::new( + buffer![u8::MAX, 1, u8::MAX], + Validity::from_iter([false, true, true]), + ) + .into_array(); + let rhs = buffer![1u8, 1, 1].into_array(); + let result = lhs + .binary(rhs, Operator::Add) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())); + + assert!(result.is_err()); + } + + #[test] + fn test_present_nullable_constant_preserves_nullable_output() { + let values = buffer![1u8, 2].into_array(); + let result = values + .binary( + ConstantArray::new(Some(1u8), values.len()).into_array(), + Operator::Add, + ) + .and_then(|a| a.execute::(&mut LEGACY_SESSION.create_execution_ctx())) + .unwrap(); + + assert_arrays_eq!( + result, + PrimitiveArray::from_option_iter([Some(2u8), Some(3)]) + ); + } } From d257d09f5b3417d9a4063df67a48221bb16b7b14 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 17 Jun 2026 08:55:17 +0100 Subject: [PATCH 030/162] --patterns flag for random-access-bench (#8446) Flag to benchmark correlated or uniform vectors separately. Useful for building flamegraphs with samply Signed-off-by: Mikhail Kot --- benchmarks/random-access-bench/src/main.rs | 71 ++++++++++++++-------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index d64482c01a3..6722265fde6 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -43,7 +43,7 @@ mod render; // --------------------------------------------------------------------------- /// Access pattern for random access benchmarks. -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, ValueEnum)] pub enum AccessPattern { /// Multiple clusters of sequential indices scattered across the dataset, /// simulating workloads with spatial locality (e.g. scanning nearby records). @@ -62,8 +62,6 @@ impl AccessPattern { } } -const ACCESS_PATTERNS: [AccessPattern; 2] = [AccessPattern::Correlated, AccessPattern::Uniform]; - /// Number of clusters for the correlated pattern. const NUM_CLUSTERS: usize = 5; @@ -190,6 +188,14 @@ struct Args { default_values_t = vec![DatasetArg::Taxi, DatasetArg::FeatureVectors, DatasetArg::NestedLists, DatasetArg::NestedStructs] )] datasets: Vec, + /// Which access patterns to benchmark. + #[arg( + long, + value_delimiter = ',', + value_enum, + default_values_t = vec![AccessPattern::Correlated, AccessPattern::Uniform] + )] + patterns: Vec, /// Whether to reopen the file on each iteration, use a cached handle, or run both. #[arg(long, value_enum, default_value_t = OpenMode::Both)] open_mode: OpenMode, @@ -201,22 +207,22 @@ async fn main() -> Result<()> { setup_logging_and_tracing(args.verbose, args.tracing)?; - let datasets: Vec> = args - .datasets - .into_iter() - .map(|d| d.into_dataset()) - .collect(); - - run_random_access( - &datasets, - args.formats, - args.time_limit, - args.open_mode, - args.display_format, - args.output_path, - args.gh_json_v3, - ) - .await + let config = RunConfig { + datasets: args + .datasets + .into_iter() + .map(|d| d.into_dataset()) + .collect(), + formats: args.formats, + patterns: args.patterns, + time_limit: args.time_limit, + open_mode: args.open_mode, + display_format: args.display_format, + output_path: args.output_path, + gh_json_v3: args.gh_json_v3, + }; + + run_random_access(config).await } // --------------------------------------------------------------------------- @@ -379,15 +385,30 @@ const BENCHMARK_ID: &str = "random-access"; /// Fixed indices used by the original taxi benchmark (preserved for historical continuity). const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000]; -async fn run_random_access( - datasets: &[Box], +/// Resolved configuration for a single random-access benchmark invocation. +struct RunConfig { + datasets: Vec>, formats: Vec, + patterns: Vec, time_limit: u64, open_mode: OpenMode, display_format: DisplayFormat, output_path: Option, gh_json_v3: Option, -) -> Result<()> { +} + +async fn run_random_access(config: RunConfig) -> Result<()> { + let RunConfig { + datasets, + formats, + patterns, + time_limit, + open_mode, + display_format, + output_path, + gh_json_v3, + } = config; + let reopen_variants: &[bool] = match open_mode { OpenMode::Cached => &[false], OpenMode::Reopen => &[true], @@ -398,7 +419,7 @@ async fn run_random_access( .iter() .map(|d| { let legacy_extra = if d.name() == "taxi" { formats.len() } else { 0 }; - (formats.len() * ACCESS_PATTERNS.len() + legacy_extra) * reopen_variants.len() + (formats.len() * patterns.len() + legacy_extra) * reopen_variants.len() }) .sum(); let progress = ProgressBar::new(total_steps as u64); @@ -408,7 +429,7 @@ async fn run_random_access( // Iteration order matters for the table renderer: row order is set by the // first time each `(dataset, pattern)` pair is observed. - for dataset in datasets { + for dataset in &datasets { for format in &formats { if dataset.name() == "taxi" { let name = measurement_name(dataset.name(), None, *format); @@ -436,7 +457,7 @@ async fn run_random_access( } } - for pattern in &ACCESS_PATTERNS { + for pattern in &patterns { let indices = generate_indices(dataset.as_ref(), *pattern); let name = measurement_name(dataset.name(), Some(*pattern), *format); for &reopen in reopen_variants { From 4652f29966efc8f85406f1f36819d1ec599b3685 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 17 Jun 2026 09:50:51 +0100 Subject: [PATCH 031/162] fmt: normalize_comments = true (#8428) Add `normalize_comments = true` https://github.com/rust-lang/rustfmt/issues/3350 --------- Signed-off-by: Joe Isaacs --- encodings/alp/src/alp/compute/compare.rs | 2 +- .../src/bitpacking/array/unpack_iter.rs | 1 - encodings/fsst/src/kernel.rs | 1 - encodings/zstd/src/array.rs | 1 - rustfmt.toml | 7 ++--- .../src/aggregate_fn/fns/is_constant/mod.rs | 1 - vortex-array/src/aggregate_fn/fns/sum/mod.rs | 1 - vortex-array/src/aliases/mod.rs | 1 - vortex-array/src/arrays/chunked/array.rs | 6 ++--- .../arrays/fixed_size_list/tests/filter.rs | 2 +- vortex-array/src/arrays/patched/mod.rs | 2 +- .../src/arrays/primitive/compute/slice.rs | 2 +- .../src/arrays/varbin/compute/compare.rs | 2 +- vortex-array/src/dtype/field.rs | 1 - vortex-array/src/expr/analysis/labeling.rs | 1 - vortex-cxx/src/lib.rs | 1 - vortex-datafusion/src/v2/table.rs | 1 - vortex-duckdb/src/convert/expr.rs | 26 +++++++++---------- vortex-duckdb/src/exporter/mod.rs | 2 +- vortex-duckdb/src/table_function.rs | 20 +++++++------- vortex-ffi/cinclude/vortex.h | 9 ------- vortex-ffi/examples/hello_vortex.rs | 2 +- vortex-ffi/src/array.rs | 4 --- vortex-ffi/src/expression.rs | 4 --- vortex-ffi/src/macros.rs | 1 - vortex-ffi/src/struct_array.rs | 1 - vortex-file/src/lib.rs | 1 - vortex-io/src/object_store/filesystem.rs | 1 - vortex-io/src/runtime/handle.rs | 1 - vortex-python/src/arrays/mod.rs | 13 +++++----- vortex-python/src/arrays/py/python.rs | 1 - vortex-python/src/expr/mod.rs | 1 - vortex-python/src/io.rs | 1 - 33 files changed, 41 insertions(+), 80 deletions(-) diff --git a/encodings/alp/src/alp/compute/compare.rs b/encodings/alp/src/alp/compute/compare.rs index 239e0558692..6cd918f8be3 100644 --- a/encodings/alp/src/alp/compute/compare.rs +++ b/encodings/alp/src/alp/compute/compare.rs @@ -269,7 +269,7 @@ mod tests { let expected = BoolArray::from_iter([true; 10]); assert_arrays_eq!(r_lte, expected); - //0.0605_f32 < 0.06051_f32; + // 0.0605_f32 < 0.06051_f32; let r_lt = alp_scalar_compare(encoded.as_view(), 0.06051_f32, CompareOperator::Lt) .unwrap() .unwrap(); diff --git a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs index c9158d8bdac..23f0f96aed4 100644 --- a/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs +++ b/encodings/fastlanes/src/bitpacking/array/unpack_iter.rs @@ -79,7 +79,6 @@ impl> UnpackStrategy for BitPackingStr /// // handle partial trailing chunk /// } /// ``` -/// pub struct UnpackedChunks> { strategy: S, bit_width: usize, diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs index 386da3b0f60..a54fe41d8cc 100644 --- a/encodings/fsst/src/kernel.rs +++ b/encodings/fsst/src/kernel.rs @@ -115,7 +115,6 @@ mod tests { #[test] fn issues_6034_test_fsst_filter_with_nulls_and_special_chars() -> VortexResult<()> { - // // Test case with special characters and nulls // Values: ["", "", "", "", "", "", "", "", "", "", "", ",", "A<<<<<<<", "", "", "", "", null, null, null, null, null, null] // Mask: only the last element is selected (true at index 22) diff --git a/encodings/zstd/src/array.rs b/encodings/zstd/src/array.rs index 0e3bccc4667..1de2b89a63f 100644 --- a/encodings/zstd/src/array.rs +++ b/encodings/zstd/src/array.rs @@ -935,7 +935,6 @@ impl ZstdData { } else if dtype.is_nullable() && matches!(slice_validity, Validity::NonNullable) { slice_validity = Validity::AllValid; } - // // END OF IMPORTANT BLOCK // diff --git a/rustfmt.toml b/rustfmt.toml index 83fe627dbf6..e6ffec93d96 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,8 +1,9 @@ condense_wildcard_suffixes = true -format_macro_matchers = true format_macro_bodies = true +format_macro_matchers = true group_imports = "StdExternalCrate" -unstable_features = true -use_field_init_shorthand = true imports_granularity = "Item" +normalize_comments = true style_edition = "2024" +unstable_features = true +use_field_init_shorthand = true diff --git a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs index d1dd6bef39c..862437ebcdc 100644 --- a/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/is_constant/mod.rs @@ -50,7 +50,6 @@ use crate::scalar_fn::fns::operators::Operator; /// Check if two arrays of the same length have equal values at every position (null-safe). /// /// Two positions are considered equal if they are both null, or both non-null with the same value. -/// // TODO(ngates): move this function out when we have any/all aggregate functions. fn arrays_value_equal(a: &ArrayRef, b: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult { debug_assert_eq!(a.len(), b.len()); diff --git a/vortex-array/src/aggregate_fn/fns/sum/mod.rs b/vortex-array/src/aggregate_fn/fns/sum/mod.rs index 9d525bec742..f48397798d9 100644 --- a/vortex-array/src/aggregate_fn/fns/sum/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/sum/mod.rs @@ -272,7 +272,6 @@ pub struct SumPartial { } /// The accumulated sum value. -/// // TODO(ngates): instead of an enum, we should use a Box to avoid dispatcher over the // input type every time? Perhaps? pub enum SumState { diff --git a/vortex-array/src/aliases/mod.rs b/vortex-array/src/aliases/mod.rs index d001e0402e2..6d0ab2524c5 100644 --- a/vortex-array/src/aliases/mod.rs +++ b/vortex-array/src/aliases/mod.rs @@ -2,7 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors //! Re-exports of third-party crates we use in macros exported from vortex-array. -//! pub mod paste { //! Re-export of [`paste`](https://docs.rs/paste/latest/paste/). diff --git a/vortex-array/src/arrays/chunked/array.rs b/vortex-array/src/arrays/chunked/array.rs index a0ed3f98d67..9faa1486e64 100644 --- a/vortex-array/src/arrays/chunked/array.rs +++ b/vortex-array/src/arrays/chunked/array.rs @@ -384,9 +384,9 @@ mod test { // Create chunks where some are empty but all non-empty chunks have all invalid values let chunks = vec![ PrimitiveArray::new(buffer![1u64, 2], Validity::AllInvalid).into_array(), - PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk + PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */ PrimitiveArray::new(buffer![3u64, 4, 5], Validity::AllInvalid).into_array(), - PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk + PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */ ]; let chunked = @@ -410,7 +410,7 @@ mod test { PrimitiveArray::new(buffer![1u64, 2], Validity::AllValid).into_array(), PrimitiveArray::new(buffer![0u64; 0], Validity::AllValid).into_array(), // empty chunk PrimitiveArray::new(buffer![3u64, 4], Validity::AllInvalid).into_array(), - PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), // empty chunk + PrimitiveArray::new(buffer![0u64; 0], Validity::AllInvalid).into_array(), /* empty chunk */ ]; let chunked = diff --git a/vortex-array/src/arrays/fixed_size_list/tests/filter.rs b/vortex-array/src/arrays/fixed_size_list/tests/filter.rs index 7efa87eedee..0228790f0fd 100644 --- a/vortex-array/src/arrays/fixed_size_list/tests/filter.rs +++ b/vortex-array/src/arrays/fixed_size_list/tests/filter.rs @@ -97,7 +97,7 @@ fn test_filter_with_nulls() { let expected = FixedSizeListArray::new( expected_elements.into_array(), 2, - Validity::from_iter([true, true]), // Both selected lists are valid, but type is still nullable. + Validity::from_iter([true, true]), /* Both selected lists are valid, but type is still nullable. */ 2, ); diff --git a/vortex-array/src/arrays/patched/mod.rs b/vortex-array/src/arrays/patched/mod.rs index 0ba25ab7929..7d6acd395e7 100644 --- a/vortex-array/src/arrays/patched/mod.rs +++ b/vortex-array/src/arrays/patched/mod.rs @@ -43,7 +43,7 @@ //! `indices` and `values` are aligned and accessed together. //! //! ```text -//! +//! //! chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 chunk 0 //! lane 0 lane 1 lane 2 lane 3 lane 4 lane 5 //! ┌────────────┬────────────┬────────────┬────────────┬────────────┬────────────┐ diff --git a/vortex-array/src/arrays/primitive/compute/slice.rs b/vortex-array/src/arrays/primitive/compute/slice.rs index 4775adba749..9ac45e3baae 100644 --- a/vortex-array/src/arrays/primitive/compute/slice.rs +++ b/vortex-array/src/arrays/primitive/compute/slice.rs @@ -20,7 +20,7 @@ impl SliceReduce for Primitive { let validity = array.validity()?.slice(range)?; // SAFETY: - //slicing an existing PrimitiveArray on element boundaries preserves the buffer + // slicing an existing PrimitiveArray on element boundaries preserves the buffer // alignment, ptype, length, and validity invariants. let array = unsafe { PrimitiveArray::new_unchecked_from_handle(values, array.ptype(), validity).into_array() diff --git a/vortex-array/src/arrays/varbin/compute/compare.rs b/vortex-array/src/arrays/varbin/compute/compare.rs index 9a5a615c81e..9aef7d0f162 100644 --- a/vortex-array/src/arrays/varbin/compute/compare.rs +++ b/vortex-array/src/arrays/varbin/compute/compare.rs @@ -58,7 +58,7 @@ impl CompareKernel for VarBin { if rhs_is_empty { let buffer = match operator { - CompareOperator::Gte => BitBuffer::new_set(len), // Every possible value is >= "" + CompareOperator::Gte => BitBuffer::new_set(len), /* Every possible value is >= "" */ CompareOperator::Lt => BitBuffer::new_unset(len), // No value is < "" CompareOperator::Eq | CompareOperator::Lte => { let lhs_offsets = lhs.offsets().clone().execute::(ctx)?; diff --git a/vortex-array/src/dtype/field.rs b/vortex-array/src/dtype/field.rs index 84f631b5074..c507b51a093 100644 --- a/vortex-array/src/dtype/field.rs +++ b/vortex-array/src/dtype/field.rs @@ -84,7 +84,6 @@ impl Display for Field { /// let dtype_i32 = DType::Primitive(PType::I32, Nullability::NonNullable); /// assert_eq!(dtype_i32, FieldPath::root().resolve(dtype_i32.clone()).unwrap()); /// ``` -/// // TODO(ngates): we should probably reverse the path. Or better yet, store a Arc<[Field]> along // with a positional index to allow cheap step_into. #[derive(Clone, Default, Debug, PartialEq, Eq, Hash)] diff --git a/vortex-array/src/expr/analysis/labeling.rs b/vortex-array/src/expr/analysis/labeling.rs index 090e2992fe2..721241a14c9 100644 --- a/vortex-array/src/expr/analysis/labeling.rs +++ b/vortex-array/src/expr/analysis/labeling.rs @@ -29,7 +29,6 @@ use crate::expr::traversal::TraversalOrder; /// - `merge_child`: Mutable function that folds child labels into an accumulator. /// Takes `(self_label, child_label)` and returns the updated accumulator. /// Called once per child, with the initial accumulator being the node's self-label. -/// pub fn label_tree( expr: &Expression, self_label: impl Fn(&Expression) -> L, diff --git a/vortex-cxx/src/lib.rs b/vortex-cxx/src/lib.rs index 21383fe977f..b6d002513f6 100644 --- a/vortex-cxx/src/lib.rs +++ b/vortex-cxx/src/lib.rs @@ -24,7 +24,6 @@ use write::*; /// By default, the C++ API uses a current-thread runtime, providing control of the threading /// model to the C++ side. -/// // TODO(ngates): in the future, we could expose an API for C++ to spawn threads that can drive // this runtime. pub(crate) static RUNTIME: LazyLock = diff --git a/vortex-datafusion/src/v2/table.rs b/vortex-datafusion/src/v2/table.rs index b881eb025ab..15849893121 100644 --- a/vortex-datafusion/src/v2/table.rs +++ b/vortex-datafusion/src/v2/table.rs @@ -147,7 +147,6 @@ impl TableProvider for VortexTable { /// /// We should not (and actually, cannot) perform I/O here, so the best we can do is return /// cardinality and byte size estimates. - /// // NOTE(ngates): it's not obvious these are actually used? I think DataFusion does join // planning over stats from the physical plan? fn statistics(&self) -> Option { diff --git a/vortex-duckdb/src/convert/expr.rs b/vortex-duckdb/src/convert/expr.rs index 2fdd6d9c033..a8342f5e560 100644 --- a/vortex-duckdb/src/convert/expr.rs +++ b/vortex-duckdb/src/convert/expr.rs @@ -117,20 +117,18 @@ pub(super) fn try_from_bound_expression_with_col_sub( try_from_expression_inner(value, Some(col_sub)) } -/* - * Called before pushdown_complex_filter or a table filter expression call. - * As we support complex filter pushdown, Duckdb pushes expressions to Vortex. - * However, it doesn't know what type of expressions we can handle. Here we list - * all expressions that are quaranteed to be converted to Vortex expressions. - * - * If we return true here, and expression is in the list for - * pushdown_complex_filter, we must handle it, or query engine will break. - * - * Example: we don't support substr() expression so we tell Duckdb we can't - * push it. - * Example: optional filters may fail to parse on our side (we return - * Ok(None)), so we don't allow pushing these. - */ +// Called before pushdown_complex_filter or a table filter expression call. +// As we support complex filter pushdown, Duckdb pushes expressions to Vortex. +// However, it doesn't know what type of expressions we can handle. Here we list +// all expressions that are quaranteed to be converted to Vortex expressions. +// +// If we return true here, and expression is in the list for +// pushdown_complex_filter, we must handle it, or query engine will break. +// +// Example: we don't support substr() expression so we tell Duckdb we can't +// push it. +// Example: optional filters may fail to parse on our side (we return +// Ok(None)), so we don't allow pushing these. pub fn can_push_expression(value: &duckdb::ExpressionRef) -> bool { let Some(value) = value.as_class() else { return false; diff --git a/vortex-duckdb/src/exporter/mod.rs b/vortex-duckdb/src/exporter/mod.rs index 1e470aae3fc..7282594ec45 100644 --- a/vortex-duckdb/src/exporter/mod.rs +++ b/vortex-duckdb/src/exporter/mod.rs @@ -480,7 +480,7 @@ mod tests { assert_eq!(copy_from_slice(&mut target, &source, 10, 64), 28,); assert_eq!( target[0], - 0xff_99_59_0d_0c_cc_80_c0_u64, // Python: hex(0xff_fe_65_64_34_33_32_03_02 >> 2), then remove the high two hexits + 0xff_99_59_0d_0c_cc_80_c0_u64, /* Python: hex(0xff_fe_65_64_34_33_32_03_02 >> 2), then remove the high two hexits */ "{:#08x} == {:#08x}", target[0], 0xff_99_59_0d_0c_cc_80_c0_u64 diff --git a/vortex-duckdb/src/table_function.rs b/vortex-duckdb/src/table_function.rs index 7b34725b123..b44af33b905 100644 --- a/vortex-duckdb/src/table_function.rs +++ b/vortex-duckdb/src/table_function.rs @@ -451,17 +451,15 @@ pub fn statistics(bind_data: &TableFunctionBind, column_index: usize) -> Option< Some(ColumnStatistics::from(&stats_aggregate, dtype)) } -/** - * Duckdb requires post-filter cardinality estimates, otherwise join planner - * may flip join sides which is a huge regression for some queries i.e. 1000x - * for tpcds 85. - * - * See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp - * - * As we don't report distinct values (same as Parquet), the only heuristic - * duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it - * here. - */ +/// Duckdb requires post-filter cardinality estimates, otherwise join planner +/// may flip join sides which is a huge regression for some queries i.e. 1000x +/// for tpcds 85. +/// +/// See duckdb/src/optimizer/join_order/relation_statistics_helper.cpp +/// +/// As we don't report distinct values (same as Parquet), the only heuristic +/// duckdb uses is a 0.2 filter if there is any non-optional filter. We mimic it +/// here. const DEFAULT_SELECTIVITY: f64 = 0.2; pub fn cardinality(bind_data: &TableFunctionBind) -> Cardinality { match bind_data.data_source.row_count() { diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index b67202a4dc8..0203202e437 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -654,7 +654,6 @@ bool vx_array_is_nullable(const vx_array *array); * const vx_array* array = vx_array_new_null(1); * assert(vx_array_has_dtype(array, DTYPE_NULL)); * vx_array_free(array); - * */ bool vx_array_has_dtype(const vx_array *array, vx_dtype_variant variant); @@ -664,7 +663,6 @@ bool vx_array_has_dtype(const vx_array *array, vx_dtype_variant variant); * const vx_array* array = vx_array_new_null(1); * assert(!vx_array_is_primitive(array, PTYPE_U32)); * vx_array_free(array); - * */ bool vx_array_is_primitive(const vx_array *array, vx_ptype ptype); @@ -722,7 +720,6 @@ const vx_array *vx_array_new_null(size_t len); * const vx_array* array = vx_array_new_primitive(PTYPE_U32, buffer, 3, * &validity, &error); * vx_array_free(array); - * */ const vx_array *vx_array_new_primitive(vx_ptype ptype, const void *ptr, @@ -753,7 +750,6 @@ const vx_array *vx_array_new_primitive(vx_ptype ptype, * const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error); * // ... push it to a sink or write it ... * vx_array_free(vx); - * */ const vx_array * vx_array_from_arrow(FFI_ArrowArray *array, FFI_ArrowSchema *schema, bool nullable, vx_error **error_out); @@ -1103,7 +1099,6 @@ void vx_expression_free(vx_expression *ptr); * vx_array_free(applied_array); * vx_expression_free(root); * vx_array_free(array); - * */ vx_expression *vx_expression_root(void); @@ -1137,7 +1132,6 @@ vx_expression *vx_expression_root(void); * vx_expression_free(threshold); * vx_expression_free(age); * vx_expression_free(root); - * */ vx_expression *vx_expression_literal(const vx_scalar *scalar, vx_error **err); @@ -1155,7 +1149,6 @@ vx_expression *vx_expression_literal(const vx_scalar *scalar, vx_error **err); * vx_expression* select = vx_expression_select(names, 2, root); * vx_expression_free(select); * vx_expression_free(root); - * */ vx_expression *vx_expression_select(const char *const *names, size_t len, const vx_expression *child); @@ -1192,7 +1185,6 @@ vx_expression *vx_expression_or(const vx_expression *const *expressions, size_t * ) { * return vx_expression_binary(VX_OPERATOR_EQ, lhs, rhs); * } - * */ vx_expression * vx_expression_binary(vx_binary_operator operator_, const vx_expression *lhs, const vx_expression *rhs); @@ -1694,7 +1686,6 @@ void vx_struct_column_builder_add_field(vx_struct_column_builder *builder, * * vx_array_free(struct_array); * vx_array_free(field_array); - * */ const vx_array *vx_struct_column_builder_finalize(vx_struct_column_builder *builder, vx_error **error); diff --git a/vortex-ffi/examples/hello_vortex.rs b/vortex-ffi/examples/hello_vortex.rs index e33c8e1f2bb..93ed17772f1 100644 --- a/vortex-ffi/examples/hello_vortex.rs +++ b/vortex-ffi/examples/hello_vortex.rs @@ -7,7 +7,7 @@ //! You can invoke this example from a checkout by running //! //! ```ignore -//!cargo run -p vortex-ffi --example hello_vortex +//! cargo run -p vortex-ffi --example hello_vortex //! ``` use std::clone::Clone; diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index 07dbe0eeb55..eacad4edb1e 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -91,7 +91,6 @@ pub unsafe extern "C" fn vx_array_is_nullable(array: *const vx_array) -> bool { /// const vx_array* array = vx_array_new_null(1); /// assert(vx_array_has_dtype(array, DTYPE_NULL)); /// vx_array_free(array); -/// #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_array_has_dtype( array: *const vx_array, @@ -109,7 +108,6 @@ pub unsafe extern "C-unwind" fn vx_array_has_dtype( /// const vx_array* array = vx_array_new_null(1); /// assert(!vx_array_is_primitive(array, PTYPE_U32)); /// vx_array_free(array); -/// #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_array_is_primitive( array: *const vx_array, @@ -318,7 +316,6 @@ unsafe fn primitive_from_raw( /// const vx_array* array = vx_array_new_primitive(PTYPE_U32, buffer, 3, /// &validity, &error); /// vx_array_free(array); -/// #[unsafe(no_mangle)] pub extern "C-unwind" fn vx_array_new_primitive( ptype: vx_ptype, @@ -370,7 +367,6 @@ pub extern "C-unwind" fn vx_array_new_primitive( /// const vx_array* vx = vx_array_from_arrow(&array, &schema, false, &error); /// // ... push it to a sink or write it ... /// vx_array_free(vx); -/// #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_array_from_arrow( array: *mut FFI_ArrowArray, diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index d5632648e82..10fecc4ad8e 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -61,7 +61,6 @@ crate::box_wrapper!( /// vx_array_free(applied_array); /// vx_expression_free(root); /// vx_array_free(array); -/// #[unsafe(no_mangle)] pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression { vx_expression::new(root()) @@ -96,7 +95,6 @@ pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression { /// vx_expression_free(threshold); /// vx_expression_free(age); /// vx_expression_free(root); -/// #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_expression_literal( scalar: *const vx_scalar, @@ -121,7 +119,6 @@ pub unsafe extern "C-unwind" fn vx_expression_literal( /// vx_expression* select = vx_expression_select(names, 2, root); /// vx_expression_free(select); /// vx_expression_free(root); -/// #[unsafe(no_mangle)] pub unsafe extern "C" fn vx_expression_select( names: *const *const c_char, @@ -245,7 +242,6 @@ impl From for Operator { /// ) { /// return vx_expression_binary(VX_OPERATOR_EQ, lhs, rhs); /// } -/// #[unsafe(no_mangle)] pub unsafe extern "C" fn vx_expression_binary( operator: vx_binary_operator, diff --git a/vortex-ffi/src/macros.rs b/vortex-ffi/src/macros.rs index 7ddf3d9d428..d4e58288b30 100644 --- a/vortex-ffi/src/macros.rs +++ b/vortex-ffi/src/macros.rs @@ -39,7 +39,6 @@ //! These macros don't require any Send + Sync bounds. We could try and define this behaviour here //! to make it clearer, but for now, it's important to just be careful when documenting the thread //! safety of the functions that use these types. -//! /// Define a native FFI type that wraps an [`std::sync::Arc`] type with unsized T. /// diff --git a/vortex-ffi/src/struct_array.rs b/vortex-ffi/src/struct_array.rs index f3023625a82..bfac6107319 100644 --- a/vortex-ffi/src/struct_array.rs +++ b/vortex-ffi/src/struct_array.rs @@ -106,7 +106,6 @@ pub unsafe extern "C" fn vx_struct_column_builder_add_field( /// /// vx_array_free(struct_array); /// vx_array_free(field_array); -/// #[unsafe(no_mangle)] pub extern "C-unwind" fn vx_struct_column_builder_finalize( builder: *mut vx_struct_column_builder, diff --git a/vortex-file/src/lib.rs b/vortex-file/src/lib.rs index 9b927abff54..7a131c807a3 100644 --- a/vortex-file/src/lib.rs +++ b/vortex-file/src/lib.rs @@ -88,7 +88,6 @@ //! goals of locality or parallelism. For example, one may write a column in a Struct Layout with //! or without chunking, or completely elide statistics to save space or if they are not needed, for //! example if the metadata is being stored in an external index. -//! mod counting; mod file; diff --git a/vortex-io/src/object_store/filesystem.rs b/vortex-io/src/object_store/filesystem.rs index 0688216fc0f..ca68f5f7efa 100644 --- a/vortex-io/src/object_store/filesystem.rs +++ b/vortex-io/src/object_store/filesystem.rs @@ -23,7 +23,6 @@ use crate::object_store::ObjectStoreReadAt; use crate::runtime::Handle; /// A [`FileSystem`] backed by an [`ObjectStore`]. -/// // TODO(ngates): we could consider spawning a driver task inside this file system such that we can // apply concurrency limits to the overall object store, rather than on a per-file basis. pub struct ObjectStoreFileSystem { diff --git a/vortex-io/src/runtime/handle.rs b/vortex-io/src/runtime/handle.rs index 8675f051f19..aa9f0c2100a 100644 --- a/vortex-io/src/runtime/handle.rs +++ b/vortex-io/src/runtime/handle.rs @@ -102,7 +102,6 @@ impl Handle { /// Spawn a new I/O future onto the runtime. /// /// See [`Executor::spawn_io`] for more details about how this future is expected to run. - /// // See [`Task`] for details on cancelling or detaching the spawned task. pub fn spawn_io(&self, f: Fut) -> Task where diff --git a/vortex-python/src/arrays/mod.rs b/vortex-python/src/arrays/mod.rs index 8621f00202e..7ab858b117b 100644 --- a/vortex-python/src/arrays/mod.rs +++ b/vortex-python/src/arrays/mod.rs @@ -324,7 +324,6 @@ impl PyArray { /// 3 /// ] /// ``` - /// fn to_arrow_array<'py>(self_: &'py Bound<'py, Self>) -> PyVortexResult> { // NOTE(ngates): for struct arrays, we could also return a RecordBatchStreamReader. let array = PyArrayRef::extract(self_.as_any().as_borrowed())?.into_inner(); @@ -467,42 +466,42 @@ impl PyArray { Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __lt__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __lt__: https://github.com/PyO3/pyo3/issues/4326 fn __lt__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::Lt)?; Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __le__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __le__: https://github.com/PyO3/pyo3/issues/4326 fn __le__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::Lte)?; Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __eq__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __eq__: https://github.com/PyO3/pyo3/issues/4326 fn __eq__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::Eq)?; Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __ne__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __ne__: https://github.com/PyO3/pyo3/issues/4326 fn __ne__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::NotEq)?; Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __ge__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __ge__: https://github.com/PyO3/pyo3/issues/4326 fn __ge__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::Gte)?; Ok(PyArrayRef::from(inner)) } - ///Rust docs are *not* copied into Python for __gt__: https://github.com/PyO3/pyo3/issues/4326 + /// Rust docs are *not* copied into Python for __gt__: https://github.com/PyO3/pyo3/issues/4326 fn __gt__(slf: Bound, other: PyArrayRef) -> PyVortexResult { let slf = PyArrayRef::extract(slf.as_any().as_borrowed())?.into_inner(); let inner = slf.binary(other.into_inner(), Operator::Gt)?; diff --git a/vortex-python/src/arrays/py/python.rs b/vortex-python/src/arrays/py/python.rs index ed348b2fe78..5dee87763d6 100644 --- a/vortex-python/src/arrays/py/python.rs +++ b/vortex-python/src/arrays/py/python.rs @@ -13,7 +13,6 @@ use crate::dtype::PyDType; use crate::error::PyVortexResult; /// Base class for implementing a Vortex encoding in Python. -/// // This class can hold everything _except_ a reference to its own object self. So when we // downcast and extract a [`crate::arrays::PythonArray`] from this Python object, we just have // to wrap it up with the object instance. diff --git a/vortex-python/src/expr/mod.rs b/vortex-python/src/expr/mod.rs index 17b342d7db5..4e2fef96ef1 100644 --- a/vortex-python/src/expr/mod.rs +++ b/vortex-python/src/expr/mod.rs @@ -45,7 +45,6 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { /// /// .. seealso:: /// :func:`.column` -/// #[pyclass(name = "Expr", module = "vortex", frozen, from_py_object)] #[derive(Clone)] pub struct PyExpr { diff --git a/vortex-python/src/io.rs b/vortex-python/src/io.rs index bad3804b31b..a793b16eb82 100644 --- a/vortex-python/src/io.rs +++ b/vortex-python/src/io.rs @@ -114,7 +114,6 @@ pub(crate) fn init(py: Python, parent: &Bound) -> PyResult<()> { /// ... secret_access_key="..." /// ... ) /// >>> a = vx.io.read_url("s3://my-bucket/data.vortex", store=store) # doctest: +SKIP -/// #[pyfunction] #[pyo3(signature = (url, *, store = None, projection = None, row_filter = None, indices = None, row_range = None))] pub fn read_url<'py>( From 31fda42e5243fe4a5ee80a6bf79782c3f2bf3649 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 17 Jun 2026 11:25:42 +0200 Subject: [PATCH 032/162] perf[gpu]: generate FixedSizeList offsets on device (#8458) Use the existing CUDA sequence kernel to materialize Arrow List offsets for FixedSizeList export directly on the device. This avoids building the offsets buffer on the CPU and copying it to CUDA memory while preserving existing i32 overflow checks. Signed-off-by: "Alexander Droste" --- vortex-cuda/src/arrow/canonical.rs | 36 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index 33cde27520b..cb116c9cda0 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -1011,7 +1011,7 @@ async fn export_fixed_size_list( } = array.into_data_parts(); let (validity_buffer, null_count) = export_arrow_validity_buffer(validity, len, 0, ctx).await?; - let offsets_buffer = fixed_size_list_offsets(len, list_size, ctx).await?; + let offsets_buffer = fixed_size_list_offsets(len, list_size, ctx)?; export_list_layout( elements, @@ -1025,7 +1025,7 @@ async fn export_fixed_size_list( .await } -async fn fixed_size_list_offsets( +fn fixed_size_list_offsets( len: usize, list_size: u32, ctx: &mut CudaExecutionCtx, @@ -1035,17 +1035,29 @@ async fn fixed_size_list_offsets( "cannot export FixedSizeList with list size {list_size}: Arrow List offsets require i32" ) })?; - let offsets = (0..=i32::try_from(len)?) - .map(|idx| { - idx.checked_mul(list_size) - .ok_or_else(|| vortex_err!("FixedSizeList Arrow List offsets exceed i32 range")) - }) - .collect::>>()?; + let len_i32 = i32::try_from(len)?; + len_i32 + .checked_mul(list_size) + .ok_or_else(|| vortex_err!("FixedSizeList Arrow List offsets exceed i32 range"))?; + + let output_len = len + .checked_add(1) + .ok_or_else(|| vortex_err!("FixedSizeList Arrow List offsets length overflows usize"))?; + let offsets = ctx.device_alloc::(output_len)?; + let base = 0i32; + let output_len_u64 = output_len as u64; + let kernel = ctx.load_function_with_suffixes("sequence", &["i32"])?; + + ctx.launch_kernel(&kernel, output_len, |args| { + args.arg(&offsets) + .arg(&base) + .arg(&list_size) + .arg(&output_len_u64); + })?; - ctx.ensure_on_device(BufferHandle::new_host( - Buffer::from(offsets).into_byte_buffer(), - )) - .await + Ok(BufferHandle::new_device(Arc::new(CudaDeviceBuffer::new( + offsets, + )))) } /// Return Arrow Device `List` offsets as an `i32` device buffer. From c9c8fb7bb85b2ba201989a2cc5cfda41f5ffd8dc Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 17 Jun 2026 11:21:13 +0100 Subject: [PATCH 033/162] Pushdown some expressions to Dict layout reader (#8341) When we access values of Dict layout reader, it canonicalizes them and stores them in a SharedArray. This means we always pay the cost of canonicalization which in turn means we can't do https://github.com/vortex-data/vortex/issues/8310 . In order to solve this issue, we need to apply some expressions to the values array before canonicalizing it. However, we can't push down arbitrary expressions as it may be beneficial to apply them over canonicalized array. One example of such expressions is LIKE over a Dict array with few codes used. Applying LIKE to whole values array is not beneficial. This PR adds a hardcoded internal is_negative_cost estimation for expressions that we want to push before canonicalization. A hint for these are expressions which don't depend on individual input size. As an example, for every string, len(string) doesn't read the string itself but reads the metadata and thus is O(1) on individual input. We don't push down fallible (like cast) or null sensitive (like IS NULL) expressions as well because we want to propagate the errors at call site rather than upfront. Signed-off-by: Mikhail Kot --- vortex-array/src/expr/analysis/annotation.rs | 23 ++ vortex-array/src/expr/transform/partition.rs | 16 +- vortex-array/src/scalar_fn/mod.rs | 18 ++ vortex-layout/src/layouts/dict/reader.rs | 235 ++++++++++++++++++- 4 files changed, 283 insertions(+), 9 deletions(-) diff --git a/vortex-array/src/expr/analysis/annotation.rs b/vortex-array/src/expr/analysis/annotation.rs index be842742a5a..32608f38d1e 100644 --- a/vortex-array/src/expr/analysis/annotation.rs +++ b/vortex-array/src/expr/analysis/annotation.rs @@ -42,6 +42,25 @@ pub fn descendent_annotations( let mut visitor = AnnotationVisitor { annotations: Default::default(), annotate, + propagate_up: true, + }; + expr.accept(&mut visitor).vortex_expect("Infallible"); + visitor.annotations +} + +/// Walk the expression tree and annotate each expression with zero or more +/// annotations. +/// +/// Returns a map of each expression to all annotations. Annotations of +/// children are not propagated to parents. +pub fn direct_annotations( + expr: &Expression, + annotate: A, +) -> Annotations<'_, A::Annotation> { + let mut visitor = AnnotationVisitor { + annotations: Default::default(), + annotate, + propagate_up: false, }; expr.accept(&mut visitor).vortex_expect("Infallible"); visitor.annotations @@ -50,6 +69,7 @@ pub fn descendent_annotations( struct AnnotationVisitor<'a, A: AnnotationFn> { annotations: Annotations<'a, A::Annotation>, annotate: A, + propagate_up: bool, } impl<'a, A: AnnotationFn> NodeVisitor<'a> for AnnotationVisitor<'a, A> { @@ -70,6 +90,9 @@ impl<'a, A: AnnotationFn> NodeVisitor<'a> for AnnotationVisitor<'a, A> { } fn visit_up(&mut self, node: &'a Expression) -> VortexResult { + if !self.propagate_up { + return Ok(TraversalOrder::Continue); + } let child_annotations = node .children() .iter() diff --git a/vortex-array/src/expr/transform/partition.rs b/vortex-array/src/expr/transform/partition.rs index ef852a6cd7f..92dca145d47 100644 --- a/vortex-array/src/expr/transform/partition.rs +++ b/vortex-array/src/expr/transform/partition.rs @@ -3,6 +3,7 @@ use std::fmt::Display; use std::fmt::Formatter; +use std::hash::Hash; use itertools::Itertools; use vortex_error::VortexExpect; @@ -49,11 +50,22 @@ where { // Annotate each expression with the annotations that any of its descendent expressions have. let annotations = descendent_annotations(&expr, annotate_fn); + partition_annotations(expr.clone(), scope, annotations) +} +pub fn partition_annotations( + expr: Expression, + scope: &DType, + annotations: Annotations, +) -> VortexResult> +where + A: Display + Clone + Eq + Hash, + FieldName: From, +{ // Now we split the original expression into sub-expressions based on the annotations, and // generate a root expression to re-assemble the results. - let mut splitter = StructFieldExpressionSplitter::::new(&annotations); - let root = expr.clone().rewrite(&mut splitter)?.value; + let mut splitter = StructFieldExpressionSplitter::::new(&annotations); + let root = expr.rewrite(&mut splitter)?.value; let mut partitions = Vec::with_capacity(splitter.sub_expressions.len()); let mut partition_annotations = Vec::with_capacity(splitter.sub_expressions.len()); diff --git a/vortex-array/src/scalar_fn/mod.rs b/vortex-array/src/scalar_fn/mod.rs index 590ccb44224..9f5bd5c4628 100644 --- a/vortex-array/src/scalar_fn/mod.rs +++ b/vortex-array/src/scalar_fn/mod.rs @@ -9,6 +9,10 @@ use vortex_session::registry::Id; +use crate::scalar_fn::fns::byte_length::ByteLength; +use crate::scalar_fn::fns::get_item::GetItem; +use crate::scalar_fn::fns::literal::Literal; + mod vtable; pub use vtable::*; @@ -48,3 +52,17 @@ mod sealed { /// This can be the **only** implementor for [`super::typed::DynScalarFn`]. impl Sealed for TypedScalarFnInstance {} } + +/// A scalar function has a negative cost if applying it to an array and +/// canonicalizing is cheaper than canonicalizing an array and applying it. +/// +/// Example of negative cost expressions are byte_length() and get_item() since +/// they don't depend on input size. +/// +/// Example of non-negative cost expression is like() as it's linear over +/// individual input. +pub fn is_negative_cost(id: ScalarFnId) -> bool { + id == ScalarFnVTable::id(&ByteLength) + || id == ScalarFnVTable::id(&GetItem) + || id == ScalarFnVTable::id(&Literal) +} diff --git a/vortex-layout/src/layouts/dict/reader.rs b/vortex-layout/src/layouts/dict/reader.rs index 6c8d91706db..b4a61b76e2f 100644 --- a/vortex-layout/src/layouts/dict/reader.rs +++ b/vortex-layout/src/layouts/dict/reader.rs @@ -19,9 +19,16 @@ use vortex_array::arrays::SharedArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; +use vortex_array::dtype::Nullability; use vortex_array::expr::Expression; +use vortex_array::expr::direct_annotations; +use vortex_array::expr::is_root; +use vortex_array::expr::label_tree; +use vortex_array::expr::pack; use vortex_array::expr::root; +use vortex_array::expr::transform::partition_annotations; use vortex_array::optimizer::ArrayOptimizer; +use vortex_array::scalar_fn::is_negative_cost; use vortex_error::VortexError; use vortex_error::VortexExpect; use vortex_error::VortexResult; @@ -101,10 +108,7 @@ impl DictReader { ) .vortex_expect("must construct dict values array evaluation") .map_err(Arc::new) - .map(move |array| { - let array = array?; - Ok(SharedArray::new(array).into_array()) - }) + .map(move |array| Ok(SharedArray::new(array?).into_array())) .boxed() .shared() }) @@ -156,6 +160,44 @@ impl DictReader { } } +// On expression pushdown, "inner" is packed as field with this name. +// "outer" expects this name as input. +const PUSHDOWN_ANNOTATION: &str = ""; + +/// Split expression into two parts: +/// +/// left is the outer part that we want to apply to array after canonicalizing. +/// right is the optional inner part that we want to apply to array before +/// canonicalizing. +/// +/// We want to push to array only if expression has a negative cost, is +/// infallible and null-insensitive. +fn split_expression_for_pushdown( + expr: Expression, + dtype: &DType, +) -> VortexResult<(Expression, Option)> { + let references_root = label_tree(&expr, is_root, |acc, &child| acc | child); + let annotations = direct_annotations(&expr, |expr| { + let signature = expr.signature(); + if !signature.is_fallible() + && !signature.is_null_sensitive() + && is_negative_cost(expr.id()) + && references_root.get(&expr).copied().unwrap_or(true) + { + vec![PUSHDOWN_ANNOTATION] + } else { + vec![] + } + }); + let partition = partition_annotations(expr.clone(), dtype, annotations)?; + if partition.partitions.is_empty() { + Ok((partition.root, None)) + } else { + debug_assert_eq!(1, partition.partitions.len()); + Ok((partition.root, Some(partition.partitions[0].clone()))) + } +} + impl LayoutReader for DictReader { fn name(&self) -> &Arc { &self.name @@ -233,13 +275,37 @@ impl LayoutReader for DictReader { mask: MaskFuture, ) -> VortexResult>> { // TODO: fix up expr partitioning with fallible & null sensitive annotations - let values_eval = self.values_array(); let codes_eval = self .codes .projection_evaluation(row_range, &root(), mask) .map_err(|err| err.with_context("While evaluating projection on codes"))?; - let expr = expr.clone(); + let (expr_outer, expr_inner) = split_expression_for_pushdown(expr.clone(), self.dtype())?; + + let values_eval = if let Some(inner) = expr_inner { + // "outer" takes a struct field with PUSHDOWN_ANNOTATION name, so + // pack inner with this name as well + let inner = pack([(PUSHDOWN_ANNOTATION, inner)], Nullability::NonNullable); + + // We can't use values_eval as it uses values_array_uncanonical + // which in turn gets populated from self.values. If + // self.values_array() is called first, it will populate + // self.values with uncompressed data. Supply uncached data + let values_len = self.values_len; + self.values + .projection_evaluation( + &(0..values_len as u64), + &root(), + MaskFuture::new_true(values_len), + ) + .vortex_expect("must construct dict values array evaluation") + .map_err(Arc::new) + .map(move |array| Ok(SharedArray::new(array?.apply(&inner)?).into_array())) + .boxed() + .shared() + } else { + self.values_array() + }; let all_values_referenced = self.layout.has_all_values_referenced(); Ok(async move { let (values, codes) = try_join!(values_eval.map_err(VortexError::from), codes_eval)?; @@ -256,7 +322,7 @@ impl LayoutReader for DictReader { .into_array() .optimize()?; - array.apply(&expr) + array.apply(&expr_outer) } .boxed()) } @@ -272,6 +338,7 @@ mod tests { use rstest::rstest; use vortex_array::ArrayContext; + use vortex_array::ArrayRef; use vortex_array::Canonical; use vortex_array::IntoArray as _; use vortex_array::LEGACY_SESSION; @@ -285,8 +352,14 @@ mod tests { use vortex_array::dtype::FieldName; use vortex_array::dtype::FieldNames; use vortex_array::dtype::Nullability; + use vortex_array::dtype::PType; + use vortex_array::expr::Expression; + use vortex_array::expr::byte_length; + use vortex_array::expr::cast; use vortex_array::expr::eq; + use vortex_array::expr::get_item; use vortex_array::expr::is_not_null; + use vortex_array::expr::like; use vortex_array::expr::lit; use vortex_array::expr::pack; use vortex_array::expr::root; @@ -294,6 +367,7 @@ mod tests { use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_error::VortexExpect; + use vortex_error::VortexResult; use vortex_io::runtime::Handle; use vortex_io::runtime::single::block_on; use vortex_io::session::RuntimeSession; @@ -303,6 +377,8 @@ mod tests { use crate::LayoutId; use crate::LayoutRef; use crate::LayoutStrategy; + use crate::layouts::dict::reader::PUSHDOWN_ANNOTATION; + use crate::layouts::dict::reader::split_expression_for_pushdown; use crate::layouts::dict::writer::DictLayoutOptions; use crate::layouts::dict::writer::DictStrategy; use crate::layouts::flat::writer::FlatLayoutStrategy; @@ -324,6 +400,33 @@ mod tests { .with_handle(handle) } + async fn write_dict_layout( + array: ArrayRef, + session: &VortexSession, + ) -> (LayoutRef, Arc) { + let strategy = DictStrategy::new( + FlatLayoutStrategy::default(), + FlatLayoutStrategy::default(), + FlatLayoutStrategy::default(), + DictLayoutOptions::default(), + ); + let segments = Arc::new(TestSegments::default()); + let (ptr, eof) = SequenceId::root().split(); + let dtype = array.dtype().clone(); + let layout = strategy + .write_stream( + ArrayContext::empty(), + Arc::::clone(&segments), + SequentialStreamAdapter::new(dtype, array.to_array_stream().sequenced(ptr)) + .sendable(), + eof, + session, + ) + .await + .unwrap(); + (layout, segments) + } + #[test] fn reading_nested_packs_works() { block_on(|handle| async move { @@ -546,4 +649,122 @@ mod tests { assert_arrays_eq!(actual_canonical, expected); }) } + + #[test] + fn reading_byte_length_pushdown_works() { + let array = VarBinArray::from_iter( + [ + Some("abc"), + Some("defg"), + None, + Some("abc"), + Some("defg"), + None, + Some("abc"), + Some("defg"), + None, + ], + DType::Utf8(Nullability::Nullable), + ) + .into_array(); + + let expected = array + .clone() + .apply(&byte_length(root())) + .unwrap() + .into_array(); + + block_on(|handle| async move { + let session = session_with_handle(handle); + let (layout, segments) = write_dict_layout(array, &session).await; + assert_eq!(layout.encoding_id(), LayoutId::new("vortex.dict")); + let actual = layout + .new_reader("".into(), segments, &session, &Default::default()) + .unwrap() + .projection_evaluation( + &(0..layout.row_count()), + &byte_length(root()), + MaskFuture::new_true(layout.row_count().try_into().unwrap()), + ) + .unwrap() + .await + .unwrap() + .into_array(); + assert_arrays_eq!(actual, expected); + }) + } + + fn pushed_inner(exprs: impl IntoIterator) -> Expression { + pack( + exprs + .into_iter() + .enumerate() + .map(|(idx, e)| (format!("_{idx}"), e)), + Nullability::NonNullable, + ) + } + + fn pushed_ref(idx: usize) -> Expression { + get_item(format!("_{idx}"), get_item("", root())) + } + + fn test_apply(original: Expression, outer: Expression, inner: Expression) -> VortexResult<()> { + let array = VarBinArray::from_iter( + [Some("abc"), Some("def"), None], + DType::Utf8(Nullability::Nullable), + ) + .into_array(); + + let pushed = array.clone().apply(&pack( + [(PUSHDOWN_ANNOTATION, inner)], + Nullability::NonNullable, + ))?; + let actual = pushed.apply(&outer)?; + let expected = array.apply(&original)?; + assert_arrays_eq!(actual, expected); + Ok(()) + } + + #[test] + fn split_expr_root() { + let (outer, inner) = split_expression_for_pushdown(root(), &DType::Null).unwrap(); + assert_eq!(outer, root()); + assert_eq!(inner, None); + } + + #[test] + fn split_expr_partial_pushdown() -> VortexResult<()> { + // cast is fallible, thus not pushed + let target = DType::Primitive(PType::I64, Nullability::Nullable); + let expr = cast(byte_length(root()), target.clone()); + let (outer, inner) = + split_expression_for_pushdown(expr.clone(), &DType::Utf8(false.into()))?; + let inner = inner.unwrap(); + // [0] = cast([1], dtype) + // [1] = byte_length(root) + assert_eq!(outer, cast(pushed_ref(0), target)); + assert_eq!(inner, pushed_inner([byte_length(root())])); + test_apply(expr, outer, inner) + } + + #[test] + fn split_expr_full_pushdown() -> VortexResult<()> { + let expr = byte_length(root()); + let (outer, inner) = + split_expression_for_pushdown(expr.clone(), &DType::Utf8(false.into()))?; + let inner = inner.unwrap(); + assert_eq!(outer, pushed_ref(0)); + assert_eq!(inner, pushed_inner([byte_length(root())])); + test_apply(expr, outer, inner) + } + + #[test] + fn split_expr_no_pushdown() { + // like is fallible, thus not pushed. lit() does not reference root() + let expr = like(root(), lit(1u64)); + let (outer, inner) = + split_expression_for_pushdown(expr.clone(), &DType::Utf8(true.into())).unwrap(); + assert_eq!(outer, expr); + assert_eq!(inner, None); + } } From 170ff3985fc0f3b29647459fb22d9b8c64c18ad6 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Wed, 17 Jun 2026 11:36:51 +0100 Subject: [PATCH 034/162] Add Delta scheme to the integer compressor (unstable_encodings) (#8212) Add a FastLanes Delta encoding as a candidate in the BtrBlocks integer scheme set, gated behind the `unstable_encodings` feature, in a new `schemes/integer/delta.rs` module. Selection uses a deferred callback that delta-encodes the array and measures the real transposed-delta residual span, then estimates the compression ratio as `full_width / delta_bits`, multiplied by a DELTA_PENALTY (0.9) factor. The penalty makes the selector require Delta to be meaningfully smaller than the best alternative before it wins, rather than picking it for a single-bit gain, since Delta breaks random access and adds a prefix-sum decode pass. Delta is registered last among the integer schemes so it loses exact estimate ties to simpler, randomly-accessible encodings. A descendant exclusion (plus the compressor's built-in self-exclusion) guarantees Delta is applied at most once along any path in the tree, so already-delta-encoded data is never delta-encoded again. Tests cover selection on a delta-favorable column, round-tripping, and the no-nested-delta invariant. The existing RLE selection test is updated to use scrambled (non-monotone) run values so it remains run-length-dominant rather than delta-favorable. --------- Signed-off-by: Joe Isaacs Co-authored-by: Claude --- vortex-btrblocks/src/builder.rs | 7 + vortex-btrblocks/src/schemes/integer/delta.rs | 177 ++++++++++++++++++ vortex-btrblocks/src/schemes/integer/mod.rs | 4 + .../schemes/integer/scheme_selection_tests.rs | 61 +++++- vortex-file/src/open.rs | 10 +- 5 files changed, 256 insertions(+), 3 deletions(-) create mode 100644 vortex-btrblocks/src/schemes/integer/delta.rs diff --git a/vortex-btrblocks/src/builder.rs b/vortex-btrblocks/src/builder.rs index 43df788473a..f61cb4930f2 100644 --- a/vortex-btrblocks/src/builder.rs +++ b/vortex-btrblocks/src/builder.rs @@ -41,6 +41,9 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[ &integer::RunEndScheme, &integer::SequenceScheme, &integer::IntRLEScheme, + // Prefer all other schemes above delta, for now (since its slower to decompress). + #[cfg(feature = "unstable_encodings")] + &integer::DeltaScheme, //////////////////////////////////////////////////////////////////////////////////////////////// // Float schemes. //////////////////////////////////////////////////////////////////////////////////////////////// @@ -180,6 +183,10 @@ impl BtrBlocksCompressorBuilder { ]; #[cfg(feature = "unstable_encodings")] excluded.push(string::OnPairScheme.id()); + // Delta has no GPU decode kernel and its prefix-sum decode is inherently sequential, so it + // is incompatible with pure-GPU decompression paths. + #[cfg(feature = "unstable_encodings")] + excluded.push(integer::DeltaScheme.id()); let builder = self.exclude_schemes(excluded); #[cfg(all(feature = "zstd", feature = "unstable_encodings"))] diff --git a/vortex-btrblocks/src/schemes/integer/delta.rs b/vortex-btrblocks/src/schemes/integer/delta.rs new file mode 100644 index 00000000000..064d39690a9 --- /dev/null +++ b/vortex-btrblocks/src/schemes/integer/delta.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! FastLanes Delta integer encoding. + +use vortex_array::ArrayRef; +use vortex_array::Canonical; +use vortex_array::ExecutionCtx; +use vortex_array::IntoArray; +use vortex_array::arrays::PrimitiveArray; +use vortex_compressor::builtins::BinaryDictScheme; +use vortex_compressor::builtins::FloatDictScheme; +use vortex_compressor::builtins::IntDictScheme; +use vortex_compressor::builtins::StringDictScheme; +use vortex_compressor::estimate::CompressionEstimate; +use vortex_compressor::estimate::DeferredEstimate; +use vortex_compressor::estimate::EstimateScore; +use vortex_compressor::estimate::EstimateVerdict; +use vortex_compressor::scheme::AncestorExclusion; +use vortex_compressor::scheme::ChildSelection; +use vortex_compressor::scheme::DescendantExclusion; +use vortex_error::VortexResult; +use vortex_fastlanes::Delta; + +use crate::ArrayAndStats; +use crate::CascadingCompressor; +use crate::CompressorContext; +use crate::GenerateStatsOptions; +use crate::Scheme; +use crate::SchemeExt; + +/// FastLanes Delta encoding for smooth / near-monotone integers. +/// +/// Delta replaces each value with its difference from an earlier value (at the FastLanes lane +/// stride), so a later cascade layer (FoR / BitPacking) packs the smaller residuals. It only +/// pays off when those residuals span meaningfully fewer bits than the values themselves. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct DeltaScheme; + +/// Multiplicative penalty applied to Delta's estimated compression ratio. +/// +/// Unlike FoR/BitPacking, Delta breaks random access and adds a prefix-sum decode pass, and it +/// carries a structural sign bit on its residuals. We therefore require Delta to be meaningfully +/// (~5%) smaller than the best alternative before it wins, rather than picking it for a +/// single-bit gain. This factor encodes that "delta tax". +const DELTA_PENALTY: f64 = 0.95; + +/// Minimum length before Delta is worth considering (one FastLanes chunk). +const MIN_DELTA_LEN: usize = 1024; + +impl Scheme for DeltaScheme { + fn scheme_name(&self) -> &'static str { + "vortex.int.delta" + } + + fn matches(&self, canonical: &Canonical) -> bool { + canonical.dtype().is_int() + } + + fn num_children(&self) -> usize { + 2 + } + + /// Delta-encode the data at most once per path: exclude Delta from the subtrees of both the + /// bases and the deltas children so we never delta-encode data that was already delta-encoded. + fn descendant_exclusions(&self) -> Vec { + vec![DescendantExclusion { + excluded: DeltaScheme.id(), + children: ChildSelection::All, + }] + } + + /// Delta over dictionary codes just adds indirection: codes are compact integers with no + /// monotone structure, so (like FoR/Sequence) skip the codes child. + fn ancestor_exclusions(&self) -> Vec { + vec![ + AncestorExclusion { + ancestor: IntDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: FloatDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: StringDictScheme.id(), + children: ChildSelection::One(1), + }, + AncestorExclusion { + ancestor: BinaryDictScheme.id(), + children: ChildSelection::One(1), + }, + ] + } + + fn expected_compression_ratio( + &self, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + _exec_ctx: &mut ExecutionCtx, + ) -> CompressionEstimate { + // Delta only pays off if a later cascade layer (FoR/BitPacking) packs the residuals. + if compress_ctx.finished_cascading() { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + // Too short to transpose into FastLanes chunks meaningfully. + if data.array_len() < MIN_DELTA_LEN { + return CompressionEstimate::Verdict(EstimateVerdict::Skip); + } + + // Estimating Delta needs the real transposed-delta span, so defer to a callback that + // delta-encodes the array and measures the residual range. + CompressionEstimate::Deferred(DeferredEstimate::Callback(Box::new( + |_compressor, data, best_so_far, _ctx, exec_ctx| { + let primitive = data.array().clone().execute::(exec_ctx)?; + let full_width = primitive.ptype().bit_width() as f64; + + // Delta's best case is residuals collapsing to a single bit. If even that, after + // the penalty, can't beat the incumbent, skip before doing the encode work. + let threshold = best_so_far.and_then(EstimateScore::finite_ratio); + if threshold.is_some_and(|t| full_width * DELTA_PENALTY <= t) { + return Ok(EstimateVerdict::Skip); + } + + // Measure the actual FastLanes transposed-delta span. This is the lane-stride + // difference that gets bit-packed, not the lag-1 difference (which the transpose + // makes optimistic), so it is what truly drives the compressed size. + let (_bases, deltas) = vortex_fastlanes::delta_compress(&primitive, exec_ctx)?; + let delta_stats = + ArrayAndStats::new(deltas.into_array(), GenerateStatsOptions::default()); + let span = delta_stats.integer_stats(exec_ctx).erased().max_minus_min(); + + // Bits needed to FoR-pack the residuals. A zero span means constant deltas, which + // SequenceScheme already captures more cheaply, so defer to it. + let delta_bits = match span.checked_ilog2() { + Some(l) => (l + 1) as f64, + None => return Ok(EstimateVerdict::Skip), + }; + + let ratio = full_width / delta_bits * DELTA_PENALTY; + if ratio <= 1.0 { + return Ok(EstimateVerdict::Skip); + } + Ok(EstimateVerdict::Ratio(ratio)) + }, + ))) + } + + fn compress( + &self, + compressor: &CascadingCompressor, + data: &ArrayAndStats, + compress_ctx: CompressorContext, + exec_ctx: &mut ExecutionCtx, + ) -> VortexResult { + let primitive = data.array().clone().execute::(exec_ctx)?; + let len = primitive.len(); + let (bases, deltas) = vortex_fastlanes::delta_compress(&primitive, exec_ctx)?; + + let compressed_bases = compressor.compress_child( + &bases.into_array(), + &compress_ctx, + self.id(), + 0, + exec_ctx, + )?; + let compressed_deltas = compressor.compress_child( + &deltas.into_array(), + &compress_ctx, + self.id(), + 1, + exec_ctx, + )?; + + Delta::try_new(compressed_bases, compressed_deltas, 0, len).map(IntoArray::into_array) + } +} diff --git a/vortex-btrblocks/src/schemes/integer/mod.rs b/vortex-btrblocks/src/schemes/integer/mod.rs index aed29f1ad3d..abe5868f5c8 100644 --- a/vortex-btrblocks/src/schemes/integer/mod.rs +++ b/vortex-btrblocks/src/schemes/integer/mod.rs @@ -4,6 +4,8 @@ //! Integer compression schemes. mod bitpacking; +#[cfg(feature = "unstable_encodings")] +mod delta; mod for_; mod rle; mod runend; @@ -15,6 +17,8 @@ mod zigzag; mod pco; pub use bitpacking::BitPackingScheme; +#[cfg(feature = "unstable_encodings")] +pub use delta::DeltaScheme; pub use for_::FoRScheme; #[cfg(feature = "pco")] pub use pco::PcoScheme; diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 2e0fb269fda..993827d2057 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -143,7 +143,11 @@ fn test_sequence_compressed() -> VortexResult<()> { fn test_rle_compressed() -> VortexResult<()> { let mut values: Vec = Vec::new(); for i in 0..1024 { - values.extend(iter::repeat_n(i, 10)); + // Scramble the per-run value so the data is run-length-dominant but not monotone: this + // keeps RunEnd the winner instead of Delta (whose residuals would be small on a smooth + // ramp). + let v = (i as u32).wrapping_mul(2_654_435_761) as i32; + values.extend(iter::repeat_n(v, 10)); } let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); let btr = BtrBlocksCompressor::default(); @@ -152,3 +156,58 @@ fn test_rle_compressed() -> VortexResult<()> { assert!(compressed.is::()); Ok(()) } + +/// A strictly-increasing column with small, irregular steps: not a perfect arithmetic sequence +/// (so Sequence skips), all-unique with no runs (so RunEnd/Dict skip), and a wide absolute range. +/// Delta's residuals are far smaller than the FoR span, so Delta should win and round-trip, and +/// it must appear at most once in the tree. +#[cfg(feature = "unstable_encodings")] +#[test] +fn test_delta_compressed() -> VortexResult<()> { + use vortex_array::assert_arrays_eq; + use vortex_fastlanes::Delta; + + let mut rng = StdRng::seed_from_u64(7u64); + let mut value = 500_000i32; + let values: Vec = (0..4096) + .map(|_| { + value += 1 + (rng.next_u32() % 6) as i32; + value + }) + .collect(); + let array = PrimitiveArray::new(Buffer::copy_from(&values), Validity::NonNullable); + + let btr = BtrBlocksCompressor::default(); + let compressed = btr.compress( + &array.clone().into_array(), + &mut SESSION.create_execution_ctx(), + )?; + assert!( + compressed.is::(), + "expected Delta, got tree:\n{}", + compressed.display_tree() + ); + // Delta must appear at most once per tree: no Delta node may be nested under another. + assert!( + !has_nested_delta(&compressed, false), + "Delta was applied more than once in the tree:\n{}", + compressed.display_tree() + ); + assert_arrays_eq!(compressed, array.into_array()); + Ok(()) +} + +/// Returns true if any `Delta` array appears below an ancestor `Delta` in the tree. +#[cfg(feature = "unstable_encodings")] +fn has_nested_delta(array: &vortex_array::ArrayRef, under_delta: bool) -> bool { + use vortex_fastlanes::Delta; + + let is_delta = array.is::(); + if is_delta && under_delta { + return true; + } + array + .children() + .iter() + .any(|child| has_nested_delta(child, under_delta || is_delta)) +} diff --git a/vortex-file/src/open.rs b/vortex-file/src/open.rs index c1793b7b517..46460107b8a 100644 --- a/vortex-file/src/open.rs +++ b/vortex-file/src/open.rs @@ -443,10 +443,16 @@ mod tests { // Create a large file (> 1MB) let mut buf = ByteBufferMut::empty(); - // 1.5M integers -> ~6MB. We use a pattern to avoid Sequence encoding. + // 1.5M integers -> ~6MB. We use high-entropy (pseudo-random) values so the data does not + // compress well under any encoding (Sequence, RunEnd, Delta, ...), keeping the written + // file comfortably above 1MB. + let mut state = 0x9E37_79B9u32; let array = Buffer::from( (0i32..1_500_000) - .map(|i| if i % 2 == 0 { i } else { -i }) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + state as i32 + }) .collect::>(), ) .into_array(); From 0c1d6310fce2b4d6657bfa6a28652e94ddab7561 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 12:04:53 +0100 Subject: [PATCH 035/162] Ignore the other pyo3 RUSTSEC advisory (#8460) ## Summary Follow up to #8381, another pyo3 issue. Signed-off-by: Adam Gutglick --- deny.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deny.toml b/deny.toml index a2a6935ba5b..1e2443c1fd8 100644 --- a/deny.toml +++ b/deny.toml @@ -21,7 +21,10 @@ ignore = [ # 0.29.0. We cannot bump until pyo3-bytes, pyo3-log, and pyo3-object_store support 0.29 (all # pin pyo3 to <0.29, and pyo3-ffi `links = "python"` forbids two pyo3 versions in the graph). # Not exploitable here: `vortex-python` never calls `nth`/`nth_back` on these iterators. - "RUSTSEC-2026-0176" + "RUSTSEC-2026-0176", + # Another issue that will be fixed once we can bump pyo3 to 0.29. + # PyCFunction::new_closure is missing a `Sync` bound. + "RUSTSEC-2026-0177" ] [licenses] From 4b867a021ac0a9254257d8cd80c254f144a51825 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jun 2026 12:11:53 +0100 Subject: [PATCH 036/162] Update dependency starlette to v1.3.1 [SECURITY] (#8435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Change | [Age](https://docs.renovatebot.com/merge-confidence/) | [Confidence](https://docs.renovatebot.com/merge-confidence/) | |---|---|---|---| | [starlette](https://redirect.github.com/Kludex/starlette) ([changelog](https://starlette.dev/release-notes/)) | `1.2.1` → `1.3.1` | ![age](https://developer.mend.io/api/mc/badges/age/pypi/starlette/1.3.1?slim=true) | ![confidence](https://developer.mend.io/api/mc/badges/confidence/pypi/starlette/1.2.1/1.3.1?slim=true) | --- > [!WARNING] > Some dependencies could not be looked up. Check the [Dependency Dashboard](../issues/357) for more information. --- ### Starlette: Unvalidated request path concatenated into authority poisons request.url.hostname [CVE-2026-54282](https://nvd.nist.gov/vuln/detail/CVE-2026-54282) / [GHSA-jp82-jpqv-5vv3](https://redirect.github.com/advisories/GHSA-jp82-jpqv-5vv3)

More information #### Details ##### Summary In affected versions, the HTTP request path is not validated before being used to reconstruct `request.url`. Because `request.url` is rebuilt by concatenating `{scheme}://{host}{path}` and re-parsing the result, a path that does not begin with `/` (for example `@google.com`) moves the authority boundary during re-parsing, so `request.url.hostname` and `request.url.netloc` become attacker-controlled. Code that reads `request.url.hostname` (rather than the `Host` header or `scope`) can therefore be misled into trusting an attacker-supplied host. ##### Details When a client requests a path that does not start with `/`: ```http GET @​google.com HTTP/1.1 Host: localhost ``` affected versions reconstruct the URL as `http://localhost@google.com`. Per [RFC 3986 §3.2.1](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.2.1), the substring before `@` in the authority is `userinfo`, so re-parsing yields `username = "localhost"` and `hostname = "google.com"`, with an empty path: ```text request.url == "http://localhost@google.com" request.url.hostname == "google.com" request.url.path == "" ``` The root cause is that the path is concatenated directly after the host without a separating `/`, and without validating that it begins with one. Only the `Host` header was validated when constructing `request.url`; the path was not. This requires an ASGI server that forwards a request-target lacking a leading `/` into `scope["path"]`. ##### Impact Any application running an affected version that uses `request.url`, `request.url.netloc`, or `request.url.hostname` for a security-sensitive decision (host-based authorization, redirect/callback base, SSRF target, cache key, audit log) may be affected, when no fronting proxy or load balancer rejects the malformed request-target first. Note that this is less exploitable than [GHSA-86qp-5c8j-p5mr](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-86qp-5c8j-p5mr): there, the poison is carried in the `Host` header, so the real path still routes to a valid endpoint while `request.url.path` lies. Here, the poison must be carried in the path itself, and that path (`@google.com`) does not match any registered route, so routing returns `404` and no endpoint handler runs. The exposure is limited to code that reads `request.url` before routing - notably middleware - or in 404/exception handlers. ##### Mitigation Upgrade to a patched version, which prevents the request path from crossing into the URL authority. The request above instead yields `http://localhost/@​google.com` with `request.url.hostname == "localhost"`. #### Severity - CVSS Score: 3.7 / 10 (Low) - Vector String: `CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:L/A:N` #### References - [https://github.com/Kludex/starlette/security/advisories/GHSA-jp82-jpqv-5vv3](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-jp82-jpqv-5vv3) - [https://github.com/advisories/GHSA-jp82-jpqv-5vv3](https://redirect.github.com/advisories/GHSA-jp82-jpqv-5vv3) This data is provided by the [GitHub Advisory Database](https://redirect.github.com/advisories/GHSA-jp82-jpqv-5vv3) ([CC-BY 4.0](https://redirect.github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Starlette: request.form() limits silently ignored for application/x-www-form-urlencoded enable DoS [CVE-2026-54283](https://nvd.nist.gov/vuln/detail/CVE-2026-54283) / [GHSA-82w8-qh3p-5jfq](https://redirect.github.com/advisories/GHSA-82w8-qh3p-5jfq)
More information #### Details ##### Summary `request.form()` accepts `max_fields` and `max_part_size` to bound resource consumption while parsing form data. These limits are enforced for `multipart/form-data`, but silently ignored for `application/x-www-form-urlencoded`. An unauthenticated attacker can therefore send a urlencoded body with an arbitrarily large number of fields or an arbitrarily large field, even when the application configured limits it believed would apply. ##### Details `request.form()` dispatches to a different parser depending on the `Content-Type`. For `multipart/form-data` the `max_files`, `max_fields`, and `max_part_size` limits are forwarded to the parser, but for `application/x-www-form-urlencoded` the parser is constructed without them. It has no `max_fields` or `max_part_size` parameter to receive them, and it appends every field with no count check and accumulates each field's name and value with no size check. The configured limits are therefore both unreachable and unenforced for url-encoded bodies. Because the url-encoded parser does its work synchronously between stream reads, the two attack shapes have different effects: - **Field count** drives CPU and event-loop blocking. A body of ~1,000,000 fields (a sub-10MB payload such as `f0=v&f1=v&...`) blocks the worker's event loop for several seconds while parsing, during which the worker serves no other request. - **Field size** drives memory. A single large field value (e.g. a 50MB value) is buffered in full to build the `FormData`, forcing memory allocation proportional to the request body. The equivalent `multipart/form-data` request is correctly rejected with `400 Too many fields` / `400 Field exceeded maximum size`. ##### Impact This Denial of service (DoS) vulnerability affects all applications built with Starlette (or FastAPI) that call `request.form()` on `application/x-www-form-urlencoded` requests. A single request with a very large number of fields blocks the event loop for several seconds, and a single request with a very large field forces unbounded memory allocation; in either case, parallel requests can render the service unusable. A reverse proxy that enforces a request body size limit reduces but does not eliminate the exposure, since a sub-10MB body is already enough to block the event loop. ##### Mitigation Upgrade to a patched version, which forwards `max_fields` and `max_part_size` to the url-encoded parser and enforces them while parsing, raising before the oversized field or excess fields are accumulated. The defaults match `multipart/form-data` (`max_fields=1000`, `max_part_size=1MB`) and can be customized via `request.form(max_fields=..., max_part_size=...)`. #### Severity - CVSS Score: 7.5 / 10 (High) - Vector String: `CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H` #### References - [https://github.com/Kludex/starlette/security/advisories/GHSA-82w8-qh3p-5jfq](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-82w8-qh3p-5jfq) - [https://github.com/advisories/GHSA-82w8-qh3p-5jfq](https://redirect.github.com/advisories/GHSA-82w8-qh3p-5jfq) This data is provided by the [GitHub Advisory Database](https://redirect.github.com/advisories/GHSA-82w8-qh3p-5jfq) ([CC-BY 4.0](https://redirect.github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Starlette: Unvalidated request path concatenated into authority poisons request.url.hostname [CVE-2026-54282](https://nvd.nist.gov/vuln/detail/CVE-2026-54282) / [GHSA-jp82-jpqv-5vv3](https://redirect.github.com/advisories/GHSA-jp82-jpqv-5vv3)
More information #### Details ##### Summary In affected versions, the HTTP request path is not validated before being used to reconstruct `request.url`. Because `request.url` is rebuilt by concatenating `{scheme}://{host}{path}` and re-parsing the result, a path that does not begin with `/` (for example `@google.com`) moves the authority boundary during re-parsing, so `request.url.hostname` and `request.url.netloc` become attacker-controlled. Code that reads `request.url.hostname` (rather than the `Host` header or `scope`) can therefore be misled into trusting an attacker-supplied host. ##### Details When a client requests a path that does not start with `/`: ```http GET @​google.com HTTP/1.1 Host: localhost ``` affected versions reconstruct the URL as `http://localhost@google.com`. Per [RFC 3986 §3.2.1](https://www.rfc-editor.org/rfc/rfc3986.html#section-3.2.1), the substring before `@` in the authority is `userinfo`, so re-parsing yields `username = "localhost"` and `hostname = "google.com"`, with an empty path: ```text request.url == "http://localhost@google.com" request.url.hostname == "google.com" request.url.path == "" ``` The root cause is that the path is concatenated directly after the host without a separating `/`, and without validating that it begins with one. Only the `Host` header was validated when constructing `request.url`; the path was not. This requires an ASGI server that forwards a request-target lacking a leading `/` into `scope["path"]`. ##### Impact Any application running an affected version that uses `request.url`, `request.url.netloc`, or `request.url.hostname` for a security-sensitive decision (host-based authorization, redirect/callback base, SSRF target, cache key, audit log) may be affected, when no fronting proxy or load balancer rejects the malformed request-target first. Note that this is less exploitable than [GHSA-86qp-5c8j-p5mr](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-86qp-5c8j-p5mr): there, the poison is carried in the `Host` header, so the real path still routes to a valid endpoint while `request.url.path` lies. Here, the poison must be carried in the path itself, and that path (`@google.com`) does not match any registered route, so routing returns `404` and no endpoint handler runs. The exposure is limited to code that reads `request.url` before routing - notably middleware - or in 404/exception handlers. ##### Mitigation Upgrade to a patched version, which prevents the request path from crossing into the URL authority. The request above instead yields `http://localhost/@​google.com` with `request.url.hostname == "localhost"`. #### Severity - CVSS Score: 3.7 / 10 (Low) - Vector String: `CVSS:3.1/AV:N/AC:H/PR:N/UI:N/S:U/C:N/I:L/A:N` #### References - [https://github.com/Kludex/starlette/security/advisories/GHSA-jp82-jpqv-5vv3](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-jp82-jpqv-5vv3) - [https://github.com/Kludex/starlette](https://redirect.github.com/Kludex/starlette) This data is provided by [OSV](https://osv.dev/vulnerability/GHSA-jp82-jpqv-5vv3) and the [GitHub Advisory Database](https://redirect.github.com/github/advisory-database) ([CC-BY 4.0](https://redirect.github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Starlette: request.form() limits silently ignored for application/x-www-form-urlencoded enable DoS [CVE-2026-54283](https://nvd.nist.gov/vuln/detail/CVE-2026-54283) / [GHSA-82w8-qh3p-5jfq](https://redirect.github.com/advisories/GHSA-82w8-qh3p-5jfq)
More information #### Details ##### Summary `request.form()` accepts `max_fields` and `max_part_size` to bound resource consumption while parsing form data. These limits are enforced for `multipart/form-data`, but silently ignored for `application/x-www-form-urlencoded`. An unauthenticated attacker can therefore send a urlencoded body with an arbitrarily large number of fields or an arbitrarily large field, even when the application configured limits it believed would apply. ##### Details `request.form()` dispatches to a different parser depending on the `Content-Type`. For `multipart/form-data` the `max_files`, `max_fields`, and `max_part_size` limits are forwarded to the parser, but for `application/x-www-form-urlencoded` the parser is constructed without them. It has no `max_fields` or `max_part_size` parameter to receive them, and it appends every field with no count check and accumulates each field's name and value with no size check. The configured limits are therefore both unreachable and unenforced for url-encoded bodies. Because the url-encoded parser does its work synchronously between stream reads, the two attack shapes have different effects: - **Field count** drives CPU and event-loop blocking. A body of ~1,000,000 fields (a sub-10MB payload such as `f0=v&f1=v&...`) blocks the worker's event loop for several seconds while parsing, during which the worker serves no other request. - **Field size** drives memory. A single large field value (e.g. a 50MB value) is buffered in full to build the `FormData`, forcing memory allocation proportional to the request body. The equivalent `multipart/form-data` request is correctly rejected with `400 Too many fields` / `400 Field exceeded maximum size`. ##### Impact This Denial of service (DoS) vulnerability affects all applications built with Starlette (or FastAPI) that call `request.form()` on `application/x-www-form-urlencoded` requests. A single request with a very large number of fields blocks the event loop for several seconds, and a single request with a very large field forces unbounded memory allocation; in either case, parallel requests can render the service unusable. A reverse proxy that enforces a request body size limit reduces but does not eliminate the exposure, since a sub-10MB body is already enough to block the event loop. ##### Mitigation Upgrade to a patched version, which forwards `max_fields` and `max_part_size` to the url-encoded parser and enforces them while parsing, raising before the oversized field or excess fields are accumulated. The defaults match `multipart/form-data` (`max_fields=1000`, `max_part_size=1MB`) and can be customized via `request.form(max_fields=..., max_part_size=...)`. #### Severity - CVSS Score: 7.5 / 10 (High) - Vector String: `CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H` #### References - [https://github.com/Kludex/starlette/security/advisories/GHSA-82w8-qh3p-5jfq](https://redirect.github.com/Kludex/starlette/security/advisories/GHSA-82w8-qh3p-5jfq) - [https://github.com/Kludex/starlette](https://redirect.github.com/Kludex/starlette) This data is provided by [OSV](https://osv.dev/vulnerability/GHSA-82w8-qh3p-5jfq) and the [GitHub Advisory Database](https://redirect.github.com/github/advisory-database) ([CC-BY 4.0](https://redirect.github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Release Notes
Kludex/starlette (starlette) ### [`v1.3.1`](https://redirect.github.com/Kludex/starlette/releases/tag/1.3.1): Version 1.3.1 [Compare Source](https://redirect.github.com/Kludex/starlette/compare/1.3.0...1.3.1) #### What's Changed - Use `StarletteDeprecationWarning` instead of `DeprecationWarning` by [@​Kludex](https://redirect.github.com/Kludex) in [#​3119](https://redirect.github.com/Kludex/starlette/pull/3119) - Enforce `max_fields` and `max_part_size` in `FormParser` by [@​Kludex](https://redirect.github.com/Kludex) in [#​3329](https://redirect.github.com/Kludex/starlette/pull/3329) - Enforce `FormParser` limits in parser callbacks by [@​Kludex](https://redirect.github.com/Kludex) in [#​3331](https://redirect.github.com/Kludex/starlette/pull/3331) **Full Changelog**: ### [`v1.3.0`](https://redirect.github.com/Kludex/starlette/releases/tag/1.3.0): Version 1.3.0 [Compare Source](https://redirect.github.com/Kludex/starlette/compare/1.2.1...1.3.0) #### What's Changed - Clamp oversized suffix ranges in `FileResponse` by [@​jiyujie2006](https://redirect.github.com/jiyujie2006) in [#​3307](https://redirect.github.com/Kludex/starlette/pull/3307) - Catch `OSError` alongside `MultiPartException` when closing temp files by [@​N3XT3R1337](https://redirect.github.com/N3XT3R1337) in [#​3191](https://redirect.github.com/Kludex/starlette/pull/3191) - Add `httpx2` to the `full` extra by [@​Kludex](https://redirect.github.com/Kludex) in [#​3323](https://redirect.github.com/Kludex/starlette/pull/3323) - Adjust testclient typing and warnings by [@​waketzheng](https://redirect.github.com/waketzheng) in [#​3322](https://redirect.github.com/Kludex/starlette/pull/3322) - Fix IndexError in URL.replace() on a URL with no authority by [@​LeSingh1](https://redirect.github.com/LeSingh1) in [#​3317](https://redirect.github.com/Kludex/starlette/pull/3317) - Annotate URLPath protocol parameter with Literal by [@​Chang-LeHung](https://redirect.github.com/Chang-LeHung) in [#​3285](https://redirect.github.com/Kludex/starlette/pull/3285) - avoid collapsing exception groups from user code by [@​graingert](https://redirect.github.com/graingert) in [#​2830](https://redirect.github.com/Kludex/starlette/pull/2830) - Use `removeprefix` to strip weak ETag indicator in `is_not_modified` by [@​gnosyslambda](https://redirect.github.com/gnosyslambda) in [#​3193](https://redirect.github.com/Kludex/starlette/pull/3193) - Build `request.url` from structured components by [@​Kludex](https://redirect.github.com/Kludex) in [#​3326](https://redirect.github.com/Kludex/starlette/pull/3326) #### New Contributors - [@​jiyujie2006](https://redirect.github.com/jiyujie2006) made their first contribution in [#​3307](https://redirect.github.com/Kludex/starlette/pull/3307) - [@​N3XT3R1337](https://redirect.github.com/N3XT3R1337) made their first contribution in [#​3191](https://redirect.github.com/Kludex/starlette/pull/3191) - [@​leestana01](https://redirect.github.com/leestana01) made their first contribution in [#​3319](https://redirect.github.com/Kludex/starlette/pull/3319) - [@​LeSingh1](https://redirect.github.com/LeSingh1) made their first contribution in [#​3317](https://redirect.github.com/Kludex/starlette/pull/3317) - [@​EmmanuelNiyonshuti](https://redirect.github.com/EmmanuelNiyonshuti) made their first contribution in [#​3204](https://redirect.github.com/Kludex/starlette/pull/3204) - [@​Chang-LeHung](https://redirect.github.com/Chang-LeHung) made their first contribution in [#​3285](https://redirect.github.com/Kludex/starlette/pull/3285) - [@​gnosyslambda](https://redirect.github.com/gnosyslambda) made their first contribution in [#​3193](https://redirect.github.com/Kludex/starlette/pull/3193) **Full Changelog**:
--- ### Configuration 📅 **Schedule**: (UTC) - Branch creation - At any time (no schedule defined) - Automerge - At any time (no schedule defined) 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR was generated by [Mend Renovate](https://mend.io/renovate/). View the [repository job log](https://developer.mend.io/github/vortex-data/vortex). Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 0d325a5287e..bfc71cb994e 100644 --- a/uv.lock +++ b/uv.lock @@ -1771,15 +1771,15 @@ wheels = [ [[package]] name = "starlette" -version = "1.2.1" +version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/44/ec35f1b6e83094b997da438a02c8c9b0ade2b1e84cfc48bd4656780760a6/starlette-1.2.1.tar.gz", hash = "sha256:9b9b5ebb992e67d6093741e63c2f59e4f6fff986f81163c087867bd7b924b3f6", size = 2701854, upload-time = "2026-05-31T01:07:51.847Z" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1c/54/196d0c1db10af76baa4f64894448505d60d3cdf70ef92cbb35f46a4e4c71/starlette-1.2.1-py3-none-any.whl", hash = "sha256:4de0082d08c8f6764a85a54cf1120d6939507a19905c7768acad2a9f875d2b89", size = 73350, upload-time = "2026-05-31T01:07:50.09Z" }, + { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] [[package]] From ddd03a09814c1c7b5e7e49452df9d57e179a410f Mon Sep 17 00:00:00 2001 From: Robert Kruszewski Date: Wed, 17 Jun 2026 12:32:46 +0100 Subject: [PATCH 037/162] Upgrade pyO3 to 0.29.0 (#8462) This lets us get over the security advisory Signed-off-by: Robert Kruszewski --- Cargo.lock | 37 +++++++++++++++++----------------- Cargo.toml | 6 +++--- deny.toml | 10 +-------- vortex-python/src/dtype/mod.rs | 2 +- 4 files changed, 23 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f0965f5a8e..5bedb6481c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6841,9 +6841,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ "chrono", "indexmap 2.14.0", @@ -6857,9 +6857,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e7364a95bf00e8377bbf9b0f09d7ff9715a29d8fcf93b47d1a967363b973178" +checksum = "b3ef68daa7316a3fac65e5e18b2203f010346de1c1c53456811a2624673ab046" dependencies = [ "futures-channel", "futures-util", @@ -6871,18 +6871,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-bytes" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d98190ac733bf1c9a3bdbd2856d85abee243f16b7ef1155feefcd613c8704a1" +checksum = "48cdf1b63af76396b70e0eca2c0d8f157c73ee85b873ec4ae7c01f8e71eef112" dependencies = [ "bytes", "pyo3", @@ -6890,9 +6890,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -6900,9 +6900,9 @@ dependencies = [ [[package]] name = "pyo3-log" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c2ec80932c5c3b2d4fbc578c9b56b2d4502098587edb8bef5b6bfcad43682e" +checksum = "f64083bd3a16a353d9d62335808e8e13d0552d2a2b83fdb084496192dcfa9fcd" dependencies = [ "arc-swap", "log", @@ -6911,9 +6911,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -6923,22 +6923,21 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.28.3" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn 2.0.117", ] [[package]] name = "pyo3-object_store" -version = "0.9.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3056dc8a77db5d44d32e7f740db0b01e5b65a43d181487a6917576959c6eb9a" +checksum = "13aaa9b876f9b02773cc18fd03dd359bddf3ca21978068024511f166c9c0c6a5" dependencies = [ "async-trait", "bytes", diff --git a/Cargo.toml b/Cargo.toml index a587ef1db16..b700b72195c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -209,10 +209,10 @@ proc-macro2 = "1.0.95" prost = "0.14" prost-build = "0.14" prost-types = "0.14" -pyo3 = { version = "0.28.0" } -pyo3-bytes = "0.6" +pyo3 = { version = "0.29.0" } +pyo3-bytes = "0.7" pyo3-log = "0.13.0" -pyo3-object_store = "0.9.0" +pyo3-object_store = "0.11.0" quote = "1.0.44" rand = "0.10.1" rand_distr = "0.6" diff --git a/deny.toml b/deny.toml index 1e2443c1fd8..b9f9c79a796 100644 --- a/deny.toml +++ b/deny.toml @@ -16,15 +16,7 @@ ignore = [ # Paste is no longer maintained because its essentially "finished". "RUSTSEC-2024-0436", # proc-macro-error-2 is unmaintained, only used by the `test_with` test dependency - "RUSTSEC-2026-0173", - # Out-of-bounds read in `nth`/`nth_back` on pyo3 list/tuple iterators, fixed only in pyo3 - # 0.29.0. We cannot bump until pyo3-bytes, pyo3-log, and pyo3-object_store support 0.29 (all - # pin pyo3 to <0.29, and pyo3-ffi `links = "python"` forbids two pyo3 versions in the graph). - # Not exploitable here: `vortex-python` never calls `nth`/`nth_back` on these iterators. - "RUSTSEC-2026-0176", - # Another issue that will be fixed once we can bump pyo3 to 0.29. - # PyCFunction::new_closure is missing a `Sync` bound. - "RUSTSEC-2026-0177" + "RUSTSEC-2026-0173" ] [licenses] diff --git a/vortex-python/src/dtype/mod.rs b/vortex-python/src/dtype/mod.rs index f84059005aa..bea25c133c1 100644 --- a/vortex-python/src/dtype/mod.rs +++ b/vortex-python/src/dtype/mod.rs @@ -194,7 +194,7 @@ impl PyDType { #[classmethod] #[pyo3(signature = (arrow_dtype, *, non_nullable = false))] fn from_arrow<'py>( - cls: &'py Bound<'py, PyType>, + cls: &Bound<'py, PyType>, #[pyo3(from_py_with = import_arrow_dtype)] arrow_dtype: DataType, non_nullable: bool, ) -> PyResult> { From b2c9e7989f5fc1d556634fa13bca846cf67fb9bc Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 12:35:44 +0100 Subject: [PATCH 038/162] Faster str interning (#8459) ## Summary Makes two small changes to the string interner: 1. Enables the `inline-more` feature of the `lasso` crate 2. Use the faster hasher we try and use everywhere Signed-off-by: Adam Gutglick --- Cargo.toml | 2 +- vortex-session/src/registry.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index b700b72195c..e551def0303 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,7 +174,7 @@ jetscii = "0.5.3" jiff = "0.2.0" jni = { version = "0.22.0" } kanal = "0.1.1" -lasso = { version = "0.7", features = ["multi-threaded"] } +lasso = { version = "0.7", features = ["multi-threaded", "inline-more"] } lending-iterator = "0.1.7" libfuzzer-sys = "0.4" libloading = "0.8" diff --git a/vortex-session/src/registry.rs b/vortex-session/src/registry.rs index 36c03828844..50bbb8c981e 100644 --- a/vortex-session/src/registry.rs +++ b/vortex-session/src/registry.rs @@ -19,10 +19,12 @@ use lasso::Spur; use lasso::ThreadedRodeo; use parking_lot::RwLock; use vortex_error::VortexExpect; +use vortex_utils::aliases::DefaultHashBuilder; use vortex_utils::aliases::dash_map::DashMap; /// Global string interner for [`Id`] values. -static INTERNER: LazyLock = LazyLock::new(ThreadedRodeo::new); +static INTERNER: LazyLock> = + LazyLock::new(|| ThreadedRodeo::with_hasher(DefaultHashBuilder::default())); /// A lightweight, copyable identifier backed by a global string interner. /// From 1726f5e418a9b1063c4ba57e570499a1fba06243 Mon Sep 17 00:00:00 2001 From: Alexander Droste Date: Wed, 17 Jun 2026 13:52:50 +0200 Subject: [PATCH 039/162] test: expand cuDF CUDA e2e coverage (#8463) Expand the CUDA cuDF end-to-end harness with boolean columns, sliced primitive and boolean columns, and nullable timestamp millisecond coverage. Signed-off-by: Alexander Droste --- vortex-cuda/README.md | 5 +++ vortex-test/e2e-cuda/src/lib.rs | 67 +++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/vortex-cuda/README.md b/vortex-cuda/README.md index 24b8f420290..d71ad593ce7 100644 --- a/vortex-cuda/README.md +++ b/vortex-cuda/README.md @@ -27,10 +27,15 @@ cmake -S cpp -B cpp/build \ -DBUILD_TESTS=ON \ -DDISABLE_DEPRECATION_WARNINGS=ON \ -DCMAKE_BUILD_TYPE=Debug \ + -DCUDF_BUILD_STATIC_DEPS=OFF \ -DCUDF_BUILD_STREAMS_TEST_UTIL=OFF \ -DCUDAToolkit_ROOT=/usr/local/cuda \ -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \ -DCMAKE_C_COMPILER=gcc \ -DCMAKE_CXX_COMPILER=g++ \ + # In large AArch64 Debug links, CALL26/JUMP26 relocations can exceed their branch range. + # Use 1 MiB stub groups so GNU ld emits veneers close enough to each relocation site. + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--stub-group-size=1048576" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--stub-group-size=1048576" \ -GNinja && cmake --build cpp/build --target INTEROP_TEST --parallel ``` diff --git a/vortex-test/e2e-cuda/src/lib.rs b/vortex-test/e2e-cuda/src/lib.rs index 7fa001c9639..89b167a44c8 100644 --- a/vortex-test/e2e-cuda/src/lib.rs +++ b/vortex-test/e2e-cuda/src/lib.rs @@ -13,12 +13,15 @@ use std::sync::LazyLock; use arrow_array::Array; use arrow_array::ArrayRef as ArrowArrayRef; +use arrow_array::BooleanArray; use arrow_array::Date32Array; use arrow_array::Decimal32Array; use arrow_array::Decimal64Array; use arrow_array::Decimal128Array; use arrow_array::DictionaryArray; +use arrow_array::Int32Array; use arrow_array::StringArray; +use arrow_array::TimestampMillisecondArray; use arrow_array::cast::AsArray; use arrow_array::ffi::FFI_ArrowArray; use arrow_array::ffi::from_ffi; @@ -32,6 +35,7 @@ use futures::executor::block_on; use vortex::array::ArrayRef as VortexArrayRef; use vortex::array::IntoArray; use vortex::array::VortexSessionExecute; +use vortex::array::arrays::BoolArray; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::DictArray as VortexDictArray; use vortex::array::arrays::FixedSizeListArray; @@ -100,6 +104,42 @@ fn primitive_array() -> Result { }) } +fn bool_array() -> VortexArrayRef { + BoolArray::from_iter([true, false, false, true, true]).into_array() +} + +fn sliced_i32_array() -> VortexArrayRef { + PrimitiveArray::from_option_iter([ + Some(-999i32), + Some(10), + None, + Some(30), + Some(40), + None, + Some(999), + ]) + .into_array() + .slice(1..6) + .expect("sliced i32 array") +} + +fn sliced_bool_array() -> VortexArrayRef { + BoolArray::from_iter([true, false, true, true, false, true, false]) + .into_array() + .slice(1..6) + .expect("sliced bool array") +} + +fn timestamp_ms_array() -> VortexArrayRef { + TemporalArray::new_timestamp( + PrimitiveArray::from_option_iter([Some(1_000i64), None, Some(3_000), Some(4_000), None]) + .into_array(), + TimeUnit::Milliseconds, + None, + ) + .into_array() +} + fn list_array() -> VortexArrayRef { ListArray::try_new( PrimitiveArray::from_iter([10i32, 11, 12, 13, 14]).into_array(), @@ -236,6 +276,9 @@ fn export_array_inner(schema_ptr: &mut FFI_ArrowSchema, array_ptr: &mut ArrowDev let array = StructArray::new( FieldNames::from_iter([ "prims", + "bools", + "sliced_i32", + "sliced_bools", "decimal32", "decimal64", "decimal128", @@ -246,12 +289,16 @@ fn export_array_inner(schema_ptr: &mut FFI_ArrowSchema, array_ptr: &mut ArrowDev // Arrow Device import path rejects NANOARROW_TYPE_BINARY, and treating arbitrary // bytes as strings would be semantically incorrect. "dates", + "timestamp_ms", "dictionary", "lists", "fixed_lists", ]), vec![ primitive, + bool_array(), + sliced_i32_array(), + sliced_bool_array(), decimal32.into_array(), decimal64.into_array(), decimal128.into_array(), @@ -259,6 +306,7 @@ fn export_array_inner(schema_ptr: &mut FFI_ArrowSchema, array_ptr: &mut ArrowDev sliced_utf8_array(), multi_buffer_utf8_array(), dates.into_array(), + timestamp_ms_array(), dictionary_array(), list_array(), fixed_size_list_array(), @@ -327,6 +375,9 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA &mut SESSION.create_execution_ctx(), ) .expect("expected primitive Arrow array"); + let bools = BooleanArray::from(vec![true, false, false, true, true]); + let sliced_i32 = Int32Array::from(vec![Some(10), None, Some(30), Some(40), None]); + let sliced_bools = BooleanArray::from(vec![false, true, true, false, true]); let decimal32 = Decimal32Array::from_iter([Some(0i32), Some(1), None, Some(3), Some(4)]) // cuDF stores decimals using the maximum precision for the physical width and preserves scale. .with_precision_and_scale(9, 2) @@ -359,6 +410,8 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA Some("short"), ]); let date = Date32Array::from(vec![Some(100i32), None, Some(300), Some(400), None]); + let timestamp_ms = + TimestampMillisecondArray::from(vec![Some(1_000i64), None, Some(3_000), Some(4_000), None]); let dictionary = Arc::new( vec![ Some("apple"), @@ -385,6 +438,9 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA let expected_fields = Fields::from_iter([ Field::new("prims", primitive.data_type().clone(), true), + Field::new("bools", bools.data_type().clone(), false), + Field::new("sliced_i32", sliced_i32.data_type().clone(), true), + Field::new("sliced_bools", sliced_bools.data_type().clone(), false), Field::new("decimal32", decimal32.data_type().clone(), true), Field::new("decimal64", decimal64.data_type().clone(), true), Field::new("decimal128", decimal128.data_type().clone(), true), @@ -396,6 +452,7 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA false, ), Field::new("dates", date.data_type().clone(), true), + Field::new("timestamp_ms", timestamp_ms.data_type().clone(), true), Field::new("dictionary", dictionary.data_type().clone(), true), cudf_list_field("lists"), cudf_list_field("fixed_lists"), @@ -407,8 +464,11 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA return 1; } - let expected_arrays: [ArrowArrayRef; 9] = [ + let expected_arrays: Vec = vec![ primitive, + Arc::new(bools), + Arc::new(sliced_i32), + Arc::new(sliced_bools), Arc::new(decimal32), Arc::new(decimal64), Arc::new(decimal128), @@ -416,6 +476,7 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA Arc::new(sliced_utf8), Arc::new(multi_buffer_utf8), Arc::new(date), + Arc::new(timestamp_ms), dictionary, ]; @@ -430,11 +491,11 @@ fn validate_array_inner(ffi_schema: &FFI_ArrowSchema, ffi_array: &mut FFI_ArrowA } } - if !list_values_eq(list.as_ref(), struct_array.column(9).as_ref()) { + if !list_values_eq(list.as_ref(), struct_array.column(13).as_ref()) { eprintln!("wrong values for lists column"); return 1; } - if !list_values_eq(fixed_size_list.as_ref(), struct_array.column(10).as_ref()) { + if !list_values_eq(fixed_size_list.as_ref(), struct_array.column(14).as_ref()) { eprintln!("wrong values for fixed_lists column"); return 1; } From 679e2c5b5f1e38e1cc23997a7608860d1f5034c2 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 13:12:09 +0100 Subject: [PATCH 040/162] Suppress TSAN false positives in crossbeam crate due to use of memory fences (#8464) ## Summary This PR adds a few additional ThreadSanitizer suppressions to deal with false positives caused by `crossbeam`. They have been surfaced by #8454 and in other cases in the past. Signed-off-by: Adam Gutglick --- vortex-ffi/tsan_suppressions.txt | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vortex-ffi/tsan_suppressions.txt b/vortex-ffi/tsan_suppressions.txt index ee8ac6e926d..6a2b65e351e 100644 --- a/vortex-ffi/tsan_suppressions.txt +++ b/vortex-ffi/tsan_suppressions.txt @@ -6,3 +6,23 @@ # where ordering is correct but uses an explicit fence and not relaxed load race:oneshot-*/src/channel.rs race:oneshot-*/src/lib.rs + +# https://github.com/google/sanitizers/issues/1415 +# crossbeam-epoch's epoch-based reclamation synchronizes through SeqCst fences, +# which TSan does not fully model. Its process-global collector frees a retired +# bag/node on one thread while another thread pops from the same GC queue, both +# inside `Global::collect`; the epoch protocol makes this safe but TSan reports a +# benign race. Surfaced through moka's cache (the vortex-file MultiFileSession +# footer cache) when one session's cache is dropped while another session's cache +# is queried on a parallel thread. +race:crossbeam_epoch*collect + +# Taken from crossbeam's CI - https://github.com/crossbeam-rs/crossbeam/blob/05f9478b333ead58c0bf8e5a37d9ef9bd3b5bf17/ci/tsan#L1 +# Push and steal operations in crossbeam-deque may cause data races, but such +# data races are safe. If a data race happens, the value read by `steal` is +# forgotten and the steal operation is then retried. +race:crossbeam_deque*push +race:crossbeam_deque*steal + +# Non-lock-free AtomicCell uses SeqLock which uses fences. +race:crossbeam_utils::atomic::atomic_cell::atomic_compare_exchange_weak From 9f494a1f0fc5980f1568c684b212214e6504a1bf Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 13:20:27 +0100 Subject: [PATCH 041/162] Immutable session backed by `HashMap` (#8454) ## Summary This PR replaces the current `DashMap` backed session with one backed by a `HashMap`. Instead of being initialized on access which might cause deadlocks or otherwise weird performance behavior. There's also an explicit `Builder` type for the session, allowing the user to explicitly include the components they want. --------- Signed-off-by: Adam Gutglick --- Cargo.lock | 1 - docs/developer-guide/internals/session.md | 5 +- encodings/alp/benches/alp_compress.rs | 4 +- encodings/alp/src/alp/array.rs | 4 +- encodings/alp/src/alp/plugin.rs | 3 +- encodings/bytebool/src/array.rs | 4 +- encodings/datetime-parts/src/canonical.rs | 3 +- .../experimental/onpair/benches/decode.rs | 4 +- .../onpair/src/compute/compare.rs | 4 +- encodings/experimental/onpair/src/tests.rs | 4 +- .../experimental/onpair/tests/big_data.rs | 4 +- .../fastlanes/benches/bitpack_compare.rs | 4 +- .../benches/bitpack_compare_sweep.rs | 4 +- .../fastlanes/benches/bitpacking_take.rs | 4 +- .../fastlanes/benches/canonicalize_bench.rs | 4 +- encodings/fastlanes/benches/cast_bitpacked.rs | 4 +- .../fastlanes/benches/compute_between.rs | 4 +- .../src/bitpacking/array/bitpack_compress.rs | 4 +- .../bitpacking/array/bitpack_decompress.rs | 4 +- .../fastlanes/src/bitpacking/array/mod.rs | 4 +- .../src/bitpacking/compute/compare.rs | 4 +- encodings/fastlanes/src/bitpacking/plugin.rs | 3 +- .../src/delta/array/delta_compress.rs | 4 +- encodings/fastlanes/src/delta/array/mod.rs | 4 +- encodings/fastlanes/src/delta/compute/cast.rs | 4 +- .../fastlanes/src/delta/vtable/operations.rs | 4 +- .../fastlanes/src/for/array/for_compress.rs | 4 +- encodings/fastlanes/src/lib.rs | 2 +- encodings/fastlanes/src/rle/compute/cast.rs | 4 +- .../fsst/benches/chunked_dict_fsst_builder.rs | 4 +- encodings/fsst/benches/fsst_compress.rs | 4 +- encodings/fsst/benches/fsst_like.rs | 4 +- encodings/fsst/benches/fsst_url_compare.rs | 4 +- encodings/fsst/src/canonical.rs | 4 +- encodings/fsst/src/compute/cast.rs | 4 +- encodings/fsst/src/compute/like.rs | 4 +- encodings/fsst/src/dfa/tests.rs | 4 +- encodings/fsst/src/kernel.rs | 4 +- encodings/parquet-variant/src/arrow.rs | 3 +- encodings/parquet-variant/src/vtable.rs | 6 +- encodings/pco/src/compute/cast.rs | 4 +- encodings/pco/src/tests.rs | 3 +- encodings/runend/benches/run_end_compress.rs | 4 +- encodings/runend/benches/run_end_decode.rs | 4 +- .../runend/benches/run_end_null_count.rs | 4 +- encodings/runend/src/array.rs | 4 +- encodings/runend/src/arrow.rs | 3 +- encodings/runend/src/compute/cast.rs | 4 +- encodings/runend/src/compute/take_from.rs | 11 +- encodings/sequence/src/compute/cast.rs | 4 +- encodings/sparse/benches/sparse_canonical.rs | 4 +- encodings/sparse/benches/sparse_pushdown.rs | 3 +- encodings/sparse/src/compute/between.rs | 3 +- encodings/sparse/src/compute/cast.rs | 4 +- encodings/sparse/src/compute/compare.rs | 3 +- encodings/sparse/src/compute/fill_null.rs | 3 +- encodings/sparse/src/compute/is_constant.rs | 5 +- encodings/sparse/src/compute/min_max.rs | 5 +- encodings/sparse/src/compute/nan_count.rs | 5 +- encodings/sparse/src/compute/null_count.rs | 5 +- encodings/sparse/src/compute/sum.rs | 5 +- encodings/zstd/benches/listview_rebuild.rs | 4 +- encodings/zstd/src/compute/cast.rs | 4 +- fuzz/src/fsst_like.rs | 4 +- fuzz/src/lib.rs | 2 +- vortex-array/benches/aggregate_max.rs | 4 +- vortex-array/benches/aggregate_sum.rs | 4 +- vortex-array/benches/binary_ops.rs | 4 +- vortex-array/benches/cast_primitive.rs | 4 +- vortex-array/benches/chunk_array_builder.rs | 4 +- vortex-array/benches/chunked_dict_builder.rs | 4 +- .../benches/chunked_fsl_canonicalize.rs | 4 +- vortex-array/benches/compare.rs | 5 +- vortex-array/benches/dict_compare.rs | 14 +- vortex-array/benches/dict_compress.rs | 4 +- vortex-array/benches/dict_mask.rs | 3 +- vortex-array/benches/expr/case_when_bench.rs | 4 +- vortex-array/benches/filter_bool.rs | 4 +- vortex-array/benches/interleave.rs | 5 +- vortex-array/benches/listview_rebuild.rs | 4 +- vortex-array/benches/scalar_at_struct.rs | 4 +- vortex-array/benches/scalar_subtract.rs | 4 +- vortex-array/benches/take_filter.rs | 4 +- vortex-array/benches/take_fsl.rs | 4 +- vortex-array/benches/take_patches.rs | 4 +- vortex-array/benches/take_primitive.rs | 4 +- vortex-array/benches/take_struct.rs | 4 +- vortex-array/benches/to_arrow.rs | 4 +- vortex-array/benches/varbinview_zip.rs | 4 +- vortex-array/src/aggregate_fn/accumulator.rs | 4 +- .../src/aggregate_fn/fns/bounded_max/mod.rs | 3 +- .../src/aggregate_fn/fns/bounded_min/mod.rs | 3 +- vortex-array/src/aggregate_fn/proto.rs | 7 +- vortex-array/src/aggregate_fn/session.rs | 5 +- vortex-array/src/arc_swap_map.rs | 28 +- vortex-array/src/arrays/bool/compute/cast.rs | 4 +- vortex-array/src/arrays/chunked/tests.rs | 4 +- .../src/arrays/chunked/vtable/canonical.rs | 13 +- .../src/arrays/constant/vtable/canonical.rs | 4 +- vortex-array/src/arrays/dict/compute/cast.rs | 4 +- .../src/arrays/dict/compute/min_max.rs | 4 +- vortex-array/src/arrays/dict/compute/mod.rs | 8 +- .../src/arrays/extension/compute/cast.rs | 4 +- .../src/arrays/extension/compute/rules.rs | 4 +- .../src/arrays/filter/execute/listview.rs | 4 +- .../src/arrays/filter/execute/take/tests.rs | 25 +- vortex-array/src/arrays/list/compute/cast.rs | 4 +- vortex-array/src/arrays/list/tests.rs | 4 +- .../src/arrays/listview/tests/common.rs | 4 +- .../src/arrays/listview/tests/density.rs | 4 +- .../src/arrays/listview/tests/filter.rs | 4 +- .../src/arrays/listview/tests/take.rs | 4 +- .../src/arrays/primitive/array/cast.rs | 4 +- vortex-array/src/arrays/shared/tests.rs | 4 +- .../src/arrays/struct_/compute/cast.rs | 7 +- .../src/arrays/struct_/compute/rules.rs | 3 +- .../src/arrays/varbin/compute/cast.rs | 4 +- .../src/arrays/varbinview/compute/cast.rs | 4 +- vortex-array/src/arrow/executor/list.rs | 4 +- vortex-array/src/arrow/executor/list_view.rs | 4 +- vortex-array/src/arrow/executor/run_end.rs | 4 +- vortex-array/src/arrow/session.rs | 7 +- vortex-array/src/builders/dict/bytes.rs | 4 +- vortex-array/src/builders/dict/primitive.rs | 4 +- vortex-array/src/canonical.rs | 4 +- vortex-array/src/dtype/mod.rs | 5 +- vortex-array/src/dtype/serde/proto.rs | 4 +- vortex-array/src/dtype/session.rs | 7 +- vortex-array/src/executor.rs | 8 +- vortex-array/src/lib.rs | 30 +- vortex-array/src/memory.rs | 15 +- vortex-array/src/optimizer/kernels.rs | 8 +- vortex-array/src/optimizer/mod.rs | 1 - vortex-array/src/scalar/tests/mod.rs | 5 +- vortex-array/src/scalar_fn/fns/between/mod.rs | 4 +- vortex-array/src/scalar_fn/fns/case_when.rs | 4 +- vortex-array/src/scalar_fn/session.rs | 5 +- vortex-array/src/session/mod.rs | 5 +- vortex-array/src/stats/expr.rs | 4 +- vortex-array/src/stats/rewrite.rs | 10 +- vortex-array/src/stats/rewrite/builtins.rs | 4 +- vortex-array/src/stats/session.rs | 9 +- vortex-btrblocks/benches/compress.rs | 4 +- vortex-btrblocks/benches/compress_listview.rs | 4 +- vortex-btrblocks/src/canonical_compressor.rs | 4 +- .../schemes/float/scheme_selection_tests.rs | 4 +- vortex-btrblocks/src/schemes/float/tests.rs | 4 +- .../schemes/integer/scheme_selection_tests.rs | 4 +- vortex-btrblocks/src/schemes/integer/tests.rs | 4 +- .../schemes/string/scheme_selection_tests.rs | 4 +- vortex-btrblocks/src/schemes/string/tests.rs | 4 +- vortex-btrblocks/tests/onpair_roundtrip.rs | 4 +- vortex-compressor/benches/dict_encode.rs | 4 +- vortex-compressor/benches/stats_calc.rs | 4 +- vortex-compressor/src/builtins/dict/float.rs | 6 +- .../src/builtins/dict/integer.rs | 6 +- vortex-compressor/src/compressor.rs | 4 +- vortex-cuda/benches/alp_cuda.rs | 3 +- vortex-cuda/benches/arrow_binary_cuda.rs | 8 +- vortex-cuda/benches/arrow_validity_cuda.rs | 8 +- vortex-cuda/benches/bitpacked_cuda.rs | 12 +- vortex-cuda/benches/date_time_parts_cuda.rs | 10 +- vortex-cuda/benches/dict_cuda.rs | 10 +- vortex-cuda/benches/dynamic_dispatch_cuda.rs | 45 ++- vortex-cuda/benches/filter_cuda.rs | 3 +- vortex-cuda/benches/for_cuda.rs | 19 +- vortex-cuda/benches/fsst_cuda.rs | 10 +- vortex-cuda/benches/list_view_cuda.rs | 15 +- vortex-cuda/benches/runend_cuda.rs | 6 +- vortex-cuda/benches/throughput_cuda.rs | 3 +- vortex-cuda/benches/zstd_cuda.rs | 8 +- vortex-cuda/ffi/src/lib.rs | 1 - vortex-cuda/src/arrow/canonical.rs | 105 +++--- vortex-cuda/src/dynamic_dispatch/mod.rs | 125 ++++--- vortex-cuda/src/executor.rs | 2 +- vortex-cuda/src/hybrid_dispatch/mod.rs | 15 +- vortex-cuda/src/kernel/arrays/constant.rs | 7 +- vortex-cuda/src/kernel/arrays/dict.rs | 45 ++- vortex-cuda/src/kernel/encodings/alp.rs | 15 +- vortex-cuda/src/kernel/encodings/bitpacked.rs | 21 +- .../src/kernel/encodings/date_time_parts.rs | 7 +- .../kernel/encodings/decimal_byte_parts.rs | 3 +- vortex-cuda/src/kernel/encodings/for_.rs | 5 +- vortex-cuda/src/kernel/encodings/fsst.rs | 5 +- vortex-cuda/src/kernel/encodings/runend.rs | 11 +- vortex-cuda/src/kernel/encodings/sequence.rs | 3 +- vortex-cuda/src/kernel/encodings/zigzag.rs | 3 +- vortex-cuda/src/kernel/encodings/zstd.rs | 9 +- .../src/kernel/encodings/zstd_buffers.rs | 9 +- vortex-cuda/src/kernel/filter/decimal.rs | 5 +- vortex-cuda/src/kernel/filter/primitive.rs | 5 +- vortex-cuda/src/kernel/filter/varbinview.rs | 3 +- vortex-cuda/src/kernel/patches/mod.rs | 3 +- vortex-cuda/src/kernel/patches/types.rs | 5 +- vortex-cuda/src/lib.rs | 14 + vortex-cuda/src/session.rs | 3 +- vortex-cuda/src/stream.rs | 5 +- vortex-datafusion/src/persistent/opener.rs | 4 +- vortex-file/src/multi/mod.rs | 1 + vortex-file/src/multi/session.rs | 9 +- vortex-file/src/open.rs | 21 +- vortex-file/src/tests.rs | 6 +- vortex-file/src/v2/file_stats_reader.rs | 6 +- vortex-file/tests/test_write_table.rs | 6 +- vortex-geo/src/extension/coordinate.rs | 4 +- vortex-geo/src/extension/point.rs | 4 +- vortex-geo/src/scalar_fn/distance.rs | 10 +- vortex-geo/src/tests/mod.rs | 3 +- vortex-io/src/session.rs | 16 +- vortex-ipc/src/lib.rs | 3 +- vortex-json/src/arrow.rs | 5 +- vortex-layout/src/layouts/dict/reader.rs | 6 +- vortex-layout/src/layouts/zoned/reader.rs | 6 +- vortex-layout/src/scan/test.rs | 6 +- vortex-layout/src/session.rs | 5 +- vortex-layout/src/test.rs | 6 +- vortex-python/src/arrays/mod.rs | 10 +- vortex-row/benches/row_encode.rs | 4 +- vortex-session/Cargo.toml | 1 - vortex-session/src/immutable.rs | 308 ++++++++++++++++++ vortex-session/src/lib.rs | 224 ++----------- vortex-tensor/src/lib.rs | 3 +- .../arrays/synthetic/encodings/patched.rs | 3 +- vortex-test/e2e-cuda/src/lib.rs | 6 +- .../common_encoding_tree_throughput.rs | 4 +- vortex/benches/single_encoding_throughput.rs | 4 +- vortex/src/lib.rs | 8 +- 227 files changed, 951 insertions(+), 1111 deletions(-) create mode 100644 vortex-session/src/immutable.rs diff --git a/Cargo.lock b/Cargo.lock index 5bedb6481c0..717125c0565 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10192,7 +10192,6 @@ dependencies = [ name = "vortex-session" version = "0.1.0" dependencies = [ - "dashmap", "lasso", "parking_lot", "vortex-error", diff --git a/docs/developer-guide/internals/session.md b/docs/developer-guide/internals/session.md index 5aff27845af..823183ae7cc 100644 --- a/docs/developer-guide/internals/session.md +++ b/docs/developer-guide/internals/session.md @@ -99,9 +99,10 @@ For tests or specialized use-cases, sessions can be assembled from individual co the `.with::()` builder: ```rust -let session = VortexSession::empty() +let session = VortexSession::builder() .with::() .with::() .with::() - .with::(); + .with::() + .build(); ``` diff --git a/encodings/alp/benches/alp_compress.rs b/encodings/alp/benches/alp_compress.rs index 70a566ab0f8..f70f69c4177 100644 --- a/encodings/alp/benches/alp_compress.rs +++ b/encodings/alp/benches/alp_compress.rs @@ -19,7 +19,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; @@ -51,8 +50,7 @@ const BENCH_ARGS: &[(usize, f64, f64)] = &[ (10_000, 0.1, 1.0), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench(types = [f32, f64], args = BENCH_ARGS)] fn compress_alp(bencher: Bencher, args: (usize, f64, f64)) { diff --git a/encodings/alp/src/alp/array.rs b/encodings/alp/src/alp/array.rs index 3f7c2ca3e6d..c85c0ea1b44 100644 --- a/encodings/alp/src/alp/array.rs +++ b/encodings/alp/src/alp/array.rs @@ -485,7 +485,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_session::VortexSession; @@ -493,8 +492,7 @@ mod tests { use crate::alp_encode; use crate::decompress_into_array; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[rstest] #[case(0)] diff --git a/encodings/alp/src/alp/plugin.rs b/encodings/alp/src/alp/plugin.rs index 25266bef821..c14133109d1 100644 --- a/encodings/alp/src/alp/plugin.rs +++ b/encodings/alp/src/alp/plugin.rs @@ -98,7 +98,6 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::PatchedArraySlotsExt; use vortex_array::buffer::BufferHandle; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_error::VortexResult; use vortex_error::vortex_err; @@ -111,7 +110,7 @@ mod tests { use crate::alp_encode; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(ALPPatchedPlugin); session }); diff --git a/encodings/bytebool/src/array.rs b/encodings/bytebool/src/array.rs index 722f48f7376..225bad39ded 100644 --- a/encodings/bytebool/src/array.rs +++ b/encodings/bytebool/src/array.rs @@ -322,10 +322,8 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::ByteBufferMut; - use vortex_session::VortexSession; use vortex_session::registry::ReadContext; use super::*; @@ -367,7 +365,7 @@ mod tests { let array = ByteBool::from_option_vec(vec![Some(true), None, Some(false), None]); let dtype = array.dtype().clone(); let len = array.len(); - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(ByteBool); let ctx = ArrayContext::empty(); diff --git a/encodings/datetime-parts/src/canonical.rs b/encodings/datetime-parts/src/canonical.rs index 3a46eb6f328..eee60bbe913 100644 --- a/encodings/datetime-parts/src/canonical.rs +++ b/encodings/datetime-parts/src/canonical.rs @@ -109,7 +109,6 @@ mod test { use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::DateTimeParts; use crate::array::DateTimePartsArraySlotsExt; @@ -133,7 +132,7 @@ mod test { ], validity.clone(), ); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let date_times = DateTimeParts::try_from_temporal( TemporalArray::new_timestamp( milliseconds.clone().into_array(), diff --git a/encodings/experimental/onpair/benches/decode.rs b/encodings/experimental/onpair/benches/decode.rs index ff8db8c22c2..d8623018d8c 100644 --- a/encodings/experimental/onpair/benches/decode.rs +++ b/encodings/experimental/onpair/benches/decode.rs @@ -41,7 +41,6 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::NativePType; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_buffer::ByteBuffer; use vortex_mask::Mask; @@ -80,8 +79,7 @@ impl DecodeInputs { use vortex_onpair::onpair_compress; use vortex_session::VortexSession; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[derive(Copy, Clone, Debug)] enum Shape { diff --git a/encodings/experimental/onpair/src/compute/compare.rs b/encodings/experimental/onpair/src/compute/compare.rs index c33f1f10a08..2e84411bb58 100644 --- a/encodings/experimental/onpair/src/compute/compare.rs +++ b/encodings/experimental/onpair/src/compute/compare.rs @@ -78,15 +78,13 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; - use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::compress::DEFAULT_DICT12_CONFIG; use crate::compress::onpair_compress; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[cfg_attr(miri, ignore)] #[rstest] diff --git a/encodings/experimental/onpair/src/tests.rs b/encodings/experimental/onpair/src/tests.rs index dd6fe4b0116..571240f3206 100644 --- a/encodings/experimental/onpair/src/tests.rs +++ b/encodings/experimental/onpair/src/tests.rs @@ -15,7 +15,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::match_each_integer_ptype; -use vortex_array::session::ArraySession; use vortex_array::test_harness::check_metadata; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; @@ -28,8 +27,7 @@ use crate::OnPairMetadata; use crate::compress::DEFAULT_DICT12_CONFIG; use crate::compress::onpair_compress; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn sample_input() -> VarBinArray { VarBinArray::from_iter( diff --git a/encodings/experimental/onpair/tests/big_data.rs b/encodings/experimental/onpair/tests/big_data.rs index c6bc163ea9f..9a38788378c 100644 --- a/encodings/experimental/onpair/tests/big_data.rs +++ b/encodings/experimental/onpair/tests/big_data.rs @@ -26,13 +26,11 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_onpair::DEFAULT_DICT12_CONFIG; use vortex_onpair::onpair_compress; use vortex_session::VortexSession; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn corpus(n: usize) -> Vec { let templates: &[&str] = &[ diff --git a/encodings/fastlanes/benches/bitpack_compare.rs b/encodings/fastlanes/benches/bitpack_compare.rs index fc6e67bb11d..e1e8a0e365b 100644 --- a/encodings/fastlanes/benches/bitpack_compare.rs +++ b/encodings/fastlanes/benches/bitpack_compare.rs @@ -23,7 +23,6 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Alignment; use vortex_buffer::BufferMut; @@ -36,8 +35,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const LENS: &[usize] = &[1024, 64 * 1024]; const BIT_WIDTHS: &[u8] = &[4, 16]; diff --git a/encodings/fastlanes/benches/bitpack_compare_sweep.rs b/encodings/fastlanes/benches/bitpack_compare_sweep.rs index 570fa9ceec6..037581d016b 100644 --- a/encodings/fastlanes/benches/bitpack_compare_sweep.rs +++ b/encodings/fastlanes/benches/bitpack_compare_sweep.rs @@ -27,7 +27,6 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::NativePType; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Alignment; use vortex_buffer::BufferMut; @@ -40,8 +39,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Number of elements per benchmarked array (64 full FastLanes blocks). const LEN: usize = 64 * 1024; diff --git a/encodings/fastlanes/benches/bitpacking_take.rs b/encodings/fastlanes/benches/bitpacking_take.rs index 2e01f1253e6..4d0a915aa99 100644 --- a/encodings/fastlanes/benches/bitpacking_take.rs +++ b/encodings/fastlanes/benches/bitpacking_take.rs @@ -15,7 +15,6 @@ use vortex_array::IntoArray as _; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer; @@ -27,8 +26,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn take_10_stratified(bencher: Bencher) { diff --git a/encodings/fastlanes/benches/canonicalize_bench.rs b/encodings/fastlanes/benches/canonicalize_bench.rs index 7b0ee2453f5..deea4de0032 100644 --- a/encodings/fastlanes/benches/canonicalize_bench.rs +++ b/encodings/fastlanes/benches/canonicalize_bench.rs @@ -12,7 +12,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; -use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_fastlanes::bitpack_compress::test_harness::make_array; use vortex_session::VortexSession; @@ -35,8 +34,7 @@ const BENCH_ARGS: &[(usize, usize, f64)] = &[ (10000, 1000, 0.00), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[cfg(not(codspeed))] #[divan::bench(args = BENCH_ARGS)] diff --git a/encodings/fastlanes/benches/cast_bitpacked.rs b/encodings/fastlanes/benches/cast_bitpacked.rs index 097230930a3..9552d61c632 100644 --- a/encodings/fastlanes/benches/cast_bitpacked.rs +++ b/encodings/fastlanes/benches/cast_bitpacked.rs @@ -27,7 +27,6 @@ use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_error::VortexExpect; @@ -39,8 +38,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const U32: DType = DType::Primitive(PType::U32, Nullability::NonNullable); diff --git a/encodings/fastlanes/benches/compute_between.rs b/encodings/fastlanes/benches/compute_between.rs index 5a1bdcee655..73af3cc07d7 100644 --- a/encodings/fastlanes/benches/compute_between.rs +++ b/encodings/fastlanes/benches/compute_between.rs @@ -16,7 +16,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::NativePType; -use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_fastlanes::bitpack_compress::bitpack_to_best_bit_width; use vortex_session::VortexSession; @@ -25,8 +24,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn generate_primitive_array( rng: &mut StdRng, diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs index eba5be1b749..890bf72f4a5 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_compress.rs @@ -437,7 +437,6 @@ mod test { use vortex_array::assert_arrays_eq; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; - use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_error::VortexError; use vortex_error::vortex_err; @@ -448,8 +447,7 @@ mod test { use crate::bitpack_compress::test_harness::make_array; use crate::bitpacking::array::BitPackedArrayExt; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_best_bit_width() { diff --git a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs index a4f8224966a..71e25ec8197 100644 --- a/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs +++ b/encodings/fastlanes/src/bitpacking/array/bitpack_decompress.rs @@ -210,7 +210,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::assert_arrays_eq; use vortex_array::dtype::Nullability; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -226,8 +225,7 @@ mod tests { bitpack_encode(array, bit_width, None, &mut SESSION.create_execution_ctx()).unwrap() } - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn unpack(bitpacked: &BitPackedArray) -> VortexResult { unpack_array(bitpacked.as_view(), &mut SESSION.create_execution_ctx()) diff --git a/encodings/fastlanes/src/bitpacking/array/mod.rs b/encodings/fastlanes/src/bitpacking/array/mod.rs index e5c64252fbc..03c0973ce52 100644 --- a/encodings/fastlanes/src/bitpacking/array/mod.rs +++ b/encodings/fastlanes/src/bitpacking/array/mod.rs @@ -327,15 +327,13 @@ mod test { use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_session::VortexSession; use crate::BitPackedData; use crate::bitpacking::array::BitPackedArrayExt; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_encode() { diff --git a/encodings/fastlanes/src/bitpacking/compute/compare.rs b/encodings/fastlanes/src/bitpacking/compute/compare.rs index e622e64a665..d5c50751bae 100644 --- a/encodings/fastlanes/src/bitpacking/compute/compare.rs +++ b/encodings/fastlanes/src/bitpacking/compute/compare.rs @@ -118,7 +118,6 @@ mod tests { use vortex_array::scalar_fn::fns::binary::CompareKernel; use vortex_array::scalar_fn::fns::operators::CompareOperator; use vortex_array::scalar_fn::fns::operators::Operator; - use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; @@ -126,8 +125,7 @@ mod tests { use crate::BitPackedArrayExt; use crate::BitPackedData; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// All six operators on a small in-range input. #[rstest] diff --git a/encodings/fastlanes/src/bitpacking/plugin.rs b/encodings/fastlanes/src/bitpacking/plugin.rs index d5ecdefd0c7..a621d085514 100644 --- a/encodings/fastlanes/src/bitpacking/plugin.rs +++ b/encodings/fastlanes/src/bitpacking/plugin.rs @@ -100,7 +100,6 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::patched::PatchedArraySlotsExt; use vortex_array::buffer::BufferHandle; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -114,7 +113,7 @@ mod tests { use crate::BitPackedData; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(BitPackedPatchedPlugin); session }); diff --git a/encodings/fastlanes/src/delta/array/delta_compress.rs b/encodings/fastlanes/src/delta/array/delta_compress.rs index e35778ad29e..fd17d481f43 100644 --- a/encodings/fastlanes/src/delta/array/delta_compress.rs +++ b/encodings/fastlanes/src/delta/array/delta_compress.rs @@ -106,7 +106,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; - use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_error::VortexResult; use vortex_session::VortexSession; @@ -116,8 +115,7 @@ mod tests { use crate::delta::array::delta_decompress::delta_decompress; use crate::delta_compress; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[rstest] #[case::u32((0u32..10_000).collect())] diff --git a/encodings/fastlanes/src/delta/array/mod.rs b/encodings/fastlanes/src/delta/array/mod.rs index 33ece0deddd..7754bb37a59 100644 --- a/encodings/fastlanes/src/delta/array/mod.rs +++ b/encodings/fastlanes/src/delta/array/mod.rs @@ -38,7 +38,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; /// use vortex_session::VortexSession; /// use vortex_fastlanes::Delta; /// -/// let session = VortexSession::empty().with::(); +/// let session = vortex_array::array_session(); /// let primitive = PrimitiveArray::from_iter([1_u32, 2, 3, 5, 10, 11]); /// let array = Delta::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); /// ``` @@ -53,7 +53,7 @@ pub(super) const SLOT_NAMES: [&str; NUM_SLOTS] = ["bases", "deltas"]; /// use vortex_session::VortexSession; /// use vortex_fastlanes::Delta; /// -/// let session = VortexSession::empty().with::(); +/// let session = vortex_array::array_session(); /// let primitive = PrimitiveArray::from_iter([-3_i32, -2, -1, 0, 1, 2]); /// let array = Delta::try_from_primitive_array(&primitive, &mut session.create_execution_ctx()).unwrap(); /// ``` diff --git a/encodings/fastlanes/src/delta/compute/cast.rs b/encodings/fastlanes/src/delta/compute/cast.rs index f782eedd524..5a9167f126f 100644 --- a/encodings/fastlanes/src/delta/compute/cast.rs +++ b/encodings/fastlanes/src/delta/compute/cast.rs @@ -51,14 +51,12 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::Delta; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_delta_unsigned_widening_wraps() { diff --git a/encodings/fastlanes/src/delta/vtable/operations.rs b/encodings/fastlanes/src/delta/vtable/operations.rs index 785cdf6aca8..53c5d726158 100644 --- a/encodings/fastlanes/src/delta/vtable/operations.rs +++ b/encodings/fastlanes/src/delta/vtable/operations.rs @@ -35,7 +35,6 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::compute::conformance::binary_numeric::test_binary_numeric_array; use vortex_array::compute::conformance::consistency::test_array_consistency; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexExpect; @@ -44,8 +43,7 @@ mod tests { use crate::Delta; use crate::DeltaArray; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn da(array: &PrimitiveArray) -> DeltaArray { Delta::try_from_primitive_array(array, &mut SESSION.create_execution_ctx()) diff --git a/encodings/fastlanes/src/for/array/for_compress.rs b/encodings/fastlanes/src/for/array/for_compress.rs index ad9566027d2..48efe87e834 100644 --- a/encodings/fastlanes/src/for/array/for_compress.rs +++ b/encodings/fastlanes/src/for/array/for_compress.rs @@ -58,7 +58,6 @@ mod test { use vortex_array::dtype::PType; use vortex_array::expr::stats::StatsProvider; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -69,8 +68,7 @@ mod test { use crate::r#for::array::for_decompress::decompress; use crate::r#for::array::for_decompress::fused_decompress; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_compress_round_trip_small() { diff --git a/encodings/fastlanes/src/lib.rs b/encodings/fastlanes/src/lib.rs index 9022b7c4e2b..85b6d0259a1 100644 --- a/encodings/fastlanes/src/lib.rs +++ b/encodings/fastlanes/src/lib.rs @@ -144,7 +144,7 @@ mod test { use super::*; pub static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty(); + let session = vortex_array::array_session(); session.arrays().register(BitPacked); session.arrays().register(Delta); session.arrays().register(FoR); diff --git a/encodings/fastlanes/src/rle/compute/cast.rs b/encodings/fastlanes/src/rle/compute/cast.rs index e01d359e59b..4f5fec886f8 100644 --- a/encodings/fastlanes/src/rle/compute/cast.rs +++ b/encodings/fastlanes/src/rle/compute/cast.rs @@ -58,7 +58,6 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -66,8 +65,7 @@ mod tests { use crate::RLEData; use crate::rle::RLEArray; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn rle(primitive: &PrimitiveArray, ctx: &mut ExecutionCtx) -> RLEArray { RLEData::encode(primitive.as_view(), ctx).unwrap() diff --git a/encodings/fsst/benches/chunked_dict_fsst_builder.rs b/encodings/fsst/benches/chunked_dict_fsst_builder.rs index 22131631a6e..2ea5cfb8f5d 100644 --- a/encodings/fsst/benches/chunked_dict_fsst_builder.rs +++ b/encodings/fsst/benches/chunked_dict_fsst_builder.rs @@ -11,7 +11,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::NativePType; -use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_fsst::test_utils::gen_dict_fsst_test_data; use vortex_session::VortexSession; @@ -29,8 +28,7 @@ const BENCH_ARGS: &[(usize, usize, usize)] = &[ (1000, 1000, 100), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_dict_fsst_chunks( len: usize, diff --git a/encodings/fsst/benches/fsst_compress.rs b/encodings/fsst/benches/fsst_compress.rs index 88e4e26a903..cf648f68c48 100644 --- a/encodings/fsst/benches/fsst_compress.rs +++ b/encodings/fsst/benches/fsst_compress.rs @@ -23,7 +23,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; use vortex_session::VortexSession; @@ -32,8 +31,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); // [(string_count, avg_len, unique_chars)] const BENCH_ARGS: &[(usize, usize, u8)] = &[ diff --git a/encodings/fsst/benches/fsst_like.rs b/encodings/fsst/benches/fsst_like.rs index 28aa3b109ae..58a49af5582 100644 --- a/encodings/fsst/benches/fsst_like.rs +++ b/encodings/fsst/benches/fsst_like.rs @@ -14,7 +14,6 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::scalar_fn::ScalarFnFactoryExt; use vortex_array::scalar_fn::fns::like::Like; use vortex_array::scalar_fn::fns::like::LikeOptions; -use vortex_array::session::ArraySession; use vortex_fsst::FSSTArray; use vortex_fsst::test_utils::NUM_STRINGS; use vortex_fsst::test_utils::make_fsst_clickbench_urls; @@ -30,8 +29,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const N: usize = NUM_STRINGS; diff --git a/encodings/fsst/benches/fsst_url_compare.rs b/encodings/fsst/benches/fsst_url_compare.rs index 656cd9f1866..00fdd2aa6f1 100644 --- a/encodings/fsst/benches/fsst_url_compare.rs +++ b/encodings/fsst/benches/fsst_url_compare.rs @@ -18,7 +18,6 @@ use vortex_array::expr::lit; use vortex_array::expr::root; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_fsst::fsst_compress; use vortex_fsst::fsst_train_compressor; use vortex_fsst::test_utils::HIGH_MATCH_DOMAIN; @@ -31,8 +30,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const NUM_URLS: usize = NUM_STRINGS; diff --git a/encodings/fsst/src/canonical.rs b/encodings/fsst/src/canonical.rs index a8b8171b043..67241ffb026 100644 --- a/encodings/fsst/src/canonical.rs +++ b/encodings/fsst/src/canonical.rs @@ -104,15 +104,13 @@ mod tests { use vortex_array::builders::VarBinViewBuilder; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::fsst_compress; use crate::fsst_train_compressor; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_data() -> (VarBinArray, Vec>>) { const STRING_COUNT: usize = 1000; diff --git a/encodings/fsst/src/compute/cast.rs b/encodings/fsst/src/compute/cast.rs index 47c324fc2a4..f6e776e16d9 100644 --- a/encodings/fsst/src/compute/cast.rs +++ b/encodings/fsst/src/compute/cast.rs @@ -98,14 +98,12 @@ mod tests { use vortex_array::compute::conformance::cast::test_cast_conformance; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::session::ArraySession; use vortex_session::VortexSession; use crate::fsst_compress; use crate::fsst_train_compressor; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_fsst_nullability() { diff --git a/encodings/fsst/src/compute/like.rs b/encodings/fsst/src/compute/like.rs index 3f704f1e034..ae2402a6e51 100644 --- a/encodings/fsst/src/compute/like.rs +++ b/encodings/fsst/src/compute/like.rs @@ -96,7 +96,6 @@ mod tests { use vortex_array::scalar_fn::fns::like::Like; use vortex_array::scalar_fn::fns::like::LikeKernel; use vortex_array::scalar_fn::fns::like::LikeOptions; - use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; @@ -105,8 +104,7 @@ mod tests { use crate::fsst_compress; use crate::fsst_train_compressor; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_fsst(strings: &[Option<&str>], nullability: Nullability) -> FSSTArray { let varbin = VarBinArray::from_iter(strings.iter().copied(), DType::Utf8(nullability)); diff --git a/encodings/fsst/src/dfa/tests.rs b/encodings/fsst/src/dfa/tests.rs index d41fedde330..af23ae581d5 100644 --- a/encodings/fsst/src/dfa/tests.rs +++ b/encodings/fsst/src/dfa/tests.rs @@ -20,7 +20,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::like::Like; use vortex_array::scalar_fn::fns::like::LikeOptions; -use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; @@ -32,8 +31,7 @@ use crate::FSSTArray; use crate::fsst_compress; use crate::fsst_train_compressor; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Helper: make a Symbol from a byte string (up to 8 bytes, zero-padded). fn sym(bytes: &[u8]) -> Symbol { diff --git a/encodings/fsst/src/kernel.rs b/encodings/fsst/src/kernel.rs index a54fe41d8cc..d45ee2978e3 100644 --- a/encodings/fsst/src/kernel.rs +++ b/encodings/fsst/src/kernel.rs @@ -36,7 +36,6 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::expr::byte_length; use vortex_array::expr::root; - use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -45,8 +44,7 @@ mod tests { use crate::fsst_compress; use crate::fsst_train_compressor; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn build_test_fsst_array() -> ArrayRef { let mut builder = VarBinBuilder::::with_capacity(10); diff --git a/encodings/parquet-variant/src/arrow.rs b/encodings/parquet-variant/src/arrow.rs index 9bee5c738d3..760cd1fb1aa 100644 --- a/encodings/parquet-variant/src/arrow.rs +++ b/encodings/parquet-variant/src/arrow.rs @@ -295,7 +295,6 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; @@ -306,7 +305,7 @@ mod tests { #[fixture] fn session() -> VortexSession { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); crate::initialize(&session); session } diff --git a/encodings/parquet-variant/src/vtable.rs b/encodings/parquet-variant/src/vtable.rs index f2ad4f64743..d73071fc3a6 100644 --- a/encodings/parquet-variant/src/vtable.rs +++ b/encodings/parquet-variant/src/vtable.rs @@ -323,7 +323,6 @@ mod tests { use vortex_array::dtype::PType; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::stream::ArrayStreamExt; use vortex_array::validity::Validity; @@ -348,7 +347,7 @@ mod tests { let dtype = array.dtype().clone(); let len = array.len(); - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(ParquetVariant); let ctx = ArrayContext::empty(); @@ -387,8 +386,7 @@ mod tests { #[fixture] fn parquet_variant_file_session() -> VortexSession { - let session = VortexSession::empty() - .with::() + let session = vortex_array::array_session() .with::() .with::(); vortex_file::register_default_encodings(&session); diff --git a/encodings/pco/src/compute/cast.rs b/encodings/pco/src/compute/cast.rs index b35526c1016..89100279a3f 100644 --- a/encodings/pco/src/compute/cast.rs +++ b/encodings/pco/src/compute/cast.rs @@ -65,15 +65,13 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; use crate::Pco; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_pco_f32_to_f64() { diff --git a/encodings/pco/src/tests.rs b/encodings/pco/src/tests.rs index 20fec7dd9e0..32656c9b7fa 100644 --- a/encodings/pco/src/tests.rs +++ b/encodings/pco/src/tests.rs @@ -18,7 +18,6 @@ use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::serde::SerializeOptions; use vortex_array::serde::SerializedArray; -use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_array::vtable::child_to_validity; @@ -33,7 +32,7 @@ use vortex_session::registry::ReadContext; use crate::PcoData; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Pco); session }); diff --git a/encodings/runend/benches/run_end_compress.rs b/encodings/runend/benches/run_end_compress.rs index a49a5cfcb50..9bb1b35ab94 100644 --- a/encodings/runend/benches/run_end_compress.rs +++ b/encodings/runend/benches/run_end_compress.rs @@ -13,7 +13,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::IntegerPType; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_runend::RunEnd; @@ -24,8 +23,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const BENCH_ARGS: &[(usize, usize)] = &[ (1000, 4), diff --git a/encodings/runend/benches/run_end_decode.rs b/encodings/runend/benches/run_end_decode.rs index f509e55aed3..a2915c0b8b7 100644 --- a/encodings/runend/benches/run_end_decode.rs +++ b/encodings/runend/benches/run_end_decode.rs @@ -10,7 +10,6 @@ use divan::Bencher; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::BufferMut; @@ -21,8 +20,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Distribution types for bool benchmarks #[derive(Clone, Copy)] diff --git a/encodings/runend/benches/run_end_null_count.rs b/encodings/runend/benches/run_end_null_count.rs index 5392e04189b..e247ad52208 100644 --- a/encodings/runend/benches/run_end_null_count.rs +++ b/encodings/runend/benches/run_end_null_count.rs @@ -12,7 +12,6 @@ use rand::rngs::StdRng; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_runend::RunEnd; use vortex_runend::RunEndArray; @@ -50,8 +49,7 @@ const BENCH_ARGS: &[(usize, usize, f64)] = &[ (100_000, 1024, 0.5), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench(args = BENCH_ARGS)] fn null_count_run_end(bencher: Bencher, (n, run_step, valid_density): (usize, usize, f64)) { diff --git a/encodings/runend/src/array.rs b/encodings/runend/src/array.rs index 7bb069bb458..99cecddc0b0 100644 --- a/encodings/runend/src/array.rs +++ b/encodings/runend/src/array.rs @@ -521,14 +521,12 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; use crate::RunEnd; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_runend_constructor() { diff --git a/encodings/runend/src/arrow.rs b/encodings/runend/src/arrow.rs index c26674770c0..aa03e3be5dc 100644 --- a/encodings/runend/src/arrow.rs +++ b/encodings/runend/src/arrow.rs @@ -95,7 +95,6 @@ mod tests { use vortex_array::scalar::PValue; use vortex_array::search_sorted::SearchSorted; use vortex_array::search_sorted::SearchSortedSide; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_array::validity::Validity; use vortex_buffer::Buffer; @@ -107,7 +106,7 @@ mod tests { use crate::ops::find_slice_end_index; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(RunEnd); session }); diff --git a/encodings/runend/src/compute/cast.rs b/encodings/runend/src/compute/cast.rs index f70bb0becb5..ff28baeb2d3 100644 --- a/encodings/runend/src/compute/cast.rs +++ b/encodings/runend/src/compute/cast.rs @@ -46,15 +46,13 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; use crate::RunEnd; use crate::RunEndArray; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_runend_i32_to_i64() { diff --git a/encodings/runend/src/compute/take_from.rs b/encodings/runend/src/compute/take_from.rs index 32b22eb96c5..8c3417459a0 100644 --- a/encodings/runend/src/compute/take_from.rs +++ b/encodings/runend/src/compute/take_from.rs @@ -61,7 +61,6 @@ mod tests { use vortex_array::kernel::ExecuteParentKernel; use vortex_buffer::buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use crate::RunEnd; use crate::RunEndArray; @@ -83,7 +82,7 @@ mod tests { #[test] fn test_execute_parent_no_offset() -> VortexResult<()> { - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let (codes, dict) = make_dict_with_runend_codes(&mut ctx); let result = RunEndTakeFrom @@ -98,7 +97,7 @@ mod tests { #[test] fn test_execute_parent_with_offset() -> VortexResult<()> { - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice codes to positions 2..5 → logical codes [0, 1, 1] → values [2, 3, 3] let sliced_codes = unsafe { @@ -122,7 +121,7 @@ mod tests { #[test] fn test_execute_parent_offset_at_run_boundary() -> VortexResult<()> { - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice codes to positions 3..7 → logical codes [1, 1, 0, 0] → values [3, 3, 2, 2] let sliced_codes = unsafe { @@ -146,7 +145,7 @@ mod tests { #[test] fn test_execute_parent_single_element_offset() -> VortexResult<()> { - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let (codes, dict) = make_dict_with_runend_codes(&mut ctx); // Slice to single element at position 4 → code=1 → value=3 let sliced_codes = unsafe { @@ -170,7 +169,7 @@ mod tests { #[test] fn test_execute_parent_returns_none_for_non_codes_child() -> VortexResult<()> { - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(vortex_array::array_session()); let (codes, dict) = make_dict_with_runend_codes(&mut ctx); let result = RunEndTakeFrom.execute_parent(codes.as_view(), dict.as_view(), 1, &mut ctx)?; diff --git a/encodings/sequence/src/compute/cast.rs b/encodings/sequence/src/compute/cast.rs index e6d64fdf7c5..3750fe20fd7 100644 --- a/encodings/sequence/src/compute/cast.rs +++ b/encodings/sequence/src/compute/cast.rs @@ -99,14 +99,12 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_session::VortexSession; use crate::Sequence; use crate::SequenceArray; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_sequence_nullability() { diff --git a/encodings/sparse/benches/sparse_canonical.rs b/encodings/sparse/benches/sparse_canonical.rs index 3861e26da06..8179bec369f 100644 --- a/encodings/sparse/benches/sparse_canonical.rs +++ b/encodings/sparse/benches/sparse_canonical.rs @@ -17,7 +17,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::dtype::Nullability::NonNullable; use vortex_array::dtype::PType::I32; use vortex_array::scalar::Scalar; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexExpect; @@ -28,8 +27,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const LIST_ARGS: &[(usize, usize, usize)] = &[ // len, patch_stride, list_size diff --git a/encodings/sparse/benches/sparse_pushdown.rs b/encodings/sparse/benches/sparse_pushdown.rs index 5abf0905f6e..d34758e7cd8 100644 --- a/encodings/sparse/benches/sparse_pushdown.rs +++ b/encodings/sparse/benches/sparse_pushdown.rs @@ -31,7 +31,6 @@ use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_error::VortexExpect; use vortex_session::VortexSession; @@ -45,7 +44,7 @@ const LEN: usize = 1_000_000; /// Session with Sparse and its pushdown kernels registered. static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); vortex_sparse::initialize(&session); session }); diff --git a/encodings/sparse/src/compute/between.rs b/encodings/sparse/src/compute/between.rs index 053a6694131..ba50a120174 100644 --- a/encodings/sparse/src/compute/between.rs +++ b/encodings/sparse/src/compute/between.rs @@ -74,7 +74,6 @@ mod tests { use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::between::BetweenOptions; use vortex_array::scalar_fn::fns::between::StrictComparison; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -82,7 +81,7 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); diff --git a/encodings/sparse/src/compute/cast.rs b/encodings/sparse/src/compute/cast.rs index 35ce9474589..1068987b9e4 100644 --- a/encodings/sparse/src/compute/cast.rs +++ b/encodings/sparse/src/compute/cast.rs @@ -48,15 +48,13 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; use crate::Sparse; use crate::SparseArray; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_sparse_i32_to_i64() { diff --git a/encodings/sparse/src/compute/compare.rs b/encodings/sparse/src/compute/compare.rs index c64e8142298..f9637da516a 100644 --- a/encodings/sparse/src/compute/compare.rs +++ b/encodings/sparse/src/compute/compare.rs @@ -65,7 +65,6 @@ mod tests { use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar::Scalar; use vortex_array::scalar_fn::fns::operators::Operator; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -74,7 +73,7 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); diff --git a/encodings/sparse/src/compute/fill_null.rs b/encodings/sparse/src/compute/fill_null.rs index fd62926b2f3..8f7d2b75a27 100644 --- a/encodings/sparse/src/compute/fill_null.rs +++ b/encodings/sparse/src/compute/fill_null.rs @@ -55,7 +55,6 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -63,7 +62,7 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); diff --git a/encodings/sparse/src/compute/is_constant.rs b/encodings/sparse/src/compute/is_constant.rs index cfd31c6698d..eee8683df0a 100644 --- a/encodings/sparse/src/compute/is_constant.rs +++ b/encodings/sparse/src/compute/is_constant.rs @@ -73,7 +73,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::is_constant::is_constant; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::buffer; use vortex_error::VortexResult; @@ -85,14 +84,14 @@ mod tests { /// Session with Sparse + its pushdown kernels. static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); /// Baseline session: Sparse registered but no pushdown kernels. static CANONICAL_SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Sparse); session }); diff --git a/encodings/sparse/src/compute/min_max.rs b/encodings/sparse/src/compute/min_max.rs index 22ff3e86c53..cf89bc28da6 100644 --- a/encodings/sparse/src/compute/min_max.rs +++ b/encodings/sparse/src/compute/min_max.rs @@ -69,7 +69,6 @@ mod tests { use vortex_array::aggregate_fn::fns::min_max::MinMaxResult; use vortex_array::aggregate_fn::fns::min_max::min_max; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -79,13 +78,13 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); static CANONICAL_SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Sparse); session }); diff --git a/encodings/sparse/src/compute/nan_count.rs b/encodings/sparse/src/compute/nan_count.rs index 7fe03e21103..9a8a3a871c1 100644 --- a/encodings/sparse/src/compute/nan_count.rs +++ b/encodings/sparse/src/compute/nan_count.rs @@ -77,7 +77,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::nan_count::nan_count; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -87,13 +86,13 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); static CANONICAL_SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Sparse); session }); diff --git a/encodings/sparse/src/compute/null_count.rs b/encodings/sparse/src/compute/null_count.rs index ad5e875ea6c..f63dad80205 100644 --- a/encodings/sparse/src/compute/null_count.rs +++ b/encodings/sparse/src/compute/null_count.rs @@ -67,7 +67,6 @@ mod tests { use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -77,13 +76,13 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); static CANONICAL_SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Sparse); session }); diff --git a/encodings/sparse/src/compute/sum.rs b/encodings/sparse/src/compute/sum.rs index 4b068aa57b4..a47884e6f0f 100644 --- a/encodings/sparse/src/compute/sum.rs +++ b/encodings/sparse/src/compute/sum.rs @@ -72,7 +72,6 @@ mod tests { use vortex_array::VortexSessionExecute; use vortex_array::aggregate_fn::fns::sum::sum; use vortex_array::scalar::Scalar; - use vortex_array::session::ArraySession; use vortex_array::session::ArraySessionExt; use vortex_buffer::buffer; use vortex_error::VortexResult; @@ -83,13 +82,13 @@ mod tests { use crate::initialize; static SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); initialize(&session); session }); static CANONICAL_SESSION: LazyLock = LazyLock::new(|| { - let session = VortexSession::empty().with::(); + let session = vortex_array::array_session(); session.arrays().register(Sparse); session }); diff --git a/encodings/zstd/benches/listview_rebuild.rs b/encodings/zstd/benches/listview_rebuild.rs index fe4e9f063d1..e3c0e6d5dd1 100644 --- a/encodings/zstd/benches/listview_rebuild.rs +++ b/encodings/zstd/benches/listview_rebuild.rs @@ -11,7 +11,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::ListViewArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewRebuildMode; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -19,8 +18,7 @@ use vortex_zstd::Zstd; use vortex_zstd::ZstdData; /// A shared session for the `ListView` rebuild benchmark, used to create execution contexts. -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench(sample_size = 1000)] fn rebuild_naive(bencher: Bencher) { diff --git a/encodings/zstd/src/compute/cast.rs b/encodings/zstd/src/compute/cast.rs index 6cbe4f37795..52f6f8020c0 100644 --- a/encodings/zstd/src/compute/cast.rs +++ b/encodings/zstd/src/compute/cast.rs @@ -83,15 +83,13 @@ mod tests { use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; use crate::Zstd; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_cast_zstd_i32_to_i64() { diff --git a/fuzz/src/fsst_like.rs b/fuzz/src/fsst_like.rs index 5ca10af310a..ec310a0b81a 100644 --- a/fuzz/src/fsst_like.rs +++ b/fuzz/src/fsst_like.rs @@ -21,7 +21,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::scalar_fn::fns::like::Like; use vortex_array::scalar_fn::fns::like::LikeOptions; -use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_fsst::FSSTArray; use vortex_fsst::fsst_compress; @@ -32,8 +31,7 @@ use crate::error::Backtrace; use crate::error::VortexFuzzError; use crate::error::VortexFuzzResult; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// A random string from a small alphabet (`a..=h`) with bounded length. #[derive(Debug)] diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index b0abf660045..8869a343f94 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -56,7 +56,7 @@ mod native_runtime { if vortex_cuda::cuda_available() { use vortex_cuda::CudaSessionExt; session = session.with::(); - vortex_cuda::initialize_cuda(&session.cuda_session()); + vortex_cuda::initialize_cuda(session.cuda_session()); } session }); diff --git a/vortex-array/benches/aggregate_max.rs b/vortex-array/benches/aggregate_max.rs index 14bf13792dd..4da7d1fc653 100644 --- a/vortex-array/benches/aggregate_max.rs +++ b/vortex-array/benches/aggregate_max.rs @@ -8,7 +8,6 @@ use rand::prelude::*; use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { @@ -17,8 +16,7 @@ fn main() { const N: usize = 100_000; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn max_i32(bencher: Bencher) { diff --git a/vortex-array/benches/aggregate_sum.rs b/vortex-array/benches/aggregate_sum.rs index b076bb80b5f..81402b24750 100644 --- a/vortex-array/benches/aggregate_sum.rs +++ b/vortex-array/benches/aggregate_sum.rs @@ -9,7 +9,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; use vortex_array::expr::stats::Stat; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { @@ -18,8 +17,7 @@ fn main() { const N: usize = 100_000; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn sum_i32(bencher: Bencher) { diff --git a/vortex-array/benches/binary_ops.rs b/vortex-array/benches/binary_ops.rs index 31cc5d2ec42..98c69413193 100644 --- a/vortex-array/benches/binary_ops.rs +++ b/vortex-array/benches/binary_ops.rs @@ -20,15 +20,13 @@ use vortex_array::arrays::ConstantArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const LEN: usize = 65_536; diff --git a/vortex-array/benches/cast_primitive.rs b/vortex-array/benches/cast_primitive.rs index 060a8290a2a..bc31081afae 100644 --- a/vortex-array/benches/cast_primitive.rs +++ b/vortex-array/benches/cast_primitive.rs @@ -14,7 +14,6 @@ use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::expr::stats::Stat; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { @@ -23,8 +22,7 @@ fn main() { const N: usize = 100_000; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn cast_u16_to_u32(bencher: Bencher) { diff --git a/vortex-array/benches/chunk_array_builder.rs b/vortex-array/benches/chunk_array_builder.rs index d74fd309453..6582071a6e1 100644 --- a/vortex-array/benches/chunk_array_builder.rs +++ b/vortex-array/benches/chunk_array_builder.rs @@ -18,7 +18,6 @@ use vortex_array::builders::ArrayBuilder; use vortex_array::builders::VarBinViewBuilder; use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::DType; -use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_session::VortexSession; @@ -33,8 +32,7 @@ const BENCH_ARGS: &[(usize, usize)] = &[ (1000, 10), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench(args = BENCH_ARGS)] fn chunked_bool_canonical_into(bencher: Bencher, (len, chunk_count): (usize, usize)) { diff --git a/vortex-array/benches/chunked_dict_builder.rs b/vortex-array/benches/chunked_dict_builder.rs index 85c8737356f..74722235e54 100644 --- a/vortex-array/benches/chunked_dict_builder.rs +++ b/vortex-array/benches/chunked_dict_builder.rs @@ -11,7 +11,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::dict_test::gen_dict_primitive_chunks; use vortex_array::builders::builder_with_capacity; use vortex_array::dtype::NativePType; -use vortex_array::session::ArraySession; use vortex_error::VortexExpect; use vortex_session::VortexSession; @@ -19,8 +18,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const BENCH_ARGS: &[(usize, usize, usize)] = &[ (1000, 10, 10), diff --git a/vortex-array/benches/chunked_fsl_canonicalize.rs b/vortex-array/benches/chunked_fsl_canonicalize.rs index ac9962046de..4277b33382e 100644 --- a/vortex-array/benches/chunked_fsl_canonicalize.rs +++ b/vortex-array/benches/chunked_fsl_canonicalize.rs @@ -18,7 +18,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::FixedSizeListArray; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -27,8 +26,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Number of lists in each chunk. const LISTS_PER_CHUNK: usize = 1_000; diff --git a/vortex-array/benches/compare.rs b/vortex-array/benches/compare.rs index 6fe5be688e9..95702c9f8f5 100644 --- a/vortex-array/benches/compare.rs +++ b/vortex-array/benches/compare.rs @@ -15,7 +15,6 @@ use vortex_array::arrays::BoolArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar_fn::fns::operators::Operator; use vortex_buffer::Buffer; -use vortex_session::VortexSession; fn main() { divan::main(); @@ -30,7 +29,7 @@ fn compare_bool(bencher: Bencher) { let arr1 = BoolArray::from_iter((0..ARRAY_SIZE).map(|_| rng.sample(range) == 0)).into_array(); let arr2 = BoolArray::from_iter((0..ARRAY_SIZE).map(|_| rng.sample(range) == 0)).into_array(); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&arr1, &arr2, session.create_execution_ctx())) @@ -58,7 +57,7 @@ fn compare_int(bencher: Bencher) { .map(|_| rng.sample(range)) .collect::>() .into_array(); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&arr1, &arr2, session.create_execution_ctx())) diff --git a/vortex-array/benches/dict_compare.rs b/vortex-array/benches/dict_compare.rs index 401393495dd..69d9d3ddf4b 100644 --- a/vortex-array/benches/dict_compare.rs +++ b/vortex-array/benches/dict_compare.rs @@ -22,15 +22,13 @@ use vortex_array::expr::eq; use vortex_array::expr::lit; use vortex_array::expr::root; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const LENGTH_AND_UNIQUE_VALUES: &[(usize, usize)] = &[ // length, unique_values @@ -59,7 +57,7 @@ fn bench_compare_primitive(bencher: divan::Bencher, (len, uniqueness): (usize, u ) .unwrap(); let value = primitive_arr.as_slice::()[0]; - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&dict, session.create_execution_ctx())) @@ -83,7 +81,7 @@ fn bench_compare_varbin(bencher: divan::Bencher, (len, uniqueness): (usize, usiz .unwrap(); let bytes = varbin_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec()); let value = from_utf8(bytes.as_slice()).unwrap(); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&dict, session.create_execution_ctx())) @@ -107,7 +105,7 @@ fn bench_compare_varbinview(bencher: divan::Bencher, (len, uniqueness): (usize, .unwrap(); let bytes = varbinview_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec()); let value = from_utf8(bytes.as_slice()).unwrap(); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&dict, session.create_execution_ctx())) @@ -146,7 +144,7 @@ fn bench_compare_sliced_dict_primitive( .unwrap(); let dict = dict.into_array().slice(0..codes_len).unwrap(); let value = primitive_arr.as_slice::()[0]; - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&dict, session.create_execution_ctx())) @@ -173,7 +171,7 @@ fn bench_compare_sliced_dict_varbinview( let dict = dict.into_array().slice(0..codes_len).unwrap(); let bytes = varbin_arr.with_iterator(|i| i.next().unwrap().unwrap().to_vec()); let value = from_utf8(bytes.as_slice()).unwrap(); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&dict, session.create_execution_ctx())) diff --git a/vortex-array/benches/dict_compress.rs b/vortex-array/benches/dict_compress.rs index 21d427bd7f8..3bd287414c0 100644 --- a/vortex-array/benches/dict_compress.rs +++ b/vortex-array/benches/dict_compress.rs @@ -17,7 +17,6 @@ use vortex_array::arrays::dict_test::gen_primitive_for_dict; use vortex_array::arrays::dict_test::gen_varbin_words; use vortex_array::builders::dict::dict_encode; use vortex_array::dtype::NativePType; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { @@ -38,8 +37,7 @@ const BENCH_ARGS: &[(usize, usize)] = &[ (10_000, 512), ]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench(types = [u8, f32, i64], args = BENCH_ARGS)] fn encode_primitives(bencher: Bencher, (len, unique_values): (usize, usize)) diff --git a/vortex-array/benches/dict_mask.rs b/vortex-array/benches/dict_mask.rs index 1951e610a51..04cf18cfd03 100644 --- a/vortex-array/benches/dict_mask.rs +++ b/vortex-array/benches/dict_mask.rs @@ -14,7 +14,6 @@ use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builtins::ArrayBuiltins; use vortex_mask::Mask; -use vortex_session::VortexSession; fn main() { divan::main(); @@ -60,7 +59,7 @@ fn bench_dict_mask(bencher: Bencher, (fraction_valid, fraction_masked): (f64, f6 let values = PrimitiveArray::from_option_iter([None, Some(42i32)]).into_array(); let array = DictArray::try_new(codes, values).unwrap().into_array(); let filter_mask = filter_mask(len, fraction_masked, &mut rng); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| (&array, &filter_mask, session.create_execution_ctx())) .bench_refs(|(array, filter_mask, ctx)| { diff --git a/vortex-array/benches/expr/case_when_bench.rs b/vortex-array/benches/expr/case_when_bench.rs index f37daa2b304..1bada8fa9e1 100644 --- a/vortex-array/benches/expr/case_when_bench.rs +++ b/vortex-array/benches/expr/case_when_bench.rs @@ -22,12 +22,10 @@ use vortex_array::expr::lit; use vortex_array::expr::lt; use vortex_array::expr::nested_case_when; use vortex_array::expr::root; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_session::VortexSession; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn main() { divan::main(); diff --git a/vortex-array/benches/filter_bool.rs b/vortex-array/benches/filter_bool.rs index 6c240db6b60..cd31805bb77 100644 --- a/vortex-array/benches/filter_bool.rs +++ b/vortex-array/benches/filter_bool.rs @@ -19,7 +19,6 @@ use rand_distr::Zipf; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; -use vortex_array::session::ArraySession; use vortex_buffer::BitBuffer; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -28,8 +27,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const SIZES: &[usize] = &[1_000, 10_000, 100_000, 250_000]; const DENSITY_SWEEP_SIZE: usize = 100_000; diff --git a/vortex-array/benches/interleave.rs b/vortex-array/benches/interleave.rs index cba7f6a910e..fdd1fafba05 100644 --- a/vortex-array/benches/interleave.rs +++ b/vortex-array/benches/interleave.rs @@ -14,7 +14,6 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::InterleaveArray; use vortex_buffer::Buffer; -use vortex_session::VortexSession; fn main() { divan::main(); @@ -54,7 +53,7 @@ fn inputs( #[divan::bench(args = [2, 4])] fn interleave_bool(bencher: Bencher, num_branches: usize) { let (values, array_indices, row_indices) = inputs(num_branches, false); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| { ( @@ -74,7 +73,7 @@ fn interleave_bool(bencher: Bencher, num_branches: usize) { #[divan::bench(args = [2, 4])] fn interleave_bool_nullable(bencher: Bencher, num_branches: usize) { let (values, array_indices, row_indices) = inputs(num_branches, true); - let session = VortexSession::empty(); + let session = vortex_array::array_session(); bencher .with_inputs(|| { ( diff --git a/vortex-array/benches/listview_rebuild.rs b/vortex-array/benches/listview_rebuild.rs index 31fdaf5e890..81157d52b5a 100644 --- a/vortex-array/benches/listview_rebuild.rs +++ b/vortex-array/benches/listview_rebuild.rs @@ -21,7 +21,6 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::arrays::listview::ListViewArrayExt; use vortex_array::arrays::listview::ListViewRebuildMode; use vortex_array::dtype::FieldNames; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -31,8 +30,7 @@ fn main() { } /// A shared session for the `ListView` rebuild benchmarks, used to create execution contexts. -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_primitive_lv(num_lists: usize, list_size: usize, step: usize) -> ListViewArray { let element_count = step * num_lists + list_size; diff --git a/vortex-array/benches/scalar_at_struct.rs b/vortex-array/benches/scalar_at_struct.rs index 5e435b1000a..3921f0a2366 100644 --- a/vortex-array/benches/scalar_at_struct.rs +++ b/vortex-array/benches/scalar_at_struct.rs @@ -15,7 +15,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; use vortex_array::dtype::FieldNames; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -27,8 +26,7 @@ fn main() { const ARRAY_SIZE: usize = 100_000; const NUM_ACCESSES: usize = 1000; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn execute_scalar_struct_simple(bencher: Bencher) { diff --git a/vortex-array/benches/scalar_subtract.rs b/vortex-array/benches/scalar_subtract.rs index 2eee8f3d9ff..ee9aba1055d 100644 --- a/vortex-array/benches/scalar_subtract.rs +++ b/vortex-array/benches/scalar_subtract.rs @@ -17,7 +17,6 @@ use vortex_array::arrays::ChunkedArray; use vortex_array::arrays::ConstantArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::scalar_fn::fns::operators::Operator; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -25,8 +24,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[divan::bench] fn scalar_subtract(bencher: Bencher) { diff --git a/vortex-array/benches/take_filter.rs b/vortex-array/benches/take_filter.rs index 0e3ff719a75..b6ec1b4040c 100644 --- a/vortex-array/benches/take_filter.rs +++ b/vortex-array/benches/take_filter.rs @@ -30,7 +30,6 @@ use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::ListArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::BitBufferMut; @@ -58,8 +57,7 @@ const INDEX_SEED: u64 = 43; const LIST_SIZE: usize = 4; const NULL_INDEX_INTERVAL: usize = 8; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn primitive_array() -> ArrayRef { PrimitiveArray::from_iter(0..ARRAY_LEN as u32).into_array() diff --git a/vortex-array/benches/take_fsl.rs b/vortex-array/benches/take_fsl.rs index 4764f5538be..438faae4983 100644 --- a/vortex-array/benches/take_fsl.rs +++ b/vortex-array/benches/take_fsl.rs @@ -20,7 +20,6 @@ use vortex_array::IntoArray; use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::FixedSizeListArray; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -29,8 +28,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Number of lists in the source array. const NUM_LISTS: usize = 500; diff --git a/vortex-array/benches/take_patches.rs b/vortex-array/benches/take_patches.rs index 4d87f8b3c63..5dc0c04c325 100644 --- a/vortex-array/benches/take_patches.rs +++ b/vortex-array/benches/take_patches.rs @@ -16,7 +16,6 @@ use vortex_array::IntoArray; use vortex_array::ToCanonical as _; use vortex_array::VortexSessionExecute; use vortex_array::patches::Patches; -use vortex_array::session::ArraySession; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -24,8 +23,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const BENCH_ARGS: &[(f64, f64)] = &[ // patches_sparsity, index_multiple diff --git a/vortex-array/benches/take_primitive.rs b/vortex-array/benches/take_primitive.rs index e211e54d5f7..383b525dcc6 100644 --- a/vortex-array/benches/take_primitive.rs +++ b/vortex-array/benches/take_primitive.rs @@ -19,7 +19,6 @@ use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::DictArray; use vortex_array::arrays::PrimitiveArray; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { @@ -32,8 +31,7 @@ const NUM_INDICES: &[usize] = &[1_000, 10_000, 100_000]; /// Size of the source vector / dictionary values. const VECTOR_SIZE: &[usize] = &[16, 256, 2048, 8192]; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); // --- DictArray canonicalization benchmarks --- diff --git a/vortex-array/benches/take_struct.rs b/vortex-array/benches/take_struct.rs index 5f15894383a..d49ce9e88a6 100644 --- a/vortex-array/benches/take_struct.rs +++ b/vortex-array/benches/take_struct.rs @@ -15,7 +15,6 @@ use vortex_array::RecursiveCanonical; use vortex_array::VortexSessionExecute; use vortex_array::arrays::StructArray; use vortex_array::dtype::FieldNames; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_session::VortexSession; @@ -24,8 +23,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const ARRAY_SIZE: usize = 100_000; const TAKE_SIZE: usize = 1000; diff --git a/vortex-array/benches/to_arrow.rs b/vortex-array/benches/to_arrow.rs index 0f84feb7baa..48e5e917476 100644 --- a/vortex-array/benches/to_arrow.rs +++ b/vortex-array/benches/to_arrow.rs @@ -25,15 +25,13 @@ use vortex_array::dtype::DecimalDType; use vortex_array::dtype::Nullability; use vortex_array::dtype::PType; use vortex_array::dtype::StructFields; -use vortex_array::session::ArraySession; use vortex_session::VortexSession; fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn schema() -> DType { let fields = StructFields::from_iter([ diff --git a/vortex-array/benches/varbinview_zip.rs b/vortex-array/benches/varbinview_zip.rs index d18012d9914..e010291e9dc 100644 --- a/vortex-array/benches/varbinview_zip.rs +++ b/vortex-array/benches/varbinview_zip.rs @@ -13,7 +13,6 @@ use vortex_array::arrays::VarBinViewArray; use vortex_array::builtins::ArrayBuiltins; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_mask::Mask; use vortex_session::VortexSession; @@ -21,8 +20,7 @@ fn main() { divan::main(); } -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Benchmarks zip on VarBinView arrays with a highly fragmented mask (worst case for per-slice lookup paths). #[divan::bench] diff --git a/vortex-array/src/aggregate_fn/accumulator.rs b/vortex-array/src/aggregate_fn/accumulator.rs index c89418e67a6..4e1aff762ad 100644 --- a/vortex-array/src/aggregate_fn/accumulator.rs +++ b/vortex-array/src/aggregate_fn/accumulator.rs @@ -264,7 +264,6 @@ impl DynAccumulator for Accumulator { mod tests { use vortex_buffer::buffer; use vortex_error::VortexResult; - use vortex_session::SessionExt; use vortex_session::VortexSession; use crate::ArrayRef; @@ -289,7 +288,6 @@ mod tests { use crate::dtype::Nullability; use crate::dtype::PType; use crate::scalar::Scalar; - use crate::session::ArraySession; /// Mean partial sentinel `{sum: 42.0, count: 1}` — distinguishable from the /// natural fan-out result `{sum: 7.0, count: 1}` that `Combined::try_accumulate` @@ -337,7 +335,7 @@ mod tests { } fn fresh_session() -> VortexSession { - VortexSession::empty().with::() + crate::array_session() } fn dict_of_seven() -> ArrayRef { diff --git a/vortex-array/src/aggregate_fn/fns/bounded_max/mod.rs b/vortex-array/src/aggregate_fn/fns/bounded_max/mod.rs index 998b6032938..37188f25797 100644 --- a/vortex-array/src/aggregate_fn/fns/bounded_max/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/bounded_max/mod.rs @@ -262,7 +262,6 @@ mod tests { use crate::arrays::VarBinViewArray; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::session::ArraySession; use crate::validity::Validity; fn max_bytes(value: usize) -> NonZeroUsize { @@ -270,7 +269,7 @@ mod tests { } fn fresh_session() -> VortexSession { - VortexSession::empty().with::() + crate::array_session() } #[test] diff --git a/vortex-array/src/aggregate_fn/fns/bounded_min/mod.rs b/vortex-array/src/aggregate_fn/fns/bounded_min/mod.rs index 7d24457e150..661b7fe1227 100644 --- a/vortex-array/src/aggregate_fn/fns/bounded_min/mod.rs +++ b/vortex-array/src/aggregate_fn/fns/bounded_min/mod.rs @@ -250,7 +250,6 @@ mod tests { use crate::arrays::VarBinViewArray; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::session::ArraySession; use crate::validity::Validity; fn max_bytes(value: usize) -> NonZeroUsize { @@ -258,7 +257,7 @@ mod tests { } fn fresh_session() -> VortexSession { - VortexSession::empty().with::() + crate::array_session() } #[test] diff --git a/vortex-array/src/aggregate_fn/proto.rs b/vortex-array/src/aggregate_fn/proto.rs index bff198b3955..e36a1080915 100644 --- a/vortex-array/src/aggregate_fn/proto.rs +++ b/vortex-array/src/aggregate_fn/proto.rs @@ -155,7 +155,7 @@ mod tests { #[test] fn aggregate_fn_serde() { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); session.aggregate_fns().register(TestAgg); let agg_fn = TestAgg.bind(EmptyOptions); @@ -170,9 +170,10 @@ mod tests { #[test] fn unknown_aggregate_fn_id_allow_unknown() { - let session = VortexSession::empty() + let session = VortexSession::builder() .with::() - .allow_unknown(); + .allow_unknown() + .build(); let proto = pb::AggregateFn { id: "vortex.test.foreign_aggregate".to_string(), diff --git a/vortex-array/src/aggregate_fn/session.rs b/vortex-array/src/aggregate_fn/session.rs index c6d7542a687..b309fd02a10 100644 --- a/vortex-array/src/aggregate_fn/session.rs +++ b/vortex-array/src/aggregate_fn/session.rs @@ -4,7 +4,6 @@ use std::any::Any; use std::sync::Arc; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; @@ -50,7 +49,7 @@ use crate::arrays::dict::compute::min_max::DictMinMaxKernel; /// The default session registers the built-in aggregate functions and kernels. Additional /// aggregate functions and kernels may be registered by extensions when they are added to a /// [`VortexSession`](vortex_session::VortexSession). -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct AggregateFnSession { registry: ArcSwapMap, @@ -222,7 +221,7 @@ impl AggregateFnSession { /// Extension trait for accessing aggregate function session data. pub trait AggregateFnSessionExt: SessionExt { /// Returns the aggregate function session data. - fn aggregate_fns(&self) -> Ref<'_, AggregateFnSession> { + fn aggregate_fns(&self) -> &AggregateFnSession { self.get::() } } diff --git a/vortex-array/src/arc_swap_map.rs b/vortex-array/src/arc_swap_map.rs index aff46c22d73..e8308d6343c 100644 --- a/vortex-array/src/arc_swap_map.rs +++ b/vortex-array/src/arc_swap_map.rs @@ -23,14 +23,27 @@ use vortex_utils::aliases::hash_map::HashMap; /// optimizer-kernel and aggregate-function registries). Because every write /// clones the entire map, it is intended for maps that are written rarely /// (typically only while a session is being configured) and read often. +/// +/// The map is held behind an [`Arc`] so that [`Clone`] shares the same +/// underlying cell: a registry mutated through one clone is observed by all +/// others. Session variables rely on this so that encodings registered after a +/// session is built remain visible to clones of that session. pub(crate) struct ArcSwapMap { - inner: ArcSwap>, + inner: Arc>>, } impl Default for ArcSwapMap { fn default() -> Self { Self { - inner: ArcSwap::from_pointee(HashMap::default()), + inner: Arc::new(ArcSwap::from_pointee(HashMap::default())), + } + } +} + +impl Clone for ArcSwapMap { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), } } } @@ -148,4 +161,15 @@ mod tests { map.insert(2, 2); assert_eq!(map.read(|m| m.values().sum::()), 3); } + + #[test] + fn clone_shares_the_same_cell() { + let map = ArcSwapMap::::default(); + let clone = map.clone(); + // A write through one handle is observed through the other. + map.insert(1, 10); + assert_eq!(clone.get(&1), Some(10)); + clone.insert(2, 20); + assert_eq!(map.get(&2), Some(20)); + } } diff --git a/vortex-array/src/arrays/bool/compute/cast.rs b/vortex-array/src/arrays/bool/compute/cast.rs index 0855af21f25..fe9332346ca 100644 --- a/vortex-array/src/arrays/bool/compute/cast.rs +++ b/vortex-array/src/arrays/bool/compute/cast.rs @@ -67,10 +67,8 @@ mod tests { use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn try_cast_bool_success() { diff --git a/vortex-array/src/arrays/chunked/tests.rs b/vortex-array/src/arrays/chunked/tests.rs index 00d113f273a..970656491ee 100644 --- a/vortex-array/src/arrays/chunked/tests.rs +++ b/vortex-array/src/arrays/chunked/tests.rs @@ -30,11 +30,9 @@ use crate::dtype::Nullability; use crate::dtype::PType; use crate::dtype::PType::I32; use crate::executor::execute_into_builder; -use crate::session::ArraySession; use crate::validity::Validity; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(crate::array_session); fn chunked_array() -> ChunkedArray { ChunkedArray::try_new( diff --git a/vortex-array/src/arrays/chunked/vtable/canonical.rs b/vortex-array/src/arrays/chunked/vtable/canonical.rs index adbe447604b..8c004f59ccb 100644 --- a/vortex-array/src/arrays/chunked/vtable/canonical.rs +++ b/vortex-array/src/arrays/chunked/vtable/canonical.rs @@ -323,12 +323,10 @@ mod tests { use crate::memory::MemorySessionExt; use crate::memory::WritableHostBuffer; use crate::scalar::Scalar; - use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for these chunked-array tests, used to create execution contexts. - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[derive(Debug)] struct CountingAllocator { @@ -640,12 +638,9 @@ mod tests { #[test] fn list_canonicalize_uses_memory_session_allocator() { let allocations = Arc::new(AtomicUsize::new(0)); - let session = VortexSession::empty(); - session - .memory_mut() - .set_allocator(Arc::new(CountingAllocator { - allocations: Arc::clone(&allocations), - })); + let session = crate::array_session().with_allocator(Arc::new(CountingAllocator { + allocations: Arc::clone(&allocations), + })); let mut ctx = ExecutionCtx::new(session); let l1 = ListArray::try_new( diff --git a/vortex-array/src/arrays/constant/vtable/canonical.rs b/vortex-array/src/arrays/constant/vtable/canonical.rs index dd891778949..e12db00c118 100644 --- a/vortex-array/src/arrays/constant/vtable/canonical.rs +++ b/vortex-array/src/arrays/constant/vtable/canonical.rs @@ -361,12 +361,10 @@ mod tests { use crate::expr::stats::Stat; use crate::expr::stats::StatsProvider; use crate::scalar::Scalar; - use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for these constant-array tests, used to create execution contexts. - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_canonicalize_null() { diff --git a/vortex-array/src/arrays/dict/compute/cast.rs b/vortex-array/src/arrays/dict/compute/cast.rs index 490f1f31549..63b20989f4c 100644 --- a/vortex-array/src/arrays/dict/compute/cast.rs +++ b/vortex-array/src/arrays/dict/compute/cast.rs @@ -66,10 +66,8 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_cast_dict_to_wider_type() { diff --git a/vortex-array/src/arrays/dict/compute/min_max.rs b/vortex-array/src/arrays/dict/compute/min_max.rs index 47a1f0eb8ee..8910ab12a53 100644 --- a/vortex-array/src/arrays/dict/compute/min_max.rs +++ b/vortex-array/src/arrays/dict/compute/min_max.rs @@ -74,10 +74,8 @@ mod tests { use crate::arrays::DictArray; use crate::arrays::PrimitiveArray; use crate::builders::dict::dict_encode; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn assert_min_max(array: &ArrayRef, expected: Option<(i32, i32)>) -> VortexResult<()> { let mut ctx = SESSION.create_execution_ctx(); diff --git a/vortex-array/src/arrays/dict/compute/mod.rs b/vortex-array/src/arrays/dict/compute/mod.rs index 28ba1d84789..5ed65c49bbc 100644 --- a/vortex-array/src/arrays/dict/compute/mod.rs +++ b/vortex-array/src/arrays/dict/compute/mod.rs @@ -81,10 +81,8 @@ mod test { use crate::dtype::Nullability; use crate::dtype::PType::I32; use crate::scalar_fn::fns::operators::Operator; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn canonicalise_nullable_primitive() { @@ -332,10 +330,8 @@ mod tests { use crate::compute::conformance::consistency::test_array_consistency; use crate::dtype::DType; use crate::dtype::Nullability; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[rstest] // Primitive arrays diff --git a/vortex-array/src/arrays/extension/compute/cast.rs b/vortex-array/src/arrays/extension/compute/cast.rs index c8e7fa17c36..9558eb35308 100644 --- a/vortex-array/src/arrays/extension/compute/cast.rs +++ b/vortex-array/src/arrays/extension/compute/cast.rs @@ -63,10 +63,8 @@ mod tests { use crate::executor::VortexSessionExecute; use crate::extension::datetime::TimeUnit; use crate::extension::datetime::Timestamp; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn cast_same_ext_dtype() { diff --git a/vortex-array/src/arrays/extension/compute/rules.rs b/vortex-array/src/arrays/extension/compute/rules.rs index 59a8c314517..611be1510a4 100644 --- a/vortex-array/src/arrays/extension/compute/rules.rs +++ b/vortex-array/src/arrays/extension/compute/rules.rs @@ -111,10 +111,8 @@ mod tests { use crate::scalar::ScalarValue; use crate::scalar_fn::fns::binary::Binary; use crate::scalar_fn::fns::operators::Operator; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] struct TestExt; diff --git a/vortex-array/src/arrays/filter/execute/listview.rs b/vortex-array/src/arrays/filter/execute/listview.rs index b3f787133ab..d6d55a72837 100644 --- a/vortex-array/src/arrays/filter/execute/listview.rs +++ b/vortex-array/src/arrays/filter/execute/listview.rs @@ -73,11 +73,9 @@ mod test { use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::filter::test_filter_conformance; - use crate::session::ArraySession; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_filter_listview_conformance() { diff --git a/vortex-array/src/arrays/filter/execute/take/tests.rs b/vortex-array/src/arrays/filter/execute/take/tests.rs index fd40865bf02..3d3a0d1a4e3 100644 --- a/vortex-array/src/arrays/filter/execute/take/tests.rs +++ b/vortex-array/src/arrays/filter/execute/take/tests.rs @@ -4,7 +4,6 @@ use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_mask::Mask; -use vortex_session::VortexSession; use crate::IntoArray; use crate::RecursiveCanonical; @@ -41,7 +40,7 @@ fn test_take_execute_kernel_maps_indices_through_filter() -> VortexResult<()> { filter.clone(), )? .into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -70,7 +69,7 @@ fn test_take_execute_kernel_nullable_fast_path_maps_indices_through_filter() -> filter.clone(), )? .into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -92,7 +91,7 @@ fn test_take_execute_kernel_fast_path_maps_indices_through_filter() -> VortexRes ) .into_array(); let parent = DictArray::try_new(buffer![2u64, 0, 3].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -113,7 +112,7 @@ fn assert_take_execute_rejects_out_of_bounds_rank( ) -> VortexResult<()> { let filter = FilterArray::new(child, filter_mask).into_array(); let parent = DictArray::try_new(codes, filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); if let Err(err) = filter.execute_parent(&parent, 1, &mut ctx) { assert!( @@ -191,7 +190,7 @@ fn test_take_execute_kernel_handles_empty_sequential_take() -> VortexResult<()> filter.clone(), )? .into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -216,7 +215,7 @@ fn assert_take_execute_maps_child_dtype( let filter = FilterArray::new(child, Mask::from_iter([true, false, true, true, false])).into_array(); let parent = DictArray::try_new(buffer![2u64, 0, 1].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -234,7 +233,7 @@ fn test_take_execute_kernel_skips_bool_filter_child() -> VortexResult<()> { ) .into_array(); let parent = DictArray::try_new(buffer![2u64, 0, 1].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter.execute_parent(&parent, 1, &mut ctx)?; @@ -256,7 +255,7 @@ fn execute_primitive_take( .into_array(); let indices = PrimitiveArray::from_iter((0..take_len).map(|idx| (idx % filtered_len) as u64)); let parent = DictArray::try_new(indices.into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); filter.execute_parent(&parent, 1, &mut ctx) } @@ -323,7 +322,7 @@ fn test_take_execute_kernel_handles_nullable_primitive_filter_child() -> VortexR ) .into_array(); let parent = DictArray::try_new(buffer![2u64, 0, 1].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter.execute_parent(&parent, 1, &mut ctx)?; @@ -345,7 +344,7 @@ fn test_take_execute_kernel_preserves_nullable_all_valid_fixed_width_child() -> ) .into_array(); let parent = DictArray::try_new(buffer![0u64, 1].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? @@ -372,7 +371,7 @@ fn test_take_execute_kernel_handles_nullable_decimal_filter_child() -> VortexRes ) .into_array(); let parent = DictArray::try_new(buffer![2u64, 0, 1].into_array(), filter.clone())?.into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter.execute_parent(&parent, 1, &mut ctx)?; @@ -465,7 +464,7 @@ fn test_take_execute_kernel_preserves_nullable_indices_dtype_fast_path() -> Vort filter.clone(), )? .into_array(); - let mut ctx = ExecutionCtx::new(VortexSession::empty()); + let mut ctx = ExecutionCtx::new(crate::array_session()); let result = filter .execute_parent(&parent, 1, &mut ctx)? diff --git a/vortex-array/src/arrays/list/compute/cast.rs b/vortex-array/src/arrays/list/compute/cast.rs index e6d71f6e80c..c297d378392 100644 --- a/vortex-array/src/arrays/list/compute/cast.rs +++ b/vortex-array/src/arrays/list/compute/cast.rs @@ -66,7 +66,6 @@ mod tests { use std::sync::LazyLock; use rstest::rstest; - use vortex_array::session::ArraySession; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -86,8 +85,7 @@ mod tests { use crate::dtype::PType; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_cast_list_success() { diff --git a/vortex-array/src/arrays/list/tests.rs b/vortex-array/src/arrays/list/tests.rs index 2ed260a5f55..b58b5e0a365 100644 --- a/vortex-array/src/arrays/list/tests.rs +++ b/vortex-array/src/arrays/list/tests.rs @@ -25,12 +25,10 @@ use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType::I32; use crate::scalar::Scalar; -use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for `List` tests, used to create execution contexts. -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_empty_list_array() { diff --git a/vortex-array/src/arrays/listview/tests/common.rs b/vortex-array/src/arrays/listview/tests/common.rs index be89b586209..1398a8defe6 100644 --- a/vortex-array/src/arrays/listview/tests/common.rs +++ b/vortex-array/src/arrays/listview/tests/common.rs @@ -12,13 +12,11 @@ use crate::IntoArray; use crate::arrays::BoolArray; use crate::arrays::ListViewArray; use crate::arrays::PrimitiveArray; -use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for `ListView` tests, used to create execution contexts via /// [`create_execution_ctx`](crate::VortexSessionExecute::create_execution_ctx). -pub static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +pub static SESSION: LazyLock = LazyLock::new(crate::array_session); /// Creates a basic ListView for testing: [[0,1,2], [3,4], [5,6], [7,8,9]] pub fn create_basic_listview() -> ListViewArray { diff --git a/vortex-array/src/arrays/listview/tests/density.rs b/vortex-array/src/arrays/listview/tests/density.rs index ad408f650b3..8538a9fe20c 100644 --- a/vortex-array/src/arrays/listview/tests/density.rs +++ b/vortex-array/src/arrays/listview/tests/density.rs @@ -7,7 +7,6 @@ use vortex_buffer::buffer; use vortex_error::VortexResult; use vortex_mask::Mask; -use vortex_session::VortexSession; use super::common::create_basic_listview; use super::common::create_empty_lists_listview; @@ -23,13 +22,12 @@ use crate::arrays::listview::tests::common::create_empty_elements_listview; use crate::expr::stats::Precision; use crate::expr::stats::Stat; use crate::scalar::ScalarValue; -use crate::session::ArraySession; use crate::validity::Validity; const EPS: f32 = 1e-6; fn test_execution_ctx() -> ExecutionCtx { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); session.create_execution_ctx() } diff --git a/vortex-array/src/arrays/listview/tests/filter.rs b/vortex-array/src/arrays/listview/tests/filter.rs index e218c94926a..756b1192aef 100644 --- a/vortex-array/src/arrays/listview/tests/filter.rs +++ b/vortex-array/src/arrays/listview/tests/filter.rs @@ -22,11 +22,9 @@ use crate::arrays::PrimitiveArray; use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::filter::test_filter_conformance; -use crate::session::ArraySession; use crate::validity::Validity; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(crate::array_session); // Conformance tests for common filter scenarios. #[rstest] diff --git a/vortex-array/src/arrays/listview/tests/take.rs b/vortex-array/src/arrays/listview/tests/take.rs index 572340259e9..4391af98ee2 100644 --- a/vortex-array/src/arrays/listview/tests/take.rs +++ b/vortex-array/src/arrays/listview/tests/take.rs @@ -21,11 +21,9 @@ use crate::arrays::PrimitiveArray; use crate::arrays::listview::ListViewArrayExt; use crate::assert_arrays_eq; use crate::compute::conformance::take::test_take_conformance; -use crate::session::ArraySession; use crate::validity::Validity; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(crate::array_session); // Conformance tests for common take scenarios. #[rstest] diff --git a/vortex-array/src/arrays/primitive/array/cast.rs b/vortex-array/src/arrays/primitive/array/cast.rs index 0d9db991d26..bdab4f0beb8 100644 --- a/vortex-array/src/arrays/primitive/array/cast.rs +++ b/vortex-array/src/arrays/primitive/array/cast.rs @@ -50,11 +50,9 @@ mod tests { use crate::dtype::DType; use crate::dtype::Nullability; use crate::dtype::PType; - use crate::session::ArraySession; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_downcast_all_invalid() { diff --git a/vortex-array/src/arrays/shared/tests.rs b/vortex-array/src/arrays/shared/tests.rs index b8301389074..fd454a8521e 100644 --- a/vortex-array/src/arrays/shared/tests.rs +++ b/vortex-array/src/arrays/shared/tests.rs @@ -3,7 +3,6 @@ use vortex_buffer::buffer; use vortex_error::VortexResult; -use vortex_session::VortexSession; use crate::Canonical; use crate::ExecutionCtx; @@ -13,7 +12,6 @@ use crate::arrays::SharedArray; use crate::arrays::shared::SharedArrayExt; use crate::hash::ArrayEq; use crate::hash::EqMode; -use crate::session::ArraySession; use crate::validity::Validity; #[test] @@ -21,7 +19,7 @@ fn shared_array_caches_on_canonicalize() -> VortexResult<()> { let array = PrimitiveArray::new(buffer![1i32, 2, 3], Validity::NonNullable).into_array(); let shared = SharedArray::new(array); - let session = VortexSession::empty().with::(); + let session = crate::array_session(); let mut ctx = ExecutionCtx::new(session); let first = shared.get_or_compute(|source| source.clone().execute::(&mut ctx))?; diff --git a/vortex-array/src/arrays/struct_/compute/cast.rs b/vortex-array/src/arrays/struct_/compute/cast.rs index f954dbe2fea..1f8ee705e9d 100644 --- a/vortex-array/src/arrays/struct_/compute/cast.rs +++ b/vortex-array/src/arrays/struct_/compute/cast.rs @@ -140,16 +140,13 @@ mod tests { use crate::dtype::Nullability; use crate::dtype::PType; use crate::dtype::StructFields; - use crate::optimizer::kernels::ArrayKernels; use crate::optimizer::kernels::ArrayKernelsExt; use crate::optimizer::kernels::ExecuteParentFn; use crate::scalar::Scalar; use crate::scalar_fn::fns::cast::Cast; - use crate::session::ArraySession; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn null_struct_cast_execute_parent( child: &ArrayRef, @@ -224,7 +221,7 @@ mod tests { .try_new_array(source.len(), target.clone(), [source]) .unwrap(); let parent_id = cast.encoding_id(); - let session = VortexSession::empty().with::(); + let session = crate::array_session(); session.kernels().register_execute_parent( parent_id, child_id, diff --git a/vortex-array/src/arrays/struct_/compute/rules.rs b/vortex-array/src/arrays/struct_/compute/rules.rs index ae3a67b50e3..287732beb7d 100644 --- a/vortex-array/src/arrays/struct_/compute/rules.rs +++ b/vortex-array/src/arrays/struct_/compute/rules.rs @@ -153,8 +153,7 @@ mod tests { use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::fns::cast::Cast; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn no_struct_cast_plugin( _child: &ArrayRef, diff --git a/vortex-array/src/arrays/varbin/compute/cast.rs b/vortex-array/src/arrays/varbin/compute/cast.rs index ca5dfde56e5..5983886ea30 100644 --- a/vortex-array/src/arrays/varbin/compute/cast.rs +++ b/vortex-array/src/arrays/varbin/compute/cast.rs @@ -81,10 +81,8 @@ mod tests { use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[rstest] #[case( diff --git a/vortex-array/src/arrays/varbinview/compute/cast.rs b/vortex-array/src/arrays/varbinview/compute/cast.rs index 449e735d1fc..476c74e7309 100644 --- a/vortex-array/src/arrays/varbinview/compute/cast.rs +++ b/vortex-array/src/arrays/varbinview/compute/cast.rs @@ -85,10 +85,8 @@ mod tests { use crate::compute::conformance::cast::test_cast_conformance; use crate::dtype::DType; use crate::dtype::Nullability; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[rstest] #[case( diff --git a/vortex-array/src/arrow/executor/list.rs b/vortex-array/src/arrow/executor/list.rs index e8bf95e14cd..7bf7a16f496 100644 --- a/vortex-array/src/arrow/executor/list.rs +++ b/vortex-array/src/arrow/executor/list.rs @@ -222,12 +222,10 @@ mod tests { use crate::arrow::executor::list::ListViewArray; use crate::dtype::DType; use crate::dtype::Nullability::NonNullable; - use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for these list-executor tests, used to create execution contexts. - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_to_arrow_list_i32() -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/list_view.rs b/vortex-array/src/arrow/executor/list_view.rs index 5a2cd517b37..7d5666b3762 100644 --- a/vortex-array/src/arrow/executor/list_view.rs +++ b/vortex-array/src/arrow/executor/list_view.rs @@ -124,12 +124,10 @@ mod tests { use crate::arrow::ArrowArrayExecutor; use crate::arrow::executor::list_view::ListViewArray; use crate::arrow::executor::list_view::PrimitiveArray; - use crate::session::ArraySession; use crate::validity::Validity; /// A shared session for these list-view-executor tests, used to create execution contexts. - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn trims_zero_copy_with_significant_trailing_waste() -> VortexResult<()> { diff --git a/vortex-array/src/arrow/executor/run_end.rs b/vortex-array/src/arrow/executor/run_end.rs index b579cab7e4d..0632f871f29 100644 --- a/vortex-array/src/arrow/executor/run_end.rs +++ b/vortex-array/src/arrow/executor/run_end.rs @@ -217,10 +217,8 @@ mod tests { use crate::dtype::PType; use crate::executor::VortexSessionExecute; use crate::scalar::Scalar; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn ree_type(ends: DataType, values_dtype: DataType) -> DataType { DataType::RunEndEncoded( diff --git a/vortex-array/src/arrow/session.rs b/vortex-array/src/arrow/session.rs index 705f7971f90..e94d403c242 100644 --- a/vortex-array/src/arrow/session.rs +++ b/vortex-array/src/arrow/session.rs @@ -39,7 +39,6 @@ use tracing::trace; use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_session::registry::Id; @@ -163,7 +162,7 @@ pub type ArrowImportVTableRef = Arc; /// keyed by Arrow extension name. The default session pre-registers the builtin UUID /// plugin; temporal extensions are handled by the canonical Arrow ↔ Vortex path and do not /// need plugins. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ArrowSession { exporters: ArcSwapMap>, exporters_by_vortex: ArcSwapMap>, @@ -610,11 +609,11 @@ impl SessionVar for ArrowSession { /// Extension trait for accessing the [`ArrowSession`] on a Vortex session. pub trait ArrowSessionExt: SessionExt { /// Get the Arrow session. - fn arrow(&self) -> Ref<'_, ArrowSession>; + fn arrow(&self) -> &ArrowSession; } impl ArrowSessionExt for S { - fn arrow(&self) -> Ref<'_, ArrowSession> { + fn arrow(&self) -> &ArrowSession { self.get::() } } diff --git a/vortex-array/src/builders/dict/bytes.rs b/vortex-array/src/builders/dict/bytes.rs index 081cef33f39..65c49fa0173 100644 --- a/vortex-array/src/builders/dict/bytes.rs +++ b/vortex-array/src/builders/dict/bytes.rs @@ -217,10 +217,8 @@ mod test { use crate::arrays::VarBinArray; use crate::arrays::dict::DictArraySlotsExt; use crate::builders::dict::dict_encode; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn encode_varbin() { diff --git a/vortex-array/src/builders/dict/primitive.rs b/vortex-array/src/builders/dict/primitive.rs index 7c0e68a456a..8d7defd6a50 100644 --- a/vortex-array/src/builders/dict/primitive.rs +++ b/vortex-array/src/builders/dict/primitive.rs @@ -168,10 +168,8 @@ mod test { use crate::assert_arrays_eq; use crate::builders::dict::dict_encode; use crate::builders::dict::primitive::PrimitiveArray; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn encode_primitive() { diff --git a/vortex-array/src/canonical.rs b/vortex-array/src/canonical.rs index 3c07650b2ef..7a2ac8a2a9f 100644 --- a/vortex-array/src/canonical.rs +++ b/vortex-array/src/canonical.rs @@ -1155,11 +1155,9 @@ mod test { use crate::canonical::StructArray; use crate::dtype::Nullability; use crate::scalar::Scalar; - use crate::session::ArraySession; /// A shared session for these canonical tests, used to create execution contexts. - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn variant_core_storage(len: usize) -> ArrayRef { ConstantArray::new( diff --git a/vortex-array/src/dtype/mod.rs b/vortex-array/src/dtype/mod.rs index a3a4765c68c..f0152c61301 100644 --- a/vortex-array/src/dtype/mod.rs +++ b/vortex-array/src/dtype/mod.rs @@ -188,8 +188,5 @@ mod test { use vortex_session::VortexSession; - use crate::dtype::session::DTypeSession; - - pub(crate) static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + pub(crate) static SESSION: LazyLock = LazyLock::new(crate::array_session); } diff --git a/vortex-array/src/dtype/serde/proto.rs b/vortex-array/src/dtype/serde/proto.rs index 35e944e0d66..78d4c62f412 100644 --- a/vortex-array/src/dtype/serde/proto.rs +++ b/vortex-array/src/dtype/serde/proto.rs @@ -229,8 +229,6 @@ impl TryFrom<&pb::FieldPath> for FieldPath { mod tests { use std::sync::Arc; - use vortex_session::VortexSession; - use super::*; use crate::dtype::DType; use crate::dtype::DecimalDType; @@ -503,7 +501,7 @@ mod tests { #[test] fn test_unknown_extension_allow_unknown() { - let session = VortexSession::empty().allow_unknown(); + let session = crate::array_session().allow_unknown(); let proto = pb::DType { dtype_type: Some(DtypeType::Extension(Box::new(pb::Extension { id: "vortex.test.foreign_ext".to_string(), diff --git a/vortex-array/src/dtype/session.rs b/vortex-array/src/dtype/session.rs index 757ffbd0ae5..e7aac12d4af 100644 --- a/vortex-array/src/dtype/session.rs +++ b/vortex-array/src/dtype/session.rs @@ -6,7 +6,6 @@ use std::any::Any; use std::sync::Arc; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_session::registry::Registry; @@ -22,7 +21,7 @@ use crate::extension::uuid::Uuid; pub type ExtDTypeRegistry = Registry; /// Session for managing extension dtypes. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct DTypeSession { registry: ExtDTypeRegistry, } @@ -69,11 +68,11 @@ impl DTypeSession { /// Extension trait for accessing the DType session. pub trait DTypeSessionExt: SessionExt { /// Get the DType session. - fn dtypes(&self) -> Ref<'_, DTypeSession>; + fn dtypes(&self) -> &DTypeSession; } impl DTypeSessionExt for S { - fn dtypes(&self) -> Ref<'_, DTypeSession> { + fn dtypes(&self) -> &DTypeSession { self.get::() } } diff --git a/vortex-array/src/executor.rs b/vortex-array/src/executor.rs index d6070ac1a4d..2bd5dfd5b37 100644 --- a/vortex-array/src/executor.rs +++ b/vortex-array/src/executor.rs @@ -26,8 +26,6 @@ use vortex_error::VortexResult; use vortex_error::vortex_bail; use vortex_error::vortex_ensure; use vortex_error::vortex_panic; -use vortex_session::Ref; -use vortex_session::SessionExt; use vortex_session::VortexSession; use crate::AnyCanonical; @@ -430,7 +428,7 @@ impl Executable for ArrayRef { for (slot_idx, slot) in array.slots().iter().enumerate() { let Some(child) = slot else { continue }; if let Some(executed_parent) = - execute_parent_for_child(&array, child, slot_idx, kernels.as_ref(), ctx)? + execute_parent_for_child(&array, child, slot_idx, kernels, ctx)? { ctx.log(format_args!( "execute_parent: slot[{}]({}) rewrote {} -> {}", @@ -542,7 +540,7 @@ fn execute_parent_for_child( parent: &ArrayRef, child: &ArrayRef, slot_idx: usize, - kernels: Option<&Ref>, + kernels: Option<&ArrayKernels>, ctx: &mut ExecutionCtx, ) -> VortexResult> { if let Some(kernels) = kernels @@ -567,7 +565,7 @@ fn try_execute_parent(array: &ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult< for (slot_idx, slot) in array.slots().iter().enumerate() { let Some(child) = slot else { continue }; if let Some(executed_parent) = - execute_parent_for_child(array, child, slot_idx, kernels.as_ref(), ctx)? + execute_parent_for_child(array, child, slot_idx, kernels, ctx)? { ctx.log(format_args!( "execute_parent: slot[{}]({}) rewrote {} -> {}", diff --git a/vortex-array/src/lib.rs b/vortex-array/src/lib.rs index 1f3189eecbb..b5ff5710489 100644 --- a/vortex-array/src/lib.rs +++ b/vortex-array/src/lib.rs @@ -27,7 +27,14 @@ pub use vortex_array_macros::array_slots; use vortex_session::VortexSession; use vortex_session::registry::Context; +use crate::aggregate_fn::session::AggregateFnSession; +use crate::arrow::ArrowSession; +use crate::dtype::session::DTypeSession; +use crate::memory::MemorySession; +use crate::optimizer::kernels::ArrayKernels; +use crate::scalar_fn::session::ScalarFnSession; use crate::session::ArraySession; +use crate::stats::session::StatsSession; pub mod accessor; pub mod aggregate_fn; @@ -78,10 +85,29 @@ pub mod flatbuffers { pub use vortex_flatbuffers::array::*; } +/// Builds a fresh [`VortexSession`] registered with all of vortex-array's built-in session +/// variables: arrays, dtypes, scalar functions, stats, optimizer kernels, aggregate functions, +/// Arrow conversion, and memory. +/// +/// Each call returns an independent session (with its own registries), so callers may register +/// additional encodings or kernels into it without affecting any other session. This does not +/// register file, layout, or runtime state — those live in higher-level crates. +pub fn array_session() -> VortexSession { + VortexSession::builder() + .with::() + .with::() + .with::() + .with::() + .with::() + .with::() + .with::() + .with::() + .build() +} + // TODO(ngates): canonicalize doesn't currently take a session, therefore we cannot invoke execute // from the new array encodings to support back-compat for legacy encodings. So we hold a session // here... -pub static LEGACY_SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +pub static LEGACY_SESSION: LazyLock = LazyLock::new(array_session); pub type ArrayContext = Context; diff --git a/vortex-array/src/memory.rs b/vortex-array/src/memory.rs index d3669e49f93..78713dfbde7 100644 --- a/vortex-array/src/memory.rs +++ b/vortex-array/src/memory.rs @@ -16,10 +16,9 @@ use vortex_buffer::ByteBufferMut; use vortex_error::VortexResult; use vortex_error::vortex_ensure; use vortex_error::vortex_err; -use vortex_session::Ref; -use vortex_session::RefMut; use vortex_session::SessionExt; use vortex_session::SessionVar; +use vortex_session::VortexSession; /// Mutable host buffer contract used by [`WritableHostBuffer`]. pub trait HostBufferMut: Send + 'static { @@ -173,7 +172,7 @@ pub trait HostAllocatorExt: HostAllocator { impl HostAllocatorExt for A {} /// Session-scoped memory configuration for Vortex arrays. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct MemorySession { allocator: HostAllocatorRef, } @@ -214,7 +213,7 @@ impl SessionVar for MemorySession { /// Extension trait for accessing session-scoped memory configuration. pub trait MemorySessionExt: SessionExt { /// Returns the memory session for this execution/session context. - fn memory(&self) -> Ref<'_, MemorySession> { + fn memory(&self) -> &MemorySession { self.get::() } @@ -223,9 +222,11 @@ pub trait MemorySessionExt: SessionExt { self.memory().allocator() } - /// Returns mutable access to the memory session. - fn memory_mut(&self) -> RefMut<'_, MemorySession> { - self.get_mut::() + /// Returns a new session configured to use `allocator` as its host allocator. + fn with_allocator(self, allocator: HostAllocatorRef) -> VortexSession { + let mut builder = self.session().to_builder(); + builder.get_mut::().set_allocator(allocator); + builder.build() } } diff --git a/vortex-array/src/optimizer/kernels.rs b/vortex-array/src/optimizer/kernels.rs index 93407a2c42d..b94393a49b3 100644 --- a/vortex-array/src/optimizer/kernels.rs +++ b/vortex-array/src/optimizer/kernels.rs @@ -30,7 +30,6 @@ use std::sync::Arc; use std::sync::LazyLock; use vortex_error::VortexResult; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_session::registry::Id; @@ -111,7 +110,7 @@ impl Borrow for ExecuteParentFnId { /// /// Each kernel kind has its own storage map, keyed by `(outer_id, child_id)`. Registering /// functions for an existing key appends them to that key's ordered list. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ArrayKernels { reduce_parent: ArcSwapMap>, execute_parent: ArcSwapMap>, @@ -213,9 +212,8 @@ impl SessionVar for ArrayKernels { /// Extension trait for accessing optimizer kernels from a /// [`VortexSession`](vortex_session::VortexSession). pub trait ArrayKernelsExt: SessionExt { - /// Returns the [`ArrayKernels`] session variable, inserting a default-constructed one if - /// none has been registered on the session yet. - fn kernels(&self) -> Ref<'_, ArrayKernels> { + /// Returns the [`ArrayKernels`] session variable. + fn kernels(&self) -> &ArrayKernels { self.get::() } } diff --git a/vortex-array/src/optimizer/mod.rs b/vortex-array/src/optimizer/mod.rs index d6e93ca0561..ca3ab1218ce 100644 --- a/vortex-array/src/optimizer/mod.rs +++ b/vortex-array/src/optimizer/mod.rs @@ -18,7 +18,6 @@ use smallvec::SmallVec; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_session::SessionExt; use vortex_session::VortexSession; use crate::ArrayRef; diff --git a/vortex-array/src/scalar/tests/mod.rs b/vortex-array/src/scalar/tests/mod.rs index a82b1908921..676363c853c 100644 --- a/vortex-array/src/scalar/tests/mod.rs +++ b/vortex-array/src/scalar/tests/mod.rs @@ -14,7 +14,4 @@ use std::sync::LazyLock; use vortex_session::VortexSession; -use crate::dtype::session::DTypeSession; - -pub(crate) static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +pub(crate) static SESSION: LazyLock = LazyLock::new(crate::array_session); diff --git a/vortex-array/src/scalar_fn/fns/between/mod.rs b/vortex-array/src/scalar_fn/fns/between/mod.rs index 0e0d9949195..013438e23b2 100644 --- a/vortex-array/src/scalar_fn/fns/between/mod.rs +++ b/vortex-array/src/scalar_fn/fns/between/mod.rs @@ -357,12 +357,10 @@ mod tests { use crate::expr::root; use crate::scalar::DecimalValue; use crate::scalar::Scalar; - use crate::session::ArraySession; use crate::test_harness::to_int_indices; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn test_display() { diff --git a/vortex-array/src/scalar_fn/fns/case_when.rs b/vortex-array/src/scalar_fn/fns/case_when.rs index c178e536b38..0a5f6b4ee14 100644 --- a/vortex-array/src/scalar_fn/fns/case_when.rs +++ b/vortex-array/src/scalar_fn/fns/case_when.rs @@ -473,10 +473,8 @@ mod tests { use crate::expr::root; use crate::expr::test_harness; use crate::scalar::Scalar; - use crate::session::ArraySession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); /// Helper to evaluate an expression using the apply+execute pattern fn evaluate_expr(expr: &Expression, array: &ArrayRef) -> ArrayRef { diff --git a/vortex-array/src/scalar_fn/session.rs b/vortex-array/src/scalar_fn/session.rs index 08a5951567b..106f214545c 100644 --- a/vortex-array/src/scalar_fn/session.rs +++ b/vortex-array/src/scalar_fn/session.rs @@ -4,7 +4,6 @@ use std::any::Any; use std::sync::Arc; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_session::registry::Registry; @@ -33,7 +32,7 @@ use crate::scalar_fn::fns::variant_get::VariantGet; pub type ScalarFnRegistry = Registry; /// Session state for scalar function vtables and rewrite rules. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ScalarFnSession { registry: ScalarFnRegistry, } @@ -92,7 +91,7 @@ impl SessionVar for ScalarFnSession { /// Extension trait for accessing scalar function session data. pub trait ScalarFnSessionExt: SessionExt { /// Returns the scalar function vtable registry. - fn scalar_fns(&self) -> Ref<'_, ScalarFnSession> { + fn scalar_fns(&self) -> &ScalarFnSession { self.get::() } } diff --git a/vortex-array/src/session/mod.rs b/vortex-array/src/session/mod.rs index 98a7417722b..b63fa64682e 100644 --- a/vortex-array/src/session/mod.rs +++ b/vortex-array/src/session/mod.rs @@ -6,7 +6,6 @@ use std::sync::Arc; use vortex_error::VortexResult; use vortex_error::vortex_bail; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_session::registry::Registry; @@ -33,7 +32,7 @@ use crate::arrays::Variant; pub type ArrayRegistry = Registry; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct ArraySession { /// The set of registered array encodings. registry: ArrayRegistry, @@ -100,7 +99,7 @@ impl SessionVar for ArraySession { /// Session data for Vortex arrays. pub trait ArraySessionExt: SessionExt { /// Returns the array encoding registry. - fn arrays(&self) -> Ref<'_, ArraySession> { + fn arrays(&self) -> &ArraySession { self.get::() } diff --git a/vortex-array/src/stats/expr.rs b/vortex-array/src/stats/expr.rs index 32fd48804de..58f0c93c508 100644 --- a/vortex-array/src/stats/expr.rs +++ b/vortex-array/src/stats/expr.rs @@ -100,11 +100,9 @@ mod tests { use crate::expr::stats::Stat; use crate::scalar::Scalar; use crate::scalar::ScalarValue; - use crate::session::ArraySession; use crate::validity::Validity; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); #[test] fn stat_expr_reads_cached_sum() -> VortexResult<()> { diff --git a/vortex-array/src/stats/rewrite.rs b/vortex-array/src/stats/rewrite.rs index 52d354df1a0..7d723155fad 100644 --- a/vortex-array/src/stats/rewrite.rs +++ b/vortex-array/src/stats/rewrite.rs @@ -156,7 +156,6 @@ fn rewrite( #[cfg(test)] mod tests { use vortex_error::VortexResult; - use vortex_session::VortexSession; use super::StatsRewriteCtx; use super::StatsRewriteRule; @@ -169,7 +168,6 @@ mod tests { use crate::scalar_fn::ScalarFnId; use crate::scalar_fn::ScalarFnVTable; use crate::scalar_fn::fns::literal::Literal; - use crate::stats::session::StatsSession; use crate::stats::session::StatsSessionExt; #[derive(Debug)] @@ -202,7 +200,7 @@ mod tests { #[test] fn combines_multiple_falsifiers_with_or() -> VortexResult<()> { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); session.stats().register_rewrite(StaticLiteralRule { falsifier: Some(lit(false)), @@ -222,7 +220,7 @@ mod tests { #[test] fn combines_multiple_satisfiers_with_or() -> VortexResult<()> { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); session.stats().register_rewrite(StaticLiteralRule { falsifier: None, @@ -242,7 +240,7 @@ mod tests { #[test] fn unregistered_expression_has_no_rewrite() -> VortexResult<()> { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); assert_eq!(lit(true).falsify(&dtype, &session)?, None); @@ -252,7 +250,7 @@ mod tests { #[test] fn non_predicate_expression_errors() { - let session = VortexSession::empty().with::(); + let session = crate::array_session(); let dtype = DType::Primitive(PType::I32, Nullability::NonNullable); assert!(lit(7).falsify(&dtype, &session).is_err()); diff --git a/vortex-array/src/stats/rewrite/builtins.rs b/vortex-array/src/stats/rewrite/builtins.rs index 2b7316c7d98..4311b7295f3 100644 --- a/vortex-array/src/stats/rewrite/builtins.rs +++ b/vortex-array/src/stats/rewrite/builtins.rs @@ -616,10 +616,8 @@ mod tests { use crate::scalar_fn::fns::dynamic::DynamicComparisonExpr; use crate::scalar_fn::fns::operators::CompareOperator; use crate::scalar_fn::internal::row_count::RowCount; - use crate::stats::session::StatsSession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(crate::array_session); fn stat(expr: Expression, stat: Stat) -> Expression { let aggregate_fn = stat.aggregate_fn().expect("stat should have aggregate fn"); diff --git a/vortex-array/src/stats/session.rs b/vortex-array/src/stats/session.rs index 2d4325b2cd7..867ec4f8fb0 100644 --- a/vortex-array/src/stats/session.rs +++ b/vortex-array/src/stats/session.rs @@ -7,7 +7,6 @@ use std::any::Any; use std::sync::Arc; use parking_lot::RwLock; -use vortex_session::Ref; use vortex_session::SessionExt; use vortex_session::SessionVar; use vortex_utils::aliases::hash_map::HashMap; @@ -20,15 +19,15 @@ use crate::stats::rewrite::register_builtins; type StatsRewriteRuleSet = Arc<[StatsRewriteRuleRef]>; /// Session state for stats APIs. -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct StatsSession { - rewrite_rules: RwLock>, + rewrite_rules: Arc>>, } impl Default for StatsSession { fn default() -> Self { let this = Self { - rewrite_rules: RwLock::new(HashMap::default()), + rewrite_rules: Arc::new(RwLock::new(HashMap::default())), }; register_builtins(&this); this @@ -77,7 +76,7 @@ impl SessionVar for StatsSession { /// Extension trait for accessing stats session data. pub(crate) trait StatsSessionExt: SessionExt { /// Returns the stats session state. - fn stats(&self) -> Ref<'_, StatsSession> { + fn stats(&self) -> &StatsSession { self.get::() } } diff --git a/vortex-btrblocks/benches/compress.rs b/vortex-btrblocks/benches/compress.rs index 8bc47d16404..6898b4a9ab4 100644 --- a/vortex-btrblocks/benches/compress.rs +++ b/vortex-btrblocks/benches/compress.rs @@ -17,14 +17,12 @@ mod benchmarks { use vortex_array::IntoArray; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; - use vortex_array::session::ArraySession; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; use vortex_session::VortexSession; use vortex_utils::aliases::hash_set::HashSet; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_clickbench_window_name() -> ArrayRef { // A test that's meant to mirror the WindowName column from ClickBench. diff --git a/vortex-btrblocks/benches/compress_listview.rs b/vortex-btrblocks/benches/compress_listview.rs index e1b0595ac27..502acf79e0f 100644 --- a/vortex-btrblocks/benches/compress_listview.rs +++ b/vortex-btrblocks/benches/compress_listview.rs @@ -21,7 +21,6 @@ mod benchmarks { use vortex_array::arrays::StructArray; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::FieldNames; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_btrblocks::BtrBlocksCompressor; use vortex_buffer::buffer_mut; @@ -30,8 +29,7 @@ mod benchmarks { const NUM_ROWS: usize = 8192; const SEED: u64 = 42; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); const SHORT_STRINGS: &[&str] = &[ "alpha_one", diff --git a/vortex-btrblocks/src/canonical_compressor.rs b/vortex-btrblocks/src/canonical_compressor.rs index 37e736fb550..ca77bc24ca8 100644 --- a/vortex-btrblocks/src/canonical_compressor.rs +++ b/vortex-btrblocks/src/canonical_compressor.rs @@ -75,7 +75,6 @@ mod tests { use vortex_array::assert_arrays_eq; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BitBuffer; use vortex_buffer::buffer; @@ -86,8 +85,7 @@ mod tests { #[cfg(feature = "zstd")] use crate::BtrBlocksCompressorBuilder; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[rstest] #[case::zctl( diff --git a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs index 9ddddef6df1..4c7a5f85fa6 100644 --- a/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/float/scheme_selection_tests.rs @@ -14,7 +14,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -22,8 +21,7 @@ use vortex_session::VortexSession; use crate::BtrBlocksCompressor; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_constant_compressed() -> VortexResult<()> { diff --git a/vortex-btrblocks/src/schemes/float/tests.rs b/vortex-btrblocks/src/schemes/float/tests.rs index 31fd88d99d8..da84831fcc0 100644 --- a/vortex-btrblocks/src/schemes/float/tests.rs +++ b/vortex-btrblocks/src/schemes/float/tests.rs @@ -12,7 +12,6 @@ use vortex_array::builders::ArrayBuilder; use vortex_array::builders::PrimitiveBuilder; use vortex_array::display::DisplayOptions; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::buffer_mut; @@ -24,8 +23,7 @@ use vortex_session::VortexSession; use crate::BtrBlocksCompressor; use crate::schemes::float::FloatRLEScheme; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_empty() -> VortexResult<()> { diff --git a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs index 993827d2057..94d2c0760e0 100644 --- a/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/integer/scheme_selection_tests.rs @@ -17,7 +17,6 @@ use vortex_array::arrays::PrimitiveArray; use vortex_array::expr::stats::Precision; use vortex_array::expr::stats::Stat; use vortex_array::expr::stats::StatsProviderExt; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_error::VortexResult; @@ -30,8 +29,7 @@ use vortex_sparse::Sparse; use crate::BtrBlocksCompressor; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_constant_compressed() -> VortexResult<()> { diff --git a/vortex-btrblocks/src/schemes/integer/tests.rs b/vortex-btrblocks/src/schemes/integer/tests.rs index 2f559af8582..5a3a0f33a40 100644 --- a/vortex-btrblocks/src/schemes/integer/tests.rs +++ b/vortex-btrblocks/src/schemes/integer/tests.rs @@ -15,7 +15,6 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::Masked; use vortex_array::arrays::PrimitiveArray; use vortex_array::assert_arrays_eq; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -29,8 +28,7 @@ use vortex_session::VortexSession; use crate::BtrBlocksCompressor; use crate::schemes::integer::IntRLEScheme; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_empty() -> VortexResult<()> { diff --git a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs index a32ac353a7f..c1473146607 100644 --- a/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs +++ b/vortex-btrblocks/src/schemes/string/scheme_selection_tests.rs @@ -12,15 +12,13 @@ use vortex_array::arrays::Dict; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_fsst::FSST; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_constant_compressed() -> VortexResult<()> { diff --git a/vortex-btrblocks/src/schemes/string/tests.rs b/vortex-btrblocks/src/schemes/string/tests.rs index 05b256eda6d..ec9c4060f64 100644 --- a/vortex-btrblocks/src/schemes/string/tests.rs +++ b/vortex-btrblocks/src/schemes/string/tests.rs @@ -11,14 +11,12 @@ use vortex_array::builders::VarBinViewBuilder; use vortex_array::display::DisplayOptions; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_error::VortexResult; use vortex_session::VortexSession; use crate::BtrBlocksCompressor; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); #[test] fn test_strings() -> VortexResult<()> { diff --git a/vortex-btrblocks/tests/onpair_roundtrip.rs b/vortex-btrblocks/tests/onpair_roundtrip.rs index 3843e02c319..31fb700a2d6 100644 --- a/vortex-btrblocks/tests/onpair_roundtrip.rs +++ b/vortex-btrblocks/tests/onpair_roundtrip.rs @@ -21,12 +21,10 @@ use vortex_array::accessor::ArrayAccessor; use vortex_array::arrays::VarBinViewArray; use vortex_array::dtype::DType; use vortex_array::dtype::Nullability; -use vortex_array::session::ArraySession; use vortex_btrblocks::BtrBlocksCompressor; use vortex_session::VortexSession; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); /// Helper: synthetic short-string corpus that the cascading compressor should /// route through OnPair. diff --git a/vortex-compressor/benches/dict_encode.rs b/vortex-compressor/benches/dict_encode.rs index 68b7cd98013..5b2b0650ffd 100644 --- a/vortex-compressor/benches/dict_encode.rs +++ b/vortex-compressor/benches/dict_encode.rs @@ -11,15 +11,13 @@ use vortex_array::VortexSessionExecute; use vortex_array::arrays::BoolArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::builders::dict::dict_encode; -use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::BufferMut; use vortex_compressor::builtins::integer_dictionary_encode; use vortex_compressor::stats::IntegerStats; use vortex_session::VortexSession; -static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); +static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn make_array() -> PrimitiveArray { let values: BufferMut = (0..50).cycle().take(64_000).collect(); diff --git a/vortex-compressor/benches/stats_calc.rs b/vortex-compressor/benches/stats_calc.rs index 69a8f970608..abbc34817f1 100644 --- a/vortex-compressor/benches/stats_calc.rs +++ b/vortex-compressor/benches/stats_calc.rs @@ -9,7 +9,6 @@ mod benchmarks { use divan::Bencher; use vortex_array::VortexSessionExecute; use vortex_array::arrays::PrimitiveArray; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::Buffer; use vortex_buffer::BufferMut; @@ -17,8 +16,7 @@ mod benchmarks { use vortex_compressor::stats::IntegerStats; use vortex_session::VortexSession; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn generate_dataset(max_run: u32, distinct: u32) -> Buffer { let mut output = BufferMut::with_capacity(64_000); diff --git a/vortex-compressor/src/builtins/dict/float.rs b/vortex-compressor/src/builtins/dict/float.rs index bf591963db3..2ab34d52e0c 100644 --- a/vortex-compressor/src/builtins/dict/float.rs +++ b/vortex-compressor/src/builtins/dict/float.rs @@ -253,11 +253,9 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use super::dictionary_encode; use crate::stats::FloatStats; @@ -265,9 +263,7 @@ mod tests { #[test] fn test_float_dict_encode() -> VortexResult<()> { - let mut ctx = VortexSession::empty() - .with::() - .create_execution_ctx(); + let mut ctx = vortex_array::array_session().create_execution_ctx(); let values = buffer![1f32, 2f32, 2f32, 0f32, 1f32]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); diff --git a/vortex-compressor/src/builtins/dict/integer.rs b/vortex-compressor/src/builtins/dict/integer.rs index 82af7f14de7..1bb9ede306c 100644 --- a/vortex-compressor/src/builtins/dict/integer.rs +++ b/vortex-compressor/src/builtins/dict/integer.rs @@ -260,20 +260,16 @@ mod tests { use vortex_array::arrays::PrimitiveArray; use vortex_array::arrays::dict::DictArraySlotsExt; use vortex_array::assert_arrays_eq; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_error::VortexResult; - use vortex_session::VortexSession; use super::dictionary_encode; use crate::stats::IntegerStats; #[test] fn test_dict_encode_integer_stats() -> VortexResult<()> { - let mut ctx = VortexSession::empty() - .with::() - .create_execution_ctx(); + let mut ctx = vortex_array::array_session().create_execution_ctx(); let data = buffer![100i32, 200, 100, 0, 100]; let validity = Validity::Array(BoolArray::from_iter([true, true, true, false, true]).into_array()); diff --git a/vortex-compressor/src/compressor.rs b/vortex-compressor/src/compressor.rs index 33502104b13..77099df91f8 100644 --- a/vortex-compressor/src/compressor.rs +++ b/vortex-compressor/src/compressor.rs @@ -596,7 +596,6 @@ mod tests { use vortex_array::arrays::Constant; use vortex_array::arrays::NullArray; use vortex_array::arrays::PrimitiveArray; - use vortex_array::session::ArraySession; use vortex_array::validity::Validity; use vortex_buffer::buffer; use vortex_session::VortexSession; @@ -613,8 +612,7 @@ mod tests { use crate::estimate::WinnerEstimate; use crate::scheme::SchemeExt; - static SESSION: LazyLock = - LazyLock::new(|| VortexSession::empty().with::()); + static SESSION: LazyLock = LazyLock::new(vortex_array::array_session); fn compressor() -> CascadingCompressor { CascadingCompressor::new(vec![&IntDictScheme, &FloatDictScheme, &StringDictScheme]) diff --git a/vortex-cuda/benches/alp_cuda.rs b/vortex-cuda/benches/alp_cuda.rs index 342f85d4902..3dc01c157be 100644 --- a/vortex-cuda/benches/alp_cuda.rs +++ b/vortex-cuda/benches/alp_cuda.rs @@ -32,7 +32,6 @@ use vortex::encodings::alp::ALPArrayExt; use vortex::encodings::alp::ALPFloat; use vortex::encodings::alp::alp_encode; use vortex::error::VortexExpect; -use vortex::session::VortexSession; use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; @@ -110,7 +109,7 @@ where let timer = timed.timer(); let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()) + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .vortex_expect("failed to create execution context") .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); diff --git a/vortex-cuda/benches/arrow_binary_cuda.rs b/vortex-cuda/benches/arrow_binary_cuda.rs index bd3154e471a..6668abd3866 100644 --- a/vortex-cuda/benches/arrow_binary_cuda.rs +++ b/vortex-cuda/benches/arrow_binary_cuda.rs @@ -29,7 +29,6 @@ use vortex::dtype::DType; use vortex::dtype::Nullability; use vortex::error::VortexExpect; use vortex::error::VortexResult; -use vortex::session::VortexSession; use vortex_cuda::CudaExecutionCtx; use vortex_cuda::CudaSession; use vortex_cuda::arrow::ArrowDeviceArray; @@ -92,9 +91,10 @@ fn benchmark_arrow_binary_export(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); let array = block_on(out_of_line_binary(len, &mut cuda_ctx)) .vortex_expect("failed to create binary fixture"); diff --git a/vortex-cuda/benches/arrow_validity_cuda.rs b/vortex-cuda/benches/arrow_validity_cuda.rs index 9221d7c0698..3e423e037a5 100644 --- a/vortex-cuda/benches/arrow_validity_cuda.rs +++ b/vortex-cuda/benches/arrow_validity_cuda.rs @@ -17,7 +17,6 @@ use futures::executor::block_on; use vortex::array::buffer::BufferHandle; use vortex::buffer::BitBuffer; use vortex::error::VortexExpect; -use vortex::session::VortexSession; use vortex_cuda::CudaSession; use vortex_cuda::arrow::test_harness; use vortex_cuda_macros::cuda_available; @@ -41,9 +40,10 @@ fn benchmark_arrow_validity_repack(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); let source = BitBuffer::collect_bool(len + INPUT_OFFSET, |idx| idx % 3 != 0); let sliced = source.slice(INPUT_OFFSET..INPUT_OFFSET + len); let (input_offset, _, input_buffer) = sliced.into_inner(); diff --git a/vortex-cuda/benches/bitpacked_cuda.rs b/vortex-cuda/benches/bitpacked_cuda.rs index 5f73f02feb5..d2f68ed8f10 100644 --- a/vortex-cuda/benches/bitpacked_cuda.rs +++ b/vortex-cuda/benches/bitpacked_cuda.rs @@ -31,7 +31,6 @@ use vortex::encodings::fastlanes::BitPackedArray; use vortex::encodings::fastlanes::BitPackedData; use vortex::encodings::fastlanes::unpack_iter::BitPacked; use vortex::error::VortexExpect; -use vortex::session::VortexSession; use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; @@ -129,10 +128,11 @@ where let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { block_on(array.clone().into_array().execute_cuda(&mut cuda_ctx)).unwrap(); @@ -184,7 +184,7 @@ where let timer = timed.timer(); let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()) + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .vortex_expect("failed to create execution context") .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) .with_launch_strategy(Arc::new(timed)); diff --git a/vortex-cuda/benches/date_time_parts_cuda.rs b/vortex-cuda/benches/date_time_parts_cuda.rs index 06d45dd2354..f5816717a55 100644 --- a/vortex-cuda/benches/date_time_parts_cuda.rs +++ b/vortex-cuda/benches/date_time_parts_cuda.rs @@ -30,7 +30,6 @@ use vortex::encodings::datetime_parts::DateTimePartsArray; use vortex::error::VortexExpect; use vortex::extension::datetime::TimeUnit; use vortex::extension::datetime::Timestamp; -use vortex::session::VortexSession; use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; @@ -68,10 +67,11 @@ fn benchmark_datetimeparts(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { // block on immediately here diff --git a/vortex-cuda/benches/dict_cuda.rs b/vortex-cuda/benches/dict_cuda.rs index 0cc785c869a..6ed91f4179e 100644 --- a/vortex-cuda/benches/dict_cuda.rs +++ b/vortex-cuda/benches/dict_cuda.rs @@ -27,7 +27,6 @@ use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::error::VortexExpect; -use vortex::session::VortexSession; use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; @@ -96,10 +95,11 @@ where let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { block_on(dict_array.clone().into_array().execute_cuda(&mut cuda_ctx)) diff --git a/vortex-cuda/benches/dynamic_dispatch_cuda.rs b/vortex-cuda/benches/dynamic_dispatch_cuda.rs index cf89c95efff..c3112e6eb0b 100644 --- a/vortex-cuda/benches/dynamic_dispatch_cuda.rs +++ b/vortex-cuda/benches/dynamic_dispatch_cuda.rs @@ -53,7 +53,6 @@ use vortex::encodings::runend::RunEnd; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; -use vortex::session::VortexSession; use vortex_cuda::CudaBufferExt; use vortex_cuda::CudaDeviceBuffer; use vortex_cuda::CudaDispatchMode; @@ -221,8 +220,8 @@ fn bench_for_bitpacked(c: &mut Criterion) { BenchmarkId::new("cuda/for_bitpacked_6bw/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -266,8 +265,8 @@ fn bench_dict_bp_codes(c: &mut Criterion) { BenchmarkId::new("cuda/dict_256vals_bp8bw_codes/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -310,8 +309,8 @@ fn bench_runend(c: &mut Criterion) { BenchmarkId::new("cuda/runend_100runs/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -387,8 +386,8 @@ fn bench_dict_bp_codes_alp_for_bp_values_dynanmic_dispatch(c: &mut Criterion) { ), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -702,7 +701,7 @@ fn bench_dict_bp_codes_alp_for_bp_values_composed_standalone(c: &mut Criterion) &(values_bp, values_reference, codes_bp), |b, (values_bp, values_reference, codes_bp)| { b.iter_custom(|iters| { - let session = VortexSession::empty(); + let session = vortex_cuda::cuda_session(); let cuda_session = session.cuda_session(); let mut cuda_ctx = CudaSession::create_execution_ctx(&session) .vortex_expect("ctx") @@ -713,7 +712,7 @@ fn bench_dict_bp_codes_alp_for_bp_values_composed_standalone(c: &mut Criterion) exponents, codes_bp, *len, - &cuda_session, + cuda_session, &mut cuda_ctx, ); @@ -777,8 +776,8 @@ fn bench_alp_for_bitpacked_f64(c: &mut Criterion) { BenchmarkId::new("cuda/alp_for_bp_6bw_f64/dispatch_f64", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -834,8 +833,8 @@ fn bench_dict_bp_codes_bp_for_values(c: &mut Criterion) { ), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -899,8 +898,8 @@ fn bench_alp_for_bitpacked(c: &mut Criterion) { BenchmarkId::new("cuda/alp_for_bp_6bw_f32/dispatch_f32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -946,8 +945,8 @@ fn bench_dict_bp_u8_codes_u32_values(c: &mut Criterion) { BenchmarkId::new("cuda/dict_widen_u8_to_u32/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -989,8 +988,8 @@ fn bench_dict_bp_u16_codes_u32_values(c: &mut Criterion) { BenchmarkId::new("cuda/dict_widen_u16_to_u32/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); @@ -1032,8 +1031,8 @@ fn bench_dict_bp_u32_codes_u32_values(c: &mut Criterion) { BenchmarkId::new("cuda/dict_nowiden_u32_to_u32/dispatch_u32", len_str), len, |b, &n| { - let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + let mut cuda_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("ctx"); let bench_runner = BenchRunner::::new(&array, n, &mut cuda_ctx); diff --git a/vortex-cuda/benches/filter_cuda.rs b/vortex-cuda/benches/filter_cuda.rs index f2d9cb74259..a9f55707a7c 100644 --- a/vortex-cuda/benches/filter_cuda.rs +++ b/vortex-cuda/benches/filter_cuda.rs @@ -27,7 +27,6 @@ use futures::executor::block_on; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; -use vortex::session::VortexSession; use vortex_cub::filter::CubFilterable; use vortex_cub::filter::cudaStream_t; use vortex_cuda::CudaDeviceBuffer; @@ -165,7 +164,7 @@ where |b, (input_data, bitmask, true_count)| { b.iter_custom(|iters| { let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()) + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .vortex_expect("failed to create execution context"); let num_items = input_data.len() as i64; diff --git a/vortex-cuda/benches/for_cuda.rs b/vortex-cuda/benches/for_cuda.rs index 44e60858a48..e8ffb3ccb7e 100644 --- a/vortex-cuda/benches/for_cuda.rs +++ b/vortex-cuda/benches/for_cuda.rs @@ -33,7 +33,6 @@ use vortex::encodings::fastlanes::FoR; use vortex::encodings::fastlanes::FoRArray; use vortex::error::VortexExpect; use vortex::scalar::Scalar; -use vortex::session::VortexSession; use vortex_cuda::CudaDispatchMode; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; @@ -90,10 +89,11 @@ where let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { block_on(for_array.clone().into_array().execute_cuda(&mut cuda_ctx)) @@ -129,10 +129,11 @@ where let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_dispatch_mode(CudaDispatchMode::StandaloneOnly) + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { block_on(for_array.clone().into_array().execute_cuda(&mut cuda_ctx)) diff --git a/vortex-cuda/benches/fsst_cuda.rs b/vortex-cuda/benches/fsst_cuda.rs index c7aeed28151..21db8744919 100644 --- a/vortex-cuda/benches/fsst_cuda.rs +++ b/vortex-cuda/benches/fsst_cuda.rs @@ -23,7 +23,6 @@ use vortex::array::arrays::PrimitiveArray; use vortex::array::match_each_integer_ptype; use vortex::encodings::fsst::FSSTArrayExt; use vortex::error::VortexExpect; -use vortex::session::VortexSession; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -41,7 +40,7 @@ fn benchmark_fsst_cuda_decompress(c: &mut Criterion) { let mut group = c.benchmark_group("cuda"); for &(n, len_str) in BENCH_SIZES { - let mut setup_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut setup_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .vortex_expect("failed to create execution context"); let fsst = make_fsst_clickbench_urls(n, setup_ctx.execution_ctx()); @@ -66,9 +65,10 @@ fn benchmark_fsst_cuda_decompress(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); for _ in 0..iters { block_on(fsst_array.clone().execute_cuda(&mut cuda_ctx)).unwrap(); diff --git a/vortex-cuda/benches/list_view_cuda.rs b/vortex-cuda/benches/list_view_cuda.rs index 0904844b900..4ce365210d6 100644 --- a/vortex-cuda/benches/list_view_cuda.rs +++ b/vortex-cuda/benches/list_view_cuda.rs @@ -25,7 +25,6 @@ use vortex::array::validity::Validity; use vortex::dtype::PType; use vortex::error::VortexExpect; use vortex::error::VortexResult; -use vortex::session::VortexSession; use vortex_cuda::CudaExecutionCtx; use vortex_cuda::CudaSession; use vortex_cuda::arrow::ArrowDeviceArray; @@ -90,9 +89,10 @@ fn benchmark_list_view_export(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); let array = block_on(contiguous_list_view(len, &mut cuda_ctx)) .vortex_expect("failed to create list-view fixture"); @@ -120,9 +120,10 @@ fn benchmark_list_view_export(c: &mut Criterion) { let timed = TimedLaunchStrategy::default(); let timer = timed.timer(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context") - .with_launch_strategy(Arc::new(timed)); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context") + .with_launch_strategy(Arc::new(timed)); let array = block_on(non_contiguous_primitive_list_view(len, &mut cuda_ctx)) .vortex_expect("failed to create list-view fixture"); diff --git a/vortex-cuda/benches/runend_cuda.rs b/vortex-cuda/benches/runend_cuda.rs index ceb79115604..194923654e0 100644 --- a/vortex-cuda/benches/runend_cuda.rs +++ b/vortex-cuda/benches/runend_cuda.rs @@ -27,7 +27,6 @@ use vortex::buffer::Buffer; use vortex::dtype::NativePType; use vortex::encodings::runend::RunEnd; use vortex::encodings::runend::RunEndArray; -use vortex::session::VortexSession; use vortex_cuda::CudaSession; use vortex_cuda::executor::CudaArrayExt; use vortex_cuda_macros::cuda_available; @@ -75,7 +74,8 @@ where group.throughput(Throughput::Bytes((len * size_of::()) as u64)); for run_len in [10, 1000, 100000] { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()).unwrap(); let runend_array = make_runend_array_typed::(len, run_len, cuda_ctx.execution_ctx()); group.bench_with_input( @@ -87,7 +87,7 @@ where let timer = timed.timer(); let mut cuda_ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()) + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .unwrap() .with_launch_strategy(Arc::new(timed)); diff --git a/vortex-cuda/benches/throughput_cuda.rs b/vortex-cuda/benches/throughput_cuda.rs index bcbaf8058aa..d40b41c5c41 100644 --- a/vortex-cuda/benches/throughput_cuda.rs +++ b/vortex-cuda/benches/throughput_cuda.rs @@ -21,7 +21,6 @@ use cudarc::driver::DevicePtr; use cudarc::driver::DevicePtrMut; use cudarc::driver::sys; use cudarc::driver::sys::CUevent_flags; -use vortex::session::VortexSession; use vortex_cuda::CudaExecutionCtx; use vortex_cuda::CudaSession; use vortex_cuda_macros::cuda_available; @@ -107,7 +106,7 @@ fn benchmark_transfer_throughput(c: &mut Criterion) { &(input_bytes, output_bytes), |b, &(in_bytes, out_bytes)| { b.iter_custom(|iters| { - let session = VortexSession::empty(); + let session = vortex_cuda::cuda_session(); let mut in_ctx = CudaSession::create_execution_ctx(&session).unwrap(); let mut out_ctx = CudaSession::create_execution_ctx(&session).unwrap(); diff --git a/vortex-cuda/benches/zstd_cuda.rs b/vortex-cuda/benches/zstd_cuda.rs index 797e8b76ec8..c0f9afaadd1 100644 --- a/vortex-cuda/benches/zstd_cuda.rs +++ b/vortex-cuda/benches/zstd_cuda.rs @@ -19,7 +19,6 @@ use vortex::encodings::zstd::ZstdDataParts; use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_err; -use vortex::session::VortexSession; use vortex_cuda::CudaSession; use vortex_cuda::ZstdKernelPrep; use vortex_cuda::nvcomp::zstd as nvcomp_zstd; @@ -126,7 +125,7 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { let mut group = c.benchmark_group("cuda"); for (num_strings, label) in BENCH_SIZES { - let mut setup_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut setup_ctx = CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) .vortex_expect("failed to create execution context"); let (zstd_array, uncompressed_size) = make_zstd_array(*num_strings, &mut setup_ctx) .vortex_expect("failed to create ZSTD array"); @@ -137,8 +136,9 @@ fn benchmark_zstd_cuda_decompress(c: &mut Criterion) { &zstd_array, |b, zstd_array| { b.iter_custom(|iters| { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) - .vortex_expect("failed to create execution context"); + let mut cuda_ctx = + CudaSession::create_execution_ctx(&vortex_cuda::cuda_session()) + .vortex_expect("failed to create execution context"); let mut total_time = Duration::ZERO; diff --git a/vortex-cuda/ffi/src/lib.rs b/vortex-cuda/ffi/src/lib.rs index 2a74cb2666c..f3f97210d5b 100644 --- a/vortex-cuda/ffi/src/lib.rs +++ b/vortex-cuda/ffi/src/lib.rs @@ -14,7 +14,6 @@ use std::ptr; use arrow_schema::ffi::FFI_ArrowSchema; use vortex::error::VortexResult; use vortex::error::vortex_ensure; -use vortex::session::SessionExt; use vortex::session::VortexSession; use vortex_cuda::CudaSession; use vortex_cuda::arrow::ArrowDeviceArray; diff --git a/vortex-cuda/src/arrow/canonical.rs b/vortex-cuda/src/arrow/canonical.rs index cb116c9cda0..6a4c97f36c7 100644 --- a/vortex-cuda/src/arrow/canonical.rs +++ b/vortex-cuda/src/arrow/canonical.rs @@ -1259,7 +1259,6 @@ mod tests { use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::extension::datetime::TimeUnit; - use vortex::session::VortexSession; use crate::CudaExecutionCtx; use crate::arrow::ARROW_DEVICE_CUDA; @@ -1594,7 +1593,7 @@ mod tests { #[case] expected_len: i64, #[case] expected_data_type: DataType, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut exported = array.export_device_array_with_schema(&mut ctx).await?; @@ -1615,7 +1614,7 @@ mod tests { #[crate::test] async fn test_export_null() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = NullArray::new(7).into_array(); @@ -1631,7 +1630,7 @@ mod tests { #[crate::test] async fn test_export_dictionary() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let out_of_line = "a dictionary value stored out-of-line"; @@ -1676,7 +1675,7 @@ mod tests { #[crate::test] async fn test_export_struct_preserves_dictionary_child() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let dictionary = DictArray::try_new( @@ -1718,7 +1717,7 @@ mod tests { #[crate::test] async fn test_export_dictionary_with_nullable_values() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = DictArray::try_new( @@ -1755,7 +1754,7 @@ mod tests { expected_data_type: DataType, expected_values: Vec, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut exported = array.export_device_array_with_schema(&mut ctx).await?; @@ -1885,7 +1884,7 @@ mod tests { #[crate::test] async fn test_export_decimal_narrowing_errors() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = DecimalArray::from_iter([i256::from_parts(0, 1)], DecimalDType::new(38, 0)) .into_array(); @@ -1900,7 +1899,7 @@ mod tests { #[crate::test] async fn test_export_decimal_narrowing_from_arrow_import() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = DecimalArray::from_iter([0i128, 1, -2], DecimalDType::new(10, 2)).into_array(); @@ -1943,7 +1942,7 @@ mod tests { #[crate::test] async fn test_export_temporal() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = TemporalArray::new_date( @@ -1966,7 +1965,7 @@ mod tests { #[crate::test] async fn test_export_bool() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = BoolArray::from_iter([true, false, true]).into_array(); @@ -1985,7 +1984,7 @@ mod tests { #[crate::test] async fn test_export_varbinview() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let out_of_line = "this is a longer string for out-of-line storage"; @@ -2001,7 +2000,7 @@ mod tests { #[crate::test] async fn test_export_binary_inline_outline_values() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let out_of_line = b"this binary payload is longer than twelve bytes"; @@ -2032,7 +2031,7 @@ mod tests { #[crate::test] async fn test_export_binary_empty_and_all_null() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let empty = VarBinViewArray::from_iter_nullable_bin(std::iter::empty::>()) @@ -2058,7 +2057,7 @@ mod tests { #[crate::test] async fn test_export_binary_invalid_data_buffer_ref_errors() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let view = BinaryView::make_view(b"this references a missing data buffer", 0, 0); @@ -2084,7 +2083,7 @@ mod tests { #[crate::test] async fn test_export_binary_i32_offset_overflow_errors() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let view = BinaryView::new_ref(i32::MAX as u32 + 1, [0; 4], 0, 0); @@ -2110,7 +2109,7 @@ mod tests { #[crate::test] async fn test_export_list() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = ListArray::try_new( @@ -2149,7 +2148,7 @@ mod tests { #[crate::test] async fn test_export_host_contiguous_list_view() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = ListViewArray::new( @@ -2176,7 +2175,7 @@ mod tests { #[crate::test] async fn test_export_host_non_contiguous_nested_list_view_falls_back_to_cpu() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = StructArray::new( @@ -2218,7 +2217,7 @@ mod tests { #[crate::test] async fn test_export_host_non_contiguous_dictionary_list_view_schema_matches_rebuilt_child() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = DictArray::try_new( @@ -2276,7 +2275,7 @@ mod tests { #[crate::test] async fn test_export_host_large_lists_dictionary_list_view_schema_matches_rebuilt_child() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = DictArray::try_new( @@ -2329,7 +2328,7 @@ mod tests { #[case] offsets_ptype: PType, #[case] sizes_ptype: PType, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = primitive_i32_on_device(0..5, &mut ctx).await?; @@ -2374,7 +2373,7 @@ mod tests { #[case] fixture: (ArrayRef, [usize; 2]), #[case] expected_data_type: DataType, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let (array, expected_data_buffer_lengths) = fixture; @@ -2405,7 +2404,7 @@ mod tests { #[case::u64(PrimitiveArray::from_iter([0u64, 2, 2, 5]).into_array())] #[crate::test] async fn test_export_list_with_non_i32_offsets(#[case] offsets: ArrayRef) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = ListArray::try_new( @@ -2437,7 +2436,7 @@ mod tests { #[case] offsets_ptype: PType, #[case] sizes_ptype: PType, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = primitive_i32_on_device([10, 11, 12, 13, 14], &mut ctx).await?; @@ -2466,7 +2465,7 @@ mod tests { #[crate::test] async fn test_export_device_non_contiguous_dictionary_list_view() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let codes = primitive_on_device([0u8, 1, 2, 0, 1], &mut ctx).await?; @@ -2514,7 +2513,7 @@ mod tests { #[crate::test] async fn test_export_device_non_contiguous_dictionary_list_view_nullable_values() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let codes = primitive_on_device([0u8, 1, 2, 0, 1], &mut ctx).await?; @@ -2543,7 +2542,7 @@ mod tests { #[crate::test] async fn test_export_device_non_contiguous_dictionary_list_view_nullable_codes_errors() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let codes = PrimitiveArray::from_option_iter([Some(0u8), None, Some(2), Some(0), Some(1)]); @@ -2581,7 +2580,7 @@ mod tests { #[case] sizes_values: &[i64], #[case] expected_error: &str, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = primitive_i32_on_device(0..4, &mut ctx).await?; @@ -2609,7 +2608,7 @@ mod tests { #[crate::test] async fn test_export_device_non_contiguous_nested_list_view_returns_error() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let field = primitive_i32_on_device(0..4, &mut ctx).await?; @@ -2643,7 +2642,7 @@ mod tests { #[crate::test] async fn test_export_device_non_contiguous_nullable_primitive_list_view_returns_error() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let elements = nullable_primitive_i32_on_device( @@ -2673,7 +2672,7 @@ mod tests { #[crate::test] async fn test_export_fixed_size_list_as_list() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = FixedSizeListArray::new( @@ -2718,7 +2717,7 @@ mod tests { // Check device metadata for data-bearing and metadata-only exports. #[crate::test] async fn test_export_device_metadata() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let expected_device_id = ctx.stream().context().ordinal() as i64; @@ -2740,7 +2739,7 @@ mod tests { // Check sliced arrays preserve the expected Arrow length/offset metadata. #[crate::test] async fn test_export_sliced_arrays() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let primitive = PrimitiveArray::from_iter(0u32..10) @@ -2766,7 +2765,7 @@ mod tests { #[crate::test] async fn test_export_sliced_varbinview_arrays() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let utf8 = VarBinViewArray::from_iter_nullable_str([ @@ -2831,7 +2830,7 @@ mod tests { #[case] arrow_offset: usize, #[case] len: usize, ) -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let logical_bits = (0..len).map(|idx| idx % 3 != 0).collect::>(); @@ -2864,7 +2863,7 @@ mod tests { #[crate::test] async fn test_repack_arrow_validity_buffer_zeroes_padding() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let len = 9; @@ -2894,7 +2893,7 @@ mod tests { #[crate::test] async fn test_export_all_false_validity_buffer_is_zeroed_on_device() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let len = 4; @@ -2920,7 +2919,7 @@ mod tests { // Check nullable primitives export Arrow null bitmaps on device. #[crate::test] async fn test_export_nullable_primitive() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut primitive = assert_nullable_export( @@ -2947,7 +2946,7 @@ mod tests { // Check nullable bool exports preserve Arrow offset metadata. #[crate::test] async fn test_export_nullable_bool() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut bools = assert_nullable_export( @@ -2968,7 +2967,7 @@ mod tests { // Check synthesized all-null bool validity is large enough for Arrow offset-based reads. #[crate::test] async fn test_export_all_null_sliced_bool_validity_covers_arrow_offset() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut bools = assert_nullable_export( @@ -2994,7 +2993,7 @@ mod tests { // Check nullable decimal exports include Arrow null bitmaps. #[crate::test] async fn test_export_nullable_decimal() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut decimal = assert_nullable_export( @@ -3024,7 +3023,7 @@ mod tests { // Check nullable temporal exports include Arrow null bitmaps. #[crate::test] async fn test_export_nullable_temporal() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut temporal = assert_nullable_export( @@ -3046,7 +3045,7 @@ mod tests { // Check nullable string-view exports include Arrow null bitmaps. #[crate::test] async fn test_export_nullable_varbinview() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut varbinview = assert_nullable_export( @@ -3069,7 +3068,7 @@ mod tests { // Check nullable struct exports include Arrow null bitmaps. #[crate::test] async fn test_export_nullable_struct() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut struct_array = assert_nullable_export( @@ -3093,7 +3092,7 @@ mod tests { // Check nested struct children expose cuDF-compatible Arrow Device layouts. #[crate::test] async fn test_export_nested_struct_child_layout() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut device_array = nested_struct_array().export_device_array(&mut ctx).await?; @@ -3146,7 +3145,7 @@ mod tests { // Check parent release recursively releases children and is safe to repeat. #[crate::test] async fn test_release_is_idempotent_and_releases_children() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut device_array = nested_struct_array().export_device_array(&mut ctx).await?; @@ -3194,7 +3193,7 @@ mod tests { #[crate::test] async fn test_export_struct() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = StructArray::new( @@ -3223,7 +3222,7 @@ mod tests { #[crate::test] async fn test_export_struct_with_schema() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = StructArray::new( @@ -3261,7 +3260,7 @@ mod tests { // Check nested struct device exports carry the matching Arrow schema. #[crate::test] async fn test_export_nested_struct_with_schema() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let mut exported = nested_struct_array() @@ -3293,7 +3292,7 @@ mod tests { #[crate::test] async fn test_export_nested_struct_decimal_with_schema() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let nested = StructArray::new( @@ -3354,7 +3353,7 @@ mod tests { #[crate::test] async fn test_export_primitive_with_schema_is_column_shaped() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = PrimitiveArray::from_iter(0u32..5).into_array(); @@ -3373,7 +3372,7 @@ mod tests { #[crate::test] async fn test_export_varbinview_with_schema_uses_utf8_view_layout() -> VortexResult<()> { - let mut ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let japanese = "こんにちは"; diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 557ec7468f5..0439436d505 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -563,7 +563,6 @@ mod tests { use vortex::encodings::zigzag::ZigZag; use vortex::error::VortexExpect; use vortex::error::VortexResult; - use vortex::session::VortexSession; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; use vortex_array::patches::Patches; @@ -636,7 +635,7 @@ mod tests { .collect(); let bitpacked = bitpacked_array_u32(bit_width, len); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let packed = bitpacked.packed().clone(); let device_input = futures::executor::block_on(cuda_ctx.ensure_on_device(packed))?; let input_ptr = device_input.cuda_device_ptr()?; @@ -752,7 +751,7 @@ mod tests { }) .collect(); - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let (input_ptr, _di) = copy_raw_to_device(&cuda_ctx, &data)?; let plan = CudaDispatchPlan::new( @@ -853,7 +852,7 @@ mod tests { .collect(); let bp = bitpacked_array_u32(bit_width, len); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx).await?; let actual = @@ -878,7 +877,7 @@ mod tests { let bp = bitpacked_array_u32(bit_width, len); let for_arr = FoR::try_new(bp.into_array(), Scalar::from(reference))?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&for_arr.into_array(), &mut cuda_ctx).await?; let actual = @@ -900,7 +899,7 @@ mod tests { expected.push(values[run]); } - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); @@ -945,7 +944,7 @@ mod tests { let dict = DictArray::try_new(codes_bp.into_array(), dict_for.into_array())?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx).await?; let actual = @@ -982,7 +981,7 @@ mod tests { None, ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&tree.into_array(), &mut cuda_ctx).await?; let actual = @@ -1015,7 +1014,7 @@ mod tests { )?; let zz = ZigZag::try_new(bp.into_array())?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&zz.into_array(), &mut cuda_ctx).await?; let actual = @@ -1039,7 +1038,7 @@ mod tests { expected.push(values[run] + reference); } - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); @@ -1073,7 +1072,7 @@ mod tests { let dict = DictArray::try_new(codes_prim.into_array(), values_prim.into_array())?; let for_arr = FoR::try_new(dict.into_array(), Scalar::from(reference))?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&for_arr.into_array(), &mut cuda_ctx).await?; let actual = @@ -1105,7 +1104,7 @@ mod tests { let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); let dict = DictArray::try_new(codes_for.into_array(), values_prim.into_array())?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx).await?; let actual = @@ -1134,7 +1133,7 @@ mod tests { let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx).await?; let actual = @@ -1165,7 +1164,7 @@ mod tests { ); // Execute through the hybrid dispatch path (handles widening). - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1197,7 +1196,7 @@ mod tests { ); // Execute through the hybrid dispatch path (handles widening). - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1214,7 +1213,7 @@ mod tests { let values: Vec = vec![10, 20, 30]; let len = 3000; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); @@ -1278,7 +1277,7 @@ mod tests { let expected: Vec = data[slice_start..slice_end].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1333,7 +1332,7 @@ mod tests { let sliced = zz.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1383,7 +1382,7 @@ mod tests { .map(|&c| dict_values[c as usize]) .collect(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1432,7 +1431,7 @@ mod tests { let sliced = bp.into_array().slice(slice_start..slice_end)?; let expected: Vec = data[slice_start..slice_end].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1485,7 +1484,7 @@ mod tests { let sliced = for_arr.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1550,7 +1549,7 @@ mod tests { let sliced = dict.into_array().slice(slice_start..slice_end)?; let expected: Vec = all_decoded[slice_start..slice_end].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1582,7 +1581,7 @@ mod tests { let seq = Sequence::try_new_typed(base, multiplier, Nullability::NonNullable, len)?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&seq.into_array(), &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( @@ -1615,7 +1614,7 @@ mod tests { let seq = Sequence::try_new_typed(base, multiplier, Nullability::NonNullable, len)?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&seq.into_array(), &mut cuda_ctx).await?; let actual_u32 = run_dynamic_dispatch_plan( @@ -1655,7 +1654,7 @@ mod tests { )?; let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1691,7 +1690,7 @@ mod tests { )?; let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1725,7 +1724,7 @@ mod tests { )?; let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1738,7 +1737,7 @@ mod tests { async fn test_empty_array() -> VortexResult<()> { let values: Vec = vec![]; let primitive = PrimitiveArray::new(Buffer::from(values), NonNullable); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&primitive.into_array(), &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); assert_eq!(result.len(), 0); @@ -1761,7 +1760,7 @@ mod tests { )?; let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1795,7 +1794,7 @@ mod tests { )?; let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -1864,7 +1863,7 @@ mod tests { use vortex::dtype::Nullability; use vortex::encodings::sequence::Sequence; - let session = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let session = CudaSession::create_execution_ctx(&crate::cuda_session())?; let cuda_session = session.cuda_session(); // Leaf encodings. @@ -1962,7 +1961,7 @@ mod tests { .into_array(); // GPU decode. - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let gpu = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2011,7 +2010,7 @@ mod tests { .execute::(&mut LEGACY_SESSION.create_execution_ctx())?; let expected: Vec = cpu_decoded.as_slice::().to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?; let result_prim = result.as_primitive(); @@ -2031,7 +2030,7 @@ mod tests { #[crate::test] async fn alp_slice_device_patches() -> VortexResult<()> { // Regression test for https://github.com/vortex-data/vortex/issues/7838#issuecomment-4452796116. - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let len = 4096; let exponents = Exponents { e: 0, f: 0 }; @@ -2098,7 +2097,7 @@ mod tests { let values: Vec = vec![100, 200, 300, 400]; let len = 2000; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let ends_arr = PrimitiveArray::new(Buffer::from(ends), NonNullable).into_array(); let values_arr = PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(); let re = RunEnd::new(ends_arr, values_arr, cuda_ctx.execution_ctx()); @@ -2161,7 +2160,7 @@ mod tests { "expected Fused for mixed-width Dict with BitPacked codes" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2205,7 +2204,7 @@ mod tests { "expected Fused for mixed-width Dict with BitPacked codes" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2249,7 +2248,7 @@ mod tests { "expected Fused for mixed-width Dict with FoR(BitPacked) codes" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2303,7 +2302,7 @@ mod tests { "expected Fused for mixed-width RunEnd with BitPacked ends" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2349,7 +2348,7 @@ mod tests { "expected Fused for mixed-width RunEnd with FoR(BitPacked) ends" ); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&array, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2381,7 +2380,7 @@ mod tests { // Slice from 1000..3000 let sliced = dict.into_array().slice(1000..3000)?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&sliced, &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2400,7 +2399,7 @@ mod tests { /// (the bit-pattern for i32(-1)), not u32(0x000000FF) = 255. #[crate::test] fn test_load_element_sign_extends_i8_to_u32() -> VortexResult<()> { - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let i8_values: Vec = vec![-1, -2, -3, 127, -128, 0, 1, 42]; let len = i8_values.len(); @@ -2438,7 +2437,7 @@ mod tests { /// Same as above but for i16 → u32 widening. #[crate::test] fn test_load_element_sign_extends_i16_to_u32() -> VortexResult<()> { - let cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let i16_values: Vec = vec![-1, -256, -32768, 32767, 0, 1, -100, 12345]; let len = i16_values.len(); @@ -2477,7 +2476,7 @@ mod tests { /// Nullable Primitive array — LOAD source with validity propagated. #[crate::test] async fn test_nullable_primitive() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let array = PrimitiveArray::from_option_iter( (0..2048u32).map(|i| if i % 3 == 0 { None } else { Some(i) }), @@ -2500,7 +2499,7 @@ mod tests { /// validity, so this produces a real nullable FoR(BitPacked) tree. #[crate::test] async fn test_nullable_for_bitpacked() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let len = 2048; let reference = 1000u32; @@ -2554,7 +2553,7 @@ mod tests { /// AllInvalid array — kernel should be skipped entirely. #[crate::test] async fn test_all_invalid_skips_kernel() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let array = PrimitiveArray::new(Buffer::from(vec![0u32; 2048]), Validity::AllInvalid); @@ -2572,7 +2571,7 @@ mod tests { /// AllValid nullable array — should fuse and produce AllValid output. #[crate::test] async fn test_all_valid_nullable() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let values: Vec = (0..2048).collect(); let array = PrimitiveArray::new(Buffer::from(values.clone()), Validity::AllValid); @@ -2610,7 +2609,7 @@ mod tests { async fn test_dict_nullable_values_fuses() -> VortexResult<()> { use vortex::buffer::buffer; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let codes = PrimitiveArray::new(buffer![0u32, 1, 2, 2, 1, 0], NonNullable); let values = PrimitiveArray::from_option_iter([Some(10u32), None, Some(30)]); @@ -2636,7 +2635,7 @@ mod tests { use vortex::array::arrays::FilterArray; use vortex::mask::Mask; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let len = 2048usize; let values: Vec> = (0..len) @@ -2670,7 +2669,7 @@ mod tests { /// Empty nullable array should preserve nullability. #[crate::test] async fn test_empty_nullable_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let array = PrimitiveArray::new(Buffer::::empty(), Validity::AllValid); let result = try_gpu_dispatch(&array.into_array(), &mut cuda_ctx).await?; @@ -2709,7 +2708,7 @@ mod tests { let array = bp.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&array, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -2757,7 +2756,7 @@ mod tests { (bp.into_array(), values) }; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&array, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -2797,7 +2796,7 @@ mod tests { let array = for_arr.into_array(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&array, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -2839,7 +2838,7 @@ mod tests { let sliced = for_arr.into_array().slice(range.clone())?; let expected = all_values[range].to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&sliced, &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan( &cuda_ctx, @@ -2886,7 +2885,7 @@ mod tests { let cpu_decoded = array.clone().execute::(&mut ctx)?; let expected: Vec = cpu_decoded.as_slice::().to_vec(); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&array, &mut cuda_ctx).await?; let actual = run_dispatch_plan_f32( &cuda_ctx, @@ -2934,7 +2933,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -2967,7 +2966,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -3000,7 +2999,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let canonical = try_gpu_dispatch(&bp.into_array(), &mut cuda_ctx).await?; let result = CanonicalCudaExt::into_host(canonical).await?.into_array(); @@ -3039,7 +3038,7 @@ mod tests { let values_prim = PrimitiveArray::new(Buffer::from(dict_values), NonNullable); let dict = DictArray::try_new(codes_bp.into_array(), values_prim.into_array())?; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&dict.into_array(), &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -3068,7 +3067,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -3100,7 +3099,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; @@ -3112,7 +3111,7 @@ mod tests { /// dispatch alongside patch application. #[crate::test] async fn test_nullable_bitpacked_with_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let len = 3000usize; let bit_width: u8 = 4; @@ -3168,7 +3167,7 @@ mod tests { )?; assert!(bp.patches().is_some(), "expected patches"); - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let plan = dispatch_plan(&bp.into_array(), &mut cuda_ctx).await?; let actual = run_dynamic_dispatch_plan(&cuda_ctx, len, &plan.dispatch_plan, plan.shared_mem_bytes)?; diff --git a/vortex-cuda/src/executor.rs b/vortex-cuda/src/executor.rs index 069941c266c..e2fa1e6dc6f 100644 --- a/vortex-cuda/src/executor.rs +++ b/vortex-cuda/src/executor.rs @@ -96,7 +96,7 @@ pub struct CudaExecutionCtx { impl CudaExecutionCtx { /// Creates a new CUDA execution context. pub(crate) fn new(stream: VortexCudaStream, ctx: ExecutionCtx) -> Self { - let cuda_session = ctx.session().cuda_session().clone(); + let cuda_session = (*ctx.session().cuda_session()).clone(); Self { stream, ctx, diff --git a/vortex-cuda/src/hybrid_dispatch/mod.rs b/vortex-cuda/src/hybrid_dispatch/mod.rs index 9f5fde8b502..a46cf517c16 100644 --- a/vortex-cuda/src/hybrid_dispatch/mod.rs +++ b/vortex-cuda/src/hybrid_dispatch/mod.rs @@ -171,7 +171,6 @@ mod tests { use vortex::error::VortexResult; use vortex::mask::Mask; use vortex::scalar::Scalar; - use vortex::session::VortexSession; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; @@ -196,7 +195,7 @@ mod tests { #[crate::test] async fn test_fused() -> VortexResult<()> { let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let values: Vec = (0..2048).map(|i| (i % 128) as u32).collect(); let bp = BitPacked::encode( &PrimitiveArray::new(Buffer::from(values), NonNullable).into_array(), @@ -226,7 +225,7 @@ mod tests { use vortex::encodings::alp::Exponents; let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let encoded: Vec = (0i32..2048).map(|i| i % 500).collect(); let bp = BitPacked::encode( &PrimitiveArray::new(Buffer::from(encoded), NonNullable).into_array(), @@ -264,7 +263,7 @@ mod tests { use vortex::encodings::alp::Exponents; let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let encoded = PrimitiveArray::new( Buffer::from((0i32..2048).map(|i| i % 500).collect::>()), NonNullable, @@ -303,7 +302,7 @@ mod tests { use vortex::encodings::fastlanes; use vortex::encodings::zstd::ZstdBuffers; - let session = VortexSession::empty(); + let session = crate::cuda_session(); fastlanes::initialize(&session); session.arrays().register(ZstdBuffers); let mut ctx = CudaSession::create_execution_ctx(&session).vortex_expect("ctx"); @@ -362,7 +361,7 @@ mod tests { #[crate::test] async fn test_filter_fused_child() -> VortexResult<()> { let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let len = 2048u32; let data: Vec = (0..len).map(|i| i % 128).collect(); @@ -405,7 +404,7 @@ mod tests { #[crate::test] async fn test_ext_storage_gpu_decode(#[case] ext: ExtensionArray) -> VortexResult<()> { let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let expected_storage = canonicalize_cpu(ext.storage_array().clone())?.into_array(); let expected = ExtensionArray::new(ext.ext_dtype().clone(), expected_storage).into_array(); @@ -428,7 +427,7 @@ mod tests { #[crate::test] async fn test_ext_canonical_storage() -> VortexResult<()> { let mut ctx = - CudaSession::create_execution_ctx(&VortexSession::empty()).vortex_expect("ctx"); + CudaSession::create_execution_ctx(&crate::cuda_session()).vortex_expect("ctx"); let ext = ExtensionArray::new( Date::new(TimeUnit::Days, Nullability::NonNullable).erased(), diff --git a/vortex-cuda/src/kernel/arrays/constant.rs b/vortex-cuda/src/kernel/arrays/constant.rs index 8e5966da243..0f769bacbf9 100644 --- a/vortex-cuda/src/kernel/arrays/constant.rs +++ b/vortex-cuda/src/kernel/arrays/constant.rs @@ -199,7 +199,6 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::scalar::Scalar; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -224,7 +223,7 @@ mod tests { async fn test_cuda_constant_materialization( #[case] constant_array: ConstantArray, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let cpu_result = crate::canonicalize_cpu(constant_array.clone())?; @@ -244,7 +243,7 @@ mod tests { #[crate::test] async fn test_cuda_constant_empty_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let constant_array = ConstantArray::new(42i32, 0); @@ -265,7 +264,7 @@ mod tests { #[crate::test] async fn test_cuda_constant_small_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Test with array smaller than one block (< 2048 elements) diff --git a/vortex-cuda/src/kernel/arrays/dict.rs b/vortex-cuda/src/kernel/arrays/dict.rs index 442b730a037..2b21d470f09 100644 --- a/vortex-cuda/src/kernel/arrays/dict.rs +++ b/vortex-cuda/src/kernel/arrays/dict.rs @@ -308,7 +308,6 @@ mod tests { use vortex::dtype::DecimalDType; use vortex::dtype::i256; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -325,7 +324,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_u32_values_u8_codes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values: [100, 200, 300, 400] @@ -357,7 +356,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_u64_values_u16_codes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values: large u64 values @@ -392,7 +391,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_i32_values_u32_codes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values: signed integers including negatives @@ -423,7 +422,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_large_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary with 256 values @@ -455,7 +454,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_values_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values with nulls: [100, null, 300, 400] @@ -487,7 +486,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_codes_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values: [100, 200, 300, 400] @@ -524,7 +523,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_both_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values with nulls: [100, null, 300, 400] @@ -568,7 +567,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_i64_values_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Dictionary values with nulls (i64) @@ -613,7 +612,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_all_valid_matches_baseline() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Non-nullable values @@ -656,7 +655,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i8_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 2 uses i8 backing type @@ -684,7 +683,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i16_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 4 uses i16 backing type @@ -712,7 +711,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i32_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 9 uses i32 backing type @@ -740,7 +739,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i64_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 18 uses i64 backing type @@ -771,7 +770,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i128_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 38 uses i128 backing type @@ -815,7 +814,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_values_u8_codes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = VarBinViewArray::from_iter_str(["cat", "dog", "bird", "fish"]); @@ -840,7 +839,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_values_u16_codes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = VarBinViewArray::from_iter_str(["alpha", "beta", "gamma", "delta", "epsilon"]); @@ -865,7 +864,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_max_inlined_12_bytes() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Exactly 12 bytes — the maximum inlined BinaryView size @@ -892,7 +891,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_outlined_views() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // 13+ bytes — outlined BinaryViews that reference data buffers @@ -922,7 +921,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_empty_strings() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = VarBinViewArray::from_iter_str(["", "a", ""]); @@ -947,7 +946,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_values_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = VarBinViewArray::from_iter_nullable_str([Some("hello"), None, Some("world")]); @@ -973,7 +972,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_string_outlined_with_validity() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Mix of inlined, outlined, and null dictionary values @@ -1006,7 +1005,7 @@ mod tests { #[crate::test] async fn test_cuda_dict_decimal_i256_values() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Precision 76 uses i256 backing type diff --git a/vortex-cuda/src/kernel/encodings/alp.rs b/vortex-cuda/src/kernel/encodings/alp.rs index 4a1e7ea931a..37c83af5d93 100644 --- a/vortex-cuda/src/kernel/encodings/alp.rs +++ b/vortex-cuda/src/kernel/encodings/alp.rs @@ -153,7 +153,6 @@ mod tests { use vortex::encodings::alp::Exponents; use vortex::encodings::alp::alp_encode; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -171,7 +170,7 @@ mod tests { /// Patches must carry `chunk_offsets` — the fused kernel requires them. #[crate::test] async fn test_cuda_alp_decompression_f32() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // For f32 with exponents (e=0, f=2): decoded = encoded * F10[2] * IF10[0] @@ -218,7 +217,7 @@ mod tests { /// preserved through the standalone ALP GPU executor. #[crate::test] async fn test_cuda_alp_nullable_with_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Values that will produce ALP exceptions at non-null positions. @@ -259,7 +258,7 @@ mod tests { /// elements are actually null. #[crate::test] async fn test_cuda_alp_all_valid_nullable() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = PrimitiveArray::new( @@ -292,7 +291,7 @@ mod tests { /// (zero patches) via the offset math rather than the NULL sentinel. #[crate::test] async fn test_cuda_alp_multi_chunk_sparse_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // 3072 values (3 chunks). Inject exceptions (values ALP can't encode @@ -337,7 +336,7 @@ mod tests { /// so this guards the fast-path for the (i64, f64) kernel variant. #[crate::test] async fn test_cuda_alp_f64_multi_chunk_with_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // 3072 values (3 chunks). Sprinkle exceptions into each chunk. @@ -381,7 +380,7 @@ mod tests { /// (existing tests have ≤ 6 patches per chunk). #[crate::test] async fn test_cuda_alp_dense_patches_single_chunk() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Build a 1024-element ALP array manually with exactly 40 patches @@ -429,7 +428,7 @@ mod tests { /// loop. Includes a patch in the tail. #[crate::test] async fn test_cuda_alp_partial_tail_chunk() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values: Buffer = (0u32..1500) diff --git a/vortex-cuda/src/kernel/encodings/bitpacked.rs b/vortex-cuda/src/kernel/encodings/bitpacked.rs index 0e528a67c1a..72161505b0a 100644 --- a/vortex-cuda/src/kernel/encodings/bitpacked.rs +++ b/vortex-cuda/src/kernel/encodings/bitpacked.rs @@ -233,7 +233,6 @@ mod tests { use vortex::buffer::buffer; use vortex::encodings::fastlanes::BitPackedArrayExt; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; @@ -251,7 +250,7 @@ mod tests { #[case] iter: impl Iterator, #[case] bw: u8, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = PrimitiveArray::new(iter.collect::>(), NonNullable); @@ -283,7 +282,7 @@ mod tests { #[crate::test] fn test_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let array = PrimitiveArray::new( @@ -326,7 +325,7 @@ mod tests { #[case::bw_7(7)] #[crate::test] fn test_cuda_bitunpack_u8(#[case] bit_width: u8) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let max_val = (1u8 << bit_width).saturating_sub(1); @@ -379,7 +378,7 @@ mod tests { #[case::bw_15(15)] #[crate::test] fn test_cuda_bitunpack_u16(#[case] bit_width: u8) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let max_val = (1u16 << bit_width).saturating_sub(1); @@ -448,7 +447,7 @@ mod tests { #[case::bw_31(31)] #[crate::test] fn test_cuda_bitunpack_u32(#[case] bit_width: u8) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let max_val = (1u32 << bit_width).saturating_sub(1); @@ -549,7 +548,7 @@ mod tests { #[case::bw_63(63)] #[crate::test] fn test_cuda_bitunpack_u64(#[case] bit_width: u8) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let max_val = (1u64 << bit_width).saturating_sub(1); @@ -586,7 +585,7 @@ mod tests { #[crate::test] fn test_cuda_bitunpack_sliced() -> VortexResult<()> { let bit_width = 32; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let max_val = (1u64 << bit_width).saturating_sub(1); @@ -668,7 +667,7 @@ mod tests { /// offset_within_chunk. #[crate::test] fn test_cuda_bitunpack_sliced_patches_offset_within_chunk() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Create an array with values that will generate patches. @@ -710,7 +709,7 @@ mod tests { /// Test slicing a bitpacked array multiple times, accumulating offset_within_chunk. #[crate::test] fn test_cuda_bitunpack_double_sliced_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Create an array with values that will generate patches. @@ -764,7 +763,7 @@ mod tests { /// Test slicing to skip an entire chunk's worth of patches. #[crate::test] fn test_cuda_bitunpack_sliced_skip_first_chunk_patches() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Create patches in first chunk only, then slice past them all. diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index 7c4f4580c30..4a1e9cc458f 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -214,7 +214,6 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::extension::datetime::TimeUnit; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -280,7 +279,7 @@ mod tests { #[case] subseconds: Vec, #[case] time_unit: TimeUnit, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let dtp_array = make_datetimeparts_array(days, seconds, subseconds, time_unit); @@ -301,7 +300,7 @@ mod tests { #[crate::test] async fn test_cuda_datetimeparts_large_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let len = 2050; @@ -327,7 +326,7 @@ mod tests { #[crate::test] async fn test_cuda_datetimeparts_with_nulls() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let days_arr = PrimitiveArray::new( diff --git a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs index 70d3c71a485..9e28737db79 100644 --- a/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs +++ b/vortex-cuda/src/kernel/encodings/decimal_byte_parts.rs @@ -70,7 +70,6 @@ mod tests { use vortex::dtype::DecimalDType; use vortex::encodings::decimal_byte_parts::DecimalByteParts; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use super::*; use crate::session::CudaSession; @@ -86,7 +85,7 @@ mod tests { #[case] precision: u8, #[case] scale: i8, ) { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("create execution context"); let decimal_dtype = DecimalDType::new(precision, scale); diff --git a/vortex-cuda/src/kernel/encodings/for_.rs b/vortex-cuda/src/kernel/encodings/for_.rs index ce13f4c3b4b..b0e231060b0 100644 --- a/vortex-cuda/src/kernel/encodings/for_.rs +++ b/vortex-cuda/src/kernel/encodings/for_.rs @@ -138,7 +138,6 @@ mod tests { use vortex::encodings::fastlanes::FoRArray; use vortex::error::VortexExpect; use vortex::scalar::Scalar; - use vortex::session::VortexSession; use vortex_array::LEGACY_SESSION; use vortex_array::VortexSessionExecute; @@ -161,7 +160,7 @@ mod tests { #[case::u64(make_for_array((0..2050).map(|i| (i % 2050) as u64).collect(), 1000000u64))] #[crate::test] async fn test_cuda_for_decompression(#[case] for_array: FoRArray) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let cpu_result = crate::canonicalize_cpu(for_array.clone())?; @@ -181,7 +180,7 @@ mod tests { #[crate::test] async fn test_signed_ffor() { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let values = (0i8..8i8) diff --git a/vortex-cuda/src/kernel/encodings/fsst.rs b/vortex-cuda/src/kernel/encodings/fsst.rs index 20a18b4e538..56ba6001cf1 100644 --- a/vortex-cuda/src/kernel/encodings/fsst.rs +++ b/vortex-cuda/src/kernel/encodings/fsst.rs @@ -209,7 +209,6 @@ mod tests { use vortex::encodings::fsst::fsst_compress; use vortex::encodings::fsst::fsst_train_compressor; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -236,7 +235,7 @@ mod tests { #[case] strings: Vec>, #[case] nullability: Nullability, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let varbin = VarBinArray::from_iter(strings, DType::Binary(nullability)); @@ -264,7 +263,7 @@ mod tests { async fn test_cuda_fsst_decompression_roundtrip_large() -> VortexResult<()> { use vortex_fsst::test_utils::make_fsst_clickbench_urls; - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let fsst_array = make_fsst_clickbench_urls(100_000, cuda_ctx.execution_ctx()).into_array(); diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index 01fc57abb86..5af16ca8e04 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -174,7 +174,6 @@ mod tests { use vortex::encodings::runend::RunEndArray; use vortex::error::VortexExpect; use vortex::error::VortexResult; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -208,7 +207,7 @@ mod tests { #[case::u64_ends_i32_values(|ctx: &mut vortex::array::ExecutionCtx| make_runend_array(vec![2u64, 5, 10], vec![1i32, 2, 3], ctx))] #[crate::test] async fn test_cuda_runend_types(#[case] build: RunEndBuilder) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let runend_array = build(cuda_ctx.execution_ctx()); @@ -229,7 +228,7 @@ mod tests { #[crate::test] async fn test_cuda_runend_large_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let num_runs = 41; @@ -259,7 +258,7 @@ mod tests { #[crate::test] async fn test_cuda_runend_single_run() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let runend_array = make_runend_array(vec![100u32], vec![42i32], cuda_ctx.execution_ctx()); @@ -281,7 +280,7 @@ mod tests { #[crate::test] async fn test_cuda_runend_many_small_runs() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Create an array where each run has length 1. @@ -308,7 +307,7 @@ mod tests { #[crate::test] async fn test_cuda_runend_nullable_values_falls_back_to_cpu() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // Build a RunEnd array whose values have Validity::Array (some nulls). diff --git a/vortex-cuda/src/kernel/encodings/sequence.rs b/vortex-cuda/src/kernel/encodings/sequence.rs index c0721b8d920..b9363fdd9dc 100644 --- a/vortex-cuda/src/kernel/encodings/sequence.rs +++ b/vortex-cuda/src/kernel/encodings/sequence.rs @@ -92,7 +92,6 @@ mod tests { use vortex::dtype::Nullability; use vortex::encodings::sequence::Sequence; use vortex::scalar::PValue; - use vortex::session::VortexSession; use crate::CanonicalCudaExt; use crate::CudaSession; @@ -125,7 +124,7 @@ mod tests { len: usize, nullability: Nullability, ) { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()).unwrap(); let array = Sequence::try_new_typed(base, multiplier, nullability, len).unwrap(); diff --git a/vortex-cuda/src/kernel/encodings/zigzag.rs b/vortex-cuda/src/kernel/encodings/zigzag.rs index 98f9534be0a..3c4cf8e08ee 100644 --- a/vortex-cuda/src/kernel/encodings/zigzag.rs +++ b/vortex-cuda/src/kernel/encodings/zigzag.rs @@ -105,7 +105,6 @@ mod tests { use vortex::buffer::Buffer; use vortex::encodings::zigzag::ZigZag; use vortex::error::VortexExpect; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -113,7 +112,7 @@ mod tests { #[crate::test] async fn test_cuda_zigzag_decompression_u32() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); // ZigZag encoding: 0->0, 1->-1, 2->1, 3->-2, 4->2, ... diff --git a/vortex-cuda/src/kernel/encodings/zstd.rs b/vortex-cuda/src/kernel/encodings/zstd.rs index abadeb1698f..703a0ddf0e1 100644 --- a/vortex-cuda/src/kernel/encodings/zstd.rs +++ b/vortex-cuda/src/kernel/encodings/zstd.rs @@ -356,7 +356,6 @@ mod tests { use vortex::array::assert_arrays_eq; use vortex::encodings::zstd::Zstd; use vortex::error::VortexResult; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -365,7 +364,7 @@ mod tests { #[crate::test] async fn test_cuda_zstd_decompression_utf8() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let strings = VarBinViewArray::from_iter_str([ @@ -391,7 +390,7 @@ mod tests { #[crate::test] async fn test_cuda_zstd_decompression_multiple_frames() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let strings = VarBinViewArray::from_iter_str([ @@ -427,7 +426,7 @@ mod tests { #[crate::test] async fn test_cuda_zstd_decompression_sliced() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let strings = VarBinViewArray::from_iter_str([ @@ -462,7 +461,7 @@ mod tests { /// correct results instead of panicking. #[crate::test] async fn test_cuda_zstd_nullable_falls_back_to_cpu() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let strings = VarBinViewArray::from_iter_nullable_str([ diff --git a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs index 8a67a338ca5..691f1ee23e7 100644 --- a/vortex-cuda/src/kernel/encodings/zstd_buffers.rs +++ b/vortex-cuda/src/kernel/encodings/zstd_buffers.rs @@ -226,7 +226,6 @@ mod tests { use vortex::array::assert_arrays_eq; use vortex::error::VortexExpect; use vortex::error::VortexResult; - use vortex::session::VortexSession; use super::*; use crate::CanonicalCudaExt; @@ -234,11 +233,11 @@ mod tests { #[crate::test] async fn test_cuda_zstd_buffers_decompression_primitive() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let input = PrimitiveArray::from_iter(0i64..1024).into_array(); - let compressed = ZstdBuffers::compress(&input, 3, &VortexSession::empty())?; + let compressed = ZstdBuffers::compress(&input, 3, &crate::cuda_session())?; let cpu_result = crate::canonicalize_cpu(compressed.clone())?; let gpu_result = ZstdBuffersExecutor @@ -253,7 +252,7 @@ mod tests { #[crate::test] async fn test_cuda_zstd_buffers_decompression_varbinview() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create execution context"); let input = VarBinViewArray::from_iter_str([ @@ -265,7 +264,7 @@ mod tests { "baz", ]) .into_array(); - let compressed = ZstdBuffers::compress(&input, 3, &VortexSession::empty())?; + let compressed = ZstdBuffers::compress(&input, 3, &crate::cuda_session())?; let cpu_result = crate::canonicalize_cpu(compressed.clone())?; let gpu_result = ZstdBuffersExecutor diff --git a/vortex-cuda/src/kernel/filter/decimal.rs b/vortex-cuda/src/kernel/filter/decimal.rs index 566b1c3313d..cd3be77f423 100644 --- a/vortex-cuda/src/kernel/filter/decimal.rs +++ b/vortex-cuda/src/kernel/filter/decimal.rs @@ -48,7 +48,6 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::mask::Mask; - use vortex::session::VortexSession; use crate::CanonicalCudaExt; use crate::FilterExecutor; @@ -89,7 +88,7 @@ mod tests { #[case] input: DecimalArray, #[case] mask: Mask, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create CUDA execution context"); let filter_array = FilterArray::try_new(input.clone().into_array(), mask.clone())?; @@ -111,7 +110,7 @@ mod tests { #[crate::test] async fn test_gpu_filter_decimal_large_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create CUDA execution context"); // Create a large array to test multi-block execution diff --git a/vortex-cuda/src/kernel/filter/primitive.rs b/vortex-cuda/src/kernel/filter/primitive.rs index 6bff0b20c70..2d943f590b6 100644 --- a/vortex-cuda/src/kernel/filter/primitive.rs +++ b/vortex-cuda/src/kernel/filter/primitive.rs @@ -46,7 +46,6 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::mask::Mask; - use vortex::session::VortexSession; use crate::CanonicalCudaExt; use crate::FilterExecutor; @@ -83,7 +82,7 @@ mod tests { #[case] input: PrimitiveArray, #[case] mask: Mask, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create CUDA execution context"); let filter_array = FilterArray::try_new(input.clone().into_array(), mask.clone())?; @@ -105,7 +104,7 @@ mod tests { #[crate::test] async fn test_gpu_filter_large_array() -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create CUDA execution context"); // Create a large array to test multi-block execution diff --git a/vortex-cuda/src/kernel/filter/varbinview.rs b/vortex-cuda/src/kernel/filter/varbinview.rs index 23a6c91b048..e4ca3fa4df4 100644 --- a/vortex-cuda/src/kernel/filter/varbinview.rs +++ b/vortex-cuda/src/kernel/filter/varbinview.rs @@ -46,7 +46,6 @@ mod tests { use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::mask::Mask; - use vortex::session::VortexSession; use crate::CanonicalCudaExt; use crate::FilterExecutor; @@ -69,7 +68,7 @@ mod tests { #[case] input: VarBinViewArray, #[case] mask: Mask, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()) + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()) .vortex_expect("failed to create CUDA execution context"); let filter_array = FilterArray::try_new(input.into_array(), mask.clone())?; diff --git a/vortex-cuda/src/kernel/patches/mod.rs b/vortex-cuda/src/kernel/patches/mod.rs index 3eb3e2ced93..993526af1fb 100644 --- a/vortex-cuda/src/kernel/patches/mod.rs +++ b/vortex-cuda/src/kernel/patches/mod.rs @@ -188,7 +188,6 @@ mod tests { use vortex::dtype::DType; use vortex::dtype::NativePType; use vortex::dtype::Nullability; - use vortex::session::VortexSession; use crate::CanonicalCudaExt; use crate::CudaDeviceBuffer; @@ -217,7 +216,7 @@ mod tests { } async fn full_test_case() { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty()).unwrap(); + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session()).unwrap(); let mut ctx = LEGACY_SESSION.create_execution_ctx(); let values = PrimitiveArray::from_iter(0..128); diff --git a/vortex-cuda/src/kernel/patches/types.rs b/vortex-cuda/src/kernel/patches/types.rs index f90d7251c09..6234250189a 100644 --- a/vortex-cuda/src/kernel/patches/types.rs +++ b/vortex-cuda/src/kernel/patches/types.rs @@ -227,7 +227,6 @@ mod tests { use vortex::array::buffer::BufferHandle; use vortex::array::validity::Validity::NonNullable; use vortex::buffer::Buffer; - use vortex::session::VortexSession; use vortex_array::IntoArray; use vortex_array::arrays::PrimitiveArray; use vortex_array::patches::Patches; @@ -274,7 +273,7 @@ mod tests { #[case] expected_offset: usize, #[case] expected_chunk_offsets: Vec, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let indices = PrimitiveArray::from_iter([100u32, 1100, 2100, 3100, 4100]); let values = PrimitiveArray::from_iter([10u32, 11, 12, 13, 14]); let chunk_offsets = PrimitiveArray::from_iter([0u32, 1, 2, 3, 4, 5]); @@ -318,7 +317,7 @@ mod tests { #[case] chunk_offsets: ArrayRef, #[case] expected: ArrayRef, ) -> VortexResult<()> { - let mut cuda_ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let mut cuda_ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let indices = PrimitiveArray::from_iter([100u32, 1100, 2100, 3100, 4100]); let values = PrimitiveArray::from_iter([10u32, 11, 12, 13, 14]); let patches = Patches::new( diff --git a/vortex-cuda/src/lib.rs b/vortex-cuda/src/lib.rs index 9e9de33a288..706b9c3c2bd 100644 --- a/vortex-cuda/src/lib.rs +++ b/vortex-cuda/src/lib.rs @@ -131,3 +131,17 @@ pub fn initialize_cuda(session: &CudaSession) { session.register_kernel(Filter.id(), &FilterExecutor); session.register_kernel(Slice.id(), &SliceExecutor); } + +/// Builds a fresh [`VortexSession`](vortex::session::VortexSession) with all array session +/// variables plus a default [`CudaSession`], for use in CUDA tests and benchmarks. +/// +/// Each call returns an independent session with its own CUDA context and stream pool, matching +/// the per-test isolation that lazily-initialized sessions previously provided. +/// +/// # Panics +/// +/// Panics if CUDA device 0 cannot be initialized (the same contract as [`CudaSession::default`]). +#[cfg(any(test, feature = "_test-harness"))] +pub fn cuda_session() -> vortex::session::VortexSession { + vortex::array::array_session().with::() +} diff --git a/vortex-cuda/src/session.rs b/vortex-cuda/src/session.rs index f3f7a2e3a49..914bc585d23 100644 --- a/vortex-cuda/src/session.rs +++ b/vortex-cuda/src/session.rs @@ -12,7 +12,6 @@ use vortex::array::ArrayId; use vortex::array::VortexSessionExecute; use vortex::error::VortexResult; use vortex::error::vortex_err; -use vortex::session::Ref; use vortex::session::SessionExt; use vortex::session::SessionVar; use vortex::utils::aliases::dash_map::DashMap; @@ -183,7 +182,7 @@ impl SessionVar for CudaSession { /// Extension trait for accessing the CUDA session from a Vortex session. pub trait CudaSessionExt: SessionExt { /// Returns the CUDA session. - fn cuda_session(&self) -> Ref<'_, CudaSession> { + fn cuda_session(&self) -> &CudaSession { self.get::() } } diff --git a/vortex-cuda/src/stream.rs b/vortex-cuda/src/stream.rs index 1120ac50e02..6342c39ab28 100644 --- a/vortex-cuda/src/stream.rs +++ b/vortex-cuda/src/stream.rs @@ -251,7 +251,6 @@ fn register_stream_callback(stream: &CudaStream) -> VortexResult VortexResult<()> { - let ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let handle = ctx.stream().copy_to_device(vec![0xab_u8])?.await?; assert_eq!(handle.len(), 1); @@ -281,7 +280,7 @@ mod tests { #[crate::test] async fn test_copy_to_device_sync_preserves_visible_len_with_padding() -> VortexResult<()> { - let ctx = CudaSession::create_execution_ctx(&VortexSession::empty())?; + let ctx = CudaSession::create_execution_ctx(&crate::cuda_session())?; let handle = ctx.stream().copy_to_device_sync(&[1_u8, 2, 3, 4, 5])?; assert_eq!(handle.len(), 5); diff --git a/vortex-datafusion/src/persistent/opener.rs b/vortex-datafusion/src/persistent/opener.rs index d50b003f1dc..2923b6c313c 100644 --- a/vortex-datafusion/src/persistent/opener.rs +++ b/vortex-datafusion/src/persistent/opener.rs @@ -237,7 +237,7 @@ impl FileOpener for VortexOpener { let this_file_schema = Arc::new(calculate_physical_schema( vxf.dtype(), &unified_file_schema, - &session.arrow(), + session.arrow(), )?); let projected_physical_schema = projection.project_schema(&unified_file_schema)?; @@ -296,7 +296,7 @@ impl FileOpener for VortexOpener { Schema::new(fields) }; let stream_schema = - calculate_physical_schema(&scan_dtype, &scan_reference_schema, &session.arrow())?; + calculate_physical_schema(&scan_dtype, &scan_reference_schema, session.arrow())?; let leftover_projection = leftover_projection .try_map_exprs(|expr| reassign_expr_columns(expr, &stream_schema))?; diff --git a/vortex-file/src/multi/mod.rs b/vortex-file/src/multi/mod.rs index 2baf53ff845..215331f0540 100644 --- a/vortex-file/src/multi/mod.rs +++ b/vortex-file/src/multi/mod.rs @@ -11,6 +11,7 @@ use async_trait::async_trait; use futures::StreamExt; use futures::TryStreamExt; use futures::stream; +pub use session::MultiFileSession; use session::MultiFileSessionExt; use tracing::debug; use vortex_error::VortexError; diff --git a/vortex-file/src/multi/session.rs b/vortex-file/src/multi/session.rs index 7023d3cd020..b6bd5477ec8 100644 --- a/vortex-file/src/multi/session.rs +++ b/vortex-file/src/multi/session.rs @@ -22,7 +22,8 @@ use crate::footer::Footer; /// /// Consider generalizing this cache into [`VortexOpenOptions`](crate::VortexOpenOptions) so /// that single-file opens also benefit from session-level footer caching. -pub(super) struct MultiFileSession { +#[derive(Clone)] +pub struct MultiFileSession { footer_cache: moka::sync::Cache, } @@ -53,12 +54,12 @@ impl Debug for MultiFileSession { impl MultiFileSession { /// Retrieve a cached footer for the given file path. - pub fn get_footer(&self, path: &str) -> Option