diff --git a/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs b/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs new file mode 100644 index 00000000000..2b665940b5f --- /dev/null +++ b/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs @@ -0,0 +1,322 @@ +// +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending +// +using Apache.Arrow; +using System.Collections; +using Array = System.Array; + +namespace Deephaven.Dh_NetClient; + +/// +/// This class hierarchy acts as a wrapper for IReadOnlyList<T> types that allows +/// us to present them simultaneously as various flavors of IList types. We use this +/// to wrap various Arrow array types. The type of wrapping depends on whether we are +/// wrapping an array containing value types or an array containing reference types. +/// +/// Apache.Arrow.StringArray is an example of an array containing reference types. +/// It implements IReadOnlyList<string>. We would wrap it as +/// ReadOnlyListAdapterForReferenceTypes<string> which would provide +/// IList and List<string> to the user. +/// +/// Apache.Arrow.Int32Array is an example of an array containing value types. +/// It implements IReadOnlyList<Nullable<Int32>>. We would wrap it as +/// ReadOnlyListAdapterForValueTypes<Int32> which would provide +/// IList and List<Nullable<Int32>> (via its base class), and +/// also IList<Int32> via the derived class. +/// +/// Background and rationale for this class hierarchy: The library initially supported a set of +/// scalar types: char, bool, int32, float, string, etc. For each simple type S, there is a +/// ColumnSource<S> that can represent its data and a Chunk<S> that can be used to +/// hold batches of data. When it came time to add support for List types, we had to decide, among +/// other things, what the appropriate ColumnSource and Chunk types would be. This is complicated +/// by the fact that there are infinitely many List types: List can be generic on S, but it can +/// also be generic on List<S> and that can be applied arbitrarily recursively: +/// List<List<S>>, and so on. (However, we are currently only supporting +/// List<S> where S is a scalar type). +/// In terms of representation, we decided that there would be a single ColumnSource<IList> +/// that could represent any list type, and a corresponding Chunk<IList> to be used with it. +/// When programmers work with these IList elements, they can use them as ILists directly, +/// or, to avoid boxing, they can cast them down to their actual concrete type. We promise that +/// the IList elements contained in these ColumnSource and Chunk Types will always implement +/// all of IList, IList<T> and, for value types, IList<Nullable<T>>. +/// +/// Note: it might have been preferable to use IReadOnlyList<T> and +/// IReadOnlyList<Nullable<T>> instead of IList. The problem is of course that +/// there is no bare (non-generic) IReadOnlyList type, so for the bare type we would still have +/// to use IList. It would have been confusing and inconsistent to use IList for the bare +/// non-generic type, but IReadOnlyList<T> for the generic type.. By using IList we can at least +/// be consistent, even though it's a big interface with a bunch of mutating methods that we don't +/// care about and will always throw exceptions for our case. +/// +public abstract class ReadOnlyListAdapterBase : IList, IList { + protected readonly IReadOnlyList _data; + + public ReadOnlyListAdapterBase(IReadOnlyList data) { + _data = data; + } + + int IList.Add(object? item) => NotImplementedForReadOnlyList(); + void ICollection.Add(T item) => NotImplementedForReadOnlyList(); + + public void Clear() => NotImplementedForReadOnlyList(); + + bool IList.Contains(object? value) => ((IList)this).IndexOf(value) >= 0; + bool ICollection.Contains(T item) => ((IList)this).IndexOf(item) >= 0; + + int IList.IndexOf(object? value) { + for (var i = 0; i != _data.Count; ++i) { + if (Equals(_data[i], value)) { + return i; + } + } + return -1; + } + + int IList.IndexOf(T value) { + for (var i = 0; i != _data.Count; ++i) { + var element = _data[i]; + if (element == null) { + if (value == null) { + return i; + } + continue; + } + if (element.Equals(value)) { + return i; + } + } + return -1; + } + + void IList.Insert(int index, object? value) => NotImplementedForReadOnlyList(); + void IList.Insert(int index, T item) => NotImplementedForReadOnlyList(); + + void IList.Remove(object? value) => NotImplementedForReadOnlyList(); + bool ICollection.Remove(T item) => NotImplementedForReadOnlyList(); + + public void RemoveAt(int index) => NotImplementedForReadOnlyList(); + + bool IList.IsFixedSize => true; + public bool IsReadOnly => true; + + void ICollection.CopyTo(Array array, int index) { + for (var i = 0; i != _data.Count; ++i) { + array.SetValue(_data[i], index + i); + } + } + + void ICollection.CopyTo(T[] array, int arrayIndex) { + for (var i = 0; i != _data.Count; ++i) { + array[arrayIndex + i] = _data[i]; + } + } + + public int Count => _data.Count; + + public bool IsSynchronized => false; + public object SyncRoot => this; + + object? IList.this[int index] { + get { + var value = _data[index]; + if (value == null) { + return null; + } + return value; + } + set => _ = NotImplementedForReadOnlyList(); + } + + T IList.this[int index] { + get => _data[index]; + set => _ = NotImplementedForReadOnlyList(); + } + + IEnumerator IEnumerable.GetEnumerator() { + return _data.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() { + return _data.GetEnumerator(); + } + + protected U NotImplementedForReadOnlyList() { + throw new NotImplementedException("This method is not implemented because the data structure is readonly"); + } +} + +/// +/// This is the leaf class used for wrapping IReadOnlyList<T> where T is a value type. +/// It provides IList and IList<Nullable<T>> to the user via its base class, +/// and provides IList<T> via the derived class. +/// +/// +public sealed class ReadOnlyListAdapterForValueTypes : ReadOnlyListAdapterBase, IList where T : struct, IEquatable { + private readonly T? _deephavenNullValue; + + public ReadOnlyListAdapterForValueTypes(IReadOnlyList data, T? deephavenNullValue) : base(data) { + _deephavenNullValue = deephavenNullValue; + } + + public int IndexOf(T item) { + for (var i = 0; i != _data.Count; ++i) { + var element = StripNull(_data[i]); + if (element.Equals(item)) { + return i; + } + } + return -1; + } + + public void Insert(int index, T item) => _ = NotImplementedForReadOnlyList(); + + public T this[int index] { + get { + var item = _data[index]; + return StripNull(item); + } + set => _ = NotImplementedForReadOnlyList(); + } + + public void Add(T item) => _ = NotImplementedForReadOnlyList(); + + public bool Contains(T item) => IndexOf(item) >= 0; + + public void CopyTo(T[] array, int arrayIndex) { + for (var i = 0; i != Count; ++i) { + array[arrayIndex + i] = this[i]; + } + } + + public bool Remove(T item) => NotImplementedForReadOnlyList(); + + public IEnumerator GetEnumerator() { + foreach (var item in _data) { + yield return StripNull(item); + } + } + + private T StripNull(T? value) { + if (!value.HasValue) { + return _deephavenNullValue ?? + throw new Exception( + $"Assertion failed: This IList contains null value but there is no Deephaven null value for T={Utility.FriendlyTypeName(typeof(T))}. Try casting to IList"); + } + return value.Value; + } +} + +/// +/// This is the leaf class used for wrapping IReadOnlyList<T> where T is a reference type. +/// It provides IList and IList<T> to the user via its base class, +/// +/// +public sealed class ReadOnlyListAdapterForReferenceTypes : ReadOnlyListAdapterBase where T : class { + public ReadOnlyListAdapterForReferenceTypes(IReadOnlyList data) : base(data) { } +} + +public class AdapterSelector : IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor, + IArrowArrayVisitor { + + public IList Result { get; private set; } = new List(); + + public void Visit(UInt16Array array) { + var adapted = new UInt16ToCharAdaptor(array); + Result = new ReadOnlyListAdapterForValueTypes(adapted, DeephavenConstants.NullChar); + } + + public void Visit(Int8Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullByte); + } + + public void Visit(Int16Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullShort); + } + + public void Visit(Int32Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullInt); + } + + public void Visit(Int64Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullLong); + } + + public void Visit(FloatArray array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullFloat); + } + + public void Visit(DoubleArray array) { + Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullDouble); + } + + public void Visit(StringArray array) { + Result = new ReadOnlyListAdapterForReferenceTypes(array); + } + + public void Visit(BooleanArray array) { + Result = new ReadOnlyListAdapterForValueTypes(array, null); + } + + public void Visit(TimestampArray array) { + Result = new ReadOnlyListAdapterForValueTypes(array, new DateTimeOffset()); + } + + public void Visit(Date64Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, new DateOnly()); + } + + public void Visit(Time64Array array) { + Result = new ReadOnlyListAdapterForValueTypes(array, new TimeOnly()); + } + + public void Visit(IArrowArray array) { + throw new NotImplementedException("Client does not support multiple levels of array nesting"); + } +} + +/// +/// "char" support in our system is a special case because it comes in over Arrow as UInt16Array, but +/// we want to present it to users as IList<char>. This class wraps IReadOnlyList<ushort?> +/// and provides IList<char?> to the user. +/// +public class UInt16ToCharAdaptor : IReadOnlyList { + private readonly IReadOnlyList _underlying; + + public UInt16ToCharAdaptor(IReadOnlyList underlying) { + _underlying = underlying; + } + + public char? this[int index] { + get { + var item = _underlying[index]; + return item.HasValue ? (char)item.Value : null; + } + } + + public IEnumerator GetEnumerator() { + foreach (var item in _underlying) { + yield return item.HasValue ? (char)item.Value : null; + } + } + + IEnumerator IEnumerable.GetEnumerator() { + foreach (var item in _underlying) { + yield return item.HasValue ? (char)item.Value : null; + } + } + + public int Count => _underlying.Count; +} + + diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs index d07864b5ff5..7487a197d34 100644 --- a/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs +++ b/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs @@ -1,6 +1,8 @@ // // Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // + +using Apache.Arrow; using Apache.Arrow.Types; namespace Deephaven.Dh_NetClient; @@ -30,21 +32,21 @@ private class ColumnSourceToArrowArrayVisitor : IColumnSourceVisitor, IColumnSourceVisitor, IColumnSourceVisitor, - IColumnSourceVisitor { + IColumnSourceVisitor, + IColumnSourceVisitor { private readonly int _numRows; private readonly Chunk _data; private readonly BooleanChunk _nulls; + public Apache.Arrow.IArrowArray? Result = null; + public ColumnSourceToArrowArrayVisitor(int numRows, Chunk data, BooleanChunk nulls) { _numRows = numRows; _data = data; _nulls = nulls; } - - public Apache.Arrow.IArrowArray? Result = null; - public void Visit(IByteColumnSource cs) { var arrowBuilder = new Apache.Arrow.Int8Array.Builder(); CopyHelper( @@ -130,13 +132,22 @@ public void Visit(IStringColumnSource cs) { Result = arrowBuilder.Build(); } + public void Visit(IListColumnSource cs) { + var type = cs is IHasElementType ihet ? ihet.ElementType : + throw new Exception($"Expected {Utility.FriendlyTypeName(cs.GetType())} to implement both {nameof(IListColumnSource)} and {nameof(IHasElementType)}"); + + var cb = ColumnBuilder.ForIListWithUnderlyingType(type); + cb.AppendChunk(_data, _nulls); + Result = cb.Build(); + } + private void CopyHelper(TBuilder arrowBuilder) where TArray : Apache.Arrow.IArrowArray where TBuilder : Apache.Arrow.IArrowArrayBuilder { var typedData = ((Chunk)_data).Data; for (var i = 0; i != _numRows; ++i) { if (!_nulls.Data[i]) { - arrowBuilder.Append(typedData[i]); + arrowBuilder.Append(typedData[i]!); } else { arrowBuilder.AppendNull(); } @@ -146,8 +157,7 @@ private void CopyHelper(TBuilder arrowBuilder) } public void Visit(IColumnSource cs) { - throw new NotImplementedException($"No ColumnSourceToArrayVisitor.Visit for {Utility.FriendlyTypeName(cs.GetType())}"); + throw new NotImplementedException($"No {nameof(ColumnSourceToArrowArrayVisitor)}.Visit for {Utility.FriendlyTypeName(cs.GetType())}"); } - } } diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs index a27d5872b89..903875fe786 100644 --- a/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs +++ b/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs @@ -13,7 +13,6 @@ global using DateTimeOffsetArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource; global using LocalDateArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource; global using LocalTimeArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource; - using Apache.Arrow; using Apache.Arrow.Types; @@ -181,10 +180,12 @@ public void FillChunk(RowSequence rows, ChunkedArray srcArray) { } sealed class ValueCopier(Chunk typedDest, BooleanChunk? nullFlags, T? deephavenNullValue) - : FillChunkHelper where T : struct { - protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) { + : FillChunkHelper where T : struct { + protected override void DoCopy(IArrowArray src, int srcStart, int destStart, int count) { var typedSrc = (IReadOnlyList)src; for (var i = 0; i < count; ++i) { + var srcOffset = srcStart + i; + var destOffset = destStart + i; var value = typedSrc[srcOffset]; var isNull = !value.HasValue || value.Value.Equals(deephavenNullValue); T destToUse; @@ -202,19 +203,23 @@ protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, i // it comes through DoGet, we're not getting null values when it comes through Barrage. nullFlags.Data[destOffset] = isNull; } - - ++srcOffset; - ++destOffset; } } } -sealed class TransformingCopier(Chunk typedDest, BooleanChunk? nullFlags, - TSrc deephavenNullValue, TDest transformedNullValue, Func transformer) +sealed class TransformingCopier( + Chunk typedDest, + BooleanChunk? nullFlags, + TSrc deephavenNullValue, + TDest transformedNullValue, + Func transformer) : FillChunkHelper where TSrc : struct where TDest : struct { - protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) { + protected override void DoCopy(IArrowArray src, int srcStart, int destStart, int count) { var typedSrc = (IReadOnlyList)src; for (var i = 0; i < count; ++i) { + var srcOffset = srcStart + i; + var destOffset = destStart + i; + var value = typedSrc[srcOffset]; bool isNull; if (!value.HasValue || value.Value.Equals(deephavenNullValue)) { @@ -228,24 +233,47 @@ protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, i if (nullFlags != null) { nullFlags.Data[destOffset] = isNull; } - - ++srcOffset; - ++destOffset; } } } sealed class ReferenceCopier(Chunk typedDest, BooleanChunk? nullFlags) : FillChunkHelper { - protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) { + protected override void DoCopy(IArrowArray src, int srcStart, int destStart, int count) { var typedSrc = (IReadOnlyList)src; for (var i = 0; i < count; ++i) { + var srcOffset = srcStart + i; + var destOffset = destStart + i; + typedDest.Data[destOffset] = typedSrc[srcOffset]; if (nullFlags != null) { nullFlags.Data[destOffset] = src.IsNull(srcOffset); } + } + } +} + +sealed class ListCopier(ListChunk typedDest, BooleanChunk? nullFlags) : FillChunkHelper { + protected override void DoCopy(IArrowArray src, int srcStart, int destStart, int count) { + var typedSrc = (ListArray)src; + for (var i = 0; i < count; ++i) { + var srcOffset = srcStart + i; + var destOffset = destStart + i; + + var isNull = src.IsNull(srcOffset); + + if (nullFlags != null) { + nullFlags.Data[destOffset] = isNull; + } + + if (isNull) { + typedDest.Data[destOffset] = null; + continue; + } - ++srcOffset; - ++destOffset; + var slicedData = typedSrc.GetSlicedValues(srcOffset); + var selector = new AdapterSelector(); + slicedData.Accept(selector); + typedDest.Data[destOffset] = selector.Result; } } } @@ -299,7 +327,8 @@ class ArrowColumnSourceMaker(ChunkedArray chunkedArray) : IArrowTypeVisitor, IArrowTypeVisitor, IArrowTypeVisitor, - IArrowTypeVisitor { + IArrowTypeVisitor, + IArrowTypeVisitor { public ArrowColumnSource? Result { get; private set; } public void Visit(UInt16Type type) { @@ -350,7 +379,86 @@ public void Visit(Time64Type type) { Result = new LocalTimeArrowColumnSource(chunkedArray); } + public void Visit(ListType type) { + var visitor = new ElementTypeVisitor(); + type.ValueDataType.Accept(visitor); + var elementType = visitor.Result!; + Result = new ListArrowColumnSource(chunkedArray, elementType); + } + + public void Visit(IArrowType type) { + throw new Exception($"Arrow type {type.Name} is not supported"); + } +} + +internal class ElementTypeVisitor : + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor, + IArrowTypeVisitor { + + public Type? Result { get; private set; } + + public void Visit(UInt16Type type) { + Result = typeof(char); + } + public void Visit(Int8Type type) { + Result = typeof(SByte); + } + public void Visit(Int16Type type) { + Result = typeof(Int16); + } + public void Visit(Int32Type type) { + Result = typeof(Int32); + } + public void Visit(Int64Type type) { + Result = typeof(Int64); + } + public void Visit(FloatType type) { + Result = typeof(float); + } + public void Visit(DoubleType type) { + Result = typeof(double); + } + public void Visit(BooleanType type) { + Result = typeof(bool); + } + public void Visit(StringType type) { + Result = typeof(string); + } + public void Visit(TimestampType type) { + Result = typeof(DateTimeOffset); + } + public void Visit(Date64Type type) { + Result = typeof(DateOnly); + } + public void Visit(Time64Type type) { + Result = typeof(TimeOnly); + } + public void Visit(IArrowType type) { throw new Exception($"Arrow type {type.Name} is not supported"); } } + +internal class ListArrowColumnSource(ChunkedArray chunkedArray, Type elementType) : ArrowColumnSource, IListColumnSource, IHasElementType { + public Type ElementType => elementType; + + public override void FillChunk(RowSequence rows, Chunk dest, BooleanChunk? nullFlags) { + var typedDest = (ListChunk)dest; + var lc = new ListCopier(typedDest, nullFlags); + lc.FillChunk(rows, chunkedArray); + } + + public override void Accept(IColumnSourceVisitor visitor) { + IColumnSource.Accept(this, visitor); + } +} diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs index 4287b146c8b..5fbbc60c312 100644 --- a/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs +++ b/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs @@ -189,11 +189,15 @@ public void Visit(ListArray array) { private IEnumerable VisitListArrayHelper(ListArray array) { // ListArray's elements are IArrowArrays. For each element, // we turn the IArrowArray into a List. - // To do this, we recursivel invoke the ToEnumerableVisitor (to handle the case + // To do this, we recursively invoke the ToEnumerableVisitor (to handle the case // where the elements are themselves lists). var innerVisitor = new ToEnumerableVisitor(); for (var i = 0; i != array.Length; ++i) { var slice = array.GetSlicedValues(i); + if (slice == null) { + yield return null; + continue; + } slice.Accept(innerVisitor); yield return new List(innerVisitor.Result.Cast()); } diff --git a/csharp/client/Dh_NetClient/chunk/Chunk.cs b/csharp/client/Dh_NetClient/chunk/Chunk.cs index 39a7c8f27da..10f7c599125 100644 --- a/csharp/client/Dh_NetClient/chunk/Chunk.cs +++ b/csharp/client/Dh_NetClient/chunk/Chunk.cs @@ -13,6 +13,7 @@ global using DateTimeOffsetChunk = Deephaven.Dh_NetClient.Chunk; global using DateOnlyChunk = Deephaven.Dh_NetClient.Chunk; global using TimeOnlyChunk = Deephaven.Dh_NetClient.Chunk; +global using ListChunk = Deephaven.Dh_NetClient.Chunk; namespace Deephaven.Dh_NetClient; @@ -22,12 +23,12 @@ public abstract class Chunk(int size) { public sealed class Chunk : Chunk { public static Chunk Create(int size) { - return new Chunk(new T[size]); + return new Chunk(new T?[size]); } - public T[] Data { get; } + public T?[] Data { get; } - private Chunk(T[] data) : base(data.Length) { + private Chunk(T?[] data) : base(data.Length) { Data = data; } } diff --git a/csharp/client/Dh_NetClient/chunk/ChunkMaker.cs b/csharp/client/Dh_NetClient/chunk/ChunkMaker.cs index bf4c11e272d..d968797b258 100644 --- a/csharp/client/Dh_NetClient/chunk/ChunkMaker.cs +++ b/csharp/client/Dh_NetClient/chunk/ChunkMaker.cs @@ -22,7 +22,8 @@ private class ChunkMakerVisitor(int chunkSize) : IColumnSourceVisitor, IColumnSourceVisitor, IColumnSourceVisitor, - IColumnSourceVisitor { + IColumnSourceVisitor, + IColumnSourceVisitor { public Chunk? Result { get; private set; } public void Visit(ICharColumnSource cs) => Make(cs); @@ -37,6 +38,7 @@ private class ChunkMakerVisitor(int chunkSize) : public void Visit(IDateTimeOffsetColumnSource cs) => Make(cs); public void Visit(IDateOnlyColumnSource cs) => Make(cs); public void Visit(ITimeOnlyColumnSource cs) => Make(cs); + public void Visit(IListColumnSource cs) => Make(cs); public void Visit(IColumnSource cs) { throw new Exception($"Assertion failed: No visitor for type {Utility.FriendlyTypeName(cs.GetType())}"); diff --git a/csharp/client/Dh_NetClient/column/ArrayColumnSource.cs b/csharp/client/Dh_NetClient/column/ArrayColumnSource.cs index 57173564d28..bd5719fbb57 100644 --- a/csharp/client/Dh_NetClient/column/ArrayColumnSource.cs +++ b/csharp/client/Dh_NetClient/column/ArrayColumnSource.cs @@ -104,7 +104,7 @@ public void Visit(IArrowType type) { } public sealed class ArrayColumnSource(int size) : ArrayColumnSource(size), IMutableColumnSource { - private readonly T[] _data = new T[size]; + private readonly T?[] _data = new T?[size]; public override void FillChunk(RowSequence rows, Chunk dest, BooleanChunk? nullFlags) { var typedChunk = (Chunk)dest; diff --git a/csharp/client/Dh_NetClient/column/ColumnSource.cs b/csharp/client/Dh_NetClient/column/ColumnSource.cs index f7373b26919..50987e45560 100644 --- a/csharp/client/Dh_NetClient/column/ColumnSource.cs +++ b/csharp/client/Dh_NetClient/column/ColumnSource.cs @@ -13,6 +13,7 @@ global using IDateTimeOffsetColumnSource = Deephaven.Dh_NetClient.IColumnSource; global using IDateOnlyColumnSource = Deephaven.Dh_NetClient.IColumnSource; global using ITimeOnlyColumnSource = Deephaven.Dh_NetClient.IColumnSource; +global using IListColumnSource = Deephaven.Dh_NetClient.IColumnSource; namespace Deephaven.Dh_NetClient; @@ -59,8 +60,13 @@ public interface IMutableColumnSource : IColumnSource { void FillFromChunk(RowSequence rows, Chunk src, BooleanChunk? nullFlags); } -public interface IMutableColumnSource : IMutableColumnSource, IColumnSource { +public interface IMutableColumnSource : IMutableColumnSource, IColumnSource; +/// +/// Used with interfaces like IColumnSource<IList> that have to expose their element type. +/// +public interface IHasElementType { + Type ElementType { get; } } /// diff --git a/csharp/client/Dh_NetClient/util/ColumnBuilder.cs b/csharp/client/Dh_NetClient/util/ColumnBuilder.cs new file mode 100644 index 00000000000..22d40144fc7 --- /dev/null +++ b/csharp/client/Dh_NetClient/util/ColumnBuilder.cs @@ -0,0 +1,445 @@ +// +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending +// +using Apache.Arrow; +using Apache.Arrow.Types; +using System.Diagnostics.CodeAnalysis; + +namespace Deephaven.Dh_NetClient; + +internal abstract class ColumnBuilder { + public static ColumnBuilder ForType(IArrowArrayBuilder? callerProvidedBuilder) { + return (ColumnBuilder)ForType(typeof(T), callerProvidedBuilder); + } + + public static ColumnBuilder ForType(Type type, IArrowArrayBuilder? callerProvidedBuilder) { + var nullableUnderlyingType = Nullable.GetUnderlyingType(type); + if (nullableUnderlyingType != null) { + var miGeneric = typeof(ColumnBuilder).GetMethod(nameof(ForNullableType)) ?? + throw new Exception($"Can't find {nameof(ForNullableType)}"); + var miInstantiated = miGeneric.MakeGenericMethod(nullableUnderlyingType); + return (ColumnBuilder)miInstantiated.Invoke(null, [callerProvidedBuilder])!; + } + + if (type == typeof(sbyte)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder + ?? new Apache.Arrow.Int8Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Int8Type.Default, DeephavenMetadataConstants.Types.Int8, + DeephavenConstants.NullByte); + } + + if (type == typeof(Int16)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.Int16Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Int16Type.Default, DeephavenMetadataConstants.Types.Int16, + DeephavenConstants.NullShort); + } + + if (type == typeof(Int32)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.Int32Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Int32Type.Default, DeephavenMetadataConstants.Types.Int32, + DeephavenConstants.NullInt); + } + + if (type == typeof(Int64)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.Int64Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Int64Type.Default, DeephavenMetadataConstants.Types.Int64, + DeephavenConstants.NullLong); + } + + if (type == typeof(float)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.FloatArray.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.FloatType.Default, DeephavenMetadataConstants.Types.Float, + DeephavenConstants.NullFloat); + } + + if (type == typeof(double)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.DoubleArray.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.DoubleType.Default, DeephavenMetadataConstants.Types.Double, + DeephavenConstants.NullDouble); + } + + if (type == typeof(bool)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.BooleanArray.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.BooleanType.Default, DeephavenMetadataConstants.Types.Bool, + null); + } + + if (type == typeof(char)) { + var builderToUse = + (Apache.Arrow.UInt16Array.Builder?)callerProvidedBuilder ?? + new Apache.Arrow.UInt16Array.Builder(); + return new CharColumnBuilder(builderToUse); + } + + if (type == typeof(string)) { + var builderToUse = + (Apache.Arrow.StringArray.Builder?)callerProvidedBuilder ?? + new Apache.Arrow.StringArray.Builder(); + return new StringColumnBuilder(builderToUse); + } + + if (type == typeof(DateTimeOffset)) { + var dataType = new Apache.Arrow.Types.TimestampType(TimeUnit.Nanosecond, "UTC"); + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.TimestampArray.Builder(dataType); + return new TypicalBuilder( + builderToUse, dataType, DeephavenMetadataConstants.Types.DateTime, null); + } + + if (type == typeof(DateOnly)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.Date64Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Date64Type.Default, DeephavenMetadataConstants.Types.LocalDate, + null); + } + + if (type == typeof(TimeOnly)) { + var builderToUse = + (IArrowArrayBuilder?)callerProvidedBuilder ?? + new Apache.Arrow.Time64Array.Builder(); + return new TypicalBuilder( + builderToUse, Apache.Arrow.Types.Time64Type.Default, DeephavenMetadataConstants.Types.LocalTime, + null); + } + + if (TryMatchTypeToIListOfUnderlying(type, out var underlyingType)) { + var miGeneric = typeof(ColumnBuilder).GetMethod(nameof(ForIListType)) ?? + throw new Exception($"Can't find {nameof(ForIListType)}"); + var miInstantiated = miGeneric.MakeGenericMethod(type, underlyingType); + return (ColumnBuilder)miInstantiated.Invoke(null, [callerProvidedBuilder])!; + } + + throw new Exception($"ColumnBuilder does not support type {Utility.FriendlyTypeName(type)}"); + } + + public static ColumnBuilder ForNullableType(IArrowArrayBuilder? callerProvidedBuilder) where T : struct { + var underlyingCb = ForType(callerProvidedBuilder); + return new NullableBuilder(underlyingCb); + } + + public static ColumnBuilder ForIListWithUnderlyingType(Type underlyingType) { + var underlyingTypeToUse = underlyingType.IsValueType ? typeof(Nullable<>).MakeGenericType(underlyingType) : underlyingType; + var ilistType = typeof(IList<>).MakeGenericType(underlyingTypeToUse); + var miGeneric = typeof(ColumnBuilder).GetMethod(nameof(ForIListType)) ?? + throw new Exception($"Can't find {nameof(ForIListType)}"); + var miInstantiated = miGeneric.MakeGenericMethod(ilistType, underlyingTypeToUse); + return (ColumnBuilder)miInstantiated.Invoke(null, [null])!; + } + + public static ColumnBuilder ForIListType( + IArrowArrayBuilder? callerProvidedBuilder) where TList : class, IList { + Apache.Arrow.ListArray.Builder builderToUse; + if (callerProvidedBuilder == null) { + // Make a temporary column builder just so I can get the correct Arrow data type + var tempCb = ForType(null); + var (underlyingArrowType, _, _) = tempCb.GetTypeInfo(); + builderToUse = new Apache.Arrow.ListArray.Builder(underlyingArrowType); + } else { + builderToUse = (Apache.Arrow.ListArray.Builder)callerProvidedBuilder; + } + return new ListBuilder(builderToUse); + } + + /// + /// Is target an IList<T> or does it inherit from IList<T> for some T? + /// If so, set underlying to T and return true. Otherwise return false. + /// + /// Type to examine + /// The underlying type if target is an IList<T> + /// True if target is an IList<T> or inherits from IList<T>, otherwise false + + private static bool TryMatchTypeToIListOfUnderlying(Type target, + [MaybeNullWhen(false)] out Type underlying) { + if (TryMatch(target, out underlying)) { + return true; + } + foreach (var iface in target.GetInterfaces()) { + if (TryMatch(iface, out underlying)) { + return true; + } + } + underlying = null; + return false; + + // Is target an IList for some T? If so, set underlying to T and return true. Otherwise return false. + static bool TryMatch(Type target, [MaybeNullWhen(false)] out Type underlying) { + if (target.IsGenericType && target.GetGenericTypeDefinition() == typeof(IList<>)) { + underlying = target.GetGenericArguments()[0]; + return true; + } + underlying = null; + return false; + } + } + + public abstract void AppendChunk(Chunk data, BooleanChunk nulls); + public abstract Apache.Arrow.IArrowArray Build(); + public abstract (Apache.Arrow.Types.IArrowType, string, string?) GetTypeInfo(); + public abstract void AppendNull(); +} + +internal abstract class ColumnBuilder : ColumnBuilder { + public abstract void Append(T item); +} + +internal sealed class TypicalBuilder : ColumnBuilder + where T : struct, IEquatable + where TArray : Apache.Arrow.IArrowArray + where TBuilder : Apache.Arrow.IArrowArrayBuilder { + private readonly Apache.Arrow.IArrowArrayBuilder _builder; + private readonly Apache.Arrow.Types.IArrowType _arrowType; + private readonly string _deephavenTypeName; + private readonly T? _deephavenNullValue; + + public TypicalBuilder(Apache.Arrow.IArrowArrayBuilder builder, + Apache.Arrow.Types.IArrowType arrowType, string deephavenTypeName, T? deephavenNullValue) { + _builder = builder; + _arrowType = arrowType; + _deephavenTypeName = deephavenTypeName; + _deephavenNullValue = deephavenNullValue; + } + + public override void Append(T item) { + if (_deephavenNullValue.HasValue && _deephavenNullValue.Value.Equals(item)) { + _builder.AppendNull(); + } else { + _builder.Append(item); + } + } + + public override void AppendNull() { + _builder.AppendNull(); + } + + public override void AppendChunk(Chunk data, BooleanChunk nulls) { + if (data.Size != nulls.Size) { + throw new ArgumentException($"Chunk size {data.Size} does not match nulls size {nulls.Size}"); + } + var typedChunk = data as Chunk + ?? throw new ArgumentException($"Expected chunk of type {Utility.FriendlyTypeName(typeof(Chunk))}, but got {Utility.FriendlyTypeName(data.GetType())}"); + + for (var i = 0; i != typedChunk.Size; i++) { + if (nulls.Data[i]) { + AppendNull(); + } else { + Append(typedChunk.Data[i]); + } + } + } + + public override (IArrowType, string, string?) GetTypeInfo() { + return (_arrowType, _deephavenTypeName, null); + } + + public override Apache.Arrow.IArrowArray Build() { + return _builder.Build(null); + } +} + +internal sealed class CharColumnBuilder : ColumnBuilder { + private readonly Apache.Arrow.UInt16Array.Builder _builder; + + public CharColumnBuilder(UInt16Array.Builder builder) { + _builder = builder; + } + + public override void Append(char item) { + if (item == DeephavenConstants.NullChar) { + _builder.AppendNull(); + } else { + _builder.Append(item); + } + } + + public override void AppendNull() { + _builder.AppendNull(); + } + + public override void AppendChunk(Chunk data, BooleanChunk nulls) { + if (data.Size != nulls.Size) { + throw new ArgumentException($"Chunk size {data.Size} does not match nulls size {nulls.Size}"); + } + var typedChunk = data as CharChunk + ?? throw new ArgumentException($"Expected chunk of type {Utility.FriendlyTypeName(typeof(CharChunk))}, but got {Utility.FriendlyTypeName(data.GetType())}"); + + for (var i = 0; i != typedChunk.Size; i++) { + if (nulls.Data[i]) { + AppendNull(); + } else { + Append(typedChunk.Data[i]); + } + } + } + + public override (IArrowType, string, string?) GetTypeInfo() { + return (Apache.Arrow.Types.UInt16Type.Default, DeephavenMetadataConstants.Types.Char16, null); + } + + public override Apache.Arrow.IArrowArray Build() { + return _builder.Build(); + } +} + +internal sealed class StringColumnBuilder : ColumnBuilder { + private readonly Apache.Arrow.StringArray.Builder _builder; + + public StringColumnBuilder(StringArray.Builder builder) { + _builder = builder; + } + + public override void Append(string item) { + _builder.Append(item); + } + + public override void AppendNull() { + _builder.AppendNull(); + } + + public override void AppendChunk(Chunk data, BooleanChunk nulls) { + if (data.Size != nulls.Size) { + throw new ArgumentException($"Chunk size {data.Size} does not match nulls size {nulls.Size}"); + } + var typedChunk = data as StringChunk + ?? throw new ArgumentException($"Expected chunk of type {Utility.FriendlyTypeName(typeof(StringChunk))}, but got {Utility.FriendlyTypeName(data.GetType())}"); + + for (var i = 0; i != typedChunk.Size; i++) { + if (nulls.Data[i]) { + AppendNull(); + } else { + Append(typedChunk.Data[i]!); + } + } + } + + + public override (IArrowType, string, string?) GetTypeInfo() { + return (Apache.Arrow.Types.StringType.Default, DeephavenMetadataConstants.Types.String, null); + } + + public override Apache.Arrow.IArrowArray Build() { + return _builder.Build(); + } +} + +internal sealed class NullableBuilder : ColumnBuilder where T : struct { + private readonly ColumnBuilder _underlyingBuilder; + + public NullableBuilder(ColumnBuilder underlyingBuilder) { + _underlyingBuilder = underlyingBuilder; + } + + public override void Append(T? item) { + if (item.HasValue) { + _underlyingBuilder.Append(item.Value); + } else { + _underlyingBuilder.AppendNull(); + } + } + + public override void AppendNull() { + _underlyingBuilder.AppendNull(); + } + + public override void AppendChunk(Chunk data, BooleanChunk nulls) { + if (data.Size != nulls.Size) { + throw new ArgumentException($"Chunk size {data.Size} does not match nulls size {nulls.Size}"); + } + var typedChunk = data as Chunk + ?? throw new ArgumentException($"Expected chunk of type {Utility.FriendlyTypeName(typeof(Chunk))}, but got {Utility.FriendlyTypeName(data.GetType())}"); + + for (var i = 0; i != typedChunk.Size; i++) { + if (nulls.Data[i]) { + AppendNull(); + } else { + Append(typedChunk.Data[i]); + } + } + } + + public override Apache.Arrow.IArrowArray Build() { + return _underlyingBuilder.Build(); + } + + public override (IArrowType, string, string?) GetTypeInfo() { + return _underlyingBuilder.GetTypeInfo(); + } +} + +internal sealed class ListBuilder : ColumnBuilder where TList : class, IList { + private readonly Apache.Arrow.ListArray.Builder _listBuilder; + private readonly ColumnBuilder _underlyingBuilder; + + public ListBuilder(Apache.Arrow.ListArray.Builder listBuilder) { + _listBuilder = listBuilder; + _underlyingBuilder = ColumnBuilder.ForType(_listBuilder.ValueBuilder); + } + + public override void Append(TList list) { + _listBuilder.Append(); + foreach (var element in list) { + if (element is not null) { + _underlyingBuilder.Append(element); + } else { + _underlyingBuilder.AppendNull(); + } + } + } + + public override void AppendNull() { + _listBuilder.AppendNull(); + } + + public override void AppendChunk(Chunk data, BooleanChunk nulls) { + if (data.Size != nulls.Size) { + throw new ArgumentException($"Chunk size {data.Size} does not match nulls size {nulls.Size}"); + } + var typedChunk = data as ListChunk + ?? throw new ArgumentException($"Expected chunk of type {typeof(ListChunk)}, but got {data.GetType()}"); + + for (var i = 0; i != typedChunk.Size; i++) { + if (nulls.Data[i]) { + AppendNull(); + } else { + var typedElement = typedChunk.Data[i] as TList ?? + throw new ArgumentException($"Expected element {i} to be of type {typeof(TList)}, but got {typedChunk.Data[i]?.GetType()}"); + Append(typedElement); + } + } + } + + public override IArrowArray Build() { + return _listBuilder.Build(); + } + + public override (IArrowType, string, string?) GetTypeInfo() { + var (underlyingArrowType, underlyingDeephavenType, _) = _underlyingBuilder.GetTypeInfo(); + + var arrowType = new Apache.Arrow.Types.ListType(underlyingArrowType); + var deephavenType = underlyingDeephavenType + "[]"; + var componentType = underlyingDeephavenType; + return (arrowType, deephavenType, componentType); + } +} diff --git a/csharp/client/Dh_NetClient/util/TableComparer.cs b/csharp/client/Dh_NetClient/util/TableComparer.cs index 88b87665167..caf52d34dd2 100644 --- a/csharp/client/Dh_NetClient/util/TableComparer.cs +++ b/csharp/client/Dh_NetClient/util/TableComparer.cs @@ -2,8 +2,6 @@ // Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // using System.Collections; -using System.Diagnostics; -using Apache.Arrow; namespace Deephaven.Dh_NetClient; diff --git a/csharp/client/Dh_NetClient/util/TableMaker.cs b/csharp/client/Dh_NetClient/util/TableMaker.cs index 87203d0479d..032a78328eb 100644 --- a/csharp/client/Dh_NetClient/util/TableMaker.cs +++ b/csharp/client/Dh_NetClient/util/TableMaker.cs @@ -1,9 +1,6 @@ // // Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending // -using Apache.Arrow; -using Apache.Arrow.Types; - namespace Deephaven.Dh_NetClient; public class TableMaker { @@ -12,6 +9,10 @@ public class TableMaker { public void AddColumn(string name, IReadOnlyList values) { var cb = ColumnBuilder.ForType(null); foreach (var value in values) { + if (value == null) { + cb.AppendNull(); + continue; + } cb.Append(value); } var array = cb.Build(); @@ -121,332 +122,6 @@ public override string ToString() { return ToString(true); } - private class ColumnBuilder { - public static ColumnBuilder ForType(IArrowArrayBuilder? callerProvidedBuilder) { - return (ColumnBuilder)ForType(typeof(T), callerProvidedBuilder); - } - - public static ColumnBuilder ForType(Type type, IArrowArrayBuilder? callerProvidedBuilder) { - var nullableUnderlyingType = Nullable.GetUnderlyingType(type); - if (nullableUnderlyingType != null) { - var miGeneric = typeof(ColumnBuilder).GetMethod(nameof(ForNullableType)) ?? - throw new Exception($"Can't find {nameof(ForNullableType)}"); - var miInstantiated = miGeneric.MakeGenericMethod(nullableUnderlyingType); - return (ColumnBuilder)miInstantiated.Invoke(null, [callerProvidedBuilder])!; - } - - if (type == typeof(sbyte)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder - ?? new Apache.Arrow.Int8Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Int8Type.Default, DeephavenMetadataConstants.Types.Int8, - DeephavenConstants.NullByte); - } - - if (type == typeof(Int16)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.Int16Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Int16Type.Default, DeephavenMetadataConstants.Types.Int16, - DeephavenConstants.NullShort); - } - - if (type == typeof(Int32)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.Int32Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Int32Type.Default, DeephavenMetadataConstants.Types.Int32, - DeephavenConstants.NullInt); - } - - if (type == typeof(Int64)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.Int64Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Int64Type.Default, DeephavenMetadataConstants.Types.Int64, - DeephavenConstants.NullLong); - } - - if (type == typeof(float)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.FloatArray.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.FloatType.Default, DeephavenMetadataConstants.Types.Float, - DeephavenConstants.NullFloat); - } - - if (type == typeof(double)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.DoubleArray.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.DoubleType.Default, DeephavenMetadataConstants.Types.Double, - DeephavenConstants.NullDouble); - } - - if (type == typeof(bool)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.BooleanArray.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.BooleanType.Default, DeephavenMetadataConstants.Types.Bool, - null); - } - - if (type == typeof(char)) { - var builderToUse = - (Apache.Arrow.UInt16Array.Builder?)callerProvidedBuilder ?? - new Apache.Arrow.UInt16Array.Builder(); - return new CharColumnBuilder(builderToUse); - } - - if (type == typeof(string)) { - var builderToUse = - (Apache.Arrow.StringArray.Builder?)callerProvidedBuilder ?? - new Apache.Arrow.StringArray.Builder(); - return new StringColumnBuilder(builderToUse); - } - - if (type == typeof(DateTimeOffset)) { - var dataType = new Apache.Arrow.Types.TimestampType(TimeUnit.Nanosecond, "UTC"); - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.TimestampArray.Builder(dataType); - return new TypicalBuilder( - builderToUse, dataType, DeephavenMetadataConstants.Types.DateTime, null); - } - - if (type == typeof(DateOnly)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.Date64Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Date64Type.Default, DeephavenMetadataConstants.Types.LocalDate, - null); - } - - if (type == typeof(TimeOnly)) { - var builderToUse = - (IArrowArrayBuilder?)callerProvidedBuilder ?? - new Apache.Arrow.Time64Array.Builder(); - return new TypicalBuilder( - builderToUse, Apache.Arrow.Types.Time64Type.Default, DeephavenMetadataConstants.Types.LocalTime, - null); - } - - var listUnderlyingType = GetIListInterfaceUnderlyingType(type); - if (listUnderlyingType != null) { - var miGeneric = typeof(ColumnBuilder).GetMethod(nameof(ForIListType)) ?? - throw new Exception($"Can't find {nameof(ForIListType)}"); - var miInstantiated = miGeneric.MakeGenericMethod(type, listUnderlyingType); - return (ColumnBuilder)miInstantiated.Invoke(null, [callerProvidedBuilder])!; - } - - throw new Exception($"ColumnBuilder does not support type {Utility.FriendlyTypeName(type)}"); - } - - public static ColumnBuilder ForNullableType(IArrowArrayBuilder? callerProvidedBuilder) where T : struct { - var underlyingCb = ForType(callerProvidedBuilder); - return new NullableBuilder(underlyingCb); - } - - public static ColumnBuilder ForIListType( - IArrowArrayBuilder? callerProvidedBuilder) where TList : IList { - Apache.Arrow.ListArray.Builder builderToUse; - if (callerProvidedBuilder == null) { - // Make a temporary column builder just so I can get the correct Arrow data type - var tempCb = ForType(null); - var (underlyingArrowType, _, _) = tempCb.GetTypeInfo(); - builderToUse = new Apache.Arrow.ListArray.Builder(underlyingArrowType); - } else { - builderToUse = (Apache.Arrow.ListArray.Builder)callerProvidedBuilder; - } - return new ListBuilder(builderToUse); - } - - private static Type? GetIListInterfaceUnderlyingType(Type ilistType) { - var temp = ilistType.GetInterfaces(); - return ilistType.GetInterfaces().Select(GetIListUnderlyingType).FirstOrDefault(t => t != null); - } - - private static Type? GetIListUnderlyingType(Type ilistType) { - if (ilistType.IsGenericType && !ilistType.IsGenericTypeDefinition) { - // Instantiated generic type only - var genericType = ilistType.GetGenericTypeDefinition(); - if (ReferenceEquals(genericType, typeof(IList<>))) { - return ilistType.GetGenericArguments()[0]; - } - } - return null; - } - } - - private abstract class ColumnBuilder : ColumnBuilder { - public abstract void Append(T item); - public abstract void AppendNull(); - - public abstract Apache.Arrow.IArrowArray Build(); - - public abstract (Apache.Arrow.Types.IArrowType, string, string?) GetTypeInfo(); - } - - private sealed class TypicalBuilder : ColumnBuilder - where T : struct, IEquatable - where TArray : Apache.Arrow.IArrowArray - where TBuilder : Apache.Arrow.IArrowArrayBuilder { - private readonly Apache.Arrow.IArrowArrayBuilder _builder; - private readonly Apache.Arrow.Types.IArrowType _arrowType; - private readonly string _deephavenTypeName; - private readonly T? _deephavenNullValue; - - public TypicalBuilder(Apache.Arrow.IArrowArrayBuilder builder, - Apache.Arrow.Types.IArrowType arrowType, string deephavenTypeName, T? deephavenNullValue) { - _builder = builder; - _arrowType = arrowType; - _deephavenTypeName = deephavenTypeName; - _deephavenNullValue = deephavenNullValue; - } - - public override void Append(T item) { - if (_deephavenNullValue.HasValue && _deephavenNullValue.Value.Equals(item)) { - _builder.AppendNull(); - } else { - _builder.Append(item); - } - } - - public override void AppendNull() { - _builder.AppendNull(); - } - - public override (IArrowType, string, string?) GetTypeInfo() { - return (_arrowType, _deephavenTypeName, null); - } - - public override Apache.Arrow.IArrowArray Build() { - return _builder.Build(null); - } - } - - private sealed class CharColumnBuilder : ColumnBuilder { - private readonly Apache.Arrow.UInt16Array.Builder _builder; - - public CharColumnBuilder(UInt16Array.Builder builder) { - _builder = builder; - } - - public override void Append(char item) { - if (item == DeephavenConstants.NullChar) { - _builder.AppendNull(); - } else { - _builder.Append(item); - } - } - - public override void AppendNull() { - _builder.AppendNull(); - } - - public override (IArrowType, string, string?) GetTypeInfo() { - return (Apache.Arrow.Types.UInt16Type.Default, DeephavenMetadataConstants.Types.Char16, null); - } - - public override Apache.Arrow.IArrowArray Build() { - return _builder.Build(); - } - } - - private sealed class StringColumnBuilder : ColumnBuilder { - private readonly Apache.Arrow.StringArray.Builder _builder; - - public StringColumnBuilder(StringArray.Builder builder) { - _builder = builder; - } - - public override void Append(string item) { - _builder.Append(item); - } - - public override void AppendNull() { - _builder.AppendNull(); - } - - public override (IArrowType, string, string?) GetTypeInfo() { - return (Apache.Arrow.Types.StringType.Default, DeephavenMetadataConstants.Types.String, null); - } - - public override Apache.Arrow.IArrowArray Build() { - return _builder.Build(); - } - } - - private sealed class NullableBuilder : ColumnBuilder where T : struct { - private readonly ColumnBuilder _underlyingBuilder; - - public NullableBuilder(ColumnBuilder underlyingBuilder) { - _underlyingBuilder = underlyingBuilder; - } - - public override void Append(T? item) { - if (item.HasValue) { - _underlyingBuilder.Append(item.Value); - } else { - _underlyingBuilder.AppendNull(); - } - } - - public override void AppendNull() { - _underlyingBuilder.AppendNull(); - } - - public override Apache.Arrow.IArrowArray Build() { - return _underlyingBuilder.Build(); - } - - public override (IArrowType, string, string?) GetTypeInfo() { - return _underlyingBuilder.GetTypeInfo(); - } - } - - private class ListBuilder : ColumnBuilder where TList : IList { - private readonly Apache.Arrow.ListArray.Builder _listBuilder; - private readonly ColumnBuilder _underlyingBuilder; - - public ListBuilder(Apache.Arrow.ListArray.Builder listBuilder) { - _listBuilder = listBuilder; - _underlyingBuilder = ColumnBuilder.ForType(_listBuilder.ValueBuilder); - } - - public override void Append(TList list) { - _listBuilder.Append(); - foreach (var element in list) { - _underlyingBuilder.Append(element); - } - } - - public override void AppendNull() { - _listBuilder.AppendNull(); - } - - public override IArrowArray Build() { - return _listBuilder.Build(); - } - - public override (IArrowType, string, string?) GetTypeInfo() { - var (underlyingArrowType, underlyingDeephavenType, _) = _underlyingBuilder.GetTypeInfo(); - - var arrowType = new Apache.Arrow.Types.ListType(underlyingArrowType); - var deephavenType = underlyingDeephavenType + "[]"; - var componentType = underlyingDeephavenType; - return (arrowType, deephavenType, componentType); - } - } - private record ColumnInfo(string Name, Apache.Arrow.IArrowArray Data, KeyValuePair[] ArrowMetadata); diff --git a/csharp/client/Dh_NetClientTests/ReadOnlyListAdapterTest.cs b/csharp/client/Dh_NetClientTests/ReadOnlyListAdapterTest.cs new file mode 100644 index 00000000000..09511e00544 --- /dev/null +++ b/csharp/client/Dh_NetClientTests/ReadOnlyListAdapterTest.cs @@ -0,0 +1,101 @@ +// +// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending +// + +using System.Collections; +using Deephaven.Dh_NetClient; + +namespace Deephaven.Dh_NetClientTests; + +public class ReadOnlyListAdaptersTest { + [Fact] + public void AsIList() { + IList list = MakeReadOnlyListAdapterForValueTypes(); + Assert.Equal(4, list.Count); + Assert.True(list.IsFixedSize); + Assert.True(list.IsReadOnly); + object?[] expected = [1, 2, null, 4]; + Assert.Equivalent(expected, list); + Assert.Equal(1, list[0]); + Assert.True(list.Contains(2)); + Assert.True(list.Contains(null)); + Assert.False(list.Contains(3)); + var dest = new object?[5]; + list.CopyTo(dest, 1); + Assert.Equivalent(new object?[]{null, 1, 2, null, 4}, dest); + + // test enumeration + var copy = list.Cast().ToArray(); + Assert.Equivalent(copy, list); + + Assert.Throws(() => list.Clear()); + Assert.Throws(() => list.Add(5)); + Assert.Throws(() => list.Remove(1)); + Assert.Throws(() => list.Insert(0, 1)); + Assert.Throws(() => list.RemoveAt(0)); + } + + [Fact] + public void AsIListOfT() { + IList list = MakeReadOnlyListAdapterForValueTypes(); + Assert.Equal(4, list.Count); + Assert.True(list.IsReadOnly); + Int32[] expected = [1, 2, DeephavenConstants.NullInt, 4]; + // Comparing expected and list directly doesn't work because of something + // in the way XUnit works. Probably because my Enumerable, Enumerable + // and Enumerable all give different answers. To be investigated. + var listAsArray = list.ToArray(); + Assert.Equal(expected.ToArray(), listAsArray); + Assert.Equal(1, list[0]); + Assert.True(list.Contains(2)); + Assert.False(list.Contains(3)); + var dest = new int[5]; + list.CopyTo(dest, 1); + Assert.Equal(new Int32[] { 0, 1, 2, DeephavenConstants.NullInt, 4 }, dest); + + // test enumeration + var copy = list.Select(x => x).ToArray(); + Assert.Equal(copy, listAsArray); + + Assert.Throws(() => list.Clear()); + Assert.Throws(() => list.Add(5)); + Assert.Throws(() => list.Remove(1)); + Assert.Throws(() => list.Insert(0, 1)); + Assert.Throws(() => list.RemoveAt(0)); + } + + [Fact] + public void AsIListOfNullableT() { + IList list = MakeReadOnlyListAdapterForValueTypes(); + Assert.Equal(4, list.Count); + Assert.True(list.IsReadOnly); + Int32?[] expected = [1, 2, null, 4]; + // Comparing expected and list directly doesn't work because of something + // in the way XUnit works. Probably because my Enumerable, Enumerable + // and Enumerable all give different answers. To be investigated. + var listAsArray = list.ToArray(); + Assert.Equal(expected, listAsArray); + Assert.Equal(1, list[0]); + Assert.True(list.Contains(2)); + Assert.False(list.Contains(3)); + var dest = new Int32?[5]; + list.CopyTo(dest, 1); + Assert.Equal(new Int32?[] { null, 1, 2, null, 4 }, dest); + + // test enumeration + var copy = list.Select(x => x).ToArray(); + Assert.Equal(copy, listAsArray); + + Assert.Throws(() => list.Clear()); + Assert.Throws(() => list.Add(5)); + Assert.Throws(() => list.Remove(1)); + Assert.Throws(() => list.Insert(0, 1)); + Assert.Throws(() => list.RemoveAt(0)); + } + + private static ReadOnlyListAdapterForValueTypes MakeReadOnlyListAdapterForValueTypes() { + var int32Data = new Int32?[] { 1, 2, null, 4 }; + var adapter = new ReadOnlyListAdapterForValueTypes(int32Data, DeephavenConstants.NullInt); + return adapter; + } +} diff --git a/csharp/client/Dh_NetClientTests/RoundTripTest.cs b/csharp/client/Dh_NetClientTests/RoundTripTest.cs index ff72778e10e..114290e512a 100644 --- a/csharp/client/Dh_NetClientTests/RoundTripTest.cs +++ b/csharp/client/Dh_NetClientTests/RoundTripTest.cs @@ -18,17 +18,34 @@ public void Elements() { [Fact] public void Nested() { var tm = new TableMaker(); - tm.AddColumn("Col1", [[0, 1, 2], [3, 4], [5]]); + var dto1 = new DateTimeOffset(1966, 3, 1, 12, 34, 56, TimeSpan.Zero); + var dto2 = new DateTimeOffset(1999, 12, 31, 3, 44, 55, TimeSpan.Zero); - var at = tm.ToArrowTable(); + tm.AddColumn("Int8", [[0, 1, 2], [3, 4, null], null, [6]]); + tm.AddColumn("Int16", [[0, 1, 2], [3, 4, null], null, [6]]); + tm.AddColumn("Int32", [[0, 1, 2], [3, 4, null], null, [6]]); + tm.AddColumn("Int64", [[0, 1, 2], [3, 4, null], null, [6]]); + tm.AddColumn("float", [[0.0f, 1.1f, 2.2f], [3.3f, 4.4f, null], null, [6.6f]]); + tm.AddColumn("double", [[0.0, 1.1, 2.2], [3.3, 4.4, null], null, [6.6]]); + tm.AddColumn("bool", [[false, true], [false, true, null], null, [true]]); + tm.AddColumn("string", [["", "hello"], ["a", "b", null], null, ["c"]]); + tm.AddColumn("char", [['a', 'X'], ['a', 'b', null], null, ['c']]); + tm.AddColumn("DateTimeOffset", [[dto1, dto2], [DateTimeOffset.MinValue, DateTimeOffset.MaxValue, null], null, [dto2]]); + tm.AddColumn("DateOnly", [ + [DateOnly.FromDateTime(dto1.DateTime), DateOnly.FromDateTime(dto2.DateTime)], + [DateOnly.MinValue, DateOnly.MaxValue, null], + null, + [DateOnly.FromDateTime(dto2.DateTime)]]); + tm.AddColumn("TimeOnly", [ + [TimeOnly.FromDateTime(dto1.DateTime), TimeOnly.FromDateTime(dto2.DateTime)], + [TimeOnly.MinValue, TimeOnly.MaxValue, null], + null, + [TimeOnly.FromDateTime(dto2.DateTime)]]); - // TODO(kosak): When you fix this, update DH-19910 - var ex = Assert.Throws(() => { - var ct = ArrowUtil.ToClientTable(at); - var at2 = ct.ToArrowTable(); - TableComparer.AssertSame(at, at2); - }); + var at = tm.ToArrowTable(); - Assert.Contains("Arrow type list is not supported", ex.Message); + var ct = ArrowUtil.ToClientTable(at); + var at2 = ct.ToArrowTable(); + TableComparer.AssertSame(at, at2); } }