diff --git a/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs b/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs
new file mode 100644
index 00000000000..fae45a95621
--- /dev/null
+++ b/csharp/client/Dh_NetClient/ReadOnlyListAdapters.cs
@@ -0,0 +1,322 @@
+//
+// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending
+//
+using Apache.Arrow;
+using System.Collections;
+using Array = System.Array;
+
+namespace Deephaven.Dh_NetClient;
+
+///
+/// This class hierarchy acts as a wrapper for IReadOnlyList<T> types that allows
+/// us to present them simultaneously as various flavors of IList types. We use this
+/// to wrap various Arrow array types. The type of wrapping depends on whether we are
+/// wrapping an array containing value types or an array containing reference types.
+///
+/// Apache.Arrow.StringArray is an example of an array containing reference types.
+/// It implements IReadOnlyList<string>. We would wrap it as
+/// ReadOnlyListAdapterForReferenceTypes<string> which would provide
+/// IList and List<string> to the user.
+///
+/// Apache.Arrow.Int32Array is an example of an array containing value types.
+/// It implements IReadOnlyList<Nullable<Int32>>. We would wrap it as
+/// ReadOnlyListAdapterForValueTypes<Int32> which would provide
+/// IList and List<Nullable<Int32>> (via its base class), and
+/// also IList<Int32> via the derived class.
+///
+/// Background and rationale for this class hierarchy:. The library initially supported a set of
+/// scalar types: char, bool, int32, float, string, etc. For each simple type S, there is a
+/// ColumnSource<S> that can represent its data and a Chunk<S> that can be used to
+/// hold batches of data. When it came time to add support for List types, we had to decide, among
+/// other things, what the appropriate ColumnSource and Chunk types would be. This is complicated
+/// by the fact that there are infinitely many List types: List can be generic on S, but it can
+/// also be generic on List<S> and that can be applied arbitrarily recursively:
+/// List<List<S>>, and so on. (However, we are currently only supporting
+/// List<S> where S is a scalar type).
+/// In terms of representation, we decided that there would be a single ColumnSource<IList>
+/// that could represent any list type, and a corresponding Chunk<IList> to be used with it.
+/// When programmers work with these IList elements, they can use them as ILists directly,
+/// or, to avoid boxing, they can cast them down to their actual concrete type. We promise that
+/// the IList elements contained in these ColumnSource and Chunk Types will always implement
+/// all ofIList, IList<T> and, for value types, IList<Nullable<T>>.
+///
+/// Note: it might have been preferable to use IReadOnlyList<T> and
+/// IReadOnlyList<Nullable<T>> instead of IList. The problem is of course that
+/// there is no bare (non-generic) IReadOnlyList type, so for the bare type we would still have
+/// to use IList. It would have been confusing and inconsistent to use IList for the bare
+/// non-generic type, but IReadOnlyList<T> for the generic type.. By using IList we can at least
+/// be consistent, even though it's a big interface with a bunch of mutating methods that we don't
+/// care about and will always throw exceptions for our case.
+///
+public abstract class ReadOnlyListAdapterBase : IList, IList {
+ protected readonly IReadOnlyList _data;
+
+ public ReadOnlyListAdapterBase(IReadOnlyList data) {
+ _data = data;
+ }
+
+ int IList.Add(object? item) => NotImplementedForReadOnlyList();
+ void ICollection.Add(T item) => NotImplementedForReadOnlyList();
+
+ public void Clear() => NotImplementedForReadOnlyList();
+
+ bool IList.Contains(object? value) => ((IList)this).IndexOf(value) >= 0;
+ bool ICollection.Contains(T item) => ((IList)this).IndexOf(item) >= 0;
+
+ int IList.IndexOf(object? value) {
+ for (var i = 0; i != _data.Count; ++i) {
+ if (Equals(_data[i], value)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ int IList.IndexOf(T value) {
+ for (var i = 0; i != _data.Count; ++i) {
+ var element = _data[i];
+ if (element == null) {
+ if (value == null) {
+ return i;
+ }
+ continue;
+ }
+ if (element.Equals(value)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ void IList.Insert(int index, object? value) => NotImplementedForReadOnlyList();
+ void IList.Insert(int index, T item) => NotImplementedForReadOnlyList();
+
+ void IList.Remove(object? value) => NotImplementedForReadOnlyList();
+ bool ICollection.Remove(T item) => NotImplementedForReadOnlyList();
+
+ public void RemoveAt(int index) => NotImplementedForReadOnlyList();
+
+ bool IList.IsFixedSize => true;
+ public bool IsReadOnly => true;
+
+ void ICollection.CopyTo(Array array, int index) {
+ for (var i = 0; i != _data.Count; ++i) {
+ array.SetValue(_data[i], index + i);
+ }
+ }
+
+ void ICollection.CopyTo(T[] array, int arrayIndex) {
+ for (var i = 0; i != _data.Count; ++i) {
+ array[arrayIndex + i] = _data[i];
+ }
+ }
+
+ public int Count => _data.Count;
+
+ public bool IsSynchronized => false;
+ public object SyncRoot => this;
+
+ object? IList.this[int index] {
+ get {
+ var value = _data[index];
+ if (value == null) {
+ return null;
+ }
+ return value;
+ }
+ set => _ = NotImplementedForReadOnlyList();
+ }
+
+ T IList.this[int index] {
+ get => _data[index];
+ set => _ = NotImplementedForReadOnlyList();
+ }
+
+ IEnumerator IEnumerable.GetEnumerator() {
+ return _data.GetEnumerator();
+ }
+
+ IEnumerator IEnumerable.GetEnumerator() {
+ return _data.GetEnumerator();
+ }
+
+ protected U NotImplementedForReadOnlyList() {
+ throw new NotImplementedException("This method is not implemented because the data structure is readonly");
+ }
+}
+
+///
+/// This is the leaf class used for wrapping IReadOnlyList<T> where T is a value type.
+/// It provides IList and IList<Nullable<T>> to the user via its base class,
+/// and provides IList<T> via the derived class.
+///
+///
+public sealed class ReadOnlyListAdapterForValueTypes : ReadOnlyListAdapterBase, IList where T : struct, IEquatable {
+ private readonly T? _deephavenNullValue;
+
+ public ReadOnlyListAdapterForValueTypes(IReadOnlyList data, T? deephavenNullValue) : base(data) {
+ _deephavenNullValue = deephavenNullValue;
+ }
+
+ public int IndexOf(T item) {
+ for (var i = 0; i != _data.Count; ++i) {
+ var element = StripNull(_data[i]);
+ if (element.Equals(item)) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public void Insert(int index, T item) => _ = NotImplementedForReadOnlyList();
+
+ public T this[int index] {
+ get {
+ var item = _data[index];
+ return StripNull(item);
+ }
+ set => _ = NotImplementedForReadOnlyList();
+ }
+
+ public void Add(T item) => _ = NotImplementedForReadOnlyList();
+
+ public bool Contains(T item) => IndexOf(item) >= 0;
+
+ public void CopyTo(T[] array, int arrayIndex) {
+ for (var i = 0; i != Count; ++i) {
+ array[arrayIndex + i] = this[i];
+ }
+ }
+
+ public bool Remove(T item) => NotImplementedForReadOnlyList();
+
+ public IEnumerator GetEnumerator() {
+ foreach (var item in _data) {
+ yield return StripNull(item);
+ }
+ }
+
+ private T StripNull(T? value) {
+ if (!value.HasValue) {
+ return _deephavenNullValue ??
+ throw new Exception(
+ $"Assertion failed: This IList contains null value but there is no Deephaven null value for T={Utility.FriendlyTypeName(typeof(T))}. Try casting to IList");
+ }
+ return value.Value;
+ }
+}
+
+///
+/// This is the leaf class used for wrapping IReadOnlyList<T> where T is a reference type.
+/// It provides IList and IList<T> to the user via its base class,
+///
+///
+public sealed class ReadOnlyListAdapterForReferenceTypes : ReadOnlyListAdapterBase where T : class {
+ public ReadOnlyListAdapterForReferenceTypes(IReadOnlyList data) : base(data) { }
+}
+
+public class AdaptorSelector : IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor,
+ IArrowArrayVisitor {
+
+ public IList Result { get; private set; } = new List();
+
+ public void Visit(UInt16Array array) {
+ var adapted = new UInt16ToCharAdaptor(array);
+ Result = new ReadOnlyListAdapterForValueTypes(adapted, DeephavenConstants.NullChar);
+ }
+
+ public void Visit(Int8Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullByte);
+ }
+
+ public void Visit(Int16Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullShort);
+ }
+
+ public void Visit(Int32Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullInt);
+ }
+
+ public void Visit(Int64Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullLong);
+ }
+
+ public void Visit(FloatArray array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullFloat);
+ }
+
+ public void Visit(DoubleArray array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, DeephavenConstants.NullDouble);
+ }
+
+ public void Visit(StringArray array) {
+ Result = new ReadOnlyListAdapterForReferenceTypes(array);
+ }
+
+ public void Visit(BooleanArray array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, null);
+ }
+
+ public void Visit(TimestampArray array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, new DateTimeOffset());
+ }
+
+ public void Visit(Date64Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, new DateOnly());
+ }
+
+ public void Visit(Time64Array array) {
+ Result = new ReadOnlyListAdapterForValueTypes(array, new TimeOnly());
+ }
+
+ public void Visit(IArrowArray array) {
+ throw new NotImplementedException("Client does not support multiple levels of array nesting");
+ }
+}
+
+///
+/// "char" support in our system is a special case because it comes in over Arrow as UInt16Array, but
+/// we want to present it to users as IList<char>. This class wraps IReadOnlyList<ushort?>
+/// and provides IList<char?> to the user.
+///
+public class UInt16ToCharAdaptor : IReadOnlyList {
+ private readonly IReadOnlyList _underlying;
+
+ public UInt16ToCharAdaptor(IReadOnlyList underlying) {
+ _underlying = underlying;
+ }
+
+ public char? this[int index] {
+ get {
+ var item = _underlying[index];
+ return item.HasValue ? (char)item.Value : null;
+ }
+ }
+
+ public IEnumerator GetEnumerator() {
+ foreach (var item in _underlying) {
+ yield return item.HasValue ? (char)item.Value : null;
+ }
+ }
+
+ IEnumerator IEnumerable.GetEnumerator() {
+ foreach (var item in _underlying) {
+ yield return item.HasValue ? (char)item.Value : null;
+ }
+ }
+
+ public int Count => _underlying.Count;
+}
+
+
diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs
index d07864b5ff5..7487a197d34 100644
--- a/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs
+++ b/csharp/client/Dh_NetClient/arrow_util/ArrowArrayConverter.cs
@@ -1,6 +1,8 @@
//
// Copyright (c) 2016-2026 Deephaven Data Labs and Patent Pending
//
+
+using Apache.Arrow;
using Apache.Arrow.Types;
namespace Deephaven.Dh_NetClient;
@@ -30,21 +32,21 @@ private class ColumnSourceToArrowArrayVisitor :
IColumnSourceVisitor,
IColumnSourceVisitor,
IColumnSourceVisitor,
- IColumnSourceVisitor {
+ IColumnSourceVisitor,
+ IColumnSourceVisitor {
private readonly int _numRows;
private readonly Chunk _data;
private readonly BooleanChunk _nulls;
+ public Apache.Arrow.IArrowArray? Result = null;
+
public ColumnSourceToArrowArrayVisitor(int numRows, Chunk data, BooleanChunk nulls) {
_numRows = numRows;
_data = data;
_nulls = nulls;
}
-
- public Apache.Arrow.IArrowArray? Result = null;
-
public void Visit(IByteColumnSource cs) {
var arrowBuilder = new Apache.Arrow.Int8Array.Builder();
CopyHelper(
@@ -130,13 +132,22 @@ public void Visit(IStringColumnSource cs) {
Result = arrowBuilder.Build();
}
+ public void Visit(IListColumnSource cs) {
+ var type = cs is IHasElementType ihet ? ihet.ElementType :
+ throw new Exception($"Expected {Utility.FriendlyTypeName(cs.GetType())} to implement both {nameof(IListColumnSource)} and {nameof(IHasElementType)}");
+
+ var cb = ColumnBuilder.ForIListWithUnderlyingType(type);
+ cb.AppendChunk(_data, _nulls);
+ Result = cb.Build();
+ }
+
private void CopyHelper(TBuilder arrowBuilder)
where TArray : Apache.Arrow.IArrowArray
where TBuilder : Apache.Arrow.IArrowArrayBuilder {
var typedData = ((Chunk)_data).Data;
for (var i = 0; i != _numRows; ++i) {
if (!_nulls.Data[i]) {
- arrowBuilder.Append(typedData[i]);
+ arrowBuilder.Append(typedData[i]!);
} else {
arrowBuilder.AppendNull();
}
@@ -146,8 +157,7 @@ private void CopyHelper(TBuilder arrowBuilder)
}
public void Visit(IColumnSource cs) {
- throw new NotImplementedException($"No ColumnSourceToArrayVisitor.Visit for {Utility.FriendlyTypeName(cs.GetType())}");
+ throw new NotImplementedException($"No {nameof(ColumnSourceToArrowArrayVisitor)}.Visit for {Utility.FriendlyTypeName(cs.GetType())}");
}
-
}
}
diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs
index a27d5872b89..6fa68f5cf65 100644
--- a/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs
+++ b/csharp/client/Dh_NetClient/arrow_util/ArrowColumnSource.cs
@@ -13,7 +13,6 @@
global using DateTimeOffsetArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource;
global using LocalDateArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource;
global using LocalTimeArrowColumnSource = Deephaven.Dh_NetClient.ArrowColumnSource;
-
using Apache.Arrow;
using Apache.Arrow.Types;
@@ -181,7 +180,7 @@ public void FillChunk(RowSequence rows, ChunkedArray srcArray) {
}
sealed class ValueCopier(Chunk typedDest, BooleanChunk? nullFlags, T? deephavenNullValue)
- : FillChunkHelper where T : struct {
+ : FillChunkHelper where T : struct {
protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) {
var typedSrc = (IReadOnlyList)src;
for (var i = 0; i < count; ++i) {
@@ -209,8 +208,12 @@ protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, i
}
}
-sealed class TransformingCopier(Chunk typedDest, BooleanChunk? nullFlags,
- TSrc deephavenNullValue, TDest transformedNullValue, Func transformer)
+sealed class TransformingCopier(
+ Chunk typedDest,
+ BooleanChunk? nullFlags,
+ TSrc deephavenNullValue,
+ TDest transformedNullValue,
+ Func transformer)
: FillChunkHelper where TSrc : struct where TDest : struct {
protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) {
var typedSrc = (IReadOnlyList)src;
@@ -250,6 +253,31 @@ protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, i
}
}
+sealed class ListCopier(ListChunk typedDest, BooleanChunk? nullFlags) : FillChunkHelper {
+ protected override void DoCopy(IArrowArray src, int srcOffset, int destOffset, int count) {
+ var typedSrc = (ListArray)src;
+ // var srcValues = (Apache.Arrow.Array)typedSrc.Values;
+ for (var i = 0; i < count; ++i, ++srcOffset, ++destOffset) {
+ if (src.IsNull(i)) {
+ if (nullFlags != null) {
+ typedDest.Data[destOffset] = null;
+ nullFlags.Data[destOffset] = true;
+ }
+ continue;
+ }
+
+ var slicedData = typedSrc.GetSlicedValues(srcOffset);
+
+ // var start = typedSrc.ValueOffsets[srcOffset];
+ // var end = typedSrc.ValueOffsets[srcOffset + 1];
+ // var slicedData = srcValues.Slice(start, end - start);
+ var sn = new AdaptorSelector();
+ slicedData.Accept(sn);
+ typedDest.Data[destOffset] = sn.Result;
+ }
+ }
+}
+
public class ChunkedArrayIterator(ChunkedArray chunkedArray) {
private int _arrayIndex = -1;
private Int64 _segmentOffset = 0;
@@ -299,7 +327,8 @@ class ArrowColumnSourceMaker(ChunkedArray chunkedArray) :
IArrowTypeVisitor,
IArrowTypeVisitor,
IArrowTypeVisitor,
- IArrowTypeVisitor {
+ IArrowTypeVisitor,
+ IArrowTypeVisitor {
public ArrowColumnSource? Result { get; private set; }
public void Visit(UInt16Type type) {
@@ -350,7 +379,86 @@ public void Visit(Time64Type type) {
Result = new LocalTimeArrowColumnSource(chunkedArray);
}
+ public void Visit(ListType type) {
+ var visitor = new ElementTypeVisitor();
+ type.ValueDataType.Accept(visitor);
+ var elementType = visitor.Result!;
+ Result = new ListArrowColumnSource(chunkedArray, elementType);
+ }
+
public void Visit(IArrowType type) {
throw new Exception($"Arrow type {type.Name} is not supported");
}
}
+
+public class ElementTypeVisitor :
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor,
+ IArrowTypeVisitor {
+
+ public Type? Result { get; private set; }
+
+ public void Visit(UInt16Type type) {
+ Result = typeof(char);
+ }
+ public void Visit(Int8Type type) {
+ Result = typeof(SByte);
+ }
+ public void Visit(Int16Type type) {
+ Result = typeof(Int16);
+ }
+ public void Visit(Int32Type type) {
+ Result = typeof(Int32);
+ }
+ public void Visit(Int64Type type) {
+ Result = typeof(Int64);
+ }
+ public void Visit(FloatType type) {
+ Result = typeof(float);
+ }
+ public void Visit(DoubleType type) {
+ Result = typeof(double);
+ }
+ public void Visit(BooleanType type) {
+ Result = typeof(bool);
+ }
+ public void Visit(StringType type) {
+ Result = typeof(string);
+ }
+ public void Visit(TimestampType type) {
+ Result = typeof(DateTimeOffset);
+ }
+ public void Visit(Date64Type type) {
+ Result = typeof(DateOnly);
+ }
+ public void Visit(Time64Type type) {
+ Result = typeof(TimeOnly);
+ }
+
+ public void Visit(IArrowType type) {
+ throw new Exception($"Arrow type {type.Name} is not supported");
+ }
+}
+
+public class ListArrowColumnSource(ChunkedArray chunkedArray, Type elementType) : ArrowColumnSource, IListColumnSource, IHasElementType {
+ public Type ElementType => elementType;
+
+ public override void FillChunk(RowSequence rows, Chunk dest, BooleanChunk? nullFlags) {
+ var typedDest = (ListChunk)dest;
+ var lc = new ListCopier(typedDest, nullFlags);
+ lc.FillChunk(rows, chunkedArray);
+ }
+
+ public override void Accept(IColumnSourceVisitor visitor) {
+ IColumnSource.Accept(this, visitor);
+ }
+}
diff --git a/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs b/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs
index 4287b146c8b..5fbbc60c312 100644
--- a/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs
+++ b/csharp/client/Dh_NetClient/arrow_util/ArrowUtil.cs
@@ -189,11 +189,15 @@ public void Visit(ListArray array) {
private IEnumerable VisitListArrayHelper(ListArray array) {
// ListArray's elements are IArrowArrays. For each element,
// we turn the IArrowArray into a List