From fd47bca76924078d89c0754f9cb9f90e235c70ef Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Sun, 24 May 2026 23:59:30 +0400 Subject: [PATCH 01/33] Bump GitHub Actions to latest major versions --- .github/workflows/NuGet.yml | 2 +- .github/workflows/Sonar.yml | 10 +++++----- .github/workflows/ci.yml | 10 +++++----- .github/workflows/codeql-analysis.yml | 8 ++++---- .github/workflows/scorecard.yml | 8 ++++---- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/NuGet.yml b/.github/workflows/NuGet.yml index 7533311e..99e2279a 100644 --- a/.github/workflows/NuGet.yml +++ b/.github/workflows/NuGet.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Download Artifacts - uses: dawidd6/action-download-artifact@v6 + uses: dawidd6/action-download-artifact@v21 with: github_token: ${{ secrets.GITHUB_TOKEN }} workflow: ci.yml diff --git a/.github/workflows/Sonar.yml b/.github/workflows/Sonar.yml index 78a815bd..c2dc1d94 100644 --- a/.github/workflows/Sonar.yml +++ b/.github/workflows/Sonar.yml @@ -28,22 +28,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'zulu' # Alternative distribution options are available. - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json - name: Cache SonarCloud packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/sonar/cache key: ${{ runner.os }}-sonar @@ -51,7 +51,7 @@ jobs: - name: Cache SonarCloud scanner id: cache-sonar-scanner - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./.sonar/scanner key: ${{ runner.os }}-sonar-scanner diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3420b0d0..a13fdc9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,13 @@ jobs: echo "NightlyBuild=true" >> $env:GITHUB_ENV - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 ref: ${{ env.BuildBranch }} - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json @@ -82,14 +82,14 @@ jobs: - name: Upload coverage reports to Codecov if: matrix.os == 'ubuntu-latest' && github.event_name != 'schedule' - uses: codecov/codecov-action@v4.0.1 + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} slug: Giorgi/DuckDB.NET - name: Upload Artifacts if: matrix.os == 'ubuntu-latest' && github.event_name != 'pull_request' && github.actor == 'Giorgi' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: nugetPackages-${{github.ref_name}} path: ./**/bin/Release/*.nupkg @@ -103,7 +103,7 @@ jobs: if: github.ref == 'refs/heads/develop' && github.event_name != 'pull_request' && github.event_name != 'schedule' steps: - name: Download nuget package artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: nugetPackages-${{github.ref_name}} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a3754128..bbf76dd3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,13 +42,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -59,7 +59,7 @@ jobs: # queries: security-extended,security-and-quality - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json @@ -82,4 +82,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 967a53f5..f1c04ef6 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -34,12 +34,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif @@ -64,7 +64,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif @@ -73,6 +73,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: results.sarif From 8666a0342342d36f1b88c577e5fd01e86c5a7601 Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Tue, 23 Jun 2026 14:16:09 +0400 Subject: [PATCH 02/33] Update to 1.5.4 --- DuckDB.NET.Bindings/Bindings.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index 13a7ae76..37082dd5 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -3,11 +3,11 @@ DuckDB Bindings for C#. -- Updated to DuckDB v1.5.3 +- Updated to DuckDB v1.5.4 DuckDB.NET.Native win-x64;win-arm64;linux-x64;linux-arm64;osx - https://github.com/duckdb/duckdb/releases/download/v1.5.3 + https://github.com/duckdb/duckdb/releases/download/v1.5.4 True ..\keyPair.snk true From 5efc5e55280f579b4abeaa7aad4a6e646ad2a4b4 Mon Sep 17 00:00:00 2001 From: Luis Quintanilla <46974588+luisquintanilla@users.noreply.github.com> Date: Wed, 24 Jun 2026 09:51:35 -0400 Subject: [PATCH 03/33] Add Apache Arrow result streaming to DuckDBCommand (#334) * Add Apache Arrow result streaming to DuckDBCommand Adds a managed Apache Arrow read surface built on DuckDB's current Arrow C Data Interface (duckdb_to_arrow_schema / duckdb_data_chunk_to_arrow), addressing #26. - Bindings: duckdb_result_get_arrow_options, duckdb_to_arrow_schema, duckdb_data_chunk_to_arrow, error-data helpers, and a DuckDBArrowOptions safe handle. - Data: DuckDBArrowArrayStream (IArrowArrayStream) that builds the schema once and converts each data chunk into an Arrow RecordBatch via Apache.Arrow's C importers. - DuckDBCommand.ExecuteArrowStream() and ExecuteArrowBatchesAsync() public APIs. - Apache.Arrow dependency added to DuckDB.NET.Data only. - Tests covering schema, scalar values, nulls, multi-chunk streaming, and the stream API. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Honor UseStreamingMode in Arrow APIs and fix result stream cleanup Pass UseStreamingMode through ExecuteArrowStream and fetch chunks via the streaming or materialized path accordingly. Dispose Arrow options and close the result if schema building fails in the stream constructor. Add tests for streaming mode, cancellation, and dispose semantics. * Refactor Arrow error handling to typed DuckDBErrorData handle - Add DuckDBErrorData SafeHandle wrapping duckdb_error_data - Move error-data P/Invokes to NativeMethods.ErrorData - Retype duckdb_to_arrow_schema/data_chunk_to_arrow returns to DuckDBErrorData - Add reusable ThrowOnError extension throwing DuckDBException - Make DuckDBArrowArrayStream internal --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- DuckDB.NET.Bindings/DuckDBWrapperObjects.cs | 22 +++ .../NativeMethods/NativeMethods.Arrow.cs | 27 +++ .../NativeMethods/NativeMethods.ErrorData.cs | 23 +++ .../Arrow/DuckDBArrowArrayStream.cs | 163 ++++++++++++++++ DuckDB.NET.Data/Data.csproj | 4 + DuckDB.NET.Data/DuckDBCommand.cs | 56 ++++++ .../Extensions/DuckDBErrorDataExtensions.cs | 15 ++ DuckDB.NET.Test/Arrow/ArrowResultTests.cs | 176 ++++++++++++++++++ 8 files changed, 486 insertions(+) create mode 100644 DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs create mode 100644 DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs create mode 100644 DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs create mode 100644 DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs create mode 100644 DuckDB.NET.Test/Arrow/ArrowResultTests.cs diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index e42f90c6..343074cf 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -68,6 +68,28 @@ protected override bool ReleaseHandle() } } +public class DuckDBArrowOptions() : SafeHandleZeroOrMinusOneIsInvalid(true) +{ + protected override bool ReleaseHandle() + { + NativeMethods.Arrow.DuckDBDestroyArrowOptions(ref handle); + return true; + } +} + +public class DuckDBErrorData() : SafeHandleZeroOrMinusOneIsInvalid(true) +{ + public bool HasError => !IsInvalid && NativeMethods.ErrorData.DuckDBErrorDataHasError(this); + + public string? Message => IsInvalid ? null : NativeMethods.ErrorData.DuckDBErrorDataMessage(this); + + protected override bool ReleaseHandle() + { + NativeMethods.ErrorData.DuckDBDestroyErrorData(ref handle); + return true; + } +} + public class DuckDBDataChunk : SafeHandleZeroOrMinusOneIsInvalid { public DuckDBDataChunk() : base(true) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs new file mode 100644 index 00000000..13392a82 --- /dev/null +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs @@ -0,0 +1,27 @@ +namespace DuckDB.NET.Native; + +public partial class NativeMethods +{ + //https://duckdb.org/docs/stable/clients/c/api#arrow-interface + public static partial class Arrow + { + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_result_get_arrow_options")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBArrowOptions DuckDBResultGetArrowOptions(ref DuckDBResult result); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_destroy_arrow_options")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial void DuckDBDestroyArrowOptions(ref IntPtr arrowOptions); + + // duckdb_error_data duckdb_to_arrow_schema(duckdb_arrow_options, duckdb_logical_type *types, + // const char **names, idx_t column_count, ArrowSchema *out_schema) + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_to_arrow_schema")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorData DuckDBToArrowSchema(DuckDBArrowOptions arrowOptions, IntPtr types, IntPtr names, ulong columnCount, IntPtr outSchema); + + // duckdb_error_data duckdb_data_chunk_to_arrow(duckdb_arrow_options, duckdb_data_chunk, ArrowArray *out_arrow_array) + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_data_chunk_to_arrow")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorData DuckDBDataChunkToArrow(DuckDBArrowOptions arrowOptions, DuckDBDataChunk chunk, IntPtr outArray); + } +} diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs new file mode 100644 index 00000000..25f0a4a4 --- /dev/null +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs @@ -0,0 +1,23 @@ +namespace DuckDB.NET.Native; + +public partial class NativeMethods +{ + //https://duckdb.org/docs/stable/clients/c/api#error-data + public static partial class ErrorData + { + [SuppressGCTransition] + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_has_error")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [return: MarshalAs(UnmanagedType.I1)] + public static partial bool DuckDBErrorDataHasError(DuckDBErrorData errorData); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_message")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [return: MarshalUsing(typeof(DuckDBOwnedStringMarshaller))] + public static partial string DuckDBErrorDataMessage(DuckDBErrorData errorData); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_destroy_error_data")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial void DuckDBDestroyErrorData(ref IntPtr errorData); + } +} diff --git a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs new file mode 100644 index 00000000..2fc5f5ea --- /dev/null +++ b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs @@ -0,0 +1,163 @@ +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.C; +using Apache.Arrow.Ipc; +using DuckDB.NET.Native; + +namespace DuckDB.NET.Data.Arrow; + +/// +/// Streams the rows of a DuckDB query result as Apache Arrow values using +/// DuckDB's Arrow C Data Interface (duckdb_to_arrow_schema / duckdb_data_chunk_to_arrow). +/// Each DuckDB data chunk is converted into one Arrow record batch and imported with no row-by-row marshaling. +/// +internal sealed class DuckDBArrowArrayStream : IArrowArrayStream +{ + private DuckDBResult result; + private readonly DuckDBArrowOptions arrowOptions; + private readonly bool streaming; + private bool disposed; + + public Schema Schema { get; } + + internal DuckDBArrowArrayStream(DuckDBResult result) + { + this.result = result; + + arrowOptions = NativeMethods.Arrow.DuckDBResultGetArrowOptions(ref this.result); + if (arrowOptions.IsInvalid) + { + this.result.Close(); + throw new InvalidOperationException("Failed to obtain Arrow options from the DuckDB result."); + } + + streaming = NativeMethods.Types.DuckDBResultIsStreaming(this.result) > 0; + + try + { + Schema = BuildSchema(); + } + catch + { + arrowOptions.Dispose(); + this.result.Close(); + throw; + } + } + + private unsafe Schema BuildSchema() + { + var columnCount = NativeMethods.Query.DuckDBColumnCount(ref result); + + var logicalTypes = new DuckDBLogicalType[columnCount]; + var typeHandles = new IntPtr[columnCount]; + var namePointers = new IntPtr[columnCount]; + + try + { + for (var index = 0UL; index < columnCount; index++) + { + var logicalType = NativeMethods.Query.DuckDBColumnLogicalType(ref result, (long)index); + logicalTypes[index] = logicalType; + typeHandles[index] = logicalType.DangerousGetHandle(); + + var name = NativeMethods.Query.DuckDBColumnName(ref result, (long)index); + namePointers[index] = Marshal.StringToCoTaskMemUTF8(name); + } + + var cSchema = CArrowSchema.Create(); + + try + { + fixed (IntPtr* typesPointer = typeHandles) + fixed (IntPtr* namesPointer = namePointers) + { + var error = NativeMethods.Arrow.DuckDBToArrowSchema(arrowOptions, (IntPtr)typesPointer, (IntPtr)namesPointer, columnCount, (IntPtr)cSchema); + error.ThrowOnError("Failed to convert the DuckDB result schema to an Arrow schema."); + } + + return CArrowSchemaImporter.ImportSchema(cSchema); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + finally + { + foreach (var pointer in namePointers) + { + if (pointer != IntPtr.Zero) + { + Marshal.FreeCoTaskMem(pointer); + } + } + + foreach (var logicalType in logicalTypes) + { + logicalType?.Dispose(); + } + } + } + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(disposed, this); + + if (cancellationToken.IsCancellationRequested) + { + return new ValueTask(Task.FromCanceled(cancellationToken)); + } + + var chunk = streaming + ? NativeMethods.StreamingResult.DuckDBStreamFetchChunk(result) + : NativeMethods.Query.DuckDBFetchChunk(result); + + if (chunk.IsInvalid) + { + chunk.Dispose(); + return new ValueTask((RecordBatch?)null); + } + + try + { + return new ValueTask(ConvertChunk(chunk)); + } + finally + { + chunk.Dispose(); + } + } + + private unsafe RecordBatch ConvertChunk(DuckDBDataChunk chunk) + { + var cArray = CArrowArray.Create(); + + try + { + var error = NativeMethods.Arrow.DuckDBDataChunkToArrow(arrowOptions, chunk, (IntPtr)cArray); + error.ThrowOnError("Failed to convert a DuckDB data chunk to an Arrow array."); + + return CArrowArrayImporter.ImportRecordBatch(cArray, Schema); + } + finally + { + CArrowArray.Free(cArray); + } + } + + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + + arrowOptions.Dispose(); + result.Close(); + } +} diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index ce657cbf..4afd3811 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -32,6 +32,10 @@ Fixes: + + + + diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index a0b67b3c..b78b7603 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -1,6 +1,11 @@ using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using DuckDB.NET.Data.Arrow; namespace DuckDB.NET.Data; @@ -109,6 +114,57 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) return reader; } + /// + /// Executes the command and returns the first result set as an Apache Arrow + /// . Each DuckDB data chunk is converted to an Arrow record batch + /// using DuckDB's Arrow C Data Interface, with no row-by-row marshaling. + /// When is enabled, record batches are produced lazily from a + /// streaming result (bounded memory), otherwise the result is materialized first. + /// The caller owns the returned stream and must dispose it. + /// + public IArrowArrayStream ExecuteArrowStream() + { + EnsureConnectionOpen(); + + var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); + + foreach (var result in results) + { + var current = result; + + if (NativeMethods.Query.DuckDBResultReturnType(current) == DuckDBResultType.QueryResult) + { + return new DuckDBArrowArrayStream(current); + } + + current.Close(); + } + + throw new InvalidOperationException("The command did not return a result set."); + } + + /// + /// Executes the command and asynchronously streams the first result set as Apache Arrow + /// values. The batches are produced lazily, one per DuckDB data chunk. + /// Set to stream from a streaming result with bounded memory. + /// + public async IAsyncEnumerable ExecuteArrowBatchesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var stream = ExecuteArrowStream(); + + try + { + while (await stream.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false) is { } batch) + { + yield return batch; + } + } + finally + { + stream.Dispose(); + } + } + public override void Prepare() { } protected override DbParameter CreateDbParameter() => new DuckDBParameter(); diff --git a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs new file mode 100644 index 00000000..30d162b9 --- /dev/null +++ b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs @@ -0,0 +1,15 @@ +namespace DuckDB.NET.Data.Extensions; + +internal static class DuckDBErrorDataExtensions +{ + public static void ThrowOnError(this DuckDBErrorData errorData, string message) + { + using (errorData) + { + if (errorData.HasError) + { + throw new DuckDBException($"{message} {errorData.Message}".TrimEnd()); + } + } + } +} diff --git a/DuckDB.NET.Test/Arrow/ArrowResultTests.cs b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs new file mode 100644 index 00000000..18e4182e --- /dev/null +++ b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs @@ -0,0 +1,176 @@ +using System; +using System.Threading; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace DuckDB.NET.Test.Arrow; + +public class ArrowResultTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_ReturnsSchemaAndScalarValues(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "select 42 as answer, 'duckdb' as name, cast(3.5 as double) as ratio"; + + var batches = await ReadAllAsync(); + + batches.Should().ContainSingle(); + + var batch = batches[0]; + batch.Schema.FieldsList.Select(f => f.Name).Should().Equal("answer", "name", "ratio"); + batch.Length.Should().Be(1); + + ((Int32Array)batch.Column("answer")).GetValue(0).Should().Be(42); + ((StringArray)batch.Column("name")).GetString(0).Should().Be("duckdb"); + ((DoubleArray)batch.Column("ratio")).GetValue(0).Should().Be(3.5); + + DisposeBatches(batches); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_HandlesNullValues(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "select unnest([1, null, 3]) as value"; + + var batches = await ReadAllAsync(); + + var column = (Int32Array)batches.Single().Column("value"); + column.Length.Should().Be(3); + column.GetValue(0).Should().Be(1); + column.GetValue(1).Should().BeNull(); + column.GetValue(2).Should().Be(3); + + DisposeBatches(batches); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_StreamsMultipleChunks(bool useStreamingMode) + { + const int rowCount = 5000; + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = $"select i from range({rowCount}) t(i)"; + + var batches = await ReadAllAsync(); + + batches.Count.Should().BeGreaterThan(1); + batches.Sum(b => b.Length).Should().Be(rowCount); + + var total = 0L; + foreach (var batch in batches) + { + var column = (Int64Array)batch.Column("i"); + for (var row = 0; row < column.Length; row++) + { + total += column.GetValue(row)!.Value; + } + } + + total.Should().Be((long)rowCount * (rowCount - 1) / 2); + + DisposeBatches(batches); + } + + [Fact] + public void ExecuteArrowStream_ExposesSchemaWithoutReading() + { + Command.CommandText = "select 1 as a, 'x' as b"; + + using var stream = Command.ExecuteArrowStream(); + + stream.Schema.FieldsList.Select(f => f.Name).Should().Equal("a", "b"); + stream.Schema.GetFieldByName("a").DataType.TypeId.Should().Be(ArrowTypeId.Int32); + stream.Schema.GetFieldByName("b").DataType.TypeId.Should().Be(ArrowTypeId.String); + } + + [Fact] + public async Task ExecuteArrowStream_ReadsBatchesUntilNull() + { + Command.CommandText = "select i from range(10) t(i)"; + + using var stream = Command.ExecuteArrowStream(); + + var rows = 0; + while (await stream.ReadNextRecordBatchAsync(CancellationToken.None) is { } batch) + { + rows += batch.Length; + batch.Dispose(); + } + + rows.Should().Be(10); + } + + [Fact] + public void ExecuteArrowStream_ThrowsWhenNoResultSet() + { + Command.CommandText = "create table t_no_result (id integer)"; + + var act = () => Command.ExecuteArrowStream(); + + act.Should().Throw(); + } + + [Fact] + public async Task ReadNextRecordBatchAsync_WithCanceledToken_ReturnsCanceledTask() + { + Command.CommandText = "select i from range(10) t(i)"; + + using var stream = Command.ExecuteArrowStream(); + + var act = async () => await stream.ReadNextRecordBatchAsync(new CancellationToken(canceled: true)); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task ReadNextRecordBatchAsync_AfterDispose_Throws() + { + Command.CommandText = "select i from range(10) t(i)"; + + var stream = Command.ExecuteArrowStream(); + stream.Dispose(); + + var act = async () => await stream.ReadNextRecordBatchAsync(CancellationToken.None); + + await act.Should().ThrowAsync(); + } + + [Fact] + public void Dispose_CalledTwice_IsNoOp() + { + Command.CommandText = "select i from range(10) t(i)"; + + var stream = Command.ExecuteArrowStream(); + + stream.Dispose(); + var act = () => stream.Dispose(); + + act.Should().NotThrow(); + } + + private async Task> ReadAllAsync() + { + var batches = new List(); + await foreach (var batch in Command.ExecuteArrowBatchesAsync()) + { + batches.Add(batch); + } + + return batches; + } + + private static void DisposeBatches(List batches) + { + foreach (var batch in batches) + { + batch.Dispose(); + } + } +} From 87da4c760e21df4827629d63875b7b6ed7d43a9e Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Wed, 24 Jun 2026 23:32:20 +0400 Subject: [PATCH 04/33] Include DuckDB error type when throwing DuckDBException Add the duckdb_error_data_error_type binding and pass the resolved error type to DuckDBException in ThrowOnError. Also drop unused usings in DuckDBArrowArrayStream, DuckDBCommand, and DuckDBDataReader. --- DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs | 5 +++++ DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs | 1 - DuckDB.NET.Data/DuckDBCommand.cs | 1 - DuckDB.NET.Data/DuckDBDataReader.cs | 3 +-- DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs index 25f0a4a4..fc7629a8 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs @@ -16,6 +16,11 @@ public static partial class ErrorData [return: MarshalUsing(typeof(DuckDBOwnedStringMarshaller))] public static partial string DuckDBErrorDataMessage(DuckDBErrorData errorData); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_error_type")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorType DuckDBErrorDataErrorType(DuckDBErrorData errorData); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_destroy_error_data")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial void DuckDBDestroyErrorData(ref IntPtr errorData); diff --git a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs index 2fc5f5ea..18b19835 100644 --- a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs +++ b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs @@ -4,7 +4,6 @@ using Apache.Arrow; using Apache.Arrow.C; using Apache.Arrow.Ipc; -using DuckDB.NET.Native; namespace DuckDB.NET.Data.Arrow; diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index b78b7603..c1e1b7d0 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -2,7 +2,6 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; using Apache.Arrow; using Apache.Arrow.Ipc; using DuckDB.NET.Data.Arrow; diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 3d74fb26..00e76d62 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -1,5 +1,4 @@ -using DuckDB.NET.Data.Common; -using DuckDB.NET.Data.DataChunk.Reader; +using DuckDB.NET.Data.DataChunk.Reader; using System.IO; using System.Runtime.CompilerServices; diff --git a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs index 30d162b9..d108bece 100644 --- a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs +++ b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs @@ -8,7 +8,8 @@ public static void ThrowOnError(this DuckDBErrorData errorData, string message) { if (errorData.HasError) { - throw new DuckDBException($"{message} {errorData.Message}".TrimEnd()); + var errorType = NativeMethods.ErrorData.DuckDBErrorDataErrorType(errorData); + throw new DuckDBException($"{message} {errorData.Message}".TrimEnd(), errorType); } } } From 6f5a68d5eed114cae96f98f8505be8127ca7827f Mon Sep 17 00:00:00 2001 From: Giorgi Dalakishvili Date: Thu, 25 Jun 2026 00:00:21 +0400 Subject: [PATCH 05/33] Use duckdb_appender_error_data for appender error handling Switch the appender from duckdb_appender_error (plain string) to duckdb_appender_error_data, which exposes both the message and the DuckDB error type. Route all appender error sites through the existing DuckDBErrorData.ThrowOnError extension (message prefix now optional), so appender exceptions carry ErrorType, consistent with the Arrow paths. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01LWKN1SKSTFDyeqSrA4dfYT --- .../NativeMethods/NativeMethods.Appender.cs | 6 ++---- DuckDB.NET.Data/DuckDBAppender.cs | 16 ++++------------ DuckDB.NET.Data/DuckDBAppenderRow.cs | 2 +- DuckDB.NET.Data/DuckDBConnection.cs | 2 +- .../Extensions/DuckDBErrorDataExtensions.cs | 4 ++-- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs index 9be82b67..f44665f8 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs @@ -23,11 +23,9 @@ public static partial class Appender [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBLogicalType DuckDBAppenderColumnType(DuckDBAppender appender, ulong index); - [SuppressGCTransition] - [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_error")] + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_error_data")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - [return: MarshalUsing(typeof(DuckDBOwnedStringMarshaller))] - public static partial string DuckDBAppenderError(DuckDBAppender appender); + public static partial DuckDBErrorData DuckDBAppenderErrorData(DuckDBAppender appender); [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_flush")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs index 5cd6e888..d32e3ca7 100644 --- a/DuckDB.NET.Data/DuckDBAppender.cs +++ b/DuckDB.NET.Data/DuckDBAppender.cs @@ -1,5 +1,6 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.DataChunk.Writer; +using DuckDB.NET.Data.Extensions; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -72,7 +73,7 @@ public void Clear() var state = NativeMethods.Appender.DuckDBAppenderClear(nativeAppender); if (!state.IsSuccess()) { - ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } rowCount = 0; @@ -102,7 +103,7 @@ public void Close() var state = NativeMethods.Appender.DuckDBAppenderClose(nativeAppender); if (!state.IsSuccess()) { - ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } } finally @@ -137,18 +138,9 @@ private void AppendDataChunk() if (!state.IsSuccess()) { - ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); } - - [DoesNotReturn] - [StackTraceHidden] - internal static void ThrowLastError(Native.DuckDBAppender appender) - { - var errorMessage = NativeMethods.Appender.DuckDBAppenderError(appender); - - throw new DuckDBException(errorMessage); - } } diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 489d03f3..186ab69a 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -135,7 +135,7 @@ public IDuckDBAppenderRow AppendDefault() if (state == DuckDBState.Error) { - DuckDBAppender.ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } columnIndex++; diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 54bfbbe6..9f20e8ad 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -161,7 +161,7 @@ public DuckDBAppender CreateAppender(string? catalog, string? schema, string tab { try { - DuckDBAppender.ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } finally { diff --git a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs index d108bece..1bbd788b 100644 --- a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs +++ b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs @@ -2,14 +2,14 @@ namespace DuckDB.NET.Data.Extensions; internal static class DuckDBErrorDataExtensions { - public static void ThrowOnError(this DuckDBErrorData errorData, string message) + public static void ThrowOnError(this DuckDBErrorData errorData, string message = "") { using (errorData) { if (errorData.HasError) { var errorType = NativeMethods.ErrorData.DuckDBErrorDataErrorType(errorData); - throw new DuckDBException($"{message} {errorData.Message}".TrimEnd(), errorType); + throw new DuckDBException($"{message} {errorData.Message}".Trim(), errorType); } } } From 1ae01e971469e4801b0367c04ea8591ef3d7e02c Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Sat, 11 Jul 2026 00:21:07 +0100 Subject: [PATCH 06/33] Skip Convert.ChangeType on the same-type object read path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NumericVectorDataReader.GetValue(offset, targetType) backs the non-generic read surface (GetValue(ordinal), this[ordinal], GetValues). It always ran the boxed value through Convert.ChangeType, even when targetType already matched the column's natural type — the common case, since those APIs read using the column's ClrType. Convert.ChangeType does not early-out on same-type for IConvertible values: it dispatches through ToXxx() and re-boxes the result, so every value was boxed twice. Add a value.GetType() == targetType fast path that returns the already boxed value directly. Benchmark (read 1,000,000 rows x 3 numeric columns via GetValue, .NET 10, M4 Pro): Before: 54.50 ms, 137.3 MB allocated After: 22.73 ms, 68.7 MB allocated -58% time and -50% allocations on the object-returning path (the remaining allocation is the unavoidable single box of the object API). The typed GetFieldValue path is unaffected. Behavior is unchanged: ChangeType returned the same value for same-type, and BigInteger (not IConvertible) already used this same early-out inside ChangeType. --- .../DataChunk/Reader/NumericVectorDataReader.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs index 9da4bea6..caefd9bc 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs @@ -71,6 +71,15 @@ internal override object GetValue(ulong offset, Type targetType) _ => base.GetValue(offset, targetType) }; + // Fast path: when the boxed value already has the requested type (the common case for + // GetValue(ordinal)/this[ordinal], which read using the column's natural ClrType), skip the + // Convert.ChangeType machinery entirely. ChangeType does not early-out on same-type for + // IConvertible values — it re-boxes via ToXxx() — so this also removes a redundant box. + if (value.GetType() == targetType) + { + return value; + } + if (targetType.IsNumeric()) { try From fe2abc5e2e752a330996f7854e876d284079dbcd Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Fri, 10 Jul 2026 23:25:16 +0100 Subject: [PATCH 07/33] Avoid LINQ and boxed-enumerator allocations in parameter binding PreparedStatement.BindParameters ran a LINQ OfType<>().Any(...) and iterated the parameters with foreach over the non-generic collection (which boxed the List struct enumerator) on every command execution. Replace both with index-based loops over the typed indexer. This removes ~128 bytes and 2-3 allocations per command execution, which matters in tight parameterized-query loops. Execution time is unchanged, as the native prepare/execute dominates. --- .../PreparedStatement/PreparedStatement.cs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index d074baab..4f7d559d 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,4 +1,3 @@ -using System.Linq; using DuckDB.NET.Data.Connection; namespace DuckDB.NET.Data.PreparedStatement; @@ -89,10 +88,26 @@ private static void BindParameters(DuckDBPreparedStatement preparedStatement, Du throw new InvalidOperationException($"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); } - if (parameterCollection.OfType().Any(p => !string.IsNullOrEmpty(p.ParameterName))) + // Index-based iteration over the typed collection avoids the per-execution allocations of + // OfType<>().Any(...) and of the boxed List enumerator that `foreach (DuckDBParameter ...)` + // over the non-generic collection would produce. BindParameters runs on every execution. + var count = parameterCollection.Count; + + var hasNamedParameters = false; + for (var i = 0; i < count; i++) { - foreach (DuckDBParameter param in parameterCollection) + if (!string.IsNullOrEmpty(parameterCollection[i].ParameterName)) { + hasNamedParameters = true; + break; + } + } + + if (hasNamedParameters) + { + for (var i = 0; i < count; i++) + { + var param = parameterCollection[i]; var state = NativeMethods.PreparedStatements.DuckDBBindParameterIndex(preparedStatement, out var index, param.ParameterName); if (state.IsSuccess()) { From 9a8aa5ab7a7467d3b5a5967bfa8c9d5b6c54f834 Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Mon, 13 Jul 2026 08:23:06 +0100 Subject: [PATCH 08/33] Add safe reusable appender row API --- DuckDB.NET.Benchmarks/AppenderBenchmark.cs | 67 ++++++ DuckDB.NET.Benchmarks/Benchmarks.csproj | 33 +++ .../MappedAppenderBenchmark.cs | 96 ++++++++ DuckDB.NET.Benchmarks/NativeLibraryLoader.cs | 48 ++++ DuckDB.NET.Benchmarks/Program.cs | 15 ++ DuckDB.NET.Data/Data.csproj | 2 + DuckDB.NET.Data/DuckDBAppender.cs | 140 +++++++++++- DuckDB.NET.Data/DuckDBAppenderRow.cs | 19 +- DuckDB.NET.Data/DuckDBMappedAppender.cs | 14 +- DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs | 17 +- DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 205 +++++++++++++++++- DuckDB.NET.slnx | 1 + 12 files changed, 638 insertions(+), 19 deletions(-) create mode 100644 DuckDB.NET.Benchmarks/AppenderBenchmark.cs create mode 100644 DuckDB.NET.Benchmarks/Benchmarks.csproj create mode 100644 DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs create mode 100644 DuckDB.NET.Benchmarks/NativeLibraryLoader.cs create mode 100644 DuckDB.NET.Benchmarks/Program.cs diff --git a/DuckDB.NET.Benchmarks/AppenderBenchmark.cs b/DuckDB.NET.Benchmarks/AppenderBenchmark.cs new file mode 100644 index 00000000..162ed2e6 --- /dev/null +++ b/DuckDB.NET.Benchmarks/AppenderBenchmark.cs @@ -0,0 +1,67 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class AppenderBenchmark +{ + private DuckDBConnection connection = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = "DROP TABLE IF EXISTS bench; CREATE TABLE bench (a INTEGER, b BIGINT, c DOUBLE, d BOOLEAN);"; + command.ExecuteNonQuery(); + } + + [Benchmark(Baseline = true)] + public void AppendRowsWithCreateRow() + { + using var appender = connection.CreateAppender("bench"); + + for (var i = 0; i < RowCount; i++) + { + appender.CreateRow() + .AppendValue(i) + .AppendValue((long)i) + .AppendValue((double)i) + .AppendValue(i % 2 == 0) + .EndRow(); + } + } + + [Benchmark] + public void AppendRowsWithAppendRow() + { + using var appender = connection.CreateAppender("bench"); + + for (var i = 0; i < RowCount; i++) + { + appender.AppendRow(i, static (row, value) => + { + row.AppendValue(value) + .AppendValue((long)value) + .AppendValue((double)value) + .AppendValue(value % 2 == 0); + }); + } + } +} diff --git a/DuckDB.NET.Benchmarks/Benchmarks.csproj b/DuckDB.NET.Benchmarks/Benchmarks.csproj new file mode 100644 index 00000000..2de0564f --- /dev/null +++ b/DuckDB.NET.Benchmarks/Benchmarks.csproj @@ -0,0 +1,33 @@ + + + + Exe + net10.0 + enable + enable + true + Full + true + ..\keyPair.snk + + + + + + + + + + + + + + + false + PreserveNewest + runtimes\%(RecursiveDir)\%(FileName)%(Extension) + + + + diff --git a/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs b/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs new file mode 100644 index 00000000..1c1bec89 --- /dev/null +++ b/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using DuckDB.NET.Data.Mapping; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class MappedAppenderBenchmark +{ + private DuckDBConnection connection = null!; + private BenchRow[] rows = null!; + private IPropertyMapping[] mappings = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + public sealed class BenchRow + { + public int Id { get; init; } + public long Score { get; init; } + public double Value { get; init; } + public bool Active { get; init; } + } + + public sealed class BenchRowMap : DuckDBAppenderMap + { + public BenchRowMap() + { + Map(row => row.Id); + Map(row => row.Score); + Map(row => row.Value); + Map(row => row.Active); + } + } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + rows = new BenchRow[RowCount]; + for (var i = 0; i < RowCount; i++) + { + rows[i] = new BenchRow { Id = i, Score = i, Value = i, Active = i % 2 == 0 }; + } + + mappings = new BenchRowMap().PropertyMappings.ToArray(); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = "DROP TABLE IF EXISTS bench_mapped; CREATE TABLE bench_mapped (a INTEGER, b BIGINT, c DOUBLE, d BOOLEAN);"; + command.ExecuteNonQuery(); + } + + // Mirrors the mapped appender loop before it adopted AppendRow. + [Benchmark(Baseline = true)] + public void AppendMappedRowsWithCreateRow() + { + using var appender = connection.CreateAppender("bench_mapped"); + + foreach (var record in rows) + { + AppendRecordWithCreateRow(appender, record); + } + } + + [Benchmark] + public void AppendMappedRowsWithAppendRow() + { + using var appender = connection.CreateAppender("bench_mapped"); + appender.AppendRecords(rows); + } + + private void AppendRecordWithCreateRow(DuckDBAppender appender, BenchRow record) + { + ArgumentNullException.ThrowIfNull(record); + + var row = appender.CreateRow(); + foreach (var mapping in mappings) + { + mapping.AppendToRow(row, record); + } + + row.EndRow(); + } +} diff --git a/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs b/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs new file mode 100644 index 00000000..429de245 --- /dev/null +++ b/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs @@ -0,0 +1,48 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace DuckDB.NET.Benchmarks; + +/// +/// Loads the platform-native DuckDB library for the benchmark process. +/// +internal static class NativeLibraryLoader +{ + [ModuleInitializer] + public static void Init() + { + if (GetRid() is not { } rid) + { + return; + } + + _ = NativeLibrary.TryLoad(Path.Join("runtimes", rid, "native", "duckdb"), Assembly.GetExecutingAssembly(), DllImportSearchPath.AssemblyDirectory, out _) || + NativeLibrary.TryLoad(Path.Join("runtimes", rid, "native", "libduckdb"), Assembly.GetExecutingAssembly(), DllImportSearchPath.AssemblyDirectory, out _); + } + + private static string? GetRid() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return Environment.Is64BitProcess ? "win-x64" : "win-x86"; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return RuntimeInformation.ProcessArchitecture switch + { + Architecture.X64 => "linux-x64", + Architecture.Arm64 => "linux-arm64", + _ => null, + }; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "osx"; + } + + return null; + } +} diff --git a/DuckDB.NET.Benchmarks/Program.cs b/DuckDB.NET.Benchmarks/Program.cs new file mode 100644 index 00000000..00251844 --- /dev/null +++ b/DuckDB.NET.Benchmarks/Program.cs @@ -0,0 +1,15 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using DuckDB.NET.Benchmarks; + +// The repo's Directory.Build.props renames the output assembly to DuckDB.NET.Benchmarks +// while the project file stays Benchmarks.csproj, so BenchmarkDotNet's default toolchain +// can't locate the csproj. Run in-process to avoid the separate build/spawn step. +var config = DefaultConfig.Instance + .AddJob(Job.Default.WithToolchain(InProcessEmitToolchain.Instance)); + +BenchmarkSwitcher + .FromTypes([typeof(AppenderBenchmark), typeof(MappedAppenderBenchmark)]) + .Run(args, config); diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index 4afd3811..184a03d9 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -30,6 +30,8 @@ Fixes: + + diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs index d32e3ca7..98a0e944 100644 --- a/DuckDB.NET.Data/DuckDBAppender.cs +++ b/DuckDB.NET.Data/DuckDBAppender.cs @@ -6,9 +6,18 @@ namespace DuckDB.NET.Data; +/// +/// Appends rows to a DuckDB table. +/// +/// +/// Instances are not thread-safe. Do not call other methods on the same appender from an +/// callback. +/// public class DuckDBAppender : IDisposable { private bool closed; + private bool isAppendingRow; + private bool isFaulted; private readonly Native.DuckDBAppender nativeAppender; private readonly string qualifiedTableName; @@ -17,6 +26,7 @@ public class DuckDBAppender : IDisposable private readonly DuckDBLogicalType[] logicalTypes; private readonly DuckDBDataChunk dataChunk; private readonly VectorDataWriterBase[] vectorWriters; + private DuckDBAppenderRow? reusableRow; internal DuckDBAppender(Native.DuckDBAppender appender, string qualifiedTableName) { @@ -43,7 +53,97 @@ internal DuckDBAppender(Native.DuckDBAppender appender, string qualifiedTableNam /// internal IReadOnlyList LogicalTypes => logicalTypes; + /// + /// Creates an independent row. The caller must append every column and call + /// . + /// public IDuckDBAppenderRow CreateRow() + { + EnsureUsable(); + return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, PrepareRow(), dataChunk, nativeAppender); + } + + /// + /// Appends a complete row using a reusable row instance. + /// + /// A callback that appends every column value. This method calls + /// after the callback returns. + /// + /// The row is valid only during the callback and must not be retained. The callback must not + /// call other methods on this appender. If the callback fails, all rows added by this appender + /// are cleared and the appender cannot be reused. + /// + public void AppendRow(Action writeRow) + { + ArgumentNullException.ThrowIfNull(writeRow); + + // Pass the callback as state so the adapter remains static and allocation-free. + AppendRow(writeRow, static (row, callback) => callback(row)); + } + + /// + /// Appends a complete row using a reusable row instance without exposing that instance as a + /// return value. The row passed to is only valid for the duration of + /// the callback and must not be retained or used after the callback returns. + /// + /// The type of value used to populate the row. + /// The value used to populate the row. + /// A callback that appends every column value. This method calls + /// after the callback returns. + /// + /// The callback must not call other methods on this appender. If the callback fails, the + /// appender is cleared and cannot be reused. + /// + public void AppendRow(TState state, Action writeRow) + { + ArgumentNullException.ThrowIfNull(writeRow); + EnsureUsable(); + + DuckDBAppenderRow? row = null; + isAppendingRow = true; + + try + { + row = CreateReusableRow(); + writeRow(row, state); + row.EndRow(); + } + catch + { + if (row is not null) + { + AbortAppendBatch(row); + } + + throw; + } + finally + { + isAppendingRow = false; + } + } + + /// + /// Creates a row whose instance may be reused by the next call. This is only safe for internal + /// callers that create, populate and end each row without exposing the row reference. + /// + internal DuckDBAppenderRow CreateReusableRow() + { + var rowIndex = PrepareRow(); + + if (reusableRow is null) + { + reusableRow = new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowIndex, dataChunk, nativeAppender); + } + else + { + reusableRow.Reset(rowIndex); + } + + return reusableRow; + } + + private ulong PrepareRow() { if (closed) { @@ -60,16 +160,23 @@ public IDuckDBAppenderRow CreateRow() } rowCount++; - return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowCount - 1, dataChunk, nativeAppender); + return rowCount - 1; } public void Clear() { + EnsureUsable(); + if (closed) { throw new InvalidOperationException("Appender is already closed"); } - + + ClearCore(); + } + + private void ClearCore() + { var state = NativeMethods.Appender.DuckDBAppenderClear(nativeAppender); if (!state.IsSuccess()) { @@ -78,10 +185,12 @@ public void Clear() rowCount = 0; NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); + InitVectorWriters(); } public void Close() { + EnsureNotAppendingRow(); closed = true; try @@ -143,4 +252,31 @@ private void AppendDataChunk() NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); } + + private void AbortAppendBatch(DuckDBAppenderRow row) + { + row.Invalidate(); + isFaulted = true; + + // The native appender cannot roll back only the current row, so discard its whole batch. + ClearCore(); + } + + private void EnsureNotAppendingRow() + { + if (isAppendingRow) + { + throw new InvalidOperationException("The appender cannot be used from inside an AppendRow callback"); + } + } + + private void EnsureUsable() + { + EnsureNotAppendingRow(); + + if (isFaulted) + { + throw new InvalidOperationException("The appender cannot be reused after an AppendRow callback failed"); + } + } } diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 186ab69a..ce4bdc60 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -7,7 +7,7 @@ public class DuckDBAppenderRow : IDuckDBAppenderRow private int columnIndex = 0; private readonly string qualifiedTableName; private readonly VectorDataWriterBase[] vectorWriters; - private readonly ulong rowIndex; + private ulong rowIndex; private readonly DuckDBDataChunk dataChunk; private readonly Native.DuckDBAppender nativeAppender; @@ -21,6 +21,23 @@ internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vec this.nativeAppender = nativeAppender; } + /// + /// Re-targets this row instance at a new row index so the appender can reuse a single + /// instead of allocating one per row. The table name, vector + /// writers, data chunk and native appender are stable for the lifetime of the appender, so only + /// the row index and column cursor need to be reset. + /// + internal void Reset(ulong rowIndex) + { + this.rowIndex = rowIndex; + columnIndex = 0; + } + + internal void Invalidate() + { + columnIndex = vectorWriters.Length; + } + public void EndRow() { if (columnIndex < vectorWriters.Length) diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index ee87f303..25a3444f 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -76,14 +76,14 @@ private void AppendRecord(T record) throw new ArgumentNullException(nameof(record)); } - var row = appender.CreateRow(); - - foreach (var mapping in mappings) + // Pass both values as state so the callback does not capture per record. + appender.AppendRow((Record: record, Mappings: mappings), static (row, state) => { - row = mapping.AppendToRow(row, record); - } - - row.EndRow(); + foreach (var mapping in state.Mappings) + { + mapping.AppendToRow(row, state.Record); + } + }); } private static DuckDBType GetExpectedDuckDBType(Type type) diff --git a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs index 0f8d576f..ff60dcf6 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs @@ -68,7 +68,7 @@ internal interface IPropertyMapping { Type PropertyType { get; } PropertyMappingType MappingType { get; } - IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record); + void AppendToRow(IDuckDBAppenderRow row, T record); } internal sealed class PropertyMapping : IPropertyMapping @@ -77,16 +77,17 @@ internal sealed class PropertyMapping : IPropertyMapping public Func Getter { get; set; } = _ => default!; public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) { var value = Getter(record); if (value is null) { - return row.AppendNullValue(); + row.AppendNullValue(); + return; } - return value switch + _ = value switch { // Reference types string v => row.AppendValue(v), @@ -125,9 +126,9 @@ internal sealed class DefaultValueMapping : IPropertyMapping public Type PropertyType { get; set; } = typeof(object); public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) { - return row.AppendDefault(); + row.AppendDefault(); } } @@ -136,8 +137,8 @@ internal sealed class NullValueMapping : IPropertyMapping public Type PropertyType { get; set; } = typeof(object); public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) { - return row.AppendNullValue(); + row.AppendNullValue(); } } diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index e81d8618..4820f28c 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -715,6 +715,209 @@ public void AppendDefault() reader.GetInt32(2).Should().Be(30); } + [Fact] + public void CreateRowReturnsIndependentRows() + { + Command.CommandText = "CREATE TABLE managedAppenderRowLifetime(a INTEGER, b INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderRowLifetime")) + { + var completed = appender.CreateRow(); + completed.AppendValue((int?)10).AppendValue((int?)11).EndRow(); + + var current = appender.CreateRow(); + + completed.Should().NotBeSameAs(current); + completed.Invoking(row => row.AppendValue((int?)99)).Should().Throw(); + current.AppendValue((int?)20).AppendValue((int?)21).EndRow(); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderRowLifetime ORDER BY a"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(10); + reader.GetInt32(1).Should().Be(11); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(20); + reader.GetInt32(1).Should().Be(21); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowStateOverloadWritesRows() + { + Command.CommandText = "CREATE TABLE managedAppenderScopedRow(a INTEGER, b VARCHAR)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderScopedRow")) + { + for (var i = 0; i < 3; i++) + { + appender.AppendRow((Id: i, Name: $"row-{i}"), static (row, value) => + { + row.AppendValue(value.Id).AppendValue(value.Name); + }); + } + } + + Command.CommandText = "SELECT a, b FROM managedAppenderScopedRow ORDER BY a"; + using var reader = Command.ExecuteReader(); + for (var i = 0; i < 3; i++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(i); + reader.GetString(1).Should().Be($"row-{i}"); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowActionOverloadWritesCompleteRow() + { + Command.CommandText = "CREATE TABLE managedAppenderActionRow(a INTEGER, b VARCHAR)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderActionRow")) + { + appender.AppendRow(row => row.AppendValue((int?)42).AppendValue("answer")); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderActionRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(42); + reader.GetString(1).Should().Be("answer"); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowActionOverloadRejectsNullCallback() + { + Command.CommandText = "CREATE TABLE managedAppenderNullAction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderNullAction"); + appender.Invoking(value => value.AppendRow((Action)null!)) + .Should().Throw() + .WithParameterName("writeRow"); + } + + [Fact] + public void AppendRowStateOverloadRejectsNullCallback() + { + Command.CommandText = "CREATE TABLE managedAppenderNullStateAction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderNullStateAction"); + appender.Invoking(value => value.AppendRow(1, (Action)null!)) + .Should().Throw() + .WithParameterName("writeRow"); + } + + [Fact] + public void IncompleteAppendRowClearsBatchAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderIncompleteScopedRow(a INTEGER, b INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderIncompleteScopedRow")) + { + appender.Invoking(value => value.AppendRow(1, static (row, state) => row.AppendValue(state))) + .Should().Throw() + .WithMessage("*specified only 1 values"); + + appender.Invoking(value => value.AppendRow((2, 3), static (row, state) => + row.AppendValue(state.Item1).AppendValue(state.Item2))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderIncompleteScopedRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowClearsBatchWhenCallbackThrows() + { + Command.CommandText = "CREATE TABLE managedAppenderThrownScopedRow(a INTEGER, b INTEGER[])"; + Command.ExecuteNonQuery(); + + IDuckDBAppenderRow failedRow = null!; + + using (var appender = Connection.CreateAppender("managedAppenderThrownScopedRow")) + { + appender.AppendRow(row => row.AppendValue((int?)1).AppendValue(new[] { 1, 2 })); + + appender.Invoking(value => value.AppendRow(row => + { + failedRow = row; + row.AppendValue((int?)2).AppendValue(new[] { 3, 4 }); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + + failedRow.Should().NotBeNull(); + failedRow.Invoking(row => row.AppendValue((int?)99)).Should().Throw(); + + appender.Invoking(value => value.AppendRow(row => row.AppendValue((int?)3).AppendValue(new[] { 5, 6 }))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderThrownScopedRow ORDER BY a"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowWritesListValue() + { + Command.CommandText = "CREATE TABLE managedAppenderScopedListRow(a INTEGER, b INTEGER[])"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderScopedListRow")) + { + appender.AppendRow(row => row.AppendValue((int?)1).AppendValue(new[] { 1, 2 })); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderScopedListRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetFieldValue>(1).Should().Equal(1, 2); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowRejectsReentrantAppenderUseAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderReentrantScopedRow(a INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderReentrantScopedRow")) + { + appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)1); + value.AppendRow(nested => nested.AppendValue((int?)2)); + })) + .Should().Throw() + .WithMessage("*inside an AppendRow callback"); + + appender.Invoking(value => value.AppendRow(row => row.AppendValue((int?)3))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a FROM managedAppenderReentrantScopedRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeFalse(); + } + [Fact] public void ClearAppender() { @@ -851,4 +1054,4 @@ private enum EnumNotValidValueTestEnum { NotValid = 12345, } -} \ No newline at end of file +} diff --git a/DuckDB.NET.slnx b/DuckDB.NET.slnx index 715e640c..b40a6d65 100644 --- a/DuckDB.NET.slnx +++ b/DuckDB.NET.slnx @@ -1,4 +1,5 @@ + From d8e201e7e074748aa8c88b76e1f48b48e5f82031 Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Mon, 20 Jul 2026 22:30:34 +0100 Subject: [PATCH 09/33] Finalize completed rows after appender failures --- DuckDB.NET.Data/DuckDBAppender.cs | 97 +++++++----- DuckDB.NET.Data/DuckDBAppenderRow.cs | 2 + DuckDB.NET.Test/DuckDBManagedAppenderTests.cs | 147 ++++++++++++++++-- 3 files changed, 191 insertions(+), 55 deletions(-) diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs index 98a0e944..ef9216e9 100644 --- a/DuckDB.NET.Data/DuckDBAppender.cs +++ b/DuckDB.NET.Data/DuckDBAppender.cs @@ -1,9 +1,6 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.DataChunk.Writer; using DuckDB.NET.Data.Extensions; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; - namespace DuckDB.NET.Data; /// @@ -70,8 +67,9 @@ public IDuckDBAppenderRow CreateRow() /// after the callback returns. /// /// The row is valid only during the callback and must not be retained. The callback must not - /// call other methods on this appender. If the callback fails, all rows added by this appender - /// are cleared and the appender cannot be reused. + /// call other methods on this appender. If the callback or automatic + /// fails, all previously completed rows are flushed, + /// the failed row is discarded, and the appender cannot be reused. /// public void AppendRow(Action writeRow) { @@ -91,8 +89,9 @@ public void AppendRow(Action writeRow) /// A callback that appends every column value. This method calls /// after the callback returns. /// - /// The callback must not call other methods on this appender. If the callback fails, the - /// appender is cleared and cannot be reused. + /// The callback must not call other methods on this appender. If the callback or automatic + /// fails, all previously completed rows are flushed, + /// the failed row is discarded, and the appender cannot be reused. /// public void AppendRow(TState state, Action writeRow) { @@ -108,11 +107,21 @@ public void AppendRow(TState state, Action w writeRow(row, state); row.EndRow(); } - catch + catch (Exception appendException) { if (row is not null) { - AbortAppendBatch(row); + try + { + FinalizeFailedAppendRow(row); + } + catch (Exception finalizationException) + { + throw new AggregateException( + "Appending the row failed and the previously completed rows could not be finalized", + appendException, + finalizationException); + } } throw; @@ -145,11 +154,6 @@ internal DuckDBAppenderRow CreateReusableRow() private ulong PrepareRow() { - if (closed) - { - throw new InvalidOperationException("Appender is already closed"); - } - if (rowCount % DuckDBGlobalData.VectorSize == 0) { AppendDataChunk(); @@ -167,16 +171,6 @@ public void Clear() { EnsureUsable(); - if (closed) - { - throw new InvalidOperationException("Appender is already closed"); - } - - ClearCore(); - } - - private void ClearCore() - { var state = NativeMethods.Appender.DuckDBAppenderClear(nativeAppender); if (!state.IsSuccess()) { @@ -190,25 +184,18 @@ private void ClearCore() public void Close() { - EnsureNotAppendingRow(); + EnsureUsable(); + CloseCore(); + } + + private void CloseCore() + { closed = true; try { AppendDataChunk(); - foreach (var logicalType in logicalTypes) - { - logicalType.Dispose(); - } - - foreach (var writer in vectorWriters) - { - writer?.Dispose(); - } - - dataChunk.Dispose(); - var state = NativeMethods.Appender.DuckDBAppenderClose(nativeAppender); if (!state.IsSuccess()) { @@ -217,8 +204,30 @@ public void Close() } finally { - nativeAppender.Close(); + try + { + DisposeManagedResources(); + } + finally + { + nativeAppender.Close(); + } + } + } + + private void DisposeManagedResources() + { + foreach (var logicalType in logicalTypes) + { + logicalType.Dispose(); + } + + foreach (var writer in vectorWriters) + { + writer?.Dispose(); } + + dataChunk.Dispose(); } public void Dispose() @@ -253,13 +262,14 @@ private void AppendDataChunk() NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); } - private void AbortAppendBatch(DuckDBAppenderRow row) + private void FinalizeFailedAppendRow(DuckDBAppenderRow row) { + // The row index is also the number of completed rows before the failed row in this chunk. + rowCount = row.ChunkRowIndex; row.Invalidate(); isFaulted = true; - // The native appender cannot roll back only the current row, so discard its whole batch. - ClearCore(); + CloseCore(); } private void EnsureNotAppendingRow() @@ -278,5 +288,10 @@ private void EnsureUsable() { throw new InvalidOperationException("The appender cannot be reused after an AppendRow callback failed"); } + + if (closed) + { + throw new InvalidOperationException("Appender is already closed"); + } } } diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index ce4bdc60..cd697b2a 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -11,6 +11,8 @@ public class DuckDBAppenderRow : IDuckDBAppenderRow private readonly DuckDBDataChunk dataChunk; private readonly Native.DuckDBAppender nativeAppender; + internal ulong ChunkRowIndex => rowIndex; + internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex, DuckDBDataChunk dataChunk, Native.DuckDBAppender nativeAppender) { diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index 4820f28c..f570f49b 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -1,4 +1,5 @@ using System.Globalization; +using DuckDB.NET.Data.Common; using FluentAssertions.Common; namespace DuckDB.NET.Test; @@ -483,23 +484,28 @@ public void WrongTypesThrowException() } [Fact] - public void ClosedAdapterThrowException() + public void ClosedAppenderRejectsFurtherOperations() { var table = "CREATE TABLE managedAppenderClosedAdapterTest(a BOOLEAN, c Date, b TINYINT);"; Command.CommandText = table; Command.ExecuteNonQuery(); - Connection.Invoking(dbConnection => - { - using var appender = dbConnection.CreateAppender("managedAppenderClosedAdapterTest"); - appender.Close(); - var row = appender.CreateRow(); - row - .AppendValue(false) - .AppendValue((byte)1) - .AppendValue((short?)1) - .EndRow(); - }).Should().Throw(); + using var appender = Connection.CreateAppender("managedAppenderClosedAdapterTest"); + appender.Close(); + + appender.Invoking(value => value.CreateRow()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.AppendRow(row => row.AppendValue(false).AppendNullValue().AppendNullValue())) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Clear()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Close()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Dispose()).Should().NotThrow(); } [Fact] @@ -817,7 +823,7 @@ public void AppendRowStateOverloadRejectsNullCallback() } [Fact] - public void IncompleteAppendRowClearsBatchAndFaultsAppender() + public void IncompleteAppendRowDiscardsFailedRowAndFaultsAppender() { Command.CommandText = "CREATE TABLE managedAppenderIncompleteScopedRow(a INTEGER, b INTEGER)"; Command.ExecuteNonQuery(); @@ -840,7 +846,7 @@ public void IncompleteAppendRowClearsBatchAndFaultsAppender() } [Fact] - public void AppendRowClearsBatchWhenCallbackThrows() + public void AppendRowFailureFlushesCompletedRowsAndFaultsAppender() { Command.CommandText = "CREATE TABLE managedAppenderThrownScopedRow(a INTEGER, b INTEGER[])"; Command.ExecuteNonQuery(); @@ -866,13 +872,126 @@ public void AppendRowClearsBatchWhenCallbackThrows() appender.Invoking(value => value.AppendRow(row => row.AppendValue((int?)3).AppendValue(new[] { 5, 6 }))) .Should().Throw() .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.CreateRow()) + .Should().Throw() + .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.Clear()) + .Should().Throw() + .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.Close()) + .Should().Throw() + .WithMessage("*cannot be reused*"); } Command.CommandText = "SELECT a, b FROM managedAppenderThrownScopedRow ORDER BY a"; using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetFieldValue>(1).Should().Equal(1, 2); reader.Read().Should().BeFalse(); } + [Fact] + public void AppendRowFailureAggregatesFinalizationFailureAndCanBeDisposed() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedFinalization(a INTEGER UNIQUE)"; + Command.ExecuteNonQuery(); + Command.CommandText = "INSERT INTO managedAppenderFailedFinalization VALUES (1)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderFailedFinalization"); + appender.AppendRow(row => row.AppendValue((int?)1)); + + var exception = appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)2); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw().Which; + + exception.InnerExceptions.Should().HaveCount(2); + exception.InnerExceptions[0].Should().BeOfType() + .Which.Message.Should().Be("callback failed"); + exception.InnerExceptions[1].Should().BeOfType() + .Which.ErrorType.Should().Be(DuckDBErrorType.Constraint); + + appender.Invoking(value => value.Dispose()).Should().NotThrow(); + appender.Invoking(value => value.Dispose()).Should().NotThrow(); + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedFinalization"; + Command.ExecuteScalar().Should().Be(1); + } + + [Fact] + public void AppendRowFailureFlushesCompletedRowsAcrossDataChunks() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedAcrossChunks(a INTEGER)"; + Command.ExecuteNonQuery(); + + var completedRowCount = checked((int)DuckDBGlobalData.VectorSize + 3); + + using (var appender = Connection.CreateAppender("managedAppenderFailedAcrossChunks")) + { + for (var i = 0; i < completedRowCount; i++) + { + appender.AppendRow(i, static (row, value) => row.AppendValue(value)); + } + + appender.Invoking(value => value.AppendRow(completedRowCount, static (row, failedValue) => + { + row.AppendValue(failedValue); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + } + + Command.CommandText = """ + SELECT count(*)::BIGINT, min(a), max(a), sum(a)::BIGINT + FROM managedAppenderFailedAcrossChunks + """; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(completedRowCount); + reader.GetInt32(1).Should().Be(0); + reader.GetInt32(2).Should().Be(completedRowCount - 1); + reader.GetInt64(3).Should().Be((long)(completedRowCount - 1) * completedRowCount / 2); + } + + [Fact] + public void AppendRowFailureLeavesCompletedRowsInCallerTransaction() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedTransaction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using (var transaction = Connection.BeginTransaction()) + { + using (var appender = Connection.CreateAppender("managedAppenderFailedTransaction")) + { + appender.AppendRow(row => row.AppendValue((int?)1)); + + appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)2); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + } + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedTransaction"; + Command.ExecuteScalar().Should().Be(1); + + transaction.Rollback(); + } + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedTransaction"; + Command.ExecuteScalar().Should().Be(0); + } + [Fact] public void AppendRowWritesListValue() { From f027d609f235e10c9f87f2194183fea60852bfc0 Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Wed, 15 Jul 2026 08:47:35 +0100 Subject: [PATCH 10/33] Reuse prepared statements in DuckDBCommand --- DuckDB.NET.Benchmarks/Benchmarks.csproj | 2 +- DuckDB.NET.Benchmarks/NativeLibraryLoader.cs | 7 +- .../PreparedCommandBenchmark.cs | 101 ++++++ DuckDB.NET.Benchmarks/Program.cs | 7 +- .../NativeMethods.PreparedStatements.cs | 6 + DuckDB.NET.Data/DuckDBCommand.cs | 287 +++++++++++++++++- DuckDB.NET.Data/DuckDBConnection.cs | 67 +++- .../PreparedStatement/PreparedStatement.cs | 172 +++++++++-- DuckDB.NET.Test/DuckDBCommandTests.cs | 220 +++++++++++++- .../Parameters/ParameterCollectionTests.cs | 4 +- DuckDB.NET.slnx | 1 + 11 files changed, 838 insertions(+), 36 deletions(-) create mode 100644 DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs diff --git a/DuckDB.NET.Benchmarks/Benchmarks.csproj b/DuckDB.NET.Benchmarks/Benchmarks.csproj index 2de0564f..0d08d156 100644 --- a/DuckDB.NET.Benchmarks/Benchmarks.csproj +++ b/DuckDB.NET.Benchmarks/Benchmarks.csproj @@ -17,7 +17,7 @@ - + diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index 37082dd5..05288209 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -5,6 +5,8 @@ - Updated to DuckDB v1.5.4 + Unofficial preview build of the DuckDB native bindings from the skuirrels DuckDB.NET fork. + Unofficial preview bindings for the consolidated DuckDB.NET performance work, bundling DuckDB v1.5.4. DuckDB.NET.Native win-x64;win-arm64;linux-x64;linux-arm64;osx https://github.com/duckdb/duckdb/releases/download/v1.5.4 diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index 184a03d9..36a1442b 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -11,6 +11,8 @@ New features: Fixes: - Fixed ObjectDisposedException in enum appender (#327) + Unofficial preview build of the DuckDB ADO.NET provider from the skuirrels DuckDB.NET fork. + Unofficial preview of the consolidated DuckDB.NET performance work for DuckDB v1.5.4. Do not reference this package alongside the official DuckDB.NET packages. True ..\keyPair.snk true @@ -38,8 +40,12 @@ Fixes: - + + + + + diff --git a/README-PREVIEW.md b/README-PREVIEW.md new file mode 100644 index 00000000..35d45f9d --- /dev/null +++ b/README-PREVIEW.md @@ -0,0 +1,25 @@ +# DuckDB.NET fork preview + +This is an unofficial preview build from the +[`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork. It +packages the consolidated performance work while the corresponding upstream +pull requests are under review. + +Install the bundled provider explicitly: + +```shell +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.1 +``` + +The package keeps the official `DuckDB.NET.Data` namespaces and assembly names, +so application source code does not need to change. Do not reference this fork +package and the official `DuckDB.NET.Data.Full` package in the same dependency +graph because they contain assemblies with the same identities. + +This preview bundles DuckDB v1.5.4 and includes the consolidated appender, +parameter binding, prepared-command, result materialisation, and scoped-writer +optimisations from the fork. When equivalent upstream changes are released, +move back to the official `DuckDB.NET.Data.Full` package. + +The original DuckDB.NET and DuckDB licences and attribution are included in the +package. diff --git a/scripts/validate-preview-packages.sh b/scripts/validate-preview-packages.sh new file mode 100755 index 00000000..9a24ed6d --- /dev/null +++ b/scripts/validate-preview-packages.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +package_directory=$(cd "$1" && pwd) +preview_version=$2 +data_package="$package_directory/Skuirrels.DuckDB.NET.Data.Full.$preview_version.nupkg" +bindings_package="$package_directory/Skuirrels.DuckDB.NET.Bindings.Full.$preview_version.nupkg" + +[[ -f "$data_package" ]] || { echo "Missing $data_package" >&2; exit 1; } +[[ -f "$bindings_package" ]] || { echo "Missing $bindings_package" >&2; exit 1; } + +validation_directory=$(mktemp -d) +trap 'rm -rf "$validation_directory"' EXIT + +unzip -p "$data_package" '*.nuspec' > "$validation_directory/data.nuspec" +unzip -p "$bindings_package" '*.nuspec' > "$validation_directory/bindings.nuspec" +unzip -Z1 "$bindings_package" > "$validation_directory/bindings-files.txt" + +grep -Fq 'Skuirrels.DuckDB.NET.Data.Full' "$validation_directory/data.nuspec" +grep -Fq "$preview_version" "$validation_directory/data.nuspec" +grep -Fq "Skuirrels.DuckDB.NET.Bindings.Full' "$validation_directory/bindings.nuspec" +grep -Fq "$preview_version" "$validation_directory/bindings.nuspec" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/data.nuspec" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/bindings.nuspec" + +for native_asset in \ + runtimes/win-x64/native/duckdb.dll \ + runtimes/win-arm64/native/duckdb.dll \ + runtimes/linux-x64/native/libduckdb.so \ + runtimes/linux-arm64/native/libduckdb.so \ + runtimes/osx/native/libduckdb.dylib +do + grep -Fxq "$native_asset" "$validation_directory/bindings-files.txt" +done + +smoke_directory="$validation_directory/smoke" +mkdir -p "$smoke_directory" + +cat > "$smoke_directory/PreviewSmoke.csproj" < + + Exe + net8.0 + enable + enable + + + + + +EOF + +cat > "$smoke_directory/NuGet.config" < + + + + + + + + + + + + + + + +EOF + +cat > "$smoke_directory/Program.cs" <<'EOF' +using DuckDB.NET.Data; + +using var connection = new DuckDBConnection("Data Source=:memory:"); +connection.Open(); + +using (var command = connection.CreateCommand()) +{ + command.CommandText = "CREATE TABLE preview_smoke(value INTEGER)"; + command.ExecuteNonQuery(); +} + +using (var appender = connection.CreateAppender("preview_smoke")) +{ + appender.AppendRowScoped(42, + static (ref DuckDBAppenderRowWriter writer, int value) => writer.AppendValue(value)); +} + +using var verification = connection.CreateCommand(); +verification.CommandText = "SELECT version(), sum(value) FROM preview_smoke"; +using var reader = verification.ExecuteReader(); +if (!reader.Read() || !reader.GetString(0).Contains("v1.5.4", StringComparison.Ordinal) || reader.GetInt64(1) != 42) +{ + throw new InvalidOperationException("Preview package smoke test failed."); +} + +Console.WriteLine($"Preview smoke passed with {reader.GetString(0)}"); +EOF + +dotnet restore "$smoke_directory/PreviewSmoke.csproj" \ + --configfile "$smoke_directory/NuGet.config" \ + --packages "$validation_directory/packages" +dotnet run --project "$smoke_directory/PreviewSmoke.csproj" --configuration Release --no-restore + +echo "Validated preview packages at version $preview_version" From 9b483f71dcd910e4a42dbb5307037ea27a98919f Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Tue, 21 Jul 2026 23:20:55 +0100 Subject: [PATCH 22/33] Point fork README to preview packages --- README-PREVIEW.md | 8 ++++++++ README.md | 32 ++++++++++++++++++-------------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/README-PREVIEW.md b/README-PREVIEW.md index 35d45f9d..a544244a 100644 --- a/README-PREVIEW.md +++ b/README-PREVIEW.md @@ -11,6 +11,11 @@ Install the bundled provider explicitly: dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.1 ``` +NuGet packages: + +- [`Skuirrels.DuckDB.NET.Data.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) +- [`Skuirrels.DuckDB.NET.Bindings.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) + The package keeps the official `DuckDB.NET.Data` namespaces and assembly names, so application source code does not need to change. Do not reference this fork package and the official `DuckDB.NET.Data.Full` package in the same dependency @@ -23,3 +28,6 @@ move back to the official `DuckDB.NET.Data.Full` package. The original DuckDB.NET and DuckDB licences and attribution are included in the package. + +Report fork-specific problems in the +[`skuirrels/DuckDB.NET` issue tracker](https://github.com/skuirrels/DuckDB.NET/issues/new). diff --git a/README.md b/README.md index ea5f8b02..c766c1d3 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,29 @@ -# DuckDB.NET +# DuckDB.NET fork preview -[DuckDB](https://duckdb.org/) bindings for C# +This is the [`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) +fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). +It provides preview packages containing the consolidated performance work while +the corresponding upstream pull requests are reviewed and released. -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Giorgi/DuckDB.NET/ci.yml?branch=main&logo=GitHub&style=for-the-badge)](https://github.com/Giorgi/DuckDB.NET/actions/workflows/ci.yml) -[![Coveralls](https://img.shields.io/coveralls/github/Giorgi/DuckDB.NET?logo=coveralls&style=for-the-badge)](https://coveralls.io/github/Giorgi/DuckDB.NET) +[![Preview release](https://img.shields.io/github/actions/workflow/status/skuirrels/DuckDB.NET/preview-release.yml?branch=develop&label=preview%20release&logo=GitHub&style=for-the-badge)](https://github.com/skuirrels/DuckDB.NET/actions/workflows/preview-release.yml) [![License](https://img.shields.io/badge/License-Mit-blue.svg?style=for-the-badge&logo=mit)](LICENSE.md) -[![Ko-Fi](https://img.shields.io/static/v1?style=for-the-badge&message=Support%20the%20Project&color=success&logo=ko-fi&label=$$)](https://ko-fi.com/U6U81LHU8) [![Discord](https://img.shields.io/badge/DuckDB-.Net-%23FFF000?logo=DuckDB&style=for-the-badge)](https://discord.com/channels/909674491309850675/1051088721996427265) -[![NuGet DuckDB.NET.Data](https://img.shields.io/nuget/dt/DuckDB.NET.Data.svg?label=DuckDB.NET.Data&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Data/) -[![NuGet DuckDB.NET.Bindings](https://img.shields.io/nuget/dt/DuckDB.NET.Bindings.svg?label=DuckDB.NET.Bindings&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Bindings/) +[![NuGet Skuirrels.DuckDB.NET.Data.Full](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Data.Full.svg?label=Skuirrels.DuckDB.NET.Data.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) +[![NuGet Skuirrels.DuckDB.NET.Bindings.Full](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=Skuirrels.DuckDB.NET.Bindings.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) -[![NuGet DuckDB.NET.Data.Full](https://img.shields.io/nuget/dt/DuckDB.NET.Data.Full.svg?label=DuckDB.NET.Data.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Data.Full/) -[![NuGet DuckDB.NET.Bindings.Full](https://img.shields.io/nuget/dt/DuckDB.NET.Bindings.Full.svg?label=DuckDB.NET.Bindings.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Bindings.Full/) - -![Project Icon](https://raw.githubusercontent.com/Giorgi/DuckDB.NET/main/Logo.jpg "DuckDB.NET Project Icon") +![Project Icon](Logo.jpg "DuckDB.NET Project Icon") ## Usage ```sh -dotnet add package DuckDB.NET.Data.Full +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.1 ``` +The fork packages retain the official `DuckDB.NET.Data` namespaces and assembly +names. Do not reference this package and the official `DuckDB.NET.Data.Full` +package in the same dependency graph. + ```cs using (var duckDBConnection = new DuckDBConnection("Data Source=file.db")) { @@ -91,9 +93,11 @@ Documentation is available at [https://duckdb.net](https://duckdb.net) ## Support -If you encounter a bug with the library [Create an Issue](https://github.com/Giorgi/DuckDB.NET/issues/new). Join the [DuckDB `dotnet` channel](https://discord.duckdb.org/) for DuckDB.NET-related topics. +For a fork-specific problem, [create an issue in this fork](https://github.com/skuirrels/DuckDB.NET/issues/new). +For upstream DuckDB.NET support, use the [upstream issue tracker](https://github.com/Giorgi/DuckDB.NET/issues/new). +You can also join the [DuckDB `dotnet` channel](https://discord.duckdb.org/) for DuckDB.NET-related topics. -## Contributors +## Upstream contributors [![Contributors](https://contrib.rocks/image?repo=Giorgi/DuckDB.NET)](https://github.com/Giorgi/DuckDB.NET/graphs/contributors) From 108f024dd39ca0f4b986f2bcc7e70bbec986ea9e Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Tue, 21 Jul 2026 23:30:46 +0100 Subject: [PATCH 23/33] Use compact preview package badges --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index c766c1d3..0779f3c1 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,10 @@ fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). It provides preview packages containing the consolidated performance work while the corresponding upstream pull requests are reviewed and released. -[![Preview release](https://img.shields.io/github/actions/workflow/status/skuirrels/DuckDB.NET/preview-release.yml?branch=develop&label=preview%20release&logo=GitHub&style=for-the-badge)](https://github.com/skuirrels/DuckDB.NET/actions/workflows/preview-release.yml) -[![License](https://img.shields.io/badge/License-Mit-blue.svg?style=for-the-badge&logo=mit)](LICENSE.md) -[![Discord](https://img.shields.io/badge/DuckDB-.Net-%23FFF000?logo=DuckDB&style=for-the-badge)](https://discord.com/channels/909674491309850675/1051088721996427265) - -[![NuGet Skuirrels.DuckDB.NET.Data.Full](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Data.Full.svg?label=Skuirrels.DuckDB.NET.Data.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) -[![NuGet Skuirrels.DuckDB.NET.Bindings.Full](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=Skuirrels.DuckDB.NET.Bindings.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) +[![NuGet (Data)](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Data.Full.svg?label=NuGet%20%28Data%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full) +[![NuGet (Bindings)](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=NuGet%20%28Bindings%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full) +[![.NET 8 and 10](https://img.shields.io/badge/.NET-8%20%7C%2010-512BD4)](https://dotnet.microsoft.com/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md) ![Project Icon](Logo.jpg "DuckDB.NET Project Icon") From e60df1295a448f594ba0a2696e11b4380de74aaa Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:56:54 +0100 Subject: [PATCH 24/33] Remove project icon from README Removed project icon from README. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 0779f3c1..69898f6b 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,6 @@ the corresponding upstream pull requests are reviewed and released. [![.NET 8 and 10](https://img.shields.io/badge/.NET-8%20%7C%2010-512BD4)](https://dotnet.microsoft.com/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md) -![Project Icon](Logo.jpg "DuckDB.NET Project Icon") - ## Usage ```sh From 3893836f24a95ed6218f05fcafc51f4c126decda Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:07:06 +0100 Subject: [PATCH 25/33] Revise README to highlight cutting-edge performance Updated description to reflect cutting-edge performance work. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69898f6b..9e9a127a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This is the [`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). -It provides preview packages containing the consolidated performance work while +It provides preview packages containing the cutting-edge performance work while the corresponding upstream pull requests are reviewed and released. [![NuGet (Data)](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Data.Full.svg?label=NuGet%20%28Data%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full) From 148dc2c55319525172f166463b4cd42221b0773a Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:08:43 +0100 Subject: [PATCH 26/33] Update README title for performance fork preview --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9e9a127a..2cccbb51 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# DuckDB.NET fork preview +# DuckDB.NET - Performance fork preview This is the [`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). From c9e2ae59166d299f9c5d9874aab89914bec1f7a5 Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:14:11 +0100 Subject: [PATCH 27/33] Optimize prepared scalar execution (#5) --- .../PreparedCommandBenchmark.cs | 9 +++ DuckDB.NET.Data/DuckDBCommand.cs | 66 ++++++++++++++++ DuckDB.NET.Test/DuckDBCommandTests.cs | 75 +++++++++++++++++++ 3 files changed, 150 insertions(+) diff --git a/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs index 6f35a47a..4472181e 100644 --- a/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs +++ b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs @@ -46,6 +46,15 @@ public int ExecutePrepared() return (int)preparedCommand.ExecuteScalar()!; } + // Preserves the former ExecuteScalar path as a direct same-process comparison. + [Benchmark] + public int ExecutePreparedViaReader() + { + preparedParameter.Value = nextValue++; + using var reader = preparedCommand.ExecuteReader(); + return reader.Read() ? (int)reader.GetValue(0) : default; + } + private DuckDBCommand CreateCommand(out DuckDBParameter changingParameter) { var command = connection.CreateCommand(); diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index 98e6be2f..f21bdaa0 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -5,6 +5,7 @@ using Apache.Arrow; using Apache.Arrow.Ipc; using DuckDB.NET.Data.Arrow; +using DuckDB.NET.Data.DataChunk.Reader; using PreparedStatementBase = DuckDB.NET.Data.PreparedStatement.PreparedStatement; using ReusablePreparedStatement = DuckDB.NET.Data.PreparedStatement.ReusablePreparedStatement; @@ -121,6 +122,11 @@ public override int ExecuteNonQuery() { EnsureConnectionOpen(); + if (preparedStatement is { } reusableStatement) + { + return ExecutePreparedScalar(reusableStatement, connection!.NativeConnection); + } + using var reader = ExecuteReader(); return reader.Read() ? reader.GetValue(0) : null; } @@ -292,6 +298,66 @@ private int ExecutePreparedNonQuery( } } + private object? ExecutePreparedScalar( + ReusablePreparedStatement reusableStatement, + DuckDBNativeConnection nativeConnection) + { + BeginPreparedExecution(); + + try + { + var result = reusableStatement.Execute(parameters, UseStreamingMode, nativeConnection); + + try + { + if (NativeMethods.Query.DuckDBResultReturnType(result) != DuckDBResultType.QueryResult || + NativeMethods.Query.DuckDBColumnCount(ref result) == 0) + { + return null; + } + + return ReadFirstValue(ref result); + } + finally + { + result.Close(); + } + } + finally + { + CompletePreparedExecution(); + } + } + + private static object? ReadFirstValue(ref DuckDBResult result) + { + var streamingResult = NativeMethods.Types.DuckDBResultIsStreaming(result) > 0; + long chunkIndex = 0; + + while (true) + { + using var chunk = streamingResult + ? NativeMethods.StreamingResult.DuckDBStreamFetchChunk(result) + : NativeMethods.Types.DuckDBResultGetChunk(result, chunkIndex++); + + if (chunk is null || chunk.IsInvalid) + { + return null; + } + + if (NativeMethods.DataChunks.DuckDBDataChunkGetSize(chunk) == 0) + { + continue; + } + + var vector = NativeMethods.DataChunks.DuckDBDataChunkGetVector(chunk, 0); + using var logicalType = NativeMethods.Query.DuckDBColumnLogicalType(ref result, 0); + using var reader = VectorDataReaderFactory.CreateReader(vector, logicalType); + + return reader.IsValid(0) ? reader.GetValue(0) : DBNull.Value; + } + } + private void BeginPreparedExecution() { activeExecutions++; diff --git a/DuckDB.NET.Test/DuckDBCommandTests.cs b/DuckDB.NET.Test/DuckDBCommandTests.cs index 298cff52..75aa356b 100644 --- a/DuckDB.NET.Test/DuckDBCommandTests.cs +++ b/DuckDB.NET.Test/DuckDBCommandTests.cs @@ -35,6 +35,81 @@ public void PreparedCommandCanBeExecutedRepeatedlyWithNewParameterValues() command.ExecuteScalar().Should().Be(22); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarHandlesValuesNullsAndEmptyResults(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + command.Parameters.Add(new DuckDBParameter("value", 42)); + command.UseStreamingMode = useStreamingMode; + command.Prepare(); + + command.ExecuteScalar().Should().Be(42); + + command.Parameters["value"].Value = DBNull.Value; + command.ExecuteScalar().Should().Be(DBNull.Value); + + command.CommandText = "SELECT 42 WHERE FALSE"; + command.Parameters.Clear(); + command.Prepare(); + command.ExecuteScalar().Should().BeNull(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarHandlesRepresentativeReaderTypes(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.UseStreamingMode = useStreamingMode; + + command.CommandText = "SELECT 'duckdb'::VARCHAR"; + command.Prepare(); + command.ExecuteScalar().Should().Be("duckdb"); + + command.CommandText = "SELECT DATE '2026-07-21'"; + command.Prepare(); + command.ExecuteScalar().Should().Be(new DateOnly(2026, 7, 21)); + + command.CommandText = "SELECT [1, 2, 3]::INTEGER[]"; + command.Prepare(); + command.ExecuteScalar().Should().BeEquivalentTo(new[] { 1, 2, 3 }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarCanBeReusedAfterMaterializationFailure(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT CASE WHEN $infinite THEN DATE 'infinity' ELSE DATE '2026-07-21' END"; + command.Parameters.Add(new DuckDBParameter("infinite", true)); + command.UseStreamingMode = useStreamingMode; + command.Prepare(); + + command.Invoking(value => value.ExecuteScalar()) + .Should().Throw() + .WithMessage("Cannot convert infinite date value*"); + + command.Parameters["infinite"].Value = false; + command.ExecuteScalar().Should().Be(new DateOnly(2026, 7, 21)); + } + + [Fact] + public void PreparedExecuteScalarReturnsNullForNonQueryStatement() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = "CREATE TABLE prepared_scalar(value INTEGER)"; + command.Prepare(); + + command.ExecuteScalar().Should().BeNull(); + } + [Fact] public void PreparedCommandClearsBindingsBeforeReuse() { From 3e58971d6432e00b06060c61f7167838fcb2cbfc Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:31:58 +0100 Subject: [PATCH 28/33] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cccbb51..c4bbae7c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ the corresponding upstream pull requests are reviewed and released. ## Usage ```sh -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.1 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.2 ``` The fork packages retain the official `DuckDB.NET.Data` namespaces and assembly From 19135e0aa654ca07280eab385d92348aaf423e79 Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:53:52 +0100 Subject: [PATCH 29/33] Prepare stable fork package release (#6) --- .github/workflows/preview-release.yml | 84 ++++++++++------- Directory.Build.props | 20 +++- DuckDB.NET.Bindings/Bindings.csproj | 2 + DuckDB.NET.Data/Data.csproj | 6 +- DuckDB.NET.Test/Test.csproj | 2 +- README-FORK.md | 33 +++++++ scripts/validate-fork-packages.sh | 126 ++++++++++++++++++++++++++ scripts/validate-preview-packages.sh | 111 +---------------------- 8 files changed, 235 insertions(+), 149 deletions(-) create mode 100644 README-FORK.md create mode 100755 scripts/validate-fork-packages.sh diff --git a/.github/workflows/preview-release.yml b/.github/workflows/preview-release.yml index 54ea63dc..52002c0b 100644 --- a/.github/workflows/preview-release.yml +++ b/.github/workflows/preview-release.yml @@ -1,4 +1,4 @@ -name: Preview Release +name: Fork Package Release on: release: @@ -9,8 +9,12 @@ permissions: id-token: write jobs: - publish-preview: - if: github.event.release.prerelease == true && startsWith(github.event.release.tag_name, 'v1.5.4-preview.') + publish-fork-package: + if: >- + (github.event.release.prerelease == true && + startsWith(github.event.release.tag_name, 'v1.5.4-preview.')) || + (github.event.release.prerelease == false && + github.event.release.tag_name == 'v1.5.4') runs-on: ubuntu-latest environment: nuget-preview @@ -21,16 +25,28 @@ jobs: fetch-depth: 0 ref: ${{ github.event.release.tag_name }} - - name: Set preview version + - name: Set package version and release mode shell: bash run: | version="${{ github.event.release.tag_name }}" version="${version#v}" - if [[ ! "$version" =~ ^1\.5\.4-preview\.[0-9]+$ ]]; then - echo "Unexpected preview version: $version" >&2 + + if [[ "$version" =~ ^1\.5\.4-preview\.[0-9]+$ ]]; then + fork_preview=true + fork_release=false + elif [[ "$version" == "1.5.4" ]]; then + fork_preview=false + fork_release=true + else + echo "Unexpected fork package version: $version" >&2 exit 1 fi - echo "PREVIEW_VERSION=$version" >> "$GITHUB_ENV" + + { + echo "PACKAGE_VERSION=$version" + echo "FORK_PREVIEW=$fork_preview" + echo "FORK_RELEASE=$fork_release" + } >> "$GITHUB_ENV" - name: Setup .NET SDK uses: actions/setup-dotnet@v5 @@ -48,9 +64,10 @@ jobs: /m:1 /p:BuildType=Full /p:CI=false - /p:ForkPreview=true - /p:Version=${{ env.PREVIEW_VERSION }} - /p:PackageVersion=${{ env.PREVIEW_VERSION }} + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" /p:UseSharedCompilation=false - name: Test @@ -62,60 +79,63 @@ jobs: --logger "console;verbosity=quiet" /p:BuildType=Full /p:CI=false - /p:ForkPreview=true + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" /p:DoesNotReturnAttribute=DoesNotReturnAttribute - - name: Pack preview packages + - name: Pack fork packages shell: bash run: | - mkdir -p artifacts/preview + mkdir -p artifacts/fork-release common_args=( --configuration Release --no-build --no-restore - --output artifacts/preview + --output artifacts/fork-release /m:1 /p:BuildType=Full /p:CI=false - /p:ForkPreview=true - /p:PreviewPack=true - /p:Version="$PREVIEW_VERSION" - /p:PackageVersion="$PREVIEW_VERSION" + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:ForkPack=true + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" ) dotnet pack DuckDB.NET.Bindings/Bindings.csproj "${common_args[@]}" dotnet restore DuckDB.NET.Data/Data.csproj \ - --source artifacts/preview \ + --source artifacts/fork-release \ --source https://api.nuget.org/v3/index.json \ /p:BuildType=Full \ /p:CI=false \ - /p:ForkPreview=true \ - /p:PreviewPack=true \ - /p:Version="$PREVIEW_VERSION" \ - /p:PackageVersion="$PREVIEW_VERSION" + /p:ForkPreview="$FORK_PREVIEW" \ + /p:ForkRelease="$FORK_RELEASE" \ + /p:ForkPack=true \ + /p:Version="$PACKAGE_VERSION" \ + /p:PackageVersion="$PACKAGE_VERSION" dotnet pack DuckDB.NET.Data/Data.csproj "${common_args[@]}" - name: Validate and smoke-test packages - run: ./scripts/validate-preview-packages.sh artifacts/preview "$PREVIEW_VERSION" + run: ./scripts/validate-fork-packages.sh artifacts/fork-release "$PACKAGE_VERSION" - name: Generate checksums - working-directory: artifacts/preview + working-directory: artifacts/fork-release run: sha256sum *.nupkg > SHA256SUMS - name: Upload workflow artifacts uses: actions/upload-artifact@v7 with: - name: duckdb-net-${{ env.PREVIEW_VERSION }} + name: duckdb-net-${{ github.event.release.tag_name }} path: | - artifacts/preview/*.nupkg - artifacts/preview/SHA256SUMS + artifacts/fork-release/*.nupkg + artifacts/fork-release/SHA256SUMS if-no-files-found: error - - name: Attach packages to GitHub prerelease + - name: Attach packages to GitHub release env: GH_TOKEN: ${{ github.token }} - run: gh release upload "${{ github.event.release.tag_name }}" artifacts/preview/* --clobber + run: gh release upload "${{ github.event.release.tag_name }}" artifacts/fork-release/* --clobber - name: Authenticate to NuGet.org uses: NuGet/login@v1 @@ -123,9 +143,9 @@ jobs: with: user: skuirrels - - name: Publish preview packages to NuGet.org + - name: Publish fork packages to NuGet.org run: >- - dotnet nuget push "artifacts/preview/*.nupkg" + dotnet nuget push "artifacts/fork-release/*.nupkg" --api-key "${{ steps.nuget-login.outputs.NUGET_API_KEY }}" --source https://api.nuget.org/v3/index.json --skip-duplicate diff --git a/Directory.Build.props b/Directory.Build.props index 90d83fac..603223b8 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -19,13 +19,15 @@ Giorgi Dalakishvili Copyright © 2020 - $(Year) Giorgi Dalakishvili - Skuirrels. + true + Skuirrels. DuckDB.NET.$(MSBuildProjectName) $(ForkPackagePrefix)DuckDB.NET.$(MSBuildProjectName) DuckDB;ADO.NET;Database;Olap;Embedded Logo.jpg README.md README-PREVIEW.md + README-FORK.md MIT true @@ -40,7 +42,7 @@ - + https://github.com/skuirrels/DuckDB.NET https://github.com/skuirrels/DuckDB.NET @@ -74,13 +76,21 @@ True + + True + + - + + - + + diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index 05288209..cb88a774 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -7,6 +7,8 @@ Unofficial preview build of the DuckDB native bindings from the skuirrels DuckDB.NET fork. Unofficial preview bindings for the consolidated DuckDB.NET performance work, bundling DuckDB v1.5.4. + Unofficial stable build of the DuckDB native bindings from the skuirrels DuckDB.NET fork. + Stable fork bindings for the consolidated DuckDB.NET performance work, bundling DuckDB v1.5.4. DuckDB.NET.Native win-x64;win-arm64;linux-x64;linux-arm64;osx https://github.com/duckdb/duckdb/releases/download/v1.5.4 diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index 36a1442b..0154d74d 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -13,6 +13,8 @@ Fixes: Unofficial preview build of the DuckDB ADO.NET provider from the skuirrels DuckDB.NET fork. Unofficial preview of the consolidated DuckDB.NET performance work for DuckDB v1.5.4. Do not reference this package alongside the official DuckDB.NET packages. + Unofficial stable build of the DuckDB ADO.NET provider from the skuirrels DuckDB.NET fork. + Stable fork release of the consolidated DuckDB.NET performance work for DuckDB v1.5.4. Do not reference this package alongside the official DuckDB.NET packages. True ..\keyPair.snk true @@ -40,11 +42,11 @@ Fixes: - + - + diff --git a/DuckDB.NET.Test/Test.csproj b/DuckDB.NET.Test/Test.csproj index 37ced068..25a2753e 100644 --- a/DuckDB.NET.Test/Test.csproj +++ b/DuckDB.NET.Test/Test.csproj @@ -1,7 +1,7 @@  - net8.0 + net8.0;net10.0 false true diff --git a/README-FORK.md b/README-FORK.md new file mode 100644 index 00000000..0ddb544c --- /dev/null +++ b/README-FORK.md @@ -0,0 +1,33 @@ +# Skuirrels DuckDB.NET fork + +This is an unofficial stable build from the +[`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork. It +packages the consolidated performance work for DuckDB v1.5.4 under distinct +`Skuirrels.DuckDB.NET.*` package IDs. + +Install the bundled provider explicitly: + +```shell +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4 +``` + +NuGet packages: + +- [`Skuirrels.DuckDB.NET.Data.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) +- [`Skuirrels.DuckDB.NET.Bindings.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) + +The package keeps the official `DuckDB.NET.Data` namespaces and assembly names, +so application source code does not need to change. Do not reference this fork +package and the official `DuckDB.NET.Data.Full` package in the same dependency +graph because they contain assemblies with the same identities. + +This release bundles DuckDB v1.5.4 and includes the consolidated appender, +parameter binding, prepared-command, result materialisation, and scoped-writer +optimisations from the fork. When equivalent upstream changes are released, +move back to the official `DuckDB.NET.Data.Full` package. + +The original DuckDB.NET and DuckDB licences and attribution are included in the +package. + +Report fork-specific problems in the +[`skuirrels/DuckDB.NET` issue tracker](https://github.com/skuirrels/DuckDB.NET/issues/new). diff --git a/scripts/validate-fork-packages.sh b/scripts/validate-fork-packages.sh new file mode 100755 index 00000000..269189cc --- /dev/null +++ b/scripts/validate-fork-packages.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +package_directory=$(cd "$1" && pwd) +package_version=$2 +data_package="$package_directory/Skuirrels.DuckDB.NET.Data.Full.$package_version.nupkg" +bindings_package="$package_directory/Skuirrels.DuckDB.NET.Bindings.Full.$package_version.nupkg" + +if [[ "$package_version" == *-preview.* ]]; then + expected_readme=README-PREVIEW.md +else + expected_readme=README-FORK.md +fi + +[[ -f "$data_package" ]] || { echo "Missing $data_package" >&2; exit 1; } +[[ -f "$bindings_package" ]] || { echo "Missing $bindings_package" >&2; exit 1; } + +validation_directory=$(mktemp -d) +trap 'rm -rf "$validation_directory"' EXIT + +unzip -p "$data_package" '*.nuspec' > "$validation_directory/data.nuspec" +unzip -p "$bindings_package" '*.nuspec' > "$validation_directory/bindings.nuspec" +unzip -Z1 "$data_package" > "$validation_directory/data-files.txt" +unzip -Z1 "$bindings_package" > "$validation_directory/bindings-files.txt" + +grep -Fq 'Skuirrels.DuckDB.NET.Data.Full' "$validation_directory/data.nuspec" +grep -Fq "$package_version" "$validation_directory/data.nuspec" +grep -Fq "Skuirrels.DuckDB.NET.Bindings.Full' "$validation_directory/bindings.nuspec" +grep -Fq "$package_version" "$validation_directory/bindings.nuspec" +grep -Fq "$expected_readme" "$validation_directory/data.nuspec" +grep -Fq "$expected_readme" "$validation_directory/bindings.nuspec" +grep -Fxq "$expected_readme" "$validation_directory/data-files.txt" +grep -Fxq "$expected_readme" "$validation_directory/bindings-files.txt" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/data.nuspec" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/bindings.nuspec" + +for native_asset in \ + runtimes/win-x64/native/duckdb.dll \ + runtimes/win-arm64/native/duckdb.dll \ + runtimes/linux-x64/native/libduckdb.so \ + runtimes/linux-arm64/native/libduckdb.so \ + runtimes/osx/native/libduckdb.dylib +do + grep -Fxq "$native_asset" "$validation_directory/bindings-files.txt" +done + +for target_framework in net8.0 net10.0 +do + smoke_directory="$validation_directory/smoke-$target_framework" + mkdir -p "$smoke_directory" + + cat > "$smoke_directory/ForkPackageSmoke.csproj" < + + Exe + $target_framework + enable + enable + + + + + +EOF + + cat > "$smoke_directory/NuGet.config" < + + + + + + + + + + + + + + + +EOF + + cat > "$smoke_directory/Program.cs" <<'EOF' +using DuckDB.NET.Data; + +using var connection = new DuckDBConnection("Data Source=:memory:"); +connection.Open(); + +using (var command = connection.CreateCommand()) +{ + command.CommandText = "CREATE TABLE fork_package_smoke(value INTEGER)"; + command.ExecuteNonQuery(); +} + +using (var appender = connection.CreateAppender("fork_package_smoke")) +{ + appender.AppendRowScoped(42, + static (ref DuckDBAppenderRowWriter writer, int value) => writer.AppendValue(value)); +} + +using var verification = connection.CreateCommand(); +verification.CommandText = "SELECT version(), sum(value) FROM fork_package_smoke"; +using var reader = verification.ExecuteReader(); +if (!reader.Read() || !reader.GetString(0).Contains("v1.5.4", StringComparison.Ordinal) || reader.GetInt64(1) != 42) +{ + throw new InvalidOperationException("Fork package smoke test failed."); +} + +Console.WriteLine($"Fork package smoke passed with {reader.GetString(0)}"); +EOF + + dotnet restore "$smoke_directory/ForkPackageSmoke.csproj" \ + --configfile "$smoke_directory/NuGet.config" \ + --packages "$validation_directory/packages-$target_framework" + dotnet run --project "$smoke_directory/ForkPackageSmoke.csproj" --configuration Release --no-restore +done + +echo "Validated fork packages at version $package_version" diff --git a/scripts/validate-preview-packages.sh b/scripts/validate-preview-packages.sh index 9a24ed6d..b770b25c 100755 --- a/scripts/validate-preview-packages.sh +++ b/scripts/validate-preview-packages.sh @@ -1,112 +1,5 @@ #!/usr/bin/env bash set -euo pipefail -if [[ $# -ne 2 ]]; then - echo "Usage: $0 " >&2 - exit 2 -fi - -package_directory=$(cd "$1" && pwd) -preview_version=$2 -data_package="$package_directory/Skuirrels.DuckDB.NET.Data.Full.$preview_version.nupkg" -bindings_package="$package_directory/Skuirrels.DuckDB.NET.Bindings.Full.$preview_version.nupkg" - -[[ -f "$data_package" ]] || { echo "Missing $data_package" >&2; exit 1; } -[[ -f "$bindings_package" ]] || { echo "Missing $bindings_package" >&2; exit 1; } - -validation_directory=$(mktemp -d) -trap 'rm -rf "$validation_directory"' EXIT - -unzip -p "$data_package" '*.nuspec' > "$validation_directory/data.nuspec" -unzip -p "$bindings_package" '*.nuspec' > "$validation_directory/bindings.nuspec" -unzip -Z1 "$bindings_package" > "$validation_directory/bindings-files.txt" - -grep -Fq 'Skuirrels.DuckDB.NET.Data.Full' "$validation_directory/data.nuspec" -grep -Fq "$preview_version" "$validation_directory/data.nuspec" -grep -Fq "Skuirrels.DuckDB.NET.Bindings.Full' "$validation_directory/bindings.nuspec" -grep -Fq "$preview_version" "$validation_directory/bindings.nuspec" -grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/data.nuspec" -grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/bindings.nuspec" - -for native_asset in \ - runtimes/win-x64/native/duckdb.dll \ - runtimes/win-arm64/native/duckdb.dll \ - runtimes/linux-x64/native/libduckdb.so \ - runtimes/linux-arm64/native/libduckdb.so \ - runtimes/osx/native/libduckdb.dylib -do - grep -Fxq "$native_asset" "$validation_directory/bindings-files.txt" -done - -smoke_directory="$validation_directory/smoke" -mkdir -p "$smoke_directory" - -cat > "$smoke_directory/PreviewSmoke.csproj" < - - Exe - net8.0 - enable - enable - - - - - -EOF - -cat > "$smoke_directory/NuGet.config" < - - - - - - - - - - - - - - - -EOF - -cat > "$smoke_directory/Program.cs" <<'EOF' -using DuckDB.NET.Data; - -using var connection = new DuckDBConnection("Data Source=:memory:"); -connection.Open(); - -using (var command = connection.CreateCommand()) -{ - command.CommandText = "CREATE TABLE preview_smoke(value INTEGER)"; - command.ExecuteNonQuery(); -} - -using (var appender = connection.CreateAppender("preview_smoke")) -{ - appender.AppendRowScoped(42, - static (ref DuckDBAppenderRowWriter writer, int value) => writer.AppendValue(value)); -} - -using var verification = connection.CreateCommand(); -verification.CommandText = "SELECT version(), sum(value) FROM preview_smoke"; -using var reader = verification.ExecuteReader(); -if (!reader.Read() || !reader.GetString(0).Contains("v1.5.4", StringComparison.Ordinal) || reader.GetInt64(1) != 42) -{ - throw new InvalidOperationException("Preview package smoke test failed."); -} - -Console.WriteLine($"Preview smoke passed with {reader.GetString(0)}"); -EOF - -dotnet restore "$smoke_directory/PreviewSmoke.csproj" \ - --configfile "$smoke_directory/NuGet.config" \ - --packages "$validation_directory/packages" -dotnet run --project "$smoke_directory/PreviewSmoke.csproj" --configuration Release --no-restore - -echo "Validated preview packages at version $preview_version" +script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +exec "$script_directory/validate-fork-packages.sh" "$@" From a1b2a355e39b06e5ba19ef680e166e8577a78a06 Mon Sep 17 00:00:00 2001 From: Skuirrels <71982362+skuirrels@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:55:49 +0100 Subject: [PATCH 30/33] Enable stable fork release workflow (#7) --- .github/workflows/preview-release.yml | 84 +++++++++++++++++---------- 1 file changed, 52 insertions(+), 32 deletions(-) diff --git a/.github/workflows/preview-release.yml b/.github/workflows/preview-release.yml index 54ea63dc..52002c0b 100644 --- a/.github/workflows/preview-release.yml +++ b/.github/workflows/preview-release.yml @@ -1,4 +1,4 @@ -name: Preview Release +name: Fork Package Release on: release: @@ -9,8 +9,12 @@ permissions: id-token: write jobs: - publish-preview: - if: github.event.release.prerelease == true && startsWith(github.event.release.tag_name, 'v1.5.4-preview.') + publish-fork-package: + if: >- + (github.event.release.prerelease == true && + startsWith(github.event.release.tag_name, 'v1.5.4-preview.')) || + (github.event.release.prerelease == false && + github.event.release.tag_name == 'v1.5.4') runs-on: ubuntu-latest environment: nuget-preview @@ -21,16 +25,28 @@ jobs: fetch-depth: 0 ref: ${{ github.event.release.tag_name }} - - name: Set preview version + - name: Set package version and release mode shell: bash run: | version="${{ github.event.release.tag_name }}" version="${version#v}" - if [[ ! "$version" =~ ^1\.5\.4-preview\.[0-9]+$ ]]; then - echo "Unexpected preview version: $version" >&2 + + if [[ "$version" =~ ^1\.5\.4-preview\.[0-9]+$ ]]; then + fork_preview=true + fork_release=false + elif [[ "$version" == "1.5.4" ]]; then + fork_preview=false + fork_release=true + else + echo "Unexpected fork package version: $version" >&2 exit 1 fi - echo "PREVIEW_VERSION=$version" >> "$GITHUB_ENV" + + { + echo "PACKAGE_VERSION=$version" + echo "FORK_PREVIEW=$fork_preview" + echo "FORK_RELEASE=$fork_release" + } >> "$GITHUB_ENV" - name: Setup .NET SDK uses: actions/setup-dotnet@v5 @@ -48,9 +64,10 @@ jobs: /m:1 /p:BuildType=Full /p:CI=false - /p:ForkPreview=true - /p:Version=${{ env.PREVIEW_VERSION }} - /p:PackageVersion=${{ env.PREVIEW_VERSION }} + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" /p:UseSharedCompilation=false - name: Test @@ -62,60 +79,63 @@ jobs: --logger "console;verbosity=quiet" /p:BuildType=Full /p:CI=false - /p:ForkPreview=true + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" /p:DoesNotReturnAttribute=DoesNotReturnAttribute - - name: Pack preview packages + - name: Pack fork packages shell: bash run: | - mkdir -p artifacts/preview + mkdir -p artifacts/fork-release common_args=( --configuration Release --no-build --no-restore - --output artifacts/preview + --output artifacts/fork-release /m:1 /p:BuildType=Full /p:CI=false - /p:ForkPreview=true - /p:PreviewPack=true - /p:Version="$PREVIEW_VERSION" - /p:PackageVersion="$PREVIEW_VERSION" + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:ForkPack=true + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" ) dotnet pack DuckDB.NET.Bindings/Bindings.csproj "${common_args[@]}" dotnet restore DuckDB.NET.Data/Data.csproj \ - --source artifacts/preview \ + --source artifacts/fork-release \ --source https://api.nuget.org/v3/index.json \ /p:BuildType=Full \ /p:CI=false \ - /p:ForkPreview=true \ - /p:PreviewPack=true \ - /p:Version="$PREVIEW_VERSION" \ - /p:PackageVersion="$PREVIEW_VERSION" + /p:ForkPreview="$FORK_PREVIEW" \ + /p:ForkRelease="$FORK_RELEASE" \ + /p:ForkPack=true \ + /p:Version="$PACKAGE_VERSION" \ + /p:PackageVersion="$PACKAGE_VERSION" dotnet pack DuckDB.NET.Data/Data.csproj "${common_args[@]}" - name: Validate and smoke-test packages - run: ./scripts/validate-preview-packages.sh artifacts/preview "$PREVIEW_VERSION" + run: ./scripts/validate-fork-packages.sh artifacts/fork-release "$PACKAGE_VERSION" - name: Generate checksums - working-directory: artifacts/preview + working-directory: artifacts/fork-release run: sha256sum *.nupkg > SHA256SUMS - name: Upload workflow artifacts uses: actions/upload-artifact@v7 with: - name: duckdb-net-${{ env.PREVIEW_VERSION }} + name: duckdb-net-${{ github.event.release.tag_name }} path: | - artifacts/preview/*.nupkg - artifacts/preview/SHA256SUMS + artifacts/fork-release/*.nupkg + artifacts/fork-release/SHA256SUMS if-no-files-found: error - - name: Attach packages to GitHub prerelease + - name: Attach packages to GitHub release env: GH_TOKEN: ${{ github.token }} - run: gh release upload "${{ github.event.release.tag_name }}" artifacts/preview/* --clobber + run: gh release upload "${{ github.event.release.tag_name }}" artifacts/fork-release/* --clobber - name: Authenticate to NuGet.org uses: NuGet/login@v1 @@ -123,9 +143,9 @@ jobs: with: user: skuirrels - - name: Publish preview packages to NuGet.org + - name: Publish fork packages to NuGet.org run: >- - dotnet nuget push "artifacts/preview/*.nupkg" + dotnet nuget push "artifacts/fork-release/*.nupkg" --api-key "${{ steps.nuget-login.outputs.NUGET_API_KEY }}" --source https://api.nuget.org/v3/index.json --skip-duplicate From 7a93b23e4d69eb69b607a83bb75af0abbedf8a75 Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Thu, 23 Jul 2026 20:46:41 +0100 Subject: [PATCH 31/33] Optimize mapped appends and typed parameter binding --- .../ListAppenderBenchmark.cs | 96 ++++++ .../PreparedCommandBenchmark.cs | 29 +- DuckDB.NET.Benchmarks/Program.cs | 5 +- .../DataChunk/Writer/ListVectorDataWriter.cs | 210 ++++++++----- .../DataChunk/Writer/VectorDataWriterBase.cs | 288 +++++++++++++++++- DuckDB.NET.Data/DuckDBMappedAppender.cs | 33 +- DuckDB.NET.Data/DuckDBParameter.cs | 104 ++++++- DuckDB.NET.Data/DuckDBParameterCollection.cs | 72 ++++- DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs | 99 ++++++ .../PreparedStatement/ClrToDuckDBConverter.cs | 193 ++++++++++++ .../PreparedStatement/DuckDBTypeMap.cs | 6 + .../PreparedStatement/PreparedStatement.cs | 138 +++++++-- DuckDB.NET.Test/DuckDBCommandTests.cs | 46 +++ .../DuckDBManagedAppenderListTests.cs | 113 ++++++- DuckDB.NET.Test/DuckDBMappedAppenderTests.cs | 42 +++ 15 files changed, 1351 insertions(+), 123 deletions(-) create mode 100644 DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs diff --git a/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs b/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs new file mode 100644 index 00000000..5b89a881 --- /dev/null +++ b/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using System.Collections.ObjectModel; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class ListAppenderBenchmark +{ + private const int ItemCount = 32; + + private DuckDBConnection connection = null!; + private int[] arrayValues = null!; + private List listValues = null!; + private ReadOnlyCollection readOnlyValues = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + arrayValues = Enumerable.Range(0, ItemCount).ToArray(); + listValues = arrayValues.ToList(); + readOnlyValues = Array.AsReadOnly(arrayValues); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = """ + DROP TABLE IF EXISTS list_from_array; + DROP TABLE IF EXISTS list_from_list; + DROP TABLE IF EXISTS list_from_read_only_collection; + DROP TABLE IF EXISTS array_from_array; + CREATE TABLE list_from_array (values INTEGER[]); + CREATE TABLE list_from_list (values INTEGER[]); + CREATE TABLE list_from_read_only_collection (values INTEGER[]); + CREATE TABLE array_from_array (values INTEGER[32]); + """; + command.ExecuteNonQuery(); + } + + [Benchmark(Baseline = true)] + public void AppendListFromArray() + { + using var appender = connection.CreateAppender("list_from_array"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(arrayValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendListFromList() + { + using var appender = connection.CreateAppender("list_from_list"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(listValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendListFromReadOnlyCollection() + { + using var appender = connection.CreateAppender("list_from_read_only_collection"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(readOnlyValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendArrayFromArray() + { + using var appender = connection.CreateAppender("array_from_array"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(arrayValues, static (row, values) => row.AppendValue(values)); + } + } +} diff --git a/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs index 6f35a47a..5a9b2e75 100644 --- a/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs +++ b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs @@ -8,9 +8,11 @@ public class PreparedCommandBenchmark { private DuckDBConnection connection = null!; private DuckDBCommand unpreparedCommand = null!; + private DuckDBCommand boxedPreparedCommand = null!; private DuckDBCommand preparedCommand = null!; private DuckDBParameter unpreparedParameter = null!; - private DuckDBParameter preparedParameter = null!; + private DuckDBParameter boxedPreparedParameter = null!; + private DuckDBParameter preparedParameter = null!; private int nextValue; [GlobalSetup] @@ -20,7 +22,9 @@ public void Setup() connection.Open(); unpreparedCommand = CreateCommand(out unpreparedParameter); - preparedCommand = CreateCommand(out preparedParameter); + boxedPreparedCommand = CreateCommand(out boxedPreparedParameter); + boxedPreparedCommand.Prepare(); + preparedCommand = CreateTypedCommand(out preparedParameter); preparedCommand.Prepare(); } @@ -28,6 +32,7 @@ public void Setup() public void Cleanup() { preparedCommand.Dispose(); + boxedPreparedCommand.Dispose(); unpreparedCommand.Dispose(); connection.Dispose(); } @@ -39,10 +44,17 @@ public int ExecuteUnprepared() return (int)unpreparedCommand.ExecuteScalar()!; } + [Benchmark] + public int ExecutePreparedBoxed() + { + boxedPreparedParameter.Value = nextValue++; + return (int)boxedPreparedCommand.ExecuteScalar()!; + } + [Benchmark] public int ExecutePrepared() { - preparedParameter.Value = nextValue++; + preparedParameter.TypedValue = nextValue++; return (int)preparedCommand.ExecuteScalar()!; } @@ -56,6 +68,17 @@ private DuckDBCommand CreateCommand(out DuckDBParameter changingParameter) command.Parameters.Add(new DuckDBParameter("third", 3)); return command; } + + private DuckDBCommand CreateTypedCommand(out DuckDBParameter changingParameter) + { + var command = connection.CreateCommand(); + command.CommandText = "SELECT $first::INTEGER + $second::INTEGER + $third::INTEGER"; + changingParameter = new DuckDBParameter("first", 1); + command.Parameters.Add(changingParameter); + command.Parameters.Add(new DuckDBParameter("second", 2)); + command.Parameters.Add(new DuckDBParameter("third", 3)); + return command; + } } [MemoryDiagnoser] diff --git a/DuckDB.NET.Benchmarks/Program.cs b/DuckDB.NET.Benchmarks/Program.cs index d2e52daf..198112cd 100644 --- a/DuckDB.NET.Benchmarks/Program.cs +++ b/DuckDB.NET.Benchmarks/Program.cs @@ -14,12 +14,15 @@ .WithToolchain(InProcessEmitToolchain.Instance) .WithLaunchCount(1) .WithWarmupCount(5) - .WithIterationCount(10)); + .WithIterationCount(10) + .WithInvocationCount(1) + .WithUnrollFactor(1)); var benchmarkTypes = new List { #if !DUCKDB_NET_BASELINE_1_5_3 typeof(AppenderBenchmark), + typeof(ListAppenderBenchmark), typeof(MappedAppenderBenchmark), #endif typeof(PreparedCommandBenchmark), diff --git a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs index c56654dc..a51269c0 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs @@ -1,13 +1,27 @@ using DuckDB.NET.Data.Common; +using System.Reflection; +using System.Runtime.CompilerServices; namespace DuckDB.NET.Data.DataChunk.Writer; internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase { + private delegate void CollectionWriter(ListVectorDataWriter writer, ICollection collection, ulong startIndex); + + private static readonly ConditionalWeakTable CollectionWriterCache = new(); + private static readonly MethodInfo WriteArrayMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteArray), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteListMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteList), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteEnumerableMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteEnumerable), BindingFlags.Static | BindingFlags.NonPublic)!; + private ulong offset = 0; private readonly ulong arraySize; private readonly DuckDBLogicalType childType; private readonly VectorDataWriterBase listItemWriter; + private Type? cachedCollectionType; + private CollectionWriterPlan? cachedCollectionWriterPlan; private bool IsList => ColumnType == DuckDBType.List; private ulong vectorReservedSize = DuckDBGlobalData.VectorSize; @@ -27,59 +41,17 @@ internal override bool AppendCollection(ICollection value, ulong rowIndex) ResizeVector(rowIndex % DuckDBGlobalData.VectorSize, count); - _ = value switch - { - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - _ => WriteItemsFallback(value), - }; + ValidateArraySize(count); + + var collectionWriter = GetCollectionWriter(value.GetType()); + if (collectionWriter is not null) + { + collectionWriter(this, value, offset); + } + else + { + WriteItemsFallback(value); + } var duckDBListEntry = new DuckDBListEntry(offset, count); var result = !IsList || AppendValueInternal(duckDBListEntry, rowIndex); @@ -92,42 +64,132 @@ internal override bool AppendCollection(ICollection value, ulong rowIndex) } return result; + } - int WriteItems(IEnumerable items) + private CollectionWriter? GetCollectionWriter(Type collectionType) + { + if (collectionType == cachedCollectionType) { - if (IsList == false && count != arraySize) - { - throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}"); - } + return cachedCollectionWriterPlan!.Writer; + } - var index = 0ul; + cachedCollectionType = collectionType; + cachedCollectionWriterPlan = CollectionWriterCache.GetValue(collectionType, CreateCollectionWriter); + return cachedCollectionWriterPlan.Writer; + } - foreach (var item in items) - { - listItemWriter.WriteValue(item, offset + (index++)); - } + private void ValidateArraySize(ulong count) + { + if (!IsList && count != arraySize) + { + throw new InvalidOperationException( + $"Column has Array size of {arraySize} but the specified value has size of {count}"); + } + } - return 0; + private void WriteItemsFallback(IEnumerable items) + { + var index = 0ul; + + foreach (var item in items) + { + listItemWriter.WriteValue(item, offset + (index++)); } + } + + private static CollectionWriterPlan CreateCollectionWriter(Type collectionType) + { + MethodInfo? openMethod = null; + Type? elementType = null; - int WriteItemsFallback(IEnumerable items) + if (collectionType.IsSZArray) + { + openMethod = WriteArrayMethod; + elementType = collectionType.GetElementType(); + } + else if (collectionType.IsGenericType && + collectionType.GetGenericTypeDefinition() == typeof(List<>)) + { + openMethod = WriteListMethod; + elementType = collectionType.GetGenericArguments()[0]; + } + else { - if (IsList == false && count != arraySize) + elementType = GetEnumerableElementType(collectionType); + openMethod = elementType is null ? null : WriteEnumerableMethod; + } + + return new CollectionWriterPlan( + openMethod is null || elementType is null + ? null + : openMethod.MakeGenericMethod(elementType).CreateDelegate()); + } + + private static Type? GetEnumerableElementType(Type collectionType) + { + Type? elementType = null; + + foreach (var interfaceType in collectionType.GetInterfaces()) + { + if (!interfaceType.IsGenericType || + interfaceType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) { - throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}"); + continue; } - var index = 0ul; - - foreach (var item in items) + var candidateType = interfaceType.GetGenericArguments()[0]; + if (elementType is not null && elementType != candidateType) { - listItemWriter.WriteValue(item, offset + (index++)); + return null; } - return 0; + elementType = candidateType; + } + + return elementType; + } + + private static void WriteArray( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (T[])collection; + + for (var index = 0; index < values.Length; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); + } + } + + private static void WriteList( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (List)collection; + + for (var index = 0; index < values.Count; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); } } + private static void WriteEnumerable( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var index = 0ul; + + foreach (var value in (IEnumerable)collection) + { + writer.listItemWriter.WriteValue(value, startIndex + index++); + } + } + + private sealed record CollectionWriterPlan(CollectionWriter? Writer); + private void ResizeVector(ulong rowIndex, ulong count) { //If writing to a list column we need to make sure that enough space is allocated. Not needed for Arrays as DuckDB does it for us. @@ -163,7 +225,9 @@ private void ResizeVector(ulong rowIndex, ulong count) public override void Dispose() { + cachedCollectionType = null; + cachedCollectionWriterPlan = null; listItemWriter.Dispose(); childType.Dispose(); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs b/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs index 03995710..293c7dd2 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs @@ -20,7 +20,7 @@ public void WriteNull(ulong rowIndex) NativeMethods.ValidityMask.DuckDBValiditySetRowValidity(validity, rowIndex, false); } - public void WriteValue(T value, ulong rowIndex) + private void WriteValueFallback(T value, ulong rowIndex) { if (value == null) { @@ -64,6 +64,292 @@ public void WriteValue(T value, ulong rowIndex) }; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void WriteValue(T value, ulong rowIndex) + { + if (typeof(T) == typeof(bool)) + { + _ = AppendBool(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(bool?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendBool(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(sbyte)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(sbyte?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(short)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(short?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(int)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(int?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(long)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(long?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(byte)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(byte?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(ushort)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(ushort?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(uint)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(uint?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(ulong)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(ulong?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(float)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(float?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(double)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(double?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(decimal)) + { + _ = AppendDecimal(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(decimal?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDecimal(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(BigInteger)) + { + _ = AppendBigInteger(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(BigInteger?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendBigInteger(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(string)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue is not null) _ = AppendString(typedValue, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(Guid)) + { + _ = AppendGuid(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(Guid?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendGuid(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateTime)) + { + _ = AppendDateTime(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateTime?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateTime(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(TimeSpan)) + { + _ = AppendTimeSpan(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(TimeSpan?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeSpan(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBDateOnly)) + { + _ = AppendDateOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBDateOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBTimeOnly)) + { + _ = AppendTimeOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBTimeOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateOnly)) + { + _ = AppendDateOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(TimeOnly)) + { + _ = AppendTimeOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(TimeOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateTimeOffset)) + { + _ = AppendDateTimeOffset(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateTimeOffset?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateTimeOffset(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + WriteValueFallback(value, rowIndex); + } + internal virtual bool AppendBool(bool value, ulong rowIndex) => ThrowException(); internal virtual bool AppendDecimal(decimal value, ulong rowIndex) => ThrowException(); diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index 25a3444f..8b65eba0 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Mapping; +using System.Threading; namespace DuckDB.NET.Data; @@ -9,16 +10,18 @@ namespace DuckDB.NET.Data; /// The AppenderMap type defining the mappings public class DuckDBMappedAppender : IDisposable where TMap : DuckDBAppenderMap, new() { + private static readonly Lazy CompiledMap = + new(CreateCompiledMap, LazyThreadSafetyMode.ExecutionAndPublication); + private readonly DuckDBAppender appender; - private readonly List> mappings; + private readonly Action writeRecord; internal DuckDBMappedAppender(DuckDBAppender appender) { this.appender = appender; - var classMap = new TMap(); - - // Get mappings as List to avoid interface enumerator boxing - mappings = classMap.PropertyMappings; + var compiledMap = CompiledMap.Value; + var mappings = compiledMap.Mappings; + writeRecord = compiledMap.WriteRecord; // Validate mappings match the table structure if (mappings.Count == 0) @@ -50,6 +53,7 @@ internal DuckDBMappedAppender(DuckDBAppender appender) $"Type mismatch at column index {index}: Mapped type is {mapping.PropertyType.Name} (expected DuckDB type: {expectedType}) but actual column type is {columnType}"); } } + } /// @@ -76,14 +80,7 @@ private void AppendRecord(T record) throw new ArgumentNullException(nameof(record)); } - // Pass both values as state so the callback does not capture per record. - appender.AppendRow((Record: record, Mappings: mappings), static (row, state) => - { - foreach (var mapping in state.Mappings) - { - mapping.AppendToRow(row, state.Record); - } - }); + appender.AppendRow(record, writeRecord); } private static DuckDBType GetExpectedDuckDBType(Type type) @@ -97,6 +94,16 @@ private static DuckDBType GetExpectedDuckDBType(Type type) }; } + private static CompiledAppenderMap CreateCompiledMap() + { + var mappings = new TMap().PropertyMappings; + return new CompiledAppenderMap(mappings, DuckDBAppenderMapCompiler.Compile(mappings)); + } + + private sealed record CompiledAppenderMap( + IReadOnlyList> Mappings, + Action WriteRecord); + /// /// Closes the appender and flushes any remaining data. /// diff --git a/DuckDB.NET.Data/DuckDBParameter.cs b/DuckDB.NET.Data/DuckDBParameter.cs index 7ace2d50..0cf12bf3 100644 --- a/DuckDB.NET.Data/DuckDBParameter.cs +++ b/DuckDB.NET.Data/DuckDBParameter.cs @@ -9,12 +9,28 @@ public class DuckDBParameter : DbParameter private const DbType DefaultDbType = DbType.String; private object? value; + private string parameterName = string.Empty; + private int bindingMetadataVersion; public override DbType DbType { get; set; } [AllowNull] [DefaultValue("")] - public override string ParameterName { get; set; } + public override string ParameterName + { + get => parameterName; + set + { + var newValue = value ?? string.Empty; + if (string.Equals(parameterName, newValue, StringComparison.Ordinal)) + { + return; + } + + parameterName = newValue; + bindingMetadataVersion++; + } + } public override object? Value { @@ -38,6 +54,8 @@ public override object? Value public override bool SourceColumnNullMapping { get; set; } public override int Size { get; set; } + internal int BindingMetadataVersion => bindingMetadataVersion; + public DuckDBParameter() : this (string.Empty, DefaultDbType, null) { } @@ -66,4 +84,86 @@ public DuckDBParameter(string name, DbType type, object? value) } public override void ResetDbType() => DbType = DefaultDbType; -} \ No newline at end of file + + internal virtual bool TryBindScalarValue( + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + => value.TryBindScalarValue(statement, index, duckDBType, DbType, out result); + + internal virtual DuckDBValue ToDuckDBValue(DuckDBLogicalType logicalType, DuckDBType duckDBType) + => value.ToDuckDBValue(logicalType, duckDBType, DbType); +} + +/// +/// A DuckDB parameter that keeps its value in typed storage so repeated prepared executions do +/// not need to box common value types. +/// +/// The CLR type stored by the parameter. +public sealed class DuckDBParameter : DuckDBParameter +{ + private static readonly DbType DefaultTypedDbType = DuckDBTypeMap.GetDbTypeForType(typeof(T)); + private T typedValue; + + /// + /// Gets or sets the strongly typed parameter value. + /// + public T TypedValue + { + get => typedValue; + set => typedValue = value; + } + + public override object? Value + { + get => typedValue; + set + { + if (value is T typed) + { + typedValue = typed; + return; + } + + if (value is null && default(T) is null) + { + typedValue = default!; + return; + } + + throw new InvalidCastException( + $"Parameter '{ParameterName}' requires a value assignable to {typeof(T).Name}."); + } + } + + public DuckDBParameter(T value) + : this(string.Empty, value) + { + } + + public DuckDBParameter(string name, T value) + : base(name, DefaultTypedDbType, null) + { + typedValue = value; + } + + public override void ResetDbType() => DbType = DefaultTypedDbType; + + internal override bool TryBindScalarValue( + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + { + if (ClrToDuckDBConverter.TryBindTypedScalarValue(typedValue, statement, index, duckDBType, out result)) + { + return true; + } + + return ((object?)typedValue).TryBindScalarValue(statement, index, duckDBType, DbType, out result); + } + + internal override DuckDBValue ToDuckDBValue(DuckDBLogicalType logicalType, DuckDBType duckDBType) + => ((object?)typedValue).ToDuckDBValue(logicalType, duckDBType, DbType); +} diff --git a/DuckDB.NET.Data/DuckDBParameterCollection.cs b/DuckDB.NET.Data/DuckDBParameterCollection.cs index 86204812..661370e6 100644 --- a/DuckDB.NET.Data/DuckDBParameterCollection.cs +++ b/DuckDB.NET.Data/DuckDBParameterCollection.cs @@ -5,11 +5,18 @@ namespace DuckDB.NET.Data; public class DuckDBParameterCollection : DbParameterCollection { private readonly List parameters = new(); + private int version; + + internal int Version => version; public new DuckDBParameter this[int index] { get => parameters[index]; - set => parameters[index] = value; + set + { + parameters[index] = value; + version++; + } } public new DuckDBParameter this[string parameterName] @@ -24,22 +31,43 @@ public class DuckDBParameterCollection : DbParameterCollection public override int Add(object value) { parameters.Add((DuckDBParameter)value); + version++; return parameters.Count - 1; } - public override void Clear() => parameters.Clear(); + public override void Clear() + { + if (parameters.Count == 0) + { + return; + } + + parameters.Clear(); + version++; + } public override bool Contains(object value) => parameters.Contains((DuckDBParameter) value); public override int IndexOf(object value) => parameters.IndexOf((DuckDBParameter) value); - public override void Insert(int index, object value) => parameters.Insert(index, (DuckDBParameter) value); + public override void Insert(int index, object value) + { + parameters.Insert(index, (DuckDBParameter)value); + version++; + } - public override void Remove(object value) => parameters.Remove((DuckDBParameter) value); + public override void Remove(object value) + { + if (parameters.Remove((DuckDBParameter)value)) + { + version++; + } + } public int Add(DuckDBParameter value) { parameters.Add(value); + version++; return parameters.Count - 1; } @@ -47,26 +75,41 @@ public int Add(DuckDBParameter value) public int IndexOf(DuckDBParameter value) => parameters.IndexOf(value); - public void Insert(int index, DuckDBParameter value) => parameters.Insert(index, value); + public void Insert(int index, DuckDBParameter value) + { + parameters.Insert(index, value); + version++; + } - public void Remove(DuckDBParameter value) => parameters.Remove(value); + public void Remove(DuckDBParameter value) + { + if (parameters.Remove(value)) + { + version++; + } + } - public override void RemoveAt(int index) => parameters.RemoveAt(index); + public override void RemoveAt(int index) + { + parameters.RemoveAt(index); + version++; + } public override void RemoveAt(string parameterName) { var index = IndexOfSafe(parameterName); parameters.RemoveAt(index); + version++; } protected override void SetParameter(int index, DbParameter value) - => parameters[index] = (DuckDBParameter)value; + => this[index] = (DuckDBParameter)value; protected override void SetParameter(string parameterName, DbParameter value) { var index = IndexOfSafe(parameterName); - parameters[index] = (DuckDBParameter)value; + this[index] = (DuckDBParameter)value; } public override int IndexOf(string parameterName) @@ -95,7 +138,14 @@ public override void AddRange(Array values) => AddRange(values.Cast()); public void AddRange(IEnumerable values) - => parameters.AddRange(values); + { + var oldCount = parameters.Count; + parameters.AddRange(values); + if (parameters.Count != oldCount) + { + version++; + } + } private int IndexOfSafe(string parameterName) { @@ -104,4 +154,4 @@ private int IndexOfSafe(string parameterName) throw new IndexOutOfRangeException($"Parameter '{parameterName}' not found"); return index; } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs index ff60dcf6..8583fcde 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs @@ -1,3 +1,5 @@ +using System.Linq.Expressions; + namespace DuckDB.NET.Data.Mapping; /// @@ -69,6 +71,7 @@ internal interface IPropertyMapping Type PropertyType { get; } PropertyMappingType MappingType { get; } void AppendToRow(IDuckDBAppenderRow row, T record); + Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record); } internal sealed class PropertyMapping : IPropertyMapping @@ -119,6 +122,12 @@ public void AppendToRow(IDuckDBAppenderRow row, T record) _ => throw new NotSupportedException($"Type {typeof(TProperty).Name} is not supported for appending") }; } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + { + var value = Expression.Invoke(Expression.Constant(Getter), record); + return DuckDBAppenderMapCompiler.CreateAppendExpression(row, value, typeof(TProperty)); + } } internal sealed class DefaultValueMapping : IPropertyMapping @@ -130,6 +139,9 @@ public void AppendToRow(IDuckDBAppenderRow row, T record) { row.AppendDefault(); } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + => Expression.Call(row, nameof(IDuckDBAppenderRow.AppendDefault), Type.EmptyTypes); } internal sealed class NullValueMapping : IPropertyMapping @@ -141,4 +153,91 @@ public void AppendToRow(IDuckDBAppenderRow row, T record) { row.AppendNullValue(); } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + => Expression.Call(row, nameof(IDuckDBAppenderRow.AppendNullValue), Type.EmptyTypes); +} + +internal static class DuckDBAppenderMapCompiler +{ + private static readonly HashSet SupportedValueTypes = + [ + typeof(bool), + typeof(sbyte), + typeof(short), + typeof(int), + typeof(long), + typeof(byte), + typeof(ushort), + typeof(uint), + typeof(ulong), + typeof(float), + typeof(double), + typeof(decimal), + typeof(DateTime), + typeof(DateTimeOffset), + typeof(TimeSpan), + typeof(Guid), + typeof(BigInteger), + typeof(DuckDBDateOnly), + typeof(DuckDBTimeOnly), + typeof(DateOnly), + typeof(TimeOnly) + ]; + + public static Action Compile(IReadOnlyList> mappings) + { + var row = Expression.Parameter(typeof(IDuckDBAppenderRow), "row"); + var record = Expression.Parameter(typeof(T), "record"); + var writes = new List(mappings.Count + 1); + + for (var index = 0; index < mappings.Count; index++) + { + writes.Add(mappings[index].BuildWriteExpression(row, record)); + } + + // IDuckDBAppenderRow methods return the row to support fluent callers. The mapped + // callback is an Action, so force the compiled block's result type to void. + writes.Add(Expression.Empty()); + + return Expression.Lambda>( + Expression.Block(writes), + row, + record).Compile(); + } + + internal static Expression CreateAppendExpression( + ParameterExpression row, + Expression value, + Type propertyType) + { + if (propertyType == typeof(string) || propertyType == typeof(byte[])) + { + return Expression.Call( + row, + nameof(IDuckDBAppenderRow.AppendValue), + Type.EmptyTypes, + value); + } + + var underlyingType = Nullable.GetUnderlyingType(propertyType); + var valueType = underlyingType ?? propertyType; + + if (!SupportedValueTypes.Contains(valueType) && !valueType.IsEnum) + { + throw new NotSupportedException($"Type {propertyType.Name} is not supported for appending"); + } + + var nullableType = typeof(Nullable<>).MakeGenericType(valueType); + var nullableValue = propertyType == nullableType + ? value + : Expression.Convert(value, nullableType); + var genericArguments = valueType.IsEnum ? [valueType] : Type.EmptyTypes; + + return Expression.Call( + row, + nameof(IDuckDBAppenderRow.AppendValue), + genericArguments, + nullableValue); + } } diff --git a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs index 8cb2440a..de55d955 100644 --- a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs +++ b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; + namespace DuckDB.NET.Data.PreparedStatement; internal static class ClrToDuckDBConverter @@ -286,6 +288,197 @@ item is DateTime && duckDBType is DuckDBType.TimestampS or DuckDBType.TimestampM } } + internal static bool TryBindTypedScalarValue( + T item, + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + { + if (item is null) + { + result = NativeMethods.PreparedStatements.DuckDBBindNull(statement, index); + return true; + } + + if (typeof(T) == typeof(bool) && duckDBType == DuckDBType.Boolean) + { + result = NativeMethods.PreparedStatements.DuckDBBindBoolean( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(sbyte) && duckDBType == DuckDBType.TinyInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt8( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(short) && duckDBType == DuckDBType.SmallInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt16( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(int) && duckDBType == DuckDBType.Integer) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt32( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(long) && duckDBType == DuckDBType.BigInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt64( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(byte) && duckDBType == DuckDBType.UnsignedTinyInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt8( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(ushort) && duckDBType == DuckDBType.UnsignedSmallInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt16( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(uint) && duckDBType == DuckDBType.UnsignedInteger) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt32( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(ulong) && duckDBType == DuckDBType.UnsignedBigInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt64( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(float) && duckDBType == DuckDBType.Float) + { + result = NativeMethods.PreparedStatements.DuckDBBindFloat( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(double) && duckDBType == DuckDBType.Double) + { + result = NativeMethods.PreparedStatements.DuckDBBindDouble( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(decimal) && duckDBType == DuckDBType.Decimal) + { + result = NativeMethods.PreparedStatements.DuckDBBindDecimal( + statement, index, ToDuckDBDecimal(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(BigInteger) && duckDBType == DuckDBType.HugeInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindHugeInt( + statement, index, new DuckDBHugeInt(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(string) && duckDBType == DuckDBType.Varchar) + { + result = NativeMethods.PreparedStatements.DuckDBBindVarchar( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(DateTime)) + { + var value = Unsafe.As(ref item); + switch (duckDBType) + { + case DuckDBType.Timestamp: + result = NativeMethods.PreparedStatements.DuckDBBindTimestamp( + statement, index, value.ToTimestampStruct(duckDBType)); + return true; + case DuckDBType.TimestampTz: + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz( + statement, index, value.ToTimestampStruct(duckDBType)); + return true; + case DuckDBType.Date: + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, ((DuckDBDateOnly)value).ToDuckDBDate()); + return true; + case DuckDBType.Time: + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, NativeMethods.DateTimeHelpers.DuckDBToTime((DuckDBTimeOnly)value)); + return true; + } + } + + if (typeof(T) == typeof(DateTimeOffset) && duckDBType == DuckDBType.TimestampTz) + { + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz( + statement, index, Unsafe.As(ref item).ToTimestampStruct()); + return true; + } + + if (typeof(T) == typeof(TimeSpan) && duckDBType == DuckDBType.Interval) + { + result = NativeMethods.PreparedStatements.DuckDBBindInterval( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(DateOnly) && duckDBType == DuckDBType.Date) + { + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, ((DuckDBDateOnly)Unsafe.As(ref item)).ToDuckDBDate()); + return true; + } + + if (typeof(T) == typeof(TimeOnly) && duckDBType == DuckDBType.Time) + { + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, + NativeMethods.DateTimeHelpers.DuckDBToTime(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(DuckDBDateOnly) && duckDBType == DuckDBType.Date) + { + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, Unsafe.As(ref item).ToDuckDBDate()); + return true; + } + + if (typeof(T) == typeof(DuckDBTimeOnly) && duckDBType == DuckDBType.Time) + { + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, + NativeMethods.DateTimeHelpers.DuckDBToTime(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(byte[]) && duckDBType == DuckDBType.Blob) + { + var value = Unsafe.As(ref item); + result = NativeMethods.PreparedStatements.DuckDBBindBlob( + statement, index, value, value.LongLength); + return true; + } + + result = default; + return false; + } + private static bool TryConvertTo(object item, out T result) where T : struct { try diff --git a/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs b/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs index 9469356c..19b9d57d 100644 --- a/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs +++ b/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs @@ -40,4 +40,10 @@ public static DbType GetDbTypeForValue(object? value) return ClrToDbTypeMap.GetValueOrDefault(type, DbType.Object); } + + internal static DbType GetDbTypeForType(Type type) + { + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + return ClrToDbTypeMap.GetValueOrDefault(underlyingType, DbType.Object); + } } diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index b67233fe..8f5b6f39 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -133,28 +133,16 @@ internal DuckDBResult Execute(DuckDBParameterCollection parameterCollection, boo return queryResult; } - private void BindParameters(DuckDBParameterCollection parameterCollection) + protected virtual void BindParameters(DuckDBParameterCollection parameterCollection) { - var expectedParameters = ParameterCount; - if (parameterCollection.Count < expectedParameters) - { - throw new InvalidOperationException($"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); - } + var expectedParameters = ValidateParameterCount(parameterCollection); // Index-based iteration over the typed collection avoids the per-execution allocations of // OfType<>().Any(...) and of the boxed List enumerator that `foreach (DuckDBParameter ...)` // over the non-generic collection would produce. BindParameters runs on every execution. var count = parameterCollection.Count; - var hasNamedParameters = false; - for (var i = 0; i < count; i++) - { - if (!string.IsNullOrEmpty(parameterCollection[i].ParameterName)) - { - hasNamedParameters = true; - break; - } - } + var hasNamedParameters = HasNamedParameters(parameterCollection); if (hasNamedParameters) { @@ -202,9 +190,9 @@ protected void BindParameter(long index, DuckDBParameter parameter, DuckDBLogica var duckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(parameterLogicalType); DuckDBState result; - if (!parameter.Value.TryBindScalarValue(Statement, index, duckDBType, parameter.DbType, out result)) + if (!parameter.TryBindScalarValue(Statement, index, duckDBType, out result)) { - using var duckDBValue = parameter.Value.ToDuckDBValue(parameterLogicalType, duckDBType, parameter.DbType); + using var duckDBValue = parameter.ToDuckDBValue(parameterLogicalType, duckDBType); result = NativeMethods.PreparedStatements.DuckDBBindValue(Statement, index, duckDBValue); } @@ -215,6 +203,31 @@ protected void BindParameter(long index, DuckDBParameter parameter, DuckDBLogica } } + protected long ValidateParameterCount(DuckDBParameterCollection parameterCollection) + { + var expectedParameters = ParameterCount; + if (parameterCollection.Count < expectedParameters) + { + throw new InvalidOperationException( + $"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); + } + + return expectedParameters; + } + + protected static bool HasNamedParameters(DuckDBParameterCollection parameterCollection) + { + for (var index = 0; index < parameterCollection.Count; index++) + { + if (!string.IsNullOrEmpty(parameterCollection[index].ParameterName)) + { + return true; + } + } + + return false; + } + public virtual void Dispose() { Statement.Dispose(); @@ -226,6 +239,10 @@ internal sealed class ReusablePreparedStatement : PreparedStatement private readonly long cachedParameterCount; private readonly DuckDBLogicalType[] cachedParameterTypes; private readonly Dictionary cachedParameterIndices = new(StringComparer.Ordinal); + private DuckDBParameterCollection? cachedParameterCollection; + private int cachedParameterCollectionVersion = -1; + private int[]? cachedParameterMetadataVersions; + private ParameterBinding[]? cachedBindingPlan; public ReusablePreparedStatement(DuckDBPreparedStatement statement) : base(statement) @@ -264,11 +281,96 @@ protected override bool TryGetParameterIndex(string parameterName, out long inde return found; } + protected override void BindParameters(DuckDBParameterCollection parameterCollection) + { + var bindingPlan = GetOrCreateBindingPlan(parameterCollection); + + for (var index = 0; index < bindingPlan.Length; index++) + { + var binding = bindingPlan[index]; + BindParameter(binding.Index, binding.Parameter); + } + } + protected override void BindParameter(long index, DuckDBParameter parameter) { BindParameter(index, parameter, cachedParameterTypes[index - 1]); } + private ParameterBinding[] GetOrCreateBindingPlan(DuckDBParameterCollection parameterCollection) + { + if (IsBindingPlanCurrent(parameterCollection)) + { + return cachedBindingPlan!; + } + + var expectedParameters = ValidateParameterCount(parameterCollection); + var hasNamedParameters = HasNamedParameters(parameterCollection); + var plan = new ParameterBinding[hasNamedParameters + ? parameterCollection.Count + : checked((int)expectedParameters)]; + var bindingCount = 0; + + if (hasNamedParameters) + { + for (var parameterIndex = 0; parameterIndex < parameterCollection.Count; parameterIndex++) + { + var parameter = parameterCollection[parameterIndex]; + if (TryGetParameterIndex(parameter.ParameterName, out var nativeIndex)) + { + plan[bindingCount++] = new ParameterBinding(nativeIndex, parameter); + } + } + + if (bindingCount != plan.Length) + { + Array.Resize(ref plan, bindingCount); + } + } + else + { + for (var parameterIndex = 0; parameterIndex < expectedParameters; parameterIndex++) + { + plan[bindingCount++] = new ParameterBinding( + parameterIndex + 1, + parameterCollection[parameterIndex]); + } + } + + cachedParameterCollection = parameterCollection; + cachedParameterCollectionVersion = parameterCollection.Version; + cachedParameterMetadataVersions = new int[parameterCollection.Count]; + for (var index = 0; index < cachedParameterMetadataVersions.Length; index++) + { + cachedParameterMetadataVersions[index] = parameterCollection[index].BindingMetadataVersion; + } + + cachedBindingPlan = plan; + return plan; + } + + private bool IsBindingPlanCurrent(DuckDBParameterCollection parameterCollection) + { + if (!ReferenceEquals(cachedParameterCollection, parameterCollection) || + cachedParameterCollectionVersion != parameterCollection.Version || + cachedBindingPlan is null || + cachedParameterMetadataVersions is null || + cachedParameterMetadataVersions.Length != parameterCollection.Count) + { + return false; + } + + for (var index = 0; index < cachedParameterMetadataVersions.Length; index++) + { + if (cachedParameterMetadataVersions[index] != parameterCollection[index].BindingMetadataVersion) + { + return false; + } + } + + return true; + } + public override void Dispose() { foreach (var parameterType in cachedParameterTypes) @@ -278,4 +380,6 @@ public override void Dispose() base.Dispose(); } + + private readonly record struct ParameterBinding(long Index, DuckDBParameter Parameter); } diff --git a/DuckDB.NET.Test/DuckDBCommandTests.cs b/DuckDB.NET.Test/DuckDBCommandTests.cs index 298cff52..2317e01e 100644 --- a/DuckDB.NET.Test/DuckDBCommandTests.cs +++ b/DuckDB.NET.Test/DuckDBCommandTests.cs @@ -35,6 +35,52 @@ public void PreparedCommandCanBeExecutedRepeatedlyWithNewParameterValues() command.ExecuteScalar().Should().Be(22); } + [Fact] + public void TypedPreparedParameterCanBeExecutedRepeatedly() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + var parameter = new DuckDBParameter("value", 10); + command.Parameters.Add(parameter); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + parameter.TypedValue = 20; + command.ExecuteScalar().Should().Be(20); + + parameter.Value = 30; + command.ExecuteScalar().Should().Be(30); + } + + [Fact] + public void TypedParameterRejectsValuesOfAnotherType() + { + DuckDBParameter parameter = new DuckDBParameter("value", 10); + + parameter.Invoking(value => value.Value = "wrong") + .Should().Throw(); + } + + [Fact] + public void PreparedBindingPlanInvalidatesWhenParameterNameChanges() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + var parameter = new DuckDBParameter("value", 10); + command.Parameters.Add(parameter); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + parameter.ParameterName = "unused"; + command.Invoking(value => value.ExecuteScalar()).Should().Throw(); + + parameter.ParameterName = "value"; + parameter.TypedValue = 20; + command.ExecuteScalar().Should().Be(20); + } + [Fact] public void PreparedCommandClearsBindingsBeforeReuse() { diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index e185cc51..ddb19475 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -1,4 +1,8 @@ -namespace DuckDB.NET.Test; +using DuckDB.NET.Data.DataChunk.Writer; +using System.Collections.ObjectModel; +using System.Reflection; + +namespace DuckDB.NET.Test; public class DuckDBManagedAppenderListTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) { @@ -199,6 +203,109 @@ public void ArrayValuesInt() ListValuesInternal("Integer", faker => faker.Random.Int(), 5); } + [Fact] + public void IndexedArrayAndListPathsCrossChunkBoundaries() + { + const int rowCount = 3_000; + + Command.CommandText = """ + CREATE TABLE indexed_collection_paths( + id INTEGER, + array_list INTEGER[], + typed_list INTEGER[], + fixed_array INTEGER[3]); + """; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("indexed_collection_paths")) + { + for (var index = 0; index < rowCount; index++) + { + int?[] arrayValues = [index, null, index + 1]; + List listValues = [index + 2, null, index + 3]; + int[] fixedValues = [index + 4, index + 5, index + 6]; + + appender.AppendRow( + (index, arrayValues, listValues, fixedValues), + static (row, values) => row + .AppendValue(values.index) + .AppendValue(values.arrayValues) + .AppendValue(values.listValues) + .AppendValue(values.fixedValues)); + } + } + + Command.CommandText = "SELECT * FROM indexed_collection_paths ORDER BY id"; + using var reader = Command.ExecuteReader(); + + for (var index = 0; index < rowCount; index++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(index); + reader.GetFieldValue>(1).Should().Equal(index, null, index + 1); + reader.GetFieldValue>(2).Should().Equal(index + 2, null, index + 3); + reader.GetFieldValue>(3).Should().Equal(index + 4, index + 5, index + 6); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void TypedCollectionFallbackSupportsReadOnlyAndDerivedLists() + { + Command.CommandText = """ + CREATE TABLE typed_collection_fallback( + id INTEGER, + read_only_values INTEGER[], + derived_values INTEGER[]); + """; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("typed_collection_fallback")) + { + for (var index = 0; index < 3_000; index++) + { + ReadOnlyCollection readOnlyValues = + Array.AsReadOnly(new[] { index, index + 1, index + 2 }); + DerivedIntList derivedValues = [index + 3, index + 4, index + 5]; + + appender.AppendRow( + (index, readOnlyValues, derivedValues), + static (row, values) => row + .AppendValue(values.index) + .AppendValue(values.readOnlyValues) + .AppendValue(values.derivedValues)); + } + } + + Command.CommandText = "SELECT * FROM typed_collection_fallback ORDER BY id"; + using var reader = Command.ExecuteReader(); + + for (var index = 0; index < 3_000; index++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(index); + reader.GetFieldValue>(1).Should().Equal(index, index + 1, index + 2); + reader.GetFieldValue>(2).Should().Equal(index + 3, index + 4, index + 5); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void NonSzArraysUseTheEnumerableFallback() + { + var values = Array.CreateInstance(typeof(int), [3], [1]); + var createWriter = typeof(ListVectorDataWriter).GetMethod( + "CreateCollectionWriter", + BindingFlags.Static | BindingFlags.NonPublic); + + var plan = createWriter!.Invoke(null, [values.GetType()]); + var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan); + + writer.Should().BeNull(); + } + [Fact] public void ListValuesEnum() { @@ -357,4 +464,6 @@ private enum TestEnum Test2 = 1, Test3 = 2, } -} \ No newline at end of file + + private sealed class DerivedIntList : List; +} diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs index 9186bd58..3497bb6e 100644 --- a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -185,4 +185,46 @@ public void MappedAppender_SupportsDefaultAndNull() reader.GetInt32(2).Should().Be(18); reader.IsDBNull(3).Should().BeTrue(); } + + public class NullableValue + { + public int Id { get; set; } + public int? Value { get; set; } + } + + public class NullableValueMap : DuckDBAppenderMap + { + public NullableValueMap() + { + Map(value => value.Id); + Map(value => value.Value); + } + } + + [Fact] + public void MappedAppender_CompiledWriterSupportsNullableValues() + { + Command.CommandText = "CREATE TABLE nullable_values(id INTEGER, value INTEGER);"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("nullable_values")) + { + appender.AppendRecords( + [ + new NullableValue { Id = 1, Value = 42 }, + new NullableValue { Id = 2, Value = null } + ]); + } + + Command.CommandText = "SELECT id, value FROM nullable_values ORDER BY id"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetInt32(1).Should().Be(42); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(2); + reader.IsDBNull(1).Should().BeTrue(); + } } From ddeecc0b8f16ac854f9c934db858950582f8149f Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Thu, 23 Jul 2026 20:50:44 +0100 Subject: [PATCH 32/33] Prepare 1.5.5 preview release --- .github/workflows/preview-release.yml | 13 +++++++------ README-PREVIEW.md | 2 +- README.md | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/preview-release.yml b/.github/workflows/preview-release.yml index 52002c0b..e2327f8b 100644 --- a/.github/workflows/preview-release.yml +++ b/.github/workflows/preview-release.yml @@ -11,10 +11,11 @@ permissions: jobs: publish-fork-package: if: >- - (github.event.release.prerelease == true && - startsWith(github.event.release.tag_name, 'v1.5.4-preview.')) || - (github.event.release.prerelease == false && - github.event.release.tag_name == 'v1.5.4') + startsWith(github.event.release.tag_name, 'v1.5.') && + ((github.event.release.prerelease == true && + contains(github.event.release.tag_name, '-preview.')) || + (github.event.release.prerelease == false && + !contains(github.event.release.tag_name, '-'))) runs-on: ubuntu-latest environment: nuget-preview @@ -31,10 +32,10 @@ jobs: version="${{ github.event.release.tag_name }}" version="${version#v}" - if [[ "$version" =~ ^1\.5\.4-preview\.[0-9]+$ ]]; then + if [[ "$version" =~ ^1\.5\.[0-9]+-preview\.[0-9]+$ ]]; then fork_preview=true fork_release=false - elif [[ "$version" == "1.5.4" ]]; then + elif [[ "$version" =~ ^1\.5\.[0-9]+$ ]]; then fork_preview=false fork_release=true else diff --git a/README-PREVIEW.md b/README-PREVIEW.md index a544244a..6a95482d 100644 --- a/README-PREVIEW.md +++ b/README-PREVIEW.md @@ -8,7 +8,7 @@ pull requests are under review. Install the bundled provider explicitly: ```shell -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.1 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5-preview.1 ``` NuGet packages: diff --git a/README.md b/README.md index c4bbae7c..c687e630 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ the corresponding upstream pull requests are reviewed and released. ## Usage ```sh -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.2 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5-preview.1 ``` The fork packages retain the official `DuckDB.NET.Data` namespaces and assembly From e2cc3ae37f77d8a8309a5357a3ef9dc1128ccef6 Mon Sep 17 00:00:00 2001 From: Skuirrels Date: Thu, 23 Jul 2026 20:54:44 +0100 Subject: [PATCH 33/33] Prepare 1.5.5 stable release --- README-FORK.md | 2 +- README-PREVIEW.md | 2 +- README.md | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README-FORK.md b/README-FORK.md index 0ddb544c..4a2fa48b 100644 --- a/README-FORK.md +++ b/README-FORK.md @@ -8,7 +8,7 @@ packages the consolidated performance work for DuckDB v1.5.4 under distinct Install the bundled provider explicitly: ```shell -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5 ``` NuGet packages: diff --git a/README-PREVIEW.md b/README-PREVIEW.md index 6a95482d..0e3b2dd2 100644 --- a/README-PREVIEW.md +++ b/README-PREVIEW.md @@ -8,7 +8,7 @@ pull requests are under review. Install the bundled provider explicitly: ```shell -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5-preview.1 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.2 ``` NuGet packages: diff --git a/README.md b/README.md index c687e630..b667f01a 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# DuckDB.NET - Performance fork preview +# DuckDB.NET - Performance fork This is the [`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). -It provides preview packages containing the cutting-edge performance work while +It provides stable fork packages containing the cutting-edge performance work while the corresponding upstream pull requests are reviewed and released. -[![NuGet (Data)](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Data.Full.svg?label=NuGet%20%28Data%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full) -[![NuGet (Bindings)](https://img.shields.io/nuget/vpre/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=NuGet%20%28Bindings%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full) +[![NuGet (Data)](https://img.shields.io/nuget/v/Skuirrels.DuckDB.NET.Data.Full.svg?label=NuGet%20%28Data%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full) +[![NuGet (Bindings)](https://img.shields.io/nuget/v/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=NuGet%20%28Bindings%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full) [![.NET 8 and 10](https://img.shields.io/badge/.NET-8%20%7C%2010-512BD4)](https://dotnet.microsoft.com/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md) ## Usage ```sh -dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5-preview.1 +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5 ``` The fork packages retain the official `DuckDB.NET.Data` namespaces and assembly