diff --git a/.github/workflows/NuGet.yml b/.github/workflows/NuGet.yml index 7533311e..99e2279a 100644 --- a/.github/workflows/NuGet.yml +++ b/.github/workflows/NuGet.yml @@ -9,7 +9,7 @@ jobs: steps: - name: Download Artifacts - uses: dawidd6/action-download-artifact@v6 + uses: dawidd6/action-download-artifact@v21 with: github_token: ${{ secrets.GITHUB_TOKEN }} workflow: ci.yml diff --git a/.github/workflows/Sonar.yml b/.github/workflows/Sonar.yml index 78a815bd..c2dc1d94 100644 --- a/.github/workflows/Sonar.yml +++ b/.github/workflows/Sonar.yml @@ -28,22 +28,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up JDK 11 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: java-version: 17 distribution: 'zulu' # Alternative distribution options are available. - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha || github.ref }} fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json - name: Cache SonarCloud packages - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ~/sonar/cache key: ${{ runner.os }}-sonar @@ -51,7 +51,7 @@ jobs: - name: Cache SonarCloud scanner id: cache-sonar-scanner - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: ./.sonar/scanner key: ${{ runner.os }}-sonar-scanner diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3420b0d0..a13fdc9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,13 @@ jobs: echo "NightlyBuild=true" >> $env:GITHUB_ENV - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 ref: ${{ env.BuildBranch }} - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json @@ -82,14 +82,14 @@ jobs: - name: Upload coverage reports to Codecov if: matrix.os == 'ubuntu-latest' && github.event_name != 'schedule' - uses: codecov/codecov-action@v4.0.1 + uses: codecov/codecov-action@v6 with: token: ${{ secrets.CODECOV_TOKEN }} slug: Giorgi/DuckDB.NET - name: Upload Artifacts if: matrix.os == 'ubuntu-latest' && github.event_name != 'pull_request' && github.actor == 'Giorgi' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: nugetPackages-${{github.ref_name}} path: ./**/bin/Release/*.nupkg @@ -103,7 +103,7 @@ jobs: if: github.ref == 'refs/heads/develop' && github.event_name != 'pull_request' && github.event_name != 'schedule' steps: - name: Download nuget package artifact - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: nugetPackages-${{github.ref_name}} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index a3754128..bbf76dd3 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -42,13 +42,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: fetch-depth: 0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -59,7 +59,7 @@ jobs: # queries: security-extended,security-and-quality - name: Setup .NET Core SDK - uses: actions/setup-dotnet@v4 + uses: actions/setup-dotnet@v5 with: global-json-file: global.json @@ -82,4 +82,4 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/preview-release.yml b/.github/workflows/preview-release.yml new file mode 100644 index 00000000..e2327f8b --- /dev/null +++ b/.github/workflows/preview-release.yml @@ -0,0 +1,152 @@ +name: Fork Package Release + +on: + release: + types: [published] + +permissions: + contents: write + id-token: write + +jobs: + publish-fork-package: + if: >- + startsWith(github.event.release.tag_name, 'v1.5.') && + ((github.event.release.prerelease == true && + contains(github.event.release.tag_name, '-preview.')) || + (github.event.release.prerelease == false && + !contains(github.event.release.tag_name, '-'))) + runs-on: ubuntu-latest + environment: nuget-preview + + steps: + - name: Checkout release tag + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ github.event.release.tag_name }} + + - name: Set package version and release mode + shell: bash + run: | + version="${{ github.event.release.tag_name }}" + version="${version#v}" + + if [[ "$version" =~ ^1\.5\.[0-9]+-preview\.[0-9]+$ ]]; then + fork_preview=true + fork_release=false + elif [[ "$version" =~ ^1\.5\.[0-9]+$ ]]; then + fork_preview=false + fork_release=true + else + echo "Unexpected fork package version: $version" >&2 + exit 1 + fi + + { + echo "PACKAGE_VERSION=$version" + echo "FORK_PREVIEW=$fork_preview" + echo "FORK_RELEASE=$fork_release" + } >> "$GITHUB_ENV" + + - name: Setup .NET SDK + uses: actions/setup-dotnet@v5 + with: + global-json-file: global.json + + - name: Restore + run: dotnet restore DuckDB.NET.slnx /p:CI=false + + - name: Build + run: >- + dotnet build DuckDB.NET.slnx + --configuration Release + --no-restore + /m:1 + /p:BuildType=Full + /p:CI=false + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" + /p:UseSharedCompilation=false + + - name: Test + run: >- + dotnet test DuckDB.NET.Test/Test.csproj + --configuration Release + --no-build + --no-restore + --logger "console;verbosity=quiet" + /p:BuildType=Full + /p:CI=false + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:DoesNotReturnAttribute=DoesNotReturnAttribute + + - name: Pack fork packages + shell: bash + run: | + mkdir -p artifacts/fork-release + common_args=( + --configuration Release + --no-build + --no-restore + --output artifacts/fork-release + /m:1 + /p:BuildType=Full + /p:CI=false + /p:ForkPreview="$FORK_PREVIEW" + /p:ForkRelease="$FORK_RELEASE" + /p:ForkPack=true + /p:Version="$PACKAGE_VERSION" + /p:PackageVersion="$PACKAGE_VERSION" + ) + dotnet pack DuckDB.NET.Bindings/Bindings.csproj "${common_args[@]}" + + dotnet restore DuckDB.NET.Data/Data.csproj \ + --source artifacts/fork-release \ + --source https://api.nuget.org/v3/index.json \ + /p:BuildType=Full \ + /p:CI=false \ + /p:ForkPreview="$FORK_PREVIEW" \ + /p:ForkRelease="$FORK_RELEASE" \ + /p:ForkPack=true \ + /p:Version="$PACKAGE_VERSION" \ + /p:PackageVersion="$PACKAGE_VERSION" + + dotnet pack DuckDB.NET.Data/Data.csproj "${common_args[@]}" + + - name: Validate and smoke-test packages + run: ./scripts/validate-fork-packages.sh artifacts/fork-release "$PACKAGE_VERSION" + + - name: Generate checksums + working-directory: artifacts/fork-release + run: sha256sum *.nupkg > SHA256SUMS + + - name: Upload workflow artifacts + uses: actions/upload-artifact@v7 + with: + name: duckdb-net-${{ github.event.release.tag_name }} + path: | + artifacts/fork-release/*.nupkg + artifacts/fork-release/SHA256SUMS + if-no-files-found: error + + - name: Attach packages to GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: gh release upload "${{ github.event.release.tag_name }}" artifacts/fork-release/* --clobber + + - name: Authenticate to NuGet.org + uses: NuGet/login@v1 + id: nuget-login + with: + user: skuirrels + + - name: Publish fork packages to NuGet.org + run: >- + dotnet nuget push "artifacts/fork-release/*.nupkg" + --api-key "${{ steps.nuget-login.outputs.NUGET_API_KEY }}" + --source https://api.nuget.org/v3/index.json + --skip-duplicate diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 967a53f5..f1c04ef6 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -34,12 +34,12 @@ jobs: steps: - name: "Checkout code" - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif @@ -64,7 +64,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: SARIF file path: results.sarif @@ -73,6 +73,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@v4 with: sarif_file: results.sarif diff --git a/Directory.Build.props b/Directory.Build.props index d00d035c..603223b8 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -19,11 +19,15 @@ Giorgi Dalakishvili Copyright © 2020 - $(Year) Giorgi Dalakishvili + true + Skuirrels. DuckDB.NET.$(MSBuildProjectName) - DuckDB.NET.$(MSBuildProjectName) + $(ForkPackagePrefix)DuckDB.NET.$(MSBuildProjectName) DuckDB;ADO.NET;Database;Olap;Embedded Logo.jpg README.md + README-PREVIEW.md + README-FORK.md MIT true @@ -38,6 +42,11 @@ + + https://github.com/skuirrels/DuckDB.NET + https://github.com/skuirrels/DuckDB.NET + + ManagedOnly This package does not include a copy of the native DuckDB library. @@ -63,8 +72,27 @@ True + + True + + + + True + + + + + + + + + false diff --git a/Driver-Benchmark-Comparison-1.5.4-pre.md b/Driver-Benchmark-Comparison-1.5.4-pre.md new file mode 100644 index 00000000..661306f8 --- /dev/null +++ b/Driver-Benchmark-Comparison-1.5.4-pre.md @@ -0,0 +1,191 @@ +# DuckDB driver benchmark comparison: consolidated fork 1.5.4 + +## Current status + +**Updated 2026-07-21:** `feature/appender-scoped-writer` is included in the +local `release/fork-1.5.4` branch at commit +`2b04e5e970b1610537f362e1cff435e0f86ee712`. The complete five-lane comparison +was rerun as `20260721T172000Z-appender-scoped-consolidated-rerun`. + +The accepted run completed 13 released DuckDB.NET methods, 14 EFCoreProvider +methods, 18 consolidated-fork methods, 12 Java methods, and 12 Go methods (ten +Go repetitions each). The consolidation lane includes both the compatible +`CreateRow().EndRow()` path and the new allocation-free `AppendRowScoped` path. +Stock .NET `develop` is not included. + +| Real-work benchmark | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Updated winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Parameterized analytics, prepared | 2.618 ms | 2.609 ms* | **2.498 ms** | 2.522 ms | 2.640 ms | **Consolidated fork** | +| Materialize 100,000 mixed rows | 9.931 ms | **9.928 ms*** | 10.180 ms | 54.588 ms | 28.105 ms | **EFCoreProvider** | +| Reusable prepared insert | 54.762 us/row† | 53.697 us/row*† | **19.517 us/row** | 26.111 us/row | 29.275 us/row | **Consolidated fork** | +| Idiomatic high-throughput insert | 259.7 ns/row | 276.0 ns/row `BulkInsert`* | **79.48 ns/row `AppendRowScoped`** | 174.25 ns/row | 183.46 ns/row | **Consolidated fork** | +| TPC-H Q1, SF 0.1 | 13.9593 ms | 13.7994 ms* | **13.7020 ms** | 14.0699 ms | 14.2022 ms | **Consolidated fork** | +| TPC-H Q6, SF 0.1 | 0.9579 ms | 0.9563 ms* | **0.9122 ms** | 0.9761 ms | 0.9914 ms | **Consolidated fork** | +| TPC-H Q12, SF 0.1 | 7.8996 ms | **7.6711 ms*** | 7.7706 ms | 7.8124 ms | 7.8385 ms | **EFCoreProvider** | +| TPC-H Q14, SF 0.1 | 1.5953 ms | 1.5309 ms* | **1.5235 ms** | 1.6955 ms | 1.5905 ms | **Consolidated fork** | + +`*` Except for its public `BulkInsert`, the EF provider lane executes the same +low-level benchmark code through its transitive `DuckDB.NET.Data.Full` 1.5.3 +dependency. These cells do not measure EF LINQ translation or change tracking. + +`†` `DuckDBCommand.Prepare()` is a no-op in the released and EF dependency .NET +lanes. It does not create a native prepared statement in those lanes. + +The raw winner changed materially: the consolidated fork now wins six of eight +real-work rows. EFCoreProvider has the lowest point estimate for materialization +and Q12, but those should be understood as DuckDB.NET 1.5.3/native-engine +results rather than provider-specific wins. The 0.03% materialization gap +between the released and EF lanes is effectively a tie. + +## Appender optimization result + +| Fork appender path | Latency | Throughput | Managed allocation | +| --- | ---: | ---: | ---: | +| Compatible `CreateRow().EndRow()` | 86.47 ns/row | 11.56 M rows/s | 64 B/row | +| New `AppendRowScoped` | **79.48 ns/row** | **12.58 M rows/s** | **0 B/row** | +| Java JDBC Appender | 174.25 ns/row | 5.74 M rows/s | Not comparable | +| Go Appender | 183.46 ns/row | 5.45 M rows/s | Not comparable | + +The scoped writer is 8.1% faster than the compatible optimized path and removes +the remaining managed row allocation. Against the other drivers, it has 54.4% +lower latency than Java and 56.7% lower latency than Go. Put another way, this +run measured 2.19x Java's and 2.31x Go's row throughput. + +The compatible path also benefits substantially from the branch: its 86.47 +ns/row is 65% below the consolidated fork's previous 247.7 ns/row point +estimate. This cross-run delta is large, but the same-run scoped-versus-compatible +comparison is the stronger evidence for the incremental scoped API benefit. + +## Prepared scalar microbenchmark + +This isolates command creation, parameter binding, execution, and one returned +value. It is not a full application workload. + +| Operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Unprepared execution | 66.11 us | 64.18 us* | 62.04 us | 69.44 us | **55.03 us** | **Go** | +| Prepared execution | 65.80 us† | 67.11 us*† | 24.32 us | 24.27 us | **17.68 us** | **Go** | +| Create and prepare | 0.090 us† | 0.084 us*† | **33.03 us** | 36.50 us | 33.42 us | **Consolidated fork** | + +The approximately 84-90 ns setup results in the released and EF dependency +lanes only allocate a command object and call the no-op `Prepare()`. They are +not comparable with the fork, Java, and Go native preparation results. + +## Parameterized analytical query + +The workload filters and aggregates a deterministic 2,000,000-row table, +groups and orders the result, and consumes every returned value. + +| Operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Unprepared | 2.618 ms | 2.594 ms* | **2.553 ms** | 2.685 ms | 2.721 ms | **Consolidated fork** | +| Prepared/reused | 2.618 ms† | 2.609 ms*† | **2.498 ms** | 2.522 ms | 2.640 ms | **Consolidated fork** | + +The prepared fork/Java difference is only 1.0%, so this ordering is directional +without alternating repeated runs. + +## Result materialization + +Each lane fully reads 100,000 ordered rows containing `BIGINT`, `DATE`, +`TIMESTAMP`, `DOUBLE`, nullable `VARCHAR`, and `BOOLEAN`, while calculating a +checksum. + +| Metric | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Latency | 9.931 ms | **9.928 ms*** | 10.180 ms | 54.588 ms | 28.105 ms | **EFCoreProvider** | +| Throughput | 10.07 M rows/s | **10.07 M rows/s*** | 9.82 M rows/s | 1.83 M rows/s | 3.56 M rows/s | **EFCoreProvider** | + +The .NET results are within 2.5%. The meaningful result is the clear .NET lead +over Go and Java in this typed materialization workload, not the tiny ordering +among the closely related .NET lanes. This workload contains no MAP or LIST +columns, so it does not exercise the fork's MAP/LIST materialization work. + +## Bulk ingestion + +Each invocation writes 10,000 precomputed mixed-type rows inside an explicit +transaction and rolls the transaction back. + +| Metric | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Prepared insert latency | 54.762 us/row† | 53.697 us/row*† | **19.517 us/row** | 26.111 us/row | 29.275 us/row | **Consolidated fork** | +| Prepared insert throughput | 18,261 rows/s† | 18,623 rows/s*† | **51,238 rows/s** | 38,298 rows/s | 34,159 rows/s | **Consolidated fork** | +| Idiomatic Appender latency | 259.7 ns/row | 276.0 ns/row `BulkInsert`* | **79.48 ns/row scoped** | 174.25 ns/row | 183.46 ns/row | **Consolidated fork** | +| Idiomatic Appender throughput | 3.85 M rows/s | 3.62 M rows/s `BulkInsert`* | **12.58 M rows/s scoped** | 5.74 M rows/s | 5.45 M rows/s | **Consolidated fork** | +| Direct compatible Appender | 259.7 ns/row | 237.8 ns/row* | **86.47 ns/row** | 174.25 ns/row | 183.46 ns/row | **Consolidated fork** | + +EFCoreProvider's public `BulkInsert` is 16.1% slower than directly calling the +Appender bundled in its DuckDB.NET 1.5.3 dependency (276.0 versus 237.8 +ns/row). The public API result is the appropriate idiomatic provider figure. + +## TPC-H analytical queries + +The suite generates scale factor 0.1 using DuckDB's `tpch` extension, then +executes and fully consumes Q1, Q6, Q12, and Q14. Every connection uses one +DuckDB thread. + +| Query | DuckDB.NET 1.5.3 | EFCoreProvider 1.13.0 | Consolidated fork 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Q1 | 13.9593 ms | 13.7994 ms* | **13.7020 ms** | 14.0699 ms | 14.2022 ms | **Consolidated fork** | +| Q6 | 0.9579 ms | 0.9563 ms* | **0.9122 ms** | 0.9761 ms | 0.9914 ms | **Consolidated fork** | +| Q12 | 7.8996 ms | **7.6711 ms*** | 7.7706 ms | 7.8124 ms | 7.8385 ms | **EFCoreProvider** | +| Q14 | 1.5953 ms | 1.5309 ms* | **1.5235 ms** | 1.6955 ms | 1.5905 ms | **Consolidated fork** | + +These are primarily native-engine workloads. Most gaps are small enough to +require repeated alternating runs before attributing them to a wrapper. Q12's +raw EF result should likewise not be interpreted as an EF provider advantage. + +## Packages and environment + +- macOS 26.5.1, Apple M4 Pro, arm64; +- .NET SDK 10.0.300; +- Java 26.0.1 and `org.duckdb:duckdb_jdbc:1.5.4.0`; +- Go 1.26.5 and `github.com/duckdb/duckdb-go/v2` v2.10504.0; +- released `DuckDB.NET.Data.Full` 1.5.3; +- `DuckDB.EFCoreProvider` 1.13.0, transitively using DuckDB.NET 1.5.3 and + Entity Framework Core 10.0.10; +- consolidated DuckDB.NET fork 1.5.4 at release commit `2b04e5e`; +- one DuckDB thread per connection. + +The released .NET and EF dependency lanes use native DuckDB 1.5.3. The +consolidated fork, Java, and Go use native DuckDB 1.5.4. This is therefore a +package-level comparison rather than a wrapper-only comparison against one +identical native library. + +For reproducibility, the benchmark ran from harness commit `a73a0eb`. Its tree +was verified byte-identical to isolated tree `b8a9257`, constructed from +release commit `2b04e5e` plus the benchmark-only commits. Java JMH was rerun +outside the filesystem sandbox because JMH requires a local coordinator socket; +its benchmark configuration was unchanged. + +## Methodology and limitations + +- One in-memory database and connection are used per benchmark instance. +- Data generation, extension installation, and input construction occur outside + measured methods. +- All returned rows and columns are consumed. +- All five lanes ran sequentially on the same machine under one accepted run + configuration; Java alone required the local-socket permission noted above. +- Go values are arithmetic means of ten repetitions. BenchmarkDotNet and JMH + values are their reported arithmetic means. +- Cross-runtime allocation counts are not compared because .NET, JVM, and Go + allocations have different representations and costs. +- BenchmarkDotNet process-priority warnings and Java's `Unsafe` deprecation + warnings did not invalidate the completed measurements. +- No failed benchmark remains in the accepted result set. The earlier partial + `20260721T160513Z-appender-scoped-consolidated` run is not used. + +## Raw reports + +- [Environment](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/environment.txt) +- [DuckDB.NET 1.5.3 reports](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/dotnet-1.5.3/results/) +- [EFCoreProvider 1.13.0 reports](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/dotnet-efcore-1.13.0/results/) +- [Consolidated fork reports](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/dotnet-fork-1.5.4/results/) +- [Java JMH JSON](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/java/results.json) +- [Go benchmark output](BenchmarkDotNet.Artifacts/DriverComparison/20260721T172000Z-appender-scoped-consolidated-rerun/go.txt) + +## Recommended follow-up + +Repeat the five lanes three to five times on an idle, fixed-power machine while +alternating their order, then report medians and variation. If the goal is to +isolate managed-wrapper performance, also run every lane against exactly the +same native DuckDB build. diff --git a/Driver-Benchmark-Comparison.md b/Driver-Benchmark-Comparison.md new file mode 100644 index 00000000..ee0c8f62 --- /dev/null +++ b/Driver-Benchmark-Comparison.md @@ -0,0 +1,303 @@ +# DuckDB .NET, EF Core, Go, and Java benchmark comparison + +## Technical summary + +- **Tuned local DuckDB.NET 1.5.4 wins reusable prepared insertion** at + 19.931 us/row, 2.64x faster than both released DuckDB.NET 1.5.3 and the + ADO.NET driver bundled by `DuckDB.EFCoreProvider` 1.9.0. +- **The EF provider's real public `DbContext.BulkInsert` path measures + 261.3 ns/row.** That is 4.8% slower than accessing its bundled Appender + directly. +- **Java wins high-throughput Appender ingestion** at 168.3 ns/row, with Go + close behind at 172.7 ns/row. +- **The three .NET lanes are effectively tied on mixed-type result + materialization** and materially faster than Go and Java in this run. + +## Key findings: side-by-side results + +Results are from run `20260716T211118Z` on an Apple M4 Pro. Lower latency is +better; higher throughput is better. Values are harness-reported means, with +the arithmetic mean of ten reported repetitions used for Go. + +| Real-work benchmark | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Parameterized analytical query, prepared | **2.417 ms** | 2.422 ms* | 2.428 ms | 2.431 ms | 2.526 ms | **DuckDB.NET 1.5.3** | +| Materialize 100,000 mixed-type rows | **9.347 ms** | 9.497 ms* | 9.470 ms | 52.774 ms | 26.292 ms | **DuckDB.NET 1.5.3** | +| Insert with reusable prepared statement | 52.573 us/row | 52.604 us/row* | **19.931 us/row** | 25.508 us/row | 28.906 us/row | **Tuned .NET 1.5.4** | +| Idiomatic high-throughput insert | 245.3 ns/row Appender | 261.3 ns/row `BulkInsert` | 242.4 ns/row Appender | **168.3 ns/row Appender** | 172.7 ns/row Appender | **Java JDBC 1.5.4** | +| TPC-H Q1, SF 0.1 | 13.1018 ms | **13.0752 ms*** | 13.1425 ms | 13.0767 ms | 13.3362 ms | **EF provider dependency path, effectively tied with Java** | +| TPC-H Q6, SF 0.1 | **0.8619 ms** | 0.8660 ms* | 0.8778 ms | 0.8755 ms | 0.8850 ms | **DuckDB.NET 1.5.3** | +| TPC-H Q12, SF 0.1 | 7.2306 ms | 7.2937 ms* | 7.2821 ms | **7.1969 ms** | 7.3073 ms | **Java JDBC 1.5.4** | +| TPC-H Q14, SF 0.1 | 1.4774 ms | 1.4779 ms* | **1.4642 ms** | 1.5230 ms | 1.5207 ms | **Tuned .NET 1.5.4** | + +## Scope, packages, and metric definitions + +The suite compares five package-level lanes: + +- released `DuckDB.NET.Data.Full` 1.5.3; +- `DuckDB.EFCoreProvider` 1.9.0; +- the locally performance-tuned DuckDB.NET 1.5.4 source in this checkout; +- DuckDB's core-team-maintained `org.duckdb:duckdb_jdbc` 1.5.4.0 driver; +- `github.com/duckdb/duckdb-go/v2` v2.10504.0, using DuckDB 1.5.4. + +`DuckDB.EFCoreProvider` 1.9.0 has a transitive dependency on +`DuckDB.NET.Data.Full` 1.5.3. The EF provider column therefore contains two +clearly separated kinds of measurements: + +- **Public EF provider path:** `DbContext.BulkInsert`, which is the provider's + appender-backed bulk-ingestion API. +- **Bundled ADO.NET dependency path:** the identical SQL, prepared-command, + materialization, and TPC-H workloads executed through the provider package's + bundled DuckDB.NET 1.5.3 dependency. These cells are marked with `*`. + +The report does not claim that the `*` cells measure EF LINQ translation or EF +change tracking. They show the performance an application receives from the +ADO.NET driver bundled by `DuckDB.EFCoreProvider` 1.9.0. + +## Parameterized analytical query + +This query filters and aggregates a deterministic 2,000,000-row orders table, +groups by year and region, orders the result, and consumes every returned +value. + +| Measured operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Unprepared | **2.377 ms** | 2.421 ms* | 2.501 ms | 2.487 ms | 2.565 ms | **DuckDB.NET 1.5.3** | +| Prepared and reused | **2.417 ms** | 2.422 ms* | 2.428 ms | 2.431 ms | 2.526 ms | **DuckDB.NET 1.5.3** | +| Prepared/unprepared ratio | 1.017x | 1.000x* | 0.971x | 0.977x | 0.985x | **No meaningful winner; scan and aggregation dominate** | + +All prepared results are within 4.5% of one another. Preparation materially +helps the tuned local, Java, and Go scalar paths, but it is not the dominant +cost once the query scans and aggregates two million rows. + +## Result materialization + +All providers fully read the same 100,000 ordered rows containing `BIGINT`, +`DATE`, `TIMESTAMP`, `DOUBLE`, nullable `VARCHAR`, and `BOOLEAN`, while building +a checksum so values cannot be skipped. + +| Measured operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Mean latency | **9.347 ms** | 9.497 ms* | 9.470 ms | 52.774 ms | 26.292 ms | **DuckDB.NET 1.5.3** | +| Materialization throughput | **10.70 million rows/s** | 10.53 million rows/s* | 10.56 million rows/s | 1.89 million rows/s | 3.80 million rows/s | **DuckDB.NET 1.5.3** | + +The three .NET observations are within 1.6%, so they should be considered +effectively tied. The meaningful result is that this typed ADO.NET +materialization path is about 2.8x faster than Go and 5.6x faster than Java in +this run. + +## Bulk ingestion + +Each invocation writes the same 10,000 precomputed mixed-type rows inside an +explicit transaction and rolls that transaction back. Input creation and +provider metadata warmup happen outside the measurement. + +### Equivalent ingestion paths + +| Measured operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Prepared insert latency | 52.573 us/row | 52.604 us/row* | **19.931 us/row** | 25.508 us/row | 28.906 us/row | **Tuned .NET 1.5.4** | +| Prepared insert throughput | 19,021 rows/s | 19,010 rows/s* | **50,173 rows/s** | 39,204 rows/s | 34,595 rows/s | **Tuned .NET 1.5.4** | +| Direct Appender latency | 245.3 ns/row | 249.4 ns/row* | 242.4 ns/row | **168.3 ns/row** | 172.7 ns/row | **Java JDBC 1.5.4** | +| Direct Appender throughput | 4.08 million rows/s | 4.01 million rows/s* | 4.13 million rows/s | **5.94 million rows/s** | 5.79 million rows/s | **Java JDBC 1.5.4** | +| Idiomatic bulk API latency | 245.3 ns/row Appender | 261.3 ns/row `BulkInsert` | 242.4 ns/row Appender | **168.3 ns/row Appender** | 172.7 ns/row Appender | **Java JDBC 1.5.4** | +| Idiomatic bulk API throughput | 4.08 million rows/s | 3.83 million rows/s `BulkInsert` | 4.13 million rows/s | **5.94 million rows/s** | 5.79 million rows/s | **Java JDBC 1.5.4** | + +The tuned local prepared insert is 2.64x faster than both the released +DuckDB.NET package and the ADO.NET dependency bundled with the EF provider. It +is 1.28x faster than Java and 1.45x faster than Go. + +Java has the lowest Appender latency, with Go 2.6% behind it. The direct .NET +Appender results are close to one another, confirming that the prepared-command +tuning is not responsible for Appender performance. + +### EF provider `BulkInsert` overhead + +This isolates the extra public-provider layer by comparing two measurements +from the same EF provider benchmark project and the same bundled DuckDB.NET +1.5.3 dependency. + +| Measurement | Direct bundled Appender | Public EF `DbContext.BulkInsert` | Difference | Winner | +| --- | ---: | ---: | ---: | --- | +| Latency per row | **249.4 ns** | 261.3 ns | `BulkInsert` is 11.9 ns, or 4.8%, slower | **Direct bundled Appender** | +| Throughput | **4.01 million rows/s** | 3.83 million rows/s | `BulkInsert` processes about 4.6% fewer rows/s | **Direct bundled Appender** | + +The EF provider's public bulk path adds only 4.8% latency over the direct +Appender dependency path in this warmed 10,000-row scenario. + +## TPC-H analytical queries + +The suite uses DuckDB's official `tpch` extension to generate scale factor 0.1 +during setup, then executes and fully consumes Q1, Q6, Q12, and Q14. Every +connection uses one DuckDB thread. + +| Query | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Q1 | 13.1018 ms | **13.0752 ms*** | 13.1425 ms | 13.0767 ms | 13.3362 ms | **EF provider dependency path, effectively tied with Java** | +| Q6 | **0.8619 ms** | 0.8660 ms* | 0.8778 ms | 0.8755 ms | 0.8850 ms | **DuckDB.NET 1.5.3** | +| Q12 | 7.2306 ms | 7.2937 ms* | 7.2821 ms | **7.1969 ms** | 7.3073 ms | **Java JDBC 1.5.4** | +| Q14 | 1.4774 ms | 1.4779 ms* | **1.4642 ms** | 1.5230 ms | 1.5207 ms | **Tuned .NET 1.5.4** | + +The winners change by query and the gaps are small. Q1's first two observations +differ by only 0.0015 ms. Treat these as essentially comparable engine-query +results rather than evidence that one wrapper is universally faster. + +## Prepared scalar microbenchmark + +The three-`BIGINT` scalar query isolates statement and parameter-binding +overhead. It is intentionally not presented as a complete application +workload. + +| Measured operation | DuckDB.NET 1.5.3 | EFCoreProvider 1.9.0 | Tuned .NET 1.5.4 | Java JDBC 1.5.4 | Go 1.5.4 | Winner | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| Unprepared execution | 63.25 us | 63.03 us* | 64.74 us | 67.77 us | **54.04 us** | **Go 1.5.4** | +| Prepared execution | 64.05 us | 62.23 us* | 21.47 us | 23.43 us | **17.71 us** | **Go 1.5.4** | +| Speedup from preparing within provider | 0.99x | 1.01x* | 3.02x | 2.89x | **3.05x** | **Go 1.5.4** | +| Create and prepare | 0.080 us, no-op | 0.079 us, no-op* | **32.396 us** | 36.005 us | 32.482 us | **Tuned .NET 1.5.4 among real prepare paths** | + +`DuckDBCommand.Prepare()` is a no-op in DuckDB.NET 1.5.3, including the copy +bundled with the EF provider. Their setup timings are therefore not comparable +with the three paths that perform real native preparation. + +## What this run establishes + +- **Tuned local DuckDB.NET 1.5.4 wins the reusable prepared-insert workload.** + It is 2.64x faster than the released .NET 1.5.3 path and the ADO.NET driver + bundled with `DuckDB.EFCoreProvider` 1.9.0. +- **`DuckDB.EFCoreProvider` has a small bulk-layer cost.** Its public + `DbContext.BulkInsert` is 4.8% slower than using its bundled Appender + directly: 261.3 versus 249.4 ns/row. +- **Java wins Appender ingestion, closely followed by Go.** Java measured + 168.3 ns/row and Go measured 172.7 ns/row. +- **The .NET lanes lead mixed-type result materialization.** Their results are + within 1.6% of each other and materially ahead of Go and Java in this run. +- **Scan-heavy analytics and TPC-H are broadly close.** Small winner changes + should not be interpreted as universal wrapper advantages. + +## Methodology and workload contract + +| Workload | Data and measured boundary | +| --- | --- | +| Scalar execution | Reused command/statement; bind one changing `BIGINT`; execute and read scalar | +| Parameterized analytics | Filter, group, aggregate, and order 2,000,000 orders; fully read result | +| Materialization | Read all six typed columns from 100,000 ordered rows and checksum values | +| Prepared ingestion | Bind and execute 10,000 precomputed rows inside one transaction | +| Appender ingestion | Append the same 10,000 rows inside one transaction | +| EF bulk ingestion | Call `DbContext.BulkInsert` for the same 10,000 mapped rows inside one transaction | +| TPC-H | Generate SF 0.1 in setup; execute and fully consume Q1, Q6, Q12, and Q14 | + +Common controls: + +- one in-memory database and one connection per benchmark instance; +- `SET threads = 1` in every provider; +- deterministic data generation before measurement; +- setup, extension installation, TPC-H generation, and input construction are + outside measured methods; +- all returned rows and columns are consumed; +- ten measured results per workload after warmup; +- benchmark setup verifies the expected package or native engine version. + +JDBC has no unprepared parameterized `Statement` API. Its unprepared lane +creates, binds, executes, reads, and closes a `PreparedStatement` per operation, +which is the closest public-API equivalent. + +The released .NET package and EF provider dependency use native DuckDB 1.5.3. +The tuned local .NET, Java, and Go lanes use native DuckDB 1.5.4. This is a real +package-to-package comparison, not a wrapper-only comparison against one +identical native library. + +## Limitations and robustness + +- The machine was not otherwise idle, and the provider lanes ran sequentially + rather than in an alternating order. +- The package comparison intentionally includes two native engine versions: + DuckDB 1.5.3 for released .NET and the EF provider dependency, and DuckDB + 1.5.4 for tuned local .NET, Java, and Go. +- Large effects such as the 2.64x prepared-insert improvement are useful + signals. Single-digit percentage differences, including the EF `BulkInsert` + overhead, should be confirmed with alternating runs on an idle, + fixed-power machine. +- Cross-runtime allocation counts are not comparable because .NET, JVM, and Go + allocations have different representations and costs. + +## Recommended next steps + +1. Repeat the complete five-lane run three to five times on an idle machine, + alternating provider order, and report the median plus variation. +2. Add a separate EF-specific application benchmark for LINQ query translation + and materialization. Keep it separate from the exact-SQL table because it + measures a different abstraction boundary. +3. Add an EF `SaveChanges` ingestion benchmark if that is a production usage + pattern. Compare it with `BulkInsert`, but do not present it as equivalent + to a direct Appender. +4. Run a wrapper-only comparison against the same native DuckDB version if the + goal changes from real package performance to isolating managed-driver + overhead. + +## Further questions + +- Does the target EF application primarily use `BulkInsert`, `SaveChanges`, or + raw SQL through the provider? +- Should the next report optimize for real package versions, as this one does, + or pin every lane to one native DuckDB engine for wrapper-only attribution? + +## Projects + +- `DuckDB.NET.1_5_3.Benchmarks` benchmarks released + `DuckDB.NET.Data.Full` 1.5.3. +- `DuckDB.EFCoreProvider.1_9_0.Benchmarks` benchmarks + `DuckDB.EFCoreProvider` 1.9.0, including its public `BulkInsert` API. +- `DuckDB.NET.Benchmarks` benchmarks the locally tuned DuckDB.NET 1.5.4 + projects in this checkout. +- `DuckDB.Java.Benchmarks` uses JMH against + `org.duckdb:duckdb_jdbc:1.5.4.0`. +- `DuckDB.Go.Benchmarks` uses Go's `testing.B` against + `github.com/duckdb/duckdb-go/v2` v2.10504.0. + +## Raw reports for this run + +- [Benchmark environment](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/environment.txt) +- [DuckDB.NET 1.5.3 analytical query](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-1.5.3/results/DuckDB.NET.Benchmarks.AnalyticalQueryBenchmark-report-github.md) +- [DuckDB.NET 1.5.3 materialization](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-1.5.3/results/DuckDB.NET.Benchmarks.ResultMaterializationBenchmark-report-github.md) +- [DuckDB.NET 1.5.3 ingestion](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-1.5.3/results/DuckDB.NET.Benchmarks.BulkIngestionBenchmark-report-github.md) +- [DuckDB.NET 1.5.3 TPC-H](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-1.5.3/results/DuckDB.NET.Benchmarks.TpchBenchmark-report-github.md) +- [EF provider dependency analytical query](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-efcore-1.9.0/results/DuckDB.NET.Benchmarks.AnalyticalQueryBenchmark-report-github.md) +- [EF provider dependency materialization](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-efcore-1.9.0/results/DuckDB.NET.Benchmarks.ResultMaterializationBenchmark-report-github.md) +- [EF provider dependency ingestion](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-efcore-1.9.0/results/DuckDB.NET.Benchmarks.BulkIngestionBenchmark-report-github.md) +- [EF provider public `BulkInsert`](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-efcore-1.9.0/results/DuckDB.NET.Benchmarks.EfCoreProviderBulkIngestionBenchmark-report-github.md) +- [EF provider dependency TPC-H](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-efcore-1.9.0/results/DuckDB.NET.Benchmarks.TpchBenchmark-report-github.md) +- [Tuned local DuckDB.NET 1.5.4 analytical query](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-local-1.5.4/results/DuckDB.NET.Benchmarks.AnalyticalQueryBenchmark-report-github.md) +- [Tuned local DuckDB.NET 1.5.4 materialization](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-local-1.5.4/results/DuckDB.NET.Benchmarks.ResultMaterializationBenchmark-report-github.md) +- [Tuned local DuckDB.NET 1.5.4 ingestion](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-local-1.5.4/results/DuckDB.NET.Benchmarks.BulkIngestionBenchmark-report-github.md) +- [Tuned local DuckDB.NET 1.5.4 TPC-H](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/dotnet-local-1.5.4/results/DuckDB.NET.Benchmarks.TpchBenchmark-report-github.md) +- [Java JMH text](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/java.txt) +- [Java JMH JSON](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/java/results.json) +- [Go benchmark output](BenchmarkDotNet.Artifacts/DriverComparison/20260716T211118Z/go.txt) + +## Run + +From the repository root: + +```bash +./run-driver-comparison.sh +``` + +Results are written beneath +`BenchmarkDotNet.Artifacts/DriverComparison//`, separated into +`dotnet-1.5.3`, `dotnet-efcore-1.9.0`, `dotnet-local-1.5.4`, Java, and Go +outputs. Each .NET lane uses one launch with five warmup and ten measured +iterations. Java uses one JMH fork with five warmup and ten measured 500 ms +iterations plus the GC profiler. Go uses ten benchmark repetitions at 500 ms +per workload. + +For repeated runs after successful Release builds: + +```bash +SKIP_DOTNET_BUILD=1 SKIP_JAVA_BUILD=1 ./run-driver-comparison.sh +``` + +Run on an otherwise idle machine with fixed power and thermal settings, and +compare several alternating provider orders for publishable claims. Elapsed +time is the meaningful cross-language comparison. Allocation counts are useful +within each runtime but should not be compared as if .NET, JVM, and Go +allocations had equal cost. diff --git a/DuckDB.EFCoreProvider.1_13_0.Benchmarks/DuckDB.EFCoreProvider.1_13_0.Benchmarks.csproj b/DuckDB.EFCoreProvider.1_13_0.Benchmarks/DuckDB.EFCoreProvider.1_13_0.Benchmarks.csproj new file mode 100644 index 00000000..768c7ff9 --- /dev/null +++ b/DuckDB.EFCoreProvider.1_13_0.Benchmarks/DuckDB.EFCoreProvider.1_13_0.Benchmarks.csproj @@ -0,0 +1,28 @@ + + + + Exe + net10.0 + enable + enable + true + DuckDB.EFCoreProvider.1_13_0.Benchmarks + DuckDB.NET.Benchmarks + $(DefineConstants);DUCKDB_NET_BASELINE_1_5_3;DUCKDB_EFCORE_PROVIDER_1_13_0 + + + + + + + + + + + + + + + + + diff --git a/DuckDB.EFCoreProvider.1_13_0.Benchmarks/EfCoreProviderBulkIngestionBenchmark.cs b/DuckDB.EFCoreProvider.1_13_0.Benchmarks/EfCoreProviderBulkIngestionBenchmark.cs new file mode 100644 index 00000000..bb99f5f6 --- /dev/null +++ b/DuckDB.EFCoreProvider.1_13_0.Benchmarks/EfCoreProviderBulkIngestionBenchmark.cs @@ -0,0 +1,106 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.EFCoreProvider.Extensions; +using DuckDB.NET.Data; +using Microsoft.EntityFrameworkCore; +using System.Reflection; + +namespace DuckDB.NET.Benchmarks; + +/// +/// Measures DuckDB.EFCoreProvider's public appender-backed BulkInsert API over +/// the same 10,000-row transaction used by the driver ingestion benchmarks. +/// +[MemoryDiagnoser] +public class EfCoreProviderBulkIngestionBenchmark +{ + private DuckDBConnection connection = null!; + private EfCoreProviderBenchmarkContext context = null!; + private EfCoreIngestRow[] rows = null!; + + [GlobalSetup] + public void Setup() + { + VerifyProviderVersion(); + + connection = PreparedCommandWorkload.OpenVerifiedConnection(); + RealisticWorkload.InitializeIngest(connection); + + var options = new DbContextOptionsBuilder() + .UseDuckDB(connection, contextOwnsConnection: false) + .Options; + + context = new EfCoreProviderBenchmarkContext(options); + rows = RealisticWorkload.CreateIngestRows() + .Select(static row => new EfCoreIngestRow + { + Id = row.Id, + EventTime = row.EventTime, + Amount = row.Amount, + Category = row.Category, + IsActive = row.IsActive, + }) + .ToArray(); + + // Resolve and cache the provider's physical-column accessor plan before + // measurement, matching the warmed steady-state used by every harness. + context.BulkInsert(Array.Empty()); + } + + [GlobalCleanup] + public void Cleanup() + { + context.Dispose(); + connection.Dispose(); + } + + [Benchmark(OperationsPerInvoke = RealisticWorkload.IngestRowCount)] + public int BulkInsertInTransaction() + { + using var transaction = context.Database.BeginTransaction(); + var inserted = context.BulkInsert(rows); + transaction.Rollback(); + return inserted; + } + + private static void VerifyProviderVersion() + { + var version = typeof(DuckDBBulkExtensions).Assembly + .GetCustomAttribute() + ?.InformationalVersion; + + if (version is null || !version.StartsWith("1.13.0", StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"The provider comparison requires DuckDB.EFCoreProvider 1.13.0, but loaded {version ?? "an unknown version"}."); + } + } +} + +internal sealed class EfCoreProviderBenchmarkContext( + DbContextOptions options) : DbContext(options) +{ + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + var entity = modelBuilder.Entity(); + entity.ToTable("benchmark_ingest"); + entity.HasKey(row => row.Id); + entity.Property(row => row.Id).HasColumnName("id").ValueGeneratedNever(); + entity.Property(row => row.EventTime).HasColumnName("event_time"); + entity.Property(row => row.Amount).HasColumnName("amount"); + entity.Property(row => row.Category).HasColumnName("category"); + entity.Property(row => row.IsActive).HasColumnName("is_active"); + } +} + +internal sealed class EfCoreIngestRow +{ + public long Id { get; init; } + + public DateTime EventTime { get; init; } + + public double Amount { get; init; } + + public string Category { get; init; } = string.Empty; + + public bool IsActive { get; init; } +} diff --git a/DuckDB.Go.Benchmarks/go.mod b/DuckDB.Go.Benchmarks/go.mod new file mode 100644 index 00000000..f2a3912d --- /dev/null +++ b/DuckDB.Go.Benchmarks/go.mod @@ -0,0 +1,30 @@ +module github.com/Giorgi/DuckDB.NET/DuckDB.Go.Benchmarks + +go 1.24.0 + +require github.com/duckdb/duckdb-go/v2 v2.10504.0 + +require ( + github.com/apache/arrow-go/v18 v18.5.1 // indirect + github.com/duckdb/duckdb-go-bindings v0.10504.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10504.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10504.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10504.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10504.0 // indirect + github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10504.0 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/google/flatbuffers v25.12.19+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.32.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 // indirect + golang.org/x/tools v0.41.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect +) diff --git a/DuckDB.Go.Benchmarks/go.sum b/DuckDB.Go.Benchmarks/go.sum new file mode 100644 index 00000000..7149f85a --- /dev/null +++ b/DuckDB.Go.Benchmarks/go.sum @@ -0,0 +1,72 @@ +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI= +github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/duckdb/duckdb-go-bindings v0.10504.0 h1:XKOXNRetaJnMvMIDqi6etZOmh/nlEHM7saaC5UxU17I= +github.com/duckdb/duckdb-go-bindings v0.10504.0/go.mod h1:M2eB9+zGq+O4opimtLL5nWwkp8vdJQJNbFW5HKdqe4U= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10504.0 h1:hnWJ9SociR98hpypZPcgB+hTuWgw0hYsYnPFr8mBBEw= +github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10504.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10504.0 h1:q3JEdS5hU8ytvmcsgFEfGGQeJ5mS8l8QFYpfUlPHDRY= +github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10504.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10504.0 h1:2gABzYp2KnclSpE4qqinTZYhGpWSxmNHQKjs9WZ7zWk= +github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10504.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10504.0 h1:iTFkIVt6Ohd/rTvNQomtbP/IIh90TFGSlOUqFEC68Xo= +github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10504.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10504.0 h1:8O774uudYeexLa+uHekujkT3t3sDjQ7LSnlKIhRNWyc= +github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10504.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ= +github.com/duckdb/duckdb-go/v2 v2.10504.0 h1:bnkcNQpz3EaJmMmfOhFRFJ3ELVDHW4u3PDhZ78ZTCMY= +github.com/duckdb/duckdb-go/v2 v2.10504.0/go.mod h1:DQ8TxrUb0RGyTTFTwPNlpQ9FFx+ocJjoZGDkuOktYp8= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= +github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= +golang.org/x/mod v0.32.0 h1:9F4d3PHLljb6x//jOyokMv3eX+YDeepZSEo3mFJy93c= +golang.org/x/mod v0.32.0/go.mod h1:SgipZ/3h2Ci89DlEtEXWUk/HteuRin+HHhN+WbNhguU= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5 h1:i0p03B68+xC1kD2QUO8JzDTPXCzhN56OLJ+IhHY8U3A= +golang.org/x/telemetry v0.0.0-20260116145544-c6413dc483f5/go.mod h1:b7fPSJ0pKZ3ccUh8gnTONJxhn3c/PS6tyzQvyqw4iA8= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/DuckDB.Go.Benchmarks/prepared_command_benchmark_test.go b/DuckDB.Go.Benchmarks/prepared_command_benchmark_test.go new file mode 100644 index 00000000..644f5e3f --- /dev/null +++ b/DuckDB.Go.Benchmarks/prepared_command_benchmark_test.go @@ -0,0 +1,120 @@ +package benchmarks + +import ( + "database/sql" + "fmt" + "testing" + + _ "github.com/duckdb/duckdb-go/v2" +) + +const ( + preparedCommandQuery = "SELECT $first::BIGINT + $second::BIGINT + $third::BIGINT" + expectedEngineVersion = "v1.5.4" +) + +var resultSink int64 + +func BenchmarkPreparedCommandExecuteUnprepared(b *testing.B) { + db := openVerifiedDatabase(b) + arguments := preparedCommandArguments() + var result int64 + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + arguments[0] = sql.Named("first", int64(i)) + if err := db.QueryRow(preparedCommandQuery, arguments...).Scan(&result); err != nil { + b.Fatal(err) + } + } + + resultSink = result +} + +func BenchmarkPreparedCommandExecutePrepared(b *testing.B) { + db := openVerifiedDatabase(b) + statement, err := db.Prepare(preparedCommandQuery) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + if err := statement.Close(); err != nil { + b.Errorf("close prepared statement: %v", err) + } + }) + + arguments := preparedCommandArguments() + var result int64 + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + arguments[0] = sql.Named("first", int64(i)) + if err := statement.QueryRow(arguments...).Scan(&result); err != nil { + b.Fatal(err) + } + } + + resultSink = result +} + +func BenchmarkPreparedCommandCreateAndPrepare(b *testing.B) { + db := openVerifiedDatabase(b) + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + statement, err := db.Prepare(preparedCommandQuery) + if err != nil { + b.Fatal(err) + } + if err := statement.Close(); err != nil { + b.Fatal(err) + } + } +} + +func openVerifiedDatabase(tb testing.TB) *sql.DB { + tb.Helper() + + db, err := sql.Open("duckdb", "") + if err != nil { + tb.Fatal(err) + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + tb.Cleanup(func() { + if err := db.Close(); err != nil { + tb.Errorf("close database: %v", err) + } + }) + + var actualVersion string + if err := db.QueryRow("SELECT version()").Scan(&actualVersion); err != nil { + tb.Fatal(err) + } + if actualVersion != expectedEngineVersion { + tb.Fatal(fmt.Errorf( + "the driver comparison requires DuckDB %s, but loaded %s", + expectedEngineVersion, + actualVersion, + )) + } + if _, err := db.Exec("SET threads = 1"); err != nil { + tb.Fatal(err) + } + + return db +} + +func preparedCommandArguments() []any { + return []any{ + sql.Named("first", int64(1)), + sql.Named("second", int64(2)), + sql.Named("third", int64(3)), + } +} diff --git a/DuckDB.Go.Benchmarks/realistic_benchmark_test.go b/DuckDB.Go.Benchmarks/realistic_benchmark_test.go new file mode 100644 index 00000000..73cb652a --- /dev/null +++ b/DuckDB.Go.Benchmarks/realistic_benchmark_test.go @@ -0,0 +1,465 @@ +package benchmarks + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "math" + "testing" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" +) + +const ( + analyticsRowCount = 2_000_000 + materializationRowCount = 100_000 + ingestRowCount = 10_000 + tpchScaleFactor = 0.1 + + analyticsQuery = ` + SELECT + year(order_date) AS order_year, + region, + count(*) AS order_count, + sum(amount) AS revenue + FROM benchmark_orders + WHERE customer_id = $customerId + AND order_date >= $fromDate + AND order_date < $toDate + GROUP BY order_year, region + ORDER BY order_year, region` + + materializationQuery = ` + SELECT id, event_date, event_time, amount, customer_name, is_active + FROM benchmark_materialization + ORDER BY id` + + ingestStatement = ` + INSERT INTO benchmark_ingest + VALUES ($id, $eventTime, $amount, $category, $isActive)` +) + +var realisticResultSink int64 + +type ingestRow struct { + id int64 + eventTime time.Time + amount float64 + category string + isActive bool +} + +func BenchmarkAnalyticalQueryExecuteUnprepared(b *testing.B) { + db := openVerifiedDatabase(b) + initializeAnalytics(b, db) + + b.ReportAllocs() + b.ResetTimer() + + var checksum int64 + for i := 0; i < b.N; i++ { + rows, err := db.Query( + analyticsQuery, + analyticsArguments(i%100)..., + ) + if err != nil { + b.Fatal(err) + } + checksum = consumeAnalytics(b, rows) + } + + realisticResultSink = checksum +} + +func BenchmarkAnalyticalQueryExecutePrepared(b *testing.B) { + db := openVerifiedDatabase(b) + initializeAnalytics(b, db) + statement, err := db.Prepare(analyticsQuery) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + if err := statement.Close(); err != nil { + b.Errorf("close analytical statement: %v", err) + } + }) + + b.ReportAllocs() + b.ResetTimer() + + var checksum int64 + for i := 0; i < b.N; i++ { + rows, err := statement.Query(analyticsArguments(i % 100)...) + if err != nil { + b.Fatal(err) + } + checksum = consumeAnalytics(b, rows) + } + + realisticResultSink = checksum +} + +func BenchmarkResultMaterializationReadOneHundredThousandMixedRows(b *testing.B) { + db := openVerifiedDatabase(b) + initializeMaterialization(b, db) + + b.ReportAllocs() + b.ResetTimer() + + var checksum int64 + for i := 0; i < b.N; i++ { + rows, err := db.Query(materializationQuery) + if err != nil { + b.Fatal(err) + } + checksum = consumeMaterialization(b, rows) + } + + realisticResultSink = checksum +} + +func BenchmarkBulkIngestionInsertPreparedInTransaction(b *testing.B) { + db := openVerifiedDatabase(b) + initializeIngest(b, db) + rows := createIngestRows() + statement, err := db.Prepare(ingestStatement) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + if err := statement.Close(); err != nil { + b.Errorf("close ingest statement: %v", err) + } + }) + + b.ReportAllocs() + b.ResetTimer() + started := time.Now() + + for i := 0; i < b.N; i++ { + transaction, err := db.Begin() + if err != nil { + b.Fatal(err) + } + transactionStatement := transaction.Stmt(statement) + + for _, row := range rows { + if _, err := transactionStatement.Exec( + sql.Named("id", row.id), + sql.Named("eventTime", row.eventTime), + sql.Named("amount", row.amount), + sql.Named("category", row.category), + sql.Named("isActive", row.isActive), + ); err != nil { + b.Fatal(err) + } + } + + if err := transactionStatement.Close(); err != nil { + b.Fatal(err) + } + if err := transaction.Rollback(); err != nil { + b.Fatal(err) + } + } + + reportRowsPerSecond(b, started, b.N*len(rows)) +} + +func BenchmarkBulkIngestionInsertWithAppenderInTransaction(b *testing.B) { + db := openVerifiedDatabase(b) + initializeIngest(b, db) + rows := createIngestRows() + connection, err := db.Conn(context.Background()) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { + if err := connection.Close(); err != nil { + b.Errorf("close appender connection: %v", err) + } + }) + + b.ReportAllocs() + b.ResetTimer() + started := time.Now() + + for i := 0; i < b.N; i++ { + transaction, err := connection.BeginTx(context.Background(), nil) + if err != nil { + b.Fatal(err) + } + + err = connection.Raw(func(rawConnection any) error { + driverConnection, ok := rawConnection.(driver.Conn) + if !ok { + return fmt.Errorf("DuckDB connection does not implement driver.Conn") + } + + appender, err := duckdb.NewAppenderFromConn(driverConnection, "", "benchmark_ingest") + if err != nil { + return err + } + + for _, row := range rows { + if err := appender.AppendRow( + row.id, + row.eventTime, + row.amount, + row.category, + row.isActive, + ); err != nil { + _ = appender.Close() + return err + } + } + + return appender.Close() + }) + if err != nil { + _ = transaction.Rollback() + b.Fatal(err) + } + if err := transaction.Rollback(); err != nil { + b.Fatal(err) + } + } + + reportRowsPerSecond(b, started, b.N*len(rows)) +} + +func BenchmarkTpchQuery01(b *testing.B) { + benchmarkTpchQuery(b, 1) +} + +func BenchmarkTpchQuery06(b *testing.B) { + benchmarkTpchQuery(b, 6) +} + +func BenchmarkTpchQuery12(b *testing.B) { + benchmarkTpchQuery(b, 12) +} + +func BenchmarkTpchQuery14(b *testing.B) { + benchmarkTpchQuery(b, 14) +} + +func initializeAnalytics(tb testing.TB, db *sql.DB) { + tb.Helper() + mustExec(tb, db, fmt.Sprintf(` + CREATE TABLE benchmark_orders AS + SELECT + i::BIGINT AS order_id, + (i %% 10000)::INTEGER AS customer_id, + DATE '2020-01-01' + ((i %% 1461)::INTEGER) AS order_date, + CASE i %% 4 + WHEN 0 THEN 'north' + WHEN 1 THEN 'south' + WHEN 2 THEN 'east' + ELSE 'west' + END::VARCHAR AS region, + ((i %% 100000)::DOUBLE / 100.0) AS amount + FROM range(%d) AS source(i)`, analyticsRowCount)) +} + +func initializeMaterialization(tb testing.TB, db *sql.DB) { + tb.Helper() + mustExec(tb, db, fmt.Sprintf(` + CREATE TABLE benchmark_materialization AS + SELECT + i::BIGINT AS id, + DATE '2020-01-01' + ((i %% 1461)::INTEGER) AS event_date, + TIMESTAMP '2020-01-01 00:00:00' + ((i %% 31536000) * INTERVAL 1 SECOND) AS event_time, + ((i %% 100000)::DOUBLE / 100.0) AS amount, + CASE + WHEN i %% 10 = 0 THEN NULL + ELSE ('customer-' || (i %% 10000)::VARCHAR) + END::VARCHAR AS customer_name, + (i %% 2 = 0) AS is_active + FROM range(%d) AS source(i)`, materializationRowCount)) +} + +func initializeIngest(tb testing.TB, db *sql.DB) { + tb.Helper() + mustExec(tb, db, ` + CREATE TABLE benchmark_ingest( + id BIGINT, + event_time TIMESTAMP, + amount DOUBLE, + category VARCHAR, + is_active BOOLEAN + )`) +} + +func initializeTpch(tb testing.TB, db *sql.DB) { + tb.Helper() + mustExec(tb, db, "INSTALL tpch") + mustExec(tb, db, "LOAD tpch") + mustExec(tb, db, fmt.Sprintf("CALL dbgen(sf = %g)", tpchScaleFactor)) +} + +func analyticsArguments(customerID int) []any { + return []any{ + sql.Named("customerId", customerID), + sql.Named("fromDate", time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)), + sql.Named("toDate", time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)), + } +} + +func consumeAnalytics(tb testing.TB, rows *sql.Rows) int64 { + tb.Helper() + defer closeRows(tb, rows) + + rowCount := 0 + checksum := int64(17) + for rows.Next() { + var year int32 + var region string + var orderCount int64 + var revenue float64 + if err := rows.Scan(&year, ®ion, &orderCount, &revenue); err != nil { + tb.Fatal(err) + } + checksum = checksum*31 + int64(year) + checksum = checksum*31 + int64(len(region)) + checksum = checksum*31 + orderCount + checksum = checksum*31 + int64(math.Float64bits(revenue)) + rowCount++ + } + if err := rows.Err(); err != nil { + tb.Fatal(err) + } + if rowCount == 0 { + tb.Fatal("the analytical workload returned no rows") + } + return checksum +} + +func consumeMaterialization(tb testing.TB, rows *sql.Rows) int64 { + tb.Helper() + defer closeRows(tb, rows) + + rowCount := 0 + checksum := int64(17) + for rows.Next() { + var id int64 + var eventDate time.Time + var eventTime time.Time + var amount float64 + var customer sql.NullString + var isActive bool + if err := rows.Scan(&id, &eventDate, &eventTime, &amount, &customer, &isActive); err != nil { + tb.Fatal(err) + } + checksum += id + eventDate.Unix() + eventTime.UnixNano() + checksum += int64(math.Float64bits(amount)) + if customer.Valid { + checksum += int64(len(customer.String)) + } + if isActive { + checksum++ + } + rowCount++ + } + if err := rows.Err(); err != nil { + tb.Fatal(err) + } + if rowCount != materializationRowCount { + tb.Fatalf("expected %d materialized rows, but read %d", materializationRowCount, rowCount) + } + return checksum +} + +func benchmarkTpchQuery(b *testing.B, queryNumber int) { + db := openVerifiedDatabase(b) + initializeTpch(b, db) + query := fmt.Sprintf("PRAGMA tpch(%d)", queryNumber) + + b.ReportAllocs() + b.ResetTimer() + + var checksum int64 + for i := 0; i < b.N; i++ { + rows, err := db.Query(query) + if err != nil { + b.Fatal(err) + } + checksum = consumeGenericRows(b, rows) + } + + realisticResultSink = checksum +} + +func consumeGenericRows(tb testing.TB, rows *sql.Rows) int64 { + tb.Helper() + defer closeRows(tb, rows) + + columns, err := rows.Columns() + if err != nil { + tb.Fatal(err) + } + values := make([]any, len(columns)) + destinations := make([]any, len(columns)) + for index := range values { + destinations[index] = &values[index] + } + + rowCount := 0 + checksum := int64(17) + for rows.Next() { + if err := rows.Scan(destinations...); err != nil { + tb.Fatal(err) + } + for _, value := range values { + checksum = checksum*31 + int64(len(fmt.Sprint(value))) + } + rowCount++ + } + if err := rows.Err(); err != nil { + tb.Fatal(err) + } + if rowCount == 0 { + tb.Fatal("the TPC-H workload returned no rows") + } + return checksum +} + +func createIngestRows() []ingestRow { + rows := make([]ingestRow, ingestRowCount) + start := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + categories := []string{"north", "south", "east", "west"} + for index := range rows { + rows[index] = ingestRow{ + id: int64(index), + eventTime: start.Add(time.Duration(index) * time.Second), + amount: float64(index%100000) / 100.0, + category: categories[index%len(categories)], + isActive: index%2 == 0, + } + } + return rows +} + +func mustExec(tb testing.TB, db *sql.DB, statement string) { + tb.Helper() + if _, err := db.Exec(statement); err != nil { + tb.Fatal(err) + } +} + +func closeRows(tb testing.TB, rows *sql.Rows) { + tb.Helper() + if err := rows.Close(); err != nil { + tb.Fatal(err) + } +} + +func reportRowsPerSecond(b *testing.B, started time.Time, rowCount int) { + elapsed := time.Since(started) + if elapsed > 0 { + b.ReportMetric(float64(rowCount)/elapsed.Seconds(), "rows/s") + b.ReportMetric(float64(elapsed.Nanoseconds())/float64(rowCount), "ns/row") + } +} diff --git a/DuckDB.Java.Benchmarks/.gitignore b/DuckDB.Java.Benchmarks/.gitignore new file mode 100644 index 00000000..b83d2226 --- /dev/null +++ b/DuckDB.Java.Benchmarks/.gitignore @@ -0,0 +1 @@ +/target/ diff --git a/DuckDB.Java.Benchmarks/pom.xml b/DuckDB.Java.Benchmarks/pom.xml new file mode 100644 index 00000000..af9660d6 --- /dev/null +++ b/DuckDB.Java.Benchmarks/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + + org.duckdb.benchmarks + duckdb-java-benchmarks + 1.0.0-SNAPSHOT + + + UTF-8 + 17 + 1.5.4.0 + 1.37 + + + + + org.duckdb + duckdb_jdbc + ${duckdb.jdbc.version} + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + + + benchmarks + + + org.apache.maven.plugins + maven-compiler-plugin + 3.14.1 + + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.6.1 + + + package + + shade + + + false + ${project.build.directory}/benchmarks-all.jar + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + org.openjdk.jmh.Main + + + + + + + + + diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/AnalyticalQueryBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/AnalyticalQueryBenchmark.java new file mode 100644 index 00000000..509f0d04 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/AnalyticalQueryBenchmark.java @@ -0,0 +1,57 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class AnalyticalQueryBenchmark { + private Connection connection; + private PreparedStatement preparedStatement; + private int nextCustomer; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = RealisticBenchmarkSupport.openConnection(); + RealisticBenchmarkSupport.initializeAnalytics(connection); + preparedStatement = connection.prepareStatement(RealisticBenchmarkSupport.ANALYTICS_QUERY); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + preparedStatement.close(); + connection.close(); + } + + @Benchmark + public long executeUnprepared() throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(RealisticBenchmarkSupport.ANALYTICS_QUERY)) { + RealisticBenchmarkSupport.bindAnalytics(statement, nextCustomer++ % 100); + return RealisticBenchmarkSupport.consumeAnalytics(statement); + } + } + + @Benchmark + public long executePrepared() throws SQLException { + RealisticBenchmarkSupport.bindAnalytics(preparedStatement, nextCustomer++ % 100); + return RealisticBenchmarkSupport.consumeAnalytics(preparedStatement); + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BenchmarkSupport.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BenchmarkSupport.java new file mode 100644 index 00000000..e8016b66 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BenchmarkSupport.java @@ -0,0 +1,55 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +final class BenchmarkSupport { + static final String QUERY = "SELECT ?::BIGINT + ?::BIGINT + ?::BIGINT"; + + private static final String EXPECTED_ENGINE_VERSION = "v1.5.4"; + + private BenchmarkSupport() { + } + + static Connection openVerifiedConnection() throws SQLException { + var connection = DriverManager.getConnection("jdbc:duckdb:"); + + try (Statement statement = connection.createStatement(); + ResultSet result = statement.executeQuery("SELECT version()")) { + if (!result.next()) { + connection.close(); + throw new SQLException("DuckDB did not return an engine version"); + } + + var actualVersion = result.getString(1); + if (!EXPECTED_ENGINE_VERSION.equals(actualVersion)) { + connection.close(); + throw new SQLException( + "The driver comparison requires DuckDB " + EXPECTED_ENGINE_VERSION + + ", but loaded " + actualVersion); + } + } + + return connection; + } + + static void bindParameters(PreparedStatement statement, long first) throws SQLException { + statement.setLong(1, first); + statement.setLong(2, 2L); + statement.setLong(3, 3L); + } + + static long executeScalar(PreparedStatement statement) throws SQLException { + try (ResultSet result = statement.executeQuery()) { + if (!result.next()) { + throw new SQLException("DuckDB did not return a scalar result"); + } + + return result.getLong(1); + } + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BulkIngestionBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BulkIngestionBenchmark.java new file mode 100644 index 00000000..afd27403 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/BulkIngestionBenchmark.java @@ -0,0 +1,85 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import org.duckdb.DuckDBConnection; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class BulkIngestionBenchmark { + private DuckDBConnection connection; + private PreparedStatement preparedStatement; + private RealisticBenchmarkSupport.IngestRow[] rows; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = (DuckDBConnection) RealisticBenchmarkSupport.openConnection(); + RealisticBenchmarkSupport.initializeIngest(connection); + preparedStatement = connection.prepareStatement(RealisticBenchmarkSupport.INGEST_STATEMENT); + rows = RealisticBenchmarkSupport.createIngestRows(); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + preparedStatement.close(); + connection.close(); + } + + @Benchmark + @OperationsPerInvocation(RealisticBenchmarkSupport.INGEST_ROW_COUNT) + public int insertPreparedInTransaction() throws SQLException { + connection.setAutoCommit(false); + try { + for (var row : rows) { + RealisticBenchmarkSupport.bindIngest(preparedStatement, row); + preparedStatement.executeUpdate(); + } + connection.rollback(); + return rows.length; + } finally { + connection.setAutoCommit(true); + } + } + + @Benchmark + @OperationsPerInvocation(RealisticBenchmarkSupport.INGEST_ROW_COUNT) + public int insertWithAppenderInTransaction() throws SQLException { + connection.setAutoCommit(false); + try { + try (var appender = connection.createAppender("benchmark_ingest")) { + for (var row : rows) { + appender.beginRow() + .append(row.id()) + .append(row.eventTime()) + .append(row.amount()) + .append(row.category()) + .append(row.isActive()) + .endRow(); + } + } + connection.rollback(); + return rows.length; + } finally { + connection.setAutoCommit(true); + } + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandExecutionBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandExecutionBenchmark.java new file mode 100644 index 00000000..99b955e2 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandExecutionBenchmark.java @@ -0,0 +1,56 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class PreparedCommandExecutionBenchmark { + private Connection connection; + private PreparedStatement preparedStatement; + private long nextValue; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = BenchmarkSupport.openVerifiedConnection(); + preparedStatement = connection.prepareStatement(BenchmarkSupport.QUERY); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + preparedStatement.close(); + connection.close(); + } + + @Benchmark + public long executeUnprepared() throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(BenchmarkSupport.QUERY)) { + BenchmarkSupport.bindParameters(statement, nextValue++); + return BenchmarkSupport.executeScalar(statement); + } + } + + @Benchmark + public long executePrepared() throws SQLException { + BenchmarkSupport.bindParameters(preparedStatement, nextValue++); + return BenchmarkSupport.executeScalar(preparedStatement); + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandSetupBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandSetupBenchmark.java new file mode 100644 index 00000000..19c916c7 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/PreparedCommandSetupBenchmark.java @@ -0,0 +1,45 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class PreparedCommandSetupBenchmark { + private Connection connection; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = BenchmarkSupport.openVerifiedConnection(); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + connection.close(); + } + + @Benchmark + public void createAndPrepare() throws SQLException { + try (PreparedStatement statement = connection.prepareStatement(BenchmarkSupport.QUERY)) { + // Preparation and disposal are the measured operation. + } + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/RealisticBenchmarkSupport.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/RealisticBenchmarkSupport.java new file mode 100644 index 00000000..cdcee6e0 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/RealisticBenchmarkSupport.java @@ -0,0 +1,219 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.time.LocalDateTime; + +final class RealisticBenchmarkSupport { + static final int ANALYTICS_ROW_COUNT = 2_000_000; + static final int MATERIALIZATION_ROW_COUNT = 100_000; + static final int INGEST_ROW_COUNT = 10_000; + static final double TPCH_SCALE_FACTOR = 0.1; + + static final LocalDate ANALYTICS_FROM_DATE = LocalDate.of(2020, 1, 1); + static final LocalDate ANALYTICS_TO_DATE = LocalDate.of(2024, 1, 1); + + static final String ANALYTICS_QUERY = """ + SELECT + year(order_date) AS order_year, + region, + count(*) AS order_count, + sum(amount) AS revenue + FROM benchmark_orders + WHERE customer_id = ? + AND order_date >= ? + AND order_date < ? + GROUP BY order_year, region + ORDER BY order_year, region + """; + + static final String MATERIALIZATION_QUERY = """ + SELECT id, event_date, event_time, amount, customer_name, is_active + FROM benchmark_materialization + ORDER BY id + """; + + static final String INGEST_STATEMENT = """ + INSERT INTO benchmark_ingest + VALUES (?, ?, ?, ?, ?) + """; + + private RealisticBenchmarkSupport() { + } + + static Connection openConnection() throws SQLException { + var connection = BenchmarkSupport.openVerifiedConnection(); + executeNonQuery(connection, "SET threads = 1"); + return connection; + } + + static void initializeAnalytics(Connection connection) throws SQLException { + executeNonQuery(connection, """ + CREATE TABLE benchmark_orders AS + SELECT + i::BIGINT AS order_id, + (i %% 10000)::INTEGER AS customer_id, + DATE '2020-01-01' + ((i %% 1461)::INTEGER) AS order_date, + CASE i %% 4 + WHEN 0 THEN 'north' + WHEN 1 THEN 'south' + WHEN 2 THEN 'east' + ELSE 'west' + END::VARCHAR AS region, + ((i %% 100000)::DOUBLE / 100.0) AS amount + FROM range(%d) AS source(i) + """.formatted(ANALYTICS_ROW_COUNT)); + } + + static void initializeMaterialization(Connection connection) throws SQLException { + executeNonQuery(connection, """ + CREATE TABLE benchmark_materialization AS + SELECT + i::BIGINT AS id, + DATE '2020-01-01' + ((i %% 1461)::INTEGER) AS event_date, + TIMESTAMP '2020-01-01 00:00:00' + ((i %% 31536000) * INTERVAL 1 SECOND) AS event_time, + ((i %% 100000)::DOUBLE / 100.0) AS amount, + CASE + WHEN i %% 10 = 0 THEN NULL + ELSE ('customer-' || (i %% 10000)::VARCHAR) + END::VARCHAR AS customer_name, + (i %% 2 = 0) AS is_active + FROM range(%d) AS source(i) + """.formatted(MATERIALIZATION_ROW_COUNT)); + } + + static void initializeIngest(Connection connection) throws SQLException { + executeNonQuery(connection, """ + CREATE TABLE benchmark_ingest( + id BIGINT, + event_time TIMESTAMP, + amount DOUBLE, + category VARCHAR, + is_active BOOLEAN + ) + """); + } + + static void initializeTpch(Connection connection) throws SQLException { + executeNonQuery(connection, "INSTALL tpch"); + executeNonQuery(connection, "LOAD tpch"); + executeNonQuery(connection, "CALL dbgen(sf = " + TPCH_SCALE_FACTOR + ")"); + } + + static void bindAnalytics(PreparedStatement statement, int customerId) throws SQLException { + statement.setInt(1, customerId); + statement.setDate(2, Date.valueOf(ANALYTICS_FROM_DATE)); + statement.setDate(3, Date.valueOf(ANALYTICS_TO_DATE)); + } + + static long consumeAnalytics(PreparedStatement statement) throws SQLException { + try (ResultSet result = statement.executeQuery()) { + var rowCount = 0; + long checksum = 17; + + while (result.next()) { + checksum = checksum * 31 + result.getInt(1); + checksum = checksum * 31 + result.getString(2).length(); + checksum = checksum * 31 + result.getLong(3); + checksum = checksum * 31 + Double.doubleToLongBits(result.getDouble(4)); + rowCount++; + } + + if (rowCount == 0) { + throw new SQLException("The analytical workload returned no rows"); + } + + return checksum; + } + } + + static long consumeMaterialization(Statement statement) throws SQLException { + try (ResultSet result = statement.executeQuery(MATERIALIZATION_QUERY)) { + var rowCount = 0; + long checksum = 17; + + while (result.next()) { + var id = result.getLong(1); + var eventDate = result.getDate(2).toLocalDate(); + var eventTime = result.getTimestamp(3).toLocalDateTime(); + var amount = result.getDouble(4); + var customer = result.getString(5); + var isActive = result.getBoolean(6); + + checksum += id + eventDate.toEpochDay() + eventTime.getNano(); + checksum += Double.doubleToLongBits(amount); + checksum += (customer == null ? 0 : customer.length()) + (isActive ? 1 : 0); + rowCount++; + } + + if (rowCount != MATERIALIZATION_ROW_COUNT) { + throw new SQLException( + "Expected " + MATERIALIZATION_ROW_COUNT + " materialized rows, but read " + rowCount); + } + + return checksum; + } + } + + static long consumeTpch(Statement statement, int queryNumber) throws SQLException { + try (ResultSet result = statement.executeQuery("PRAGMA tpch(" + queryNumber + ")")) { + var rowCount = 0; + long checksum = 17; + var columnCount = result.getMetaData().getColumnCount(); + + while (result.next()) { + for (var column = 1; column <= columnCount; column++) { + var value = result.getObject(column); + checksum = checksum * 31 + (value == null ? 0 : value.hashCode()); + } + rowCount++; + } + + if (rowCount == 0) { + throw new SQLException("The TPC-H workload returned no rows"); + } + + return checksum; + } + } + + static IngestRow[] createIngestRows() { + var rows = new IngestRow[INGEST_ROW_COUNT]; + var start = LocalDateTime.of(2024, 1, 1, 0, 0); + var categories = new String[] {"north", "south", "east", "west"}; + + for (var index = 0; index < rows.length; index++) { + rows[index] = new IngestRow( + index, + start.plusSeconds(index), + index % 100000 / 100.0, + categories[index % categories.length], + index % 2 == 0); + } + + return rows; + } + + static void bindIngest(PreparedStatement statement, IngestRow row) throws SQLException { + statement.setLong(1, row.id()); + statement.setTimestamp(2, Timestamp.valueOf(row.eventTime())); + statement.setDouble(3, row.amount()); + statement.setString(4, row.category()); + statement.setBoolean(5, row.isActive()); + } + + static void executeNonQuery(Connection connection, String sql) throws SQLException { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + record IngestRow(long id, LocalDateTime eventTime, double amount, String category, boolean isActive) { + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/ResultMaterializationBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/ResultMaterializationBenchmark.java new file mode 100644 index 00000000..7dd1cf4e --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/ResultMaterializationBenchmark.java @@ -0,0 +1,47 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class ResultMaterializationBenchmark { + private Connection connection; + private Statement statement; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = RealisticBenchmarkSupport.openConnection(); + RealisticBenchmarkSupport.initializeMaterialization(connection); + statement = connection.createStatement(); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + statement.close(); + connection.close(); + } + + @Benchmark + public long readOneHundredThousandMixedRows() throws SQLException { + return RealisticBenchmarkSupport.consumeMaterialization(statement); + } +} diff --git a/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/TpchBenchmark.java b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/TpchBenchmark.java new file mode 100644 index 00000000..c91f6138 --- /dev/null +++ b/DuckDB.Java.Benchmarks/src/main/java/org/duckdb/benchmarks/TpchBenchmark.java @@ -0,0 +1,62 @@ +package org.duckdb.benchmarks; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +@State(Scope.Thread) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class TpchBenchmark { + private Connection connection; + private Statement statement; + + @Setup(Level.Trial) + public void setup() throws SQLException { + connection = RealisticBenchmarkSupport.openConnection(); + RealisticBenchmarkSupport.initializeTpch(connection); + statement = connection.createStatement(); + } + + @TearDown(Level.Trial) + public void cleanup() throws SQLException { + statement.close(); + connection.close(); + } + + @Benchmark + public long query01() throws SQLException { + return RealisticBenchmarkSupport.consumeTpch(statement, 1); + } + + @Benchmark + public long query06() throws SQLException { + return RealisticBenchmarkSupport.consumeTpch(statement, 6); + } + + @Benchmark + public long query12() throws SQLException { + return RealisticBenchmarkSupport.consumeTpch(statement, 12); + } + + @Benchmark + public long query14() throws SQLException { + return RealisticBenchmarkSupport.consumeTpch(statement, 14); + } +} diff --git a/DuckDB.NET.1_5_3.Benchmarks/DuckDB.NET.1_5_3.Benchmarks.csproj b/DuckDB.NET.1_5_3.Benchmarks/DuckDB.NET.1_5_3.Benchmarks.csproj new file mode 100644 index 00000000..b9955fa5 --- /dev/null +++ b/DuckDB.NET.1_5_3.Benchmarks/DuckDB.NET.1_5_3.Benchmarks.csproj @@ -0,0 +1,28 @@ + + + + Exe + net10.0 + enable + enable + true + DuckDB.NET.1_5_3.Benchmarks + DuckDB.NET.Benchmarks + $(DefineConstants);DUCKDB_NET_BASELINE_1_5_3 + + + + + + + + + + + + + + + + + diff --git a/DuckDB.NET.Benchmarks/AppenderBenchmark.cs b/DuckDB.NET.Benchmarks/AppenderBenchmark.cs new file mode 100644 index 00000000..162ed2e6 --- /dev/null +++ b/DuckDB.NET.Benchmarks/AppenderBenchmark.cs @@ -0,0 +1,67 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class AppenderBenchmark +{ + private DuckDBConnection connection = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = "DROP TABLE IF EXISTS bench; CREATE TABLE bench (a INTEGER, b BIGINT, c DOUBLE, d BOOLEAN);"; + command.ExecuteNonQuery(); + } + + [Benchmark(Baseline = true)] + public void AppendRowsWithCreateRow() + { + using var appender = connection.CreateAppender("bench"); + + for (var i = 0; i < RowCount; i++) + { + appender.CreateRow() + .AppendValue(i) + .AppendValue((long)i) + .AppendValue((double)i) + .AppendValue(i % 2 == 0) + .EndRow(); + } + } + + [Benchmark] + public void AppendRowsWithAppendRow() + { + using var appender = connection.CreateAppender("bench"); + + for (var i = 0; i < RowCount; i++) + { + appender.AppendRow(i, static (row, value) => + { + row.AppendValue(value) + .AppendValue((long)value) + .AppendValue((double)value) + .AppendValue(value % 2 == 0); + }); + } + } +} diff --git a/DuckDB.NET.Benchmarks/Benchmarks.csproj b/DuckDB.NET.Benchmarks/Benchmarks.csproj new file mode 100644 index 00000000..0d08d156 --- /dev/null +++ b/DuckDB.NET.Benchmarks/Benchmarks.csproj @@ -0,0 +1,33 @@ + + + + Exe + net10.0 + enable + enable + true + Full + true + ..\keyPair.snk + + + + + + + + + + + + + + + false + PreserveNewest + runtimes\%(RecursiveDir)\%(FileName)%(Extension) + + + + diff --git a/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs b/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs new file mode 100644 index 00000000..5b89a881 --- /dev/null +++ b/DuckDB.NET.Benchmarks/ListAppenderBenchmark.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using System.Collections.ObjectModel; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class ListAppenderBenchmark +{ + private const int ItemCount = 32; + + private DuckDBConnection connection = null!; + private int[] arrayValues = null!; + private List listValues = null!; + private ReadOnlyCollection readOnlyValues = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + arrayValues = Enumerable.Range(0, ItemCount).ToArray(); + listValues = arrayValues.ToList(); + readOnlyValues = Array.AsReadOnly(arrayValues); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = """ + DROP TABLE IF EXISTS list_from_array; + DROP TABLE IF EXISTS list_from_list; + DROP TABLE IF EXISTS list_from_read_only_collection; + DROP TABLE IF EXISTS array_from_array; + CREATE TABLE list_from_array (values INTEGER[]); + CREATE TABLE list_from_list (values INTEGER[]); + CREATE TABLE list_from_read_only_collection (values INTEGER[]); + CREATE TABLE array_from_array (values INTEGER[32]); + """; + command.ExecuteNonQuery(); + } + + [Benchmark(Baseline = true)] + public void AppendListFromArray() + { + using var appender = connection.CreateAppender("list_from_array"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(arrayValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendListFromList() + { + using var appender = connection.CreateAppender("list_from_list"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(listValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendListFromReadOnlyCollection() + { + using var appender = connection.CreateAppender("list_from_read_only_collection"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(readOnlyValues, static (row, values) => row.AppendValue(values)); + } + } + + [Benchmark] + public void AppendArrayFromArray() + { + using var appender = connection.CreateAppender("array_from_array"); + + for (var index = 0; index < RowCount; index++) + { + appender.AppendRow(arrayValues, static (row, values) => row.AppendValue(values)); + } + } +} diff --git a/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs b/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs new file mode 100644 index 00000000..1c1bec89 --- /dev/null +++ b/DuckDB.NET.Benchmarks/MappedAppenderBenchmark.cs @@ -0,0 +1,96 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; +using DuckDB.NET.Data.Mapping; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class MappedAppenderBenchmark +{ + private DuckDBConnection connection = null!; + private BenchRow[] rows = null!; + private IPropertyMapping[] mappings = null!; + + [Params(1_000_000)] + public int RowCount { get; set; } + + public sealed class BenchRow + { + public int Id { get; init; } + public long Score { get; init; } + public double Value { get; init; } + public bool Active { get; init; } + } + + public sealed class BenchRowMap : DuckDBAppenderMap + { + public BenchRowMap() + { + Map(row => row.Id); + Map(row => row.Score); + Map(row => row.Value); + Map(row => row.Active); + } + } + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + rows = new BenchRow[RowCount]; + for (var i = 0; i < RowCount; i++) + { + rows[i] = new BenchRow { Id = i, Score = i, Value = i, Active = i % 2 == 0 }; + } + + mappings = new BenchRowMap().PropertyMappings.ToArray(); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [IterationSetup] + public void IterationSetup() + { + using var command = connection.CreateCommand(); + command.CommandText = "DROP TABLE IF EXISTS bench_mapped; CREATE TABLE bench_mapped (a INTEGER, b BIGINT, c DOUBLE, d BOOLEAN);"; + command.ExecuteNonQuery(); + } + + // Mirrors the mapped appender loop before it adopted AppendRow. + [Benchmark(Baseline = true)] + public void AppendMappedRowsWithCreateRow() + { + using var appender = connection.CreateAppender("bench_mapped"); + + foreach (var record in rows) + { + AppendRecordWithCreateRow(appender, record); + } + } + + [Benchmark] + public void AppendMappedRowsWithAppendRow() + { + using var appender = connection.CreateAppender("bench_mapped"); + appender.AppendRecords(rows); + } + + private void AppendRecordWithCreateRow(DuckDBAppender appender, BenchRow record) + { + ArgumentNullException.ThrowIfNull(record); + + var row = appender.CreateRow(); + foreach (var mapping in mappings) + { + mapping.AppendToRow(row, record); + } + + row.EndRow(); + } +} diff --git a/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs b/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs new file mode 100644 index 00000000..f9af56f3 --- /dev/null +++ b/DuckDB.NET.Benchmarks/NativeLibraryLoader.cs @@ -0,0 +1,53 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace DuckDB.NET.Benchmarks; + +/// +/// Loads the platform-native DuckDB library for the benchmark process. +/// +internal static class NativeLibraryLoader +{ + [ModuleInitializer] + public static void Init() + { + if (GetRid() is not { } rid) + { + return; + } + + _ = NativeLibrary.TryLoad(Path.Join("runtimes", rid, "native", "duckdb"), Assembly.GetExecutingAssembly(), DllImportSearchPath.AssemblyDirectory, out _) || + NativeLibrary.TryLoad(Path.Join("runtimes", rid, "native", "libduckdb"), Assembly.GetExecutingAssembly(), DllImportSearchPath.AssemblyDirectory, out _); + } + + private static string? GetRid() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return RuntimeInformation.ProcessArchitecture switch + { + Architecture.X64 => "win-x64", + Architecture.Arm64 => "win-arm64", + _ => null, + }; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return RuntimeInformation.ProcessArchitecture switch + { + Architecture.X64 => "linux-x64", + Architecture.Arm64 => "linux-arm64", + _ => null, + }; + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return "osx"; + } + + return null; + } +} diff --git a/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs new file mode 100644 index 00000000..5db4c0b3 --- /dev/null +++ b/DuckDB.NET.Benchmarks/PreparedCommandBenchmark.cs @@ -0,0 +1,133 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class PreparedCommandBenchmark +{ + private DuckDBConnection connection = null!; + private DuckDBCommand unpreparedCommand = null!; + private DuckDBCommand boxedPreparedCommand = null!; + private DuckDBCommand preparedCommand = null!; + private DuckDBParameter unpreparedParameter = null!; + private DuckDBParameter boxedPreparedParameter = null!; + private DuckDBParameter preparedParameter = null!; + private int nextValue; + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + unpreparedCommand = CreateCommand(out unpreparedParameter); + boxedPreparedCommand = CreateCommand(out boxedPreparedParameter); + boxedPreparedCommand.Prepare(); + preparedCommand = CreateTypedCommand(out preparedParameter); + preparedCommand.Prepare(); + } + + [GlobalCleanup] + public void Cleanup() + { + preparedCommand.Dispose(); + boxedPreparedCommand.Dispose(); + unpreparedCommand.Dispose(); + connection.Dispose(); + } + + [Benchmark(Baseline = true)] + public int ExecuteUnprepared() + { + unpreparedParameter.Value = nextValue++; + return (int)unpreparedCommand.ExecuteScalar()!; + } + + [Benchmark] + public int ExecutePreparedBoxed() + { + boxedPreparedParameter.Value = nextValue++; + return (int)boxedPreparedCommand.ExecuteScalar()!; + } + + [Benchmark] + public int ExecutePrepared() + { + preparedParameter.TypedValue = nextValue++; + return (int)preparedCommand.ExecuteScalar()!; + } + + // Preserves the former ExecuteScalar path as a direct same-process comparison. + [Benchmark] + public int ExecutePreparedViaReader() + { + preparedParameter.Value = nextValue++; + using var reader = preparedCommand.ExecuteReader(); + return reader.Read() ? (int)reader.GetValue(0) : default; + } + + private DuckDBCommand CreateCommand(out DuckDBParameter changingParameter) + { + var command = connection.CreateCommand(); + command.CommandText = "SELECT $first::INTEGER + $second::INTEGER + $third::INTEGER"; + changingParameter = new DuckDBParameter("first", 1); + command.Parameters.Add(changingParameter); + command.Parameters.Add(new DuckDBParameter("second", 2)); + command.Parameters.Add(new DuckDBParameter("third", 3)); + return command; + } + + private DuckDBCommand CreateTypedCommand(out DuckDBParameter changingParameter) + { + var command = connection.CreateCommand(); + command.CommandText = "SELECT $first::INTEGER + $second::INTEGER + $third::INTEGER"; + changingParameter = new DuckDBParameter("first", 1); + command.Parameters.Add(changingParameter); + command.Parameters.Add(new DuckDBParameter("second", 2)); + command.Parameters.Add(new DuckDBParameter("third", 3)); + return command; + } +} + +[MemoryDiagnoser] +public class PreparedCommandSetupBenchmark +{ + private DuckDBConnection connection = null!; + + [GlobalSetup] + public void Setup() + { + connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + } + + [GlobalCleanup] + public void Cleanup() + { + connection.Dispose(); + } + + [Benchmark(Baseline = true)] + public void CreateUnpreparedCommand() + { + using var command = CreateCommand(); + } + + [Benchmark] + public void CreateAndPrepareCommand() + { + using var command = CreateCommand(); + command.Prepare(); + } + + private DuckDBCommand CreateCommand() + { + var command = connection.CreateCommand(); + command.CommandText = "SELECT $first::INTEGER + $second::INTEGER + $third::INTEGER"; + command.Parameters.Add(new DuckDBParameter("first", 1)); + command.Parameters.Add(new DuckDBParameter("second", 2)); + command.Parameters.Add(new DuckDBParameter("third", 3)); + return command; + } +} diff --git a/DuckDB.NET.Benchmarks/PreparedCommandWorkload.cs b/DuckDB.NET.Benchmarks/PreparedCommandWorkload.cs new file mode 100644 index 00000000..69c6bb2a --- /dev/null +++ b/DuckDB.NET.Benchmarks/PreparedCommandWorkload.cs @@ -0,0 +1,33 @@ +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +internal static class PreparedCommandWorkload +{ + public const string Query = "SELECT $first::BIGINT + $second::BIGINT + $third::BIGINT"; + +#if DUCKDB_NET_BASELINE_1_5_3 + private const string ExpectedEngineVersion = "v1.5.3"; +#else + private const string ExpectedEngineVersion = "v1.5.4"; +#endif + + public static DuckDBConnection OpenVerifiedConnection() + { + var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + using var versionCommand = connection.CreateCommand(); + versionCommand.CommandText = "SELECT version()"; + var actualVersion = versionCommand.ExecuteScalar() as string; + + if (!string.Equals(actualVersion, ExpectedEngineVersion, StringComparison.Ordinal)) + { + connection.Dispose(); + throw new InvalidOperationException( + $"The driver comparison requires DuckDB {ExpectedEngineVersion}, but loaded {actualVersion ?? "an unknown version"}."); + } + + return connection; + } +} diff --git a/DuckDB.NET.Benchmarks/Program.cs b/DuckDB.NET.Benchmarks/Program.cs new file mode 100644 index 00000000..198112cd --- /dev/null +++ b/DuckDB.NET.Benchmarks/Program.cs @@ -0,0 +1,42 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using DuckDB.NET.Benchmarks; + +// The repo's Directory.Build.props renames the output assembly to DuckDB.NET.Benchmarks +// while the project file stays Benchmarks.csproj, so BenchmarkDotNet's default toolchain +// can't locate the csproj. Run in-process to avoid the separate build/spawn step. +var config = DefaultConfig.Instance + .AddJob( + Job.Default + .WithId("DriverComparison") + .WithToolchain(InProcessEmitToolchain.Instance) + .WithLaunchCount(1) + .WithWarmupCount(5) + .WithIterationCount(10) + .WithInvocationCount(1) + .WithUnrollFactor(1)); + +var benchmarkTypes = new List +{ +#if !DUCKDB_NET_BASELINE_1_5_3 + typeof(AppenderBenchmark), + typeof(ListAppenderBenchmark), + typeof(MappedAppenderBenchmark), +#endif + typeof(PreparedCommandBenchmark), + typeof(PreparedCommandSetupBenchmark), + typeof(AnalyticalQueryBenchmark), + typeof(ResultMaterializationBenchmark), + typeof(BulkIngestionBenchmark), + typeof(TpchBenchmark), +}; + +#if DUCKDB_EFCORE_PROVIDER_1_13_0 +benchmarkTypes.Add(typeof(EfCoreProviderBulkIngestionBenchmark)); +#endif + +BenchmarkSwitcher + .FromTypes(benchmarkTypes.ToArray()) + .Run(args, config); diff --git a/DuckDB.NET.Benchmarks/RealisticBenchmarks.cs b/DuckDB.NET.Benchmarks/RealisticBenchmarks.cs new file mode 100644 index 00000000..933b4ae3 --- /dev/null +++ b/DuckDB.NET.Benchmarks/RealisticBenchmarks.cs @@ -0,0 +1,221 @@ +using BenchmarkDotNet.Attributes; +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +[MemoryDiagnoser] +public class AnalyticalQueryBenchmark +{ + private DuckDBConnection connection = null!; + private DuckDBCommand unpreparedCommand = null!; + private DuckDBCommand preparedCommand = null!; + private DuckDBParameter unpreparedCustomer = null!; + private DuckDBParameter preparedCustomer = null!; + private int nextCustomer; + + [GlobalSetup] + public void Setup() + { + connection = RealisticWorkload.OpenConnection(); + RealisticWorkload.InitializeAnalytics(connection); + + unpreparedCommand = RealisticWorkload.CreateAnalyticsCommand(connection, out unpreparedCustomer); + preparedCommand = RealisticWorkload.CreateAnalyticsCommand(connection, out preparedCustomer); + preparedCommand.Prepare(); + } + + [GlobalCleanup] + public void Cleanup() + { + preparedCommand.Dispose(); + unpreparedCommand.Dispose(); + connection.Dispose(); + } + + [Benchmark(Baseline = true)] + public long ExecuteUnprepared() + { + unpreparedCustomer.Value = nextCustomer++ % 100; + return RealisticWorkload.ConsumeAnalytics(unpreparedCommand); + } + + [Benchmark] + public long ExecutePrepared() + { + preparedCustomer.Value = nextCustomer++ % 100; + return RealisticWorkload.ConsumeAnalytics(preparedCommand); + } +} + +[MemoryDiagnoser] +public class ResultMaterializationBenchmark +{ + private DuckDBConnection connection = null!; + private DuckDBCommand command = null!; + + [GlobalSetup] + public void Setup() + { + connection = RealisticWorkload.OpenConnection(); + RealisticWorkload.InitializeMaterialization(connection); + command = connection.CreateCommand(); + command.CommandText = RealisticWorkload.MaterializationQuery; + } + + [GlobalCleanup] + public void Cleanup() + { + command.Dispose(); + connection.Dispose(); + } + + [Benchmark] + public long ReadOneHundredThousandMixedRows() => RealisticWorkload.ConsumeMaterialization(command); +} + +[MemoryDiagnoser] +public class BulkIngestionBenchmark +{ + private DuckDBConnection connection = null!; + private DuckDBCommand preparedInsert = null!; + private DuckDBParameter idParameter = null!; + private DuckDBParameter eventTimeParameter = null!; + private DuckDBParameter amountParameter = null!; + private DuckDBParameter categoryParameter = null!; + private DuckDBParameter isActiveParameter = null!; + private IngestRow[] rows = null!; + + [GlobalSetup] + public void Setup() + { + connection = RealisticWorkload.OpenConnection(); + RealisticWorkload.InitializeIngest(connection); + rows = RealisticWorkload.CreateIngestRows(); + + preparedInsert = connection.CreateCommand(); + preparedInsert.CommandText = RealisticWorkload.IngestStatement; + idParameter = new DuckDBParameter("id", 0L); + eventTimeParameter = new DuckDBParameter("eventTime", rows[0].EventTime); + amountParameter = new DuckDBParameter("amount", 0.0); + categoryParameter = new DuckDBParameter("category", rows[0].Category); + isActiveParameter = new DuckDBParameter("isActive", false); + preparedInsert.Parameters.Add(idParameter); + preparedInsert.Parameters.Add(eventTimeParameter); + preparedInsert.Parameters.Add(amountParameter); + preparedInsert.Parameters.Add(categoryParameter); + preparedInsert.Parameters.Add(isActiveParameter); + preparedInsert.Prepare(); + } + + [GlobalCleanup] + public void Cleanup() + { + preparedInsert.Dispose(); + connection.Dispose(); + } + + [Benchmark(Baseline = true, OperationsPerInvoke = RealisticWorkload.IngestRowCount)] + public int InsertPreparedInTransaction() + { + using var transaction = connection.BeginTransaction(); + preparedInsert.Transaction = transaction; + + foreach (var row in rows) + { + idParameter.Value = row.Id; + eventTimeParameter.Value = row.EventTime; + amountParameter.Value = row.Amount; + categoryParameter.Value = row.Category; + isActiveParameter.Value = row.IsActive; + preparedInsert.ExecuteNonQuery(); + } + + transaction.Rollback(); + preparedInsert.Transaction = null; + return rows.Length; + } + + [Benchmark(OperationsPerInvoke = RealisticWorkload.IngestRowCount)] + public int InsertWithAppenderInTransaction() + { + using var transaction = connection.BeginTransaction(); + + using (var appender = connection.CreateAppender("benchmark_ingest")) + { + foreach (var row in rows) + { + appender.CreateRow() + .AppendValue(row.Id) + .AppendValue(row.EventTime) + .AppendValue(row.Amount) + .AppendValue(row.Category) + .AppendValue(row.IsActive) + .EndRow(); + } + } + + transaction.Rollback(); + return rows.Length; + } + +#if !DUCKDB_NET_BASELINE_1_5_3 + [Benchmark(OperationsPerInvoke = RealisticWorkload.IngestRowCount)] + public int InsertWithScopedAppenderInTransaction() + { + using var transaction = connection.BeginTransaction(); + + using (var appender = connection.CreateAppender("benchmark_ingest")) + { + foreach (var row in rows) + { + appender.AppendRowScoped(row, static (ref DuckDBAppenderRowWriter writer, IngestRow value) => + { + writer.AppendValue(value.Id); + writer.AppendValue(value.EventTime); + writer.AppendValue(value.Amount); + writer.AppendValue(value.Category); + writer.AppendValue(value.IsActive); + }); + } + } + + transaction.Rollback(); + return rows.Length; + } +#endif +} + +[MemoryDiagnoser] +public class TpchBenchmark +{ + private DuckDBConnection connection = null!; + + [GlobalSetup] + public void Setup() + { + connection = RealisticWorkload.OpenConnection(); + RealisticWorkload.InitializeTpch(connection); + } + + [GlobalCleanup] + public void Cleanup() => connection.Dispose(); + + [Benchmark] + public long Query01() => Execute(1); + + [Benchmark] + public long Query06() => Execute(6); + + [Benchmark] + public long Query12() => Execute(12); + + [Benchmark] + public long Query14() => Execute(14); + + private long Execute(int queryNumber) + { + using var command = connection.CreateCommand(); + command.CommandText = $"PRAGMA tpch({queryNumber})"; + return RealisticWorkload.ConsumeTpch(command); + } +} diff --git a/DuckDB.NET.Benchmarks/RealisticWorkload.cs b/DuckDB.NET.Benchmarks/RealisticWorkload.cs new file mode 100644 index 00000000..c9ed3d7d --- /dev/null +++ b/DuckDB.NET.Benchmarks/RealisticWorkload.cs @@ -0,0 +1,226 @@ +using System.Data; +using DuckDB.NET.Data; + +namespace DuckDB.NET.Benchmarks; + +internal static class RealisticWorkload +{ + public const int AnalyticsRowCount = 2_000_000; + public const int MaterializationRowCount = 100_000; + public const int IngestRowCount = 10_000; + public const double TpchScaleFactor = 0.1; + + public const string AnalyticsQuery = """ + SELECT + year(order_date) AS order_year, + region, + count(*) AS order_count, + sum(amount) AS revenue + FROM benchmark_orders + WHERE customer_id = $customerId + AND order_date >= $fromDate + AND order_date < $toDate + GROUP BY order_year, region + ORDER BY order_year, region + """; + + public const string MaterializationQuery = """ + SELECT id, event_date, event_time, amount, customer_name, is_active + FROM benchmark_materialization + ORDER BY id + """; + + public const string IngestStatement = """ + INSERT INTO benchmark_ingest + VALUES ($id, $eventTime, $amount, $category, $isActive) + """; + + public static readonly DateOnly AnalyticsFromDate = new(2020, 1, 1); + public static readonly DateOnly AnalyticsToDate = new(2024, 1, 1); + + public static DuckDBConnection OpenConnection() + { + var connection = PreparedCommandWorkload.OpenVerifiedConnection(); + ExecuteNonQuery(connection, "SET threads = 1"); + return connection; + } + + public static void InitializeAnalytics(DuckDBConnection connection) + { + ExecuteNonQuery(connection, $$""" + CREATE TABLE benchmark_orders AS + SELECT + i::BIGINT AS order_id, + (i % 10000)::INTEGER AS customer_id, + DATE '2020-01-01' + ((i % 1461)::INTEGER) AS order_date, + CASE i % 4 + WHEN 0 THEN 'north' + WHEN 1 THEN 'south' + WHEN 2 THEN 'east' + ELSE 'west' + END::VARCHAR AS region, + ((i % 100000)::DOUBLE / 100.0) AS amount + FROM range({{AnalyticsRowCount}}) AS source(i) + """); + } + + public static void InitializeMaterialization(DuckDBConnection connection) + { + ExecuteNonQuery(connection, $$""" + CREATE TABLE benchmark_materialization AS + SELECT + i::BIGINT AS id, + DATE '2020-01-01' + ((i % 1461)::INTEGER) AS event_date, + TIMESTAMP '2020-01-01 00:00:00' + ((i % 31536000) * INTERVAL 1 SECOND) AS event_time, + ((i % 100000)::DOUBLE / 100.0) AS amount, + CASE + WHEN i % 10 = 0 THEN NULL + ELSE ('customer-' || (i % 10000)::VARCHAR) + END::VARCHAR AS customer_name, + (i % 2 = 0) AS is_active + FROM range({{MaterializationRowCount}}) AS source(i) + """); + } + + public static void InitializeIngest(DuckDBConnection connection) + { + ExecuteNonQuery(connection, """ + CREATE TABLE benchmark_ingest( + id BIGINT, + event_time TIMESTAMP, + amount DOUBLE, + category VARCHAR, + is_active BOOLEAN + ) + """); + } + + public static void InitializeTpch(DuckDBConnection connection) + { + ExecuteNonQuery(connection, "INSTALL tpch"); + ExecuteNonQuery(connection, "LOAD tpch"); + ExecuteNonQuery(connection, $"CALL dbgen(sf = {TpchScaleFactor.ToString(System.Globalization.CultureInfo.InvariantCulture)})"); + } + + public static DuckDBCommand CreateAnalyticsCommand(DuckDBConnection connection, out DuckDBParameter customerParameter) + { + var command = connection.CreateCommand(); + command.CommandText = AnalyticsQuery; + customerParameter = new DuckDBParameter("customerId", 0); + command.Parameters.Add(customerParameter); + command.Parameters.Add(new DuckDBParameter("fromDate", AnalyticsFromDate)); + command.Parameters.Add(new DuckDBParameter("toDate", AnalyticsToDate)); + return command; + } + + public static long ConsumeAnalytics(DuckDBCommand command) + { + using var reader = command.ExecuteReader(); + var rowCount = 0; + long checksum = 17; + + while (reader.Read()) + { + checksum = unchecked(checksum * 31 + reader.GetInt32(0)); + checksum = unchecked(checksum * 31 + reader.GetString(1).Length); + checksum = unchecked(checksum * 31 + reader.GetInt64(2)); + checksum = unchecked(checksum * 31 + BitConverter.DoubleToInt64Bits(reader.GetDouble(3))); + rowCount++; + } + + if (rowCount == 0) + { + throw new InvalidOperationException("The analytical workload returned no rows."); + } + + return checksum; + } + + public static long ConsumeMaterialization(DuckDBCommand command) + { + using var reader = command.ExecuteReader(); + var rowCount = 0; + long checksum = 17; + + while (reader.Read()) + { + var id = reader.GetInt64(0); + var eventDate = reader.GetFieldValue(1); + var eventTime = reader.GetDateTime(2); + var amount = reader.GetDouble(3); + var customerLength = reader.IsDBNull(4) ? 0 : reader.GetString(4).Length; + var isActive = reader.GetBoolean(5); + + checksum = unchecked(checksum + id + eventDate.DayNumber + eventTime.Ticks); + checksum = unchecked(checksum + BitConverter.DoubleToInt64Bits(amount)); + checksum = unchecked(checksum + customerLength + (isActive ? 1 : 0)); + rowCount++; + } + + if (rowCount != MaterializationRowCount) + { + throw new InvalidOperationException( + $"Expected {MaterializationRowCount} materialized rows, but read {rowCount}."); + } + + return checksum; + } + + public static long ConsumeTpch(DuckDBCommand command) + { + using var reader = command.ExecuteReader(); + var rowCount = 0; + long checksum = 17; + + while (reader.Read()) + { + for (var column = 0; column < reader.FieldCount; column++) + { + var value = reader.GetValue(column); + checksum = unchecked(checksum * 31 + (value is DBNull ? 0 : value.GetHashCode())); + } + + rowCount++; + } + + if (rowCount == 0) + { + throw new InvalidOperationException("The TPC-H workload returned no rows."); + } + + return checksum; + } + + public static IngestRow[] CreateIngestRows() + { + var rows = new IngestRow[IngestRowCount]; + var start = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + var categories = new[] { "north", "south", "east", "west" }; + + for (var index = 0; index < rows.Length; index++) + { + rows[index] = new IngestRow( + index, + start.AddSeconds(index), + index % 100000 / 100.0, + categories[index % categories.Length], + index % 2 == 0); + } + + return rows; + } + + public static void ExecuteNonQuery(DuckDBConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +} + +internal readonly record struct IngestRow( + long Id, + DateTime EventTime, + double Amount, + string Category, + bool IsActive); diff --git a/DuckDB.NET.Bindings/Bindings.csproj b/DuckDB.NET.Bindings/Bindings.csproj index 13a7ae76..cb88a774 100644 --- a/DuckDB.NET.Bindings/Bindings.csproj +++ b/DuckDB.NET.Bindings/Bindings.csproj @@ -3,11 +3,15 @@ DuckDB Bindings for C#. -- Updated to DuckDB v1.5.3 +- Updated to DuckDB v1.5.4 + Unofficial preview build of the DuckDB native bindings from the skuirrels DuckDB.NET fork. + Unofficial preview bindings for the consolidated DuckDB.NET performance work, bundling DuckDB v1.5.4. + Unofficial stable build of the DuckDB native bindings from the skuirrels DuckDB.NET fork. + Stable fork bindings for the consolidated DuckDB.NET performance work, bundling DuckDB v1.5.4. DuckDB.NET.Native win-x64;win-arm64;linux-x64;linux-arm64;osx - https://github.com/duckdb/duckdb/releases/download/v1.5.3 + https://github.com/duckdb/duckdb/releases/download/v1.5.4 True ..\keyPair.snk true diff --git a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs index e42f90c6..343074cf 100644 --- a/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs +++ b/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs @@ -68,6 +68,28 @@ protected override bool ReleaseHandle() } } +public class DuckDBArrowOptions() : SafeHandleZeroOrMinusOneIsInvalid(true) +{ + protected override bool ReleaseHandle() + { + NativeMethods.Arrow.DuckDBDestroyArrowOptions(ref handle); + return true; + } +} + +public class DuckDBErrorData() : SafeHandleZeroOrMinusOneIsInvalid(true) +{ + public bool HasError => !IsInvalid && NativeMethods.ErrorData.DuckDBErrorDataHasError(this); + + public string? Message => IsInvalid ? null : NativeMethods.ErrorData.DuckDBErrorDataMessage(this); + + protected override bool ReleaseHandle() + { + NativeMethods.ErrorData.DuckDBDestroyErrorData(ref handle); + return true; + } +} + public class DuckDBDataChunk : SafeHandleZeroOrMinusOneIsInvalid { public DuckDBDataChunk() : base(true) diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs index 9be82b67..f44665f8 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Appender.cs @@ -23,11 +23,9 @@ public static partial class Appender [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBLogicalType DuckDBAppenderColumnType(DuckDBAppender appender, ulong index); - [SuppressGCTransition] - [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_error")] + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_error_data")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - [return: MarshalUsing(typeof(DuckDBOwnedStringMarshaller))] - public static partial string DuckDBAppenderError(DuckDBAppender appender); + public static partial DuckDBErrorData DuckDBAppenderErrorData(DuckDBAppender appender); [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_appender_flush")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs new file mode 100644 index 00000000..13392a82 --- /dev/null +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.Arrow.cs @@ -0,0 +1,27 @@ +namespace DuckDB.NET.Native; + +public partial class NativeMethods +{ + //https://duckdb.org/docs/stable/clients/c/api#arrow-interface + public static partial class Arrow + { + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_result_get_arrow_options")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBArrowOptions DuckDBResultGetArrowOptions(ref DuckDBResult result); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_destroy_arrow_options")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial void DuckDBDestroyArrowOptions(ref IntPtr arrowOptions); + + // duckdb_error_data duckdb_to_arrow_schema(duckdb_arrow_options, duckdb_logical_type *types, + // const char **names, idx_t column_count, ArrowSchema *out_schema) + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_to_arrow_schema")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorData DuckDBToArrowSchema(DuckDBArrowOptions arrowOptions, IntPtr types, IntPtr names, ulong columnCount, IntPtr outSchema); + + // duckdb_error_data duckdb_data_chunk_to_arrow(duckdb_arrow_options, duckdb_data_chunk, ArrowArray *out_arrow_array) + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_data_chunk_to_arrow")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorData DuckDBDataChunkToArrow(DuckDBArrowOptions arrowOptions, DuckDBDataChunk chunk, IntPtr outArray); + } +} diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs new file mode 100644 index 00000000..fc7629a8 --- /dev/null +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.ErrorData.cs @@ -0,0 +1,28 @@ +namespace DuckDB.NET.Native; + +public partial class NativeMethods +{ + //https://duckdb.org/docs/stable/clients/c/api#error-data + public static partial class ErrorData + { + [SuppressGCTransition] + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_has_error")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [return: MarshalAs(UnmanagedType.I1)] + public static partial bool DuckDBErrorDataHasError(DuckDBErrorData errorData); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_message")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + [return: MarshalUsing(typeof(DuckDBOwnedStringMarshaller))] + public static partial string DuckDBErrorDataMessage(DuckDBErrorData errorData); + + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_error_data_error_type")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBErrorType DuckDBErrorDataErrorType(DuckDBErrorData errorData); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_destroy_error_data")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial void DuckDBDestroyErrorData(ref IntPtr errorData); + } +} diff --git a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.PreparedStatements.cs b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.PreparedStatements.cs index 68f20cf3..e2c72ff9 100644 --- a/DuckDB.NET.Bindings/NativeMethods/NativeMethods.PreparedStatements.cs +++ b/DuckDB.NET.Bindings/NativeMethods/NativeMethods.PreparedStatements.cs @@ -62,6 +62,10 @@ public static partial class PreparedStatements [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBBindHugeInt(DuckDBPreparedStatement preparedStatement, long index, DuckDBHugeInt val); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_bind_decimal")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBState DuckDBBindDecimal(DuckDBPreparedStatement preparedStatement, long index, DuckDBDecimal val); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_bind_uint8")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBBindUInt8(DuckDBPreparedStatement preparedStatement, long index, byte val); @@ -98,6 +102,14 @@ public static partial class PreparedStatements [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBBindTimestamp(DuckDBPreparedStatement preparedStatement, long index, DuckDBTimestampStruct val); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_bind_timestamp_tz")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBState DuckDBBindTimestampTz(DuckDBPreparedStatement preparedStatement, long index, DuckDBTimestampStruct val); + + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_bind_interval")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBState DuckDBBindInterval(DuckDBPreparedStatement preparedStatement, long index, DuckDBInterval val); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_bind_varchar", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBBindVarchar(DuckDBPreparedStatement preparedStatement, long index, string val); @@ -110,6 +122,12 @@ public static partial class PreparedStatements [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBBindNull(DuckDBPreparedStatement preparedStatement, long index); + // Clears values from a reusable prepared statement before rebinding it. This prevents a + // parameter omitted by a later named collection from retaining its previous value. + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_clear_bindings")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + public static partial DuckDBState DuckDBClearBindings(DuckDBPreparedStatement preparedStatement); + [LibraryImport(DuckDbLibrary, EntryPoint = "duckdb_execute_prepared")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] public static partial DuckDBState DuckDBExecutePrepared(DuckDBPreparedStatement preparedStatement, out DuckDBResult result); diff --git a/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs new file mode 100644 index 00000000..18b19835 --- /dev/null +++ b/DuckDB.NET.Data/Arrow/DuckDBArrowArrayStream.cs @@ -0,0 +1,162 @@ +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Apache.Arrow; +using Apache.Arrow.C; +using Apache.Arrow.Ipc; + +namespace DuckDB.NET.Data.Arrow; + +/// +/// Streams the rows of a DuckDB query result as Apache Arrow values using +/// DuckDB's Arrow C Data Interface (duckdb_to_arrow_schema / duckdb_data_chunk_to_arrow). +/// Each DuckDB data chunk is converted into one Arrow record batch and imported with no row-by-row marshaling. +/// +internal sealed class DuckDBArrowArrayStream : IArrowArrayStream +{ + private DuckDBResult result; + private readonly DuckDBArrowOptions arrowOptions; + private readonly bool streaming; + private bool disposed; + + public Schema Schema { get; } + + internal DuckDBArrowArrayStream(DuckDBResult result) + { + this.result = result; + + arrowOptions = NativeMethods.Arrow.DuckDBResultGetArrowOptions(ref this.result); + if (arrowOptions.IsInvalid) + { + this.result.Close(); + throw new InvalidOperationException("Failed to obtain Arrow options from the DuckDB result."); + } + + streaming = NativeMethods.Types.DuckDBResultIsStreaming(this.result) > 0; + + try + { + Schema = BuildSchema(); + } + catch + { + arrowOptions.Dispose(); + this.result.Close(); + throw; + } + } + + private unsafe Schema BuildSchema() + { + var columnCount = NativeMethods.Query.DuckDBColumnCount(ref result); + + var logicalTypes = new DuckDBLogicalType[columnCount]; + var typeHandles = new IntPtr[columnCount]; + var namePointers = new IntPtr[columnCount]; + + try + { + for (var index = 0UL; index < columnCount; index++) + { + var logicalType = NativeMethods.Query.DuckDBColumnLogicalType(ref result, (long)index); + logicalTypes[index] = logicalType; + typeHandles[index] = logicalType.DangerousGetHandle(); + + var name = NativeMethods.Query.DuckDBColumnName(ref result, (long)index); + namePointers[index] = Marshal.StringToCoTaskMemUTF8(name); + } + + var cSchema = CArrowSchema.Create(); + + try + { + fixed (IntPtr* typesPointer = typeHandles) + fixed (IntPtr* namesPointer = namePointers) + { + var error = NativeMethods.Arrow.DuckDBToArrowSchema(arrowOptions, (IntPtr)typesPointer, (IntPtr)namesPointer, columnCount, (IntPtr)cSchema); + error.ThrowOnError("Failed to convert the DuckDB result schema to an Arrow schema."); + } + + return CArrowSchemaImporter.ImportSchema(cSchema); + } + finally + { + CArrowSchema.Free(cSchema); + } + } + finally + { + foreach (var pointer in namePointers) + { + if (pointer != IntPtr.Zero) + { + Marshal.FreeCoTaskMem(pointer); + } + } + + foreach (var logicalType in logicalTypes) + { + logicalType?.Dispose(); + } + } + } + + public ValueTask ReadNextRecordBatchAsync(CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(disposed, this); + + if (cancellationToken.IsCancellationRequested) + { + return new ValueTask(Task.FromCanceled(cancellationToken)); + } + + var chunk = streaming + ? NativeMethods.StreamingResult.DuckDBStreamFetchChunk(result) + : NativeMethods.Query.DuckDBFetchChunk(result); + + if (chunk.IsInvalid) + { + chunk.Dispose(); + return new ValueTask((RecordBatch?)null); + } + + try + { + return new ValueTask(ConvertChunk(chunk)); + } + finally + { + chunk.Dispose(); + } + } + + private unsafe RecordBatch ConvertChunk(DuckDBDataChunk chunk) + { + var cArray = CArrowArray.Create(); + + try + { + var error = NativeMethods.Arrow.DuckDBDataChunkToArrow(arrowOptions, chunk, (IntPtr)cArray); + error.ThrowOnError("Failed to convert a DuckDB data chunk to an Arrow array."); + + return CArrowArrayImporter.ImportRecordBatch(cArray, Schema); + } + finally + { + CArrowArray.Free(cArray); + } + } + + public void Dispose() + { + if (disposed) + { + return; + } + + disposed = true; + + arrowOptions.Dispose(); + result.Close(); + } +} diff --git a/DuckDB.NET.Data/Data.csproj b/DuckDB.NET.Data/Data.csproj index ce657cbf..0154d74d 100644 --- a/DuckDB.NET.Data/Data.csproj +++ b/DuckDB.NET.Data/Data.csproj @@ -11,6 +11,10 @@ New features: Fixes: - Fixed ObjectDisposedException in enum appender (#327) + Unofficial preview build of the DuckDB ADO.NET provider from the skuirrels DuckDB.NET fork. + Unofficial preview of the consolidated DuckDB.NET performance work for DuckDB v1.5.4. Do not reference this package alongside the official DuckDB.NET packages. + Unofficial stable build of the DuckDB ADO.NET provider from the skuirrels DuckDB.NET fork. + Stable fork release of the consolidated DuckDB.NET performance work for DuckDB v1.5.4. Do not reference this package alongside the official DuckDB.NET packages. True ..\keyPair.snk true @@ -30,10 +34,20 @@ Fixes: + + + + + + + + + + diff --git a/DuckDB.NET.Data/DataChunk/Reader/ListVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/ListVectorDataReader.cs index 26990932..a1bc8a61 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/ListVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/ListVectorDataReader.cs @@ -4,6 +4,8 @@ internal sealed class ListVectorDataReader : VectorDataReaderBase { private readonly ulong arraySize; private readonly VectorDataReaderBase listDataReader; + private Type? cachedListType; + private IListFactory? cachedListFactory; public bool IsList => DuckDBType == DuckDBType.List; @@ -51,8 +53,7 @@ private object GetList(Type returnType, ulong listOffset, ulong length) var allowNulls = listType.AllowsNullValue(out _, out var nullableType); - var list = Activator.CreateInstance(returnType) as IList - ?? throw new ArgumentException($"The type '{returnType.Name}' specified in parameter {nameof(returnType)} cannot be instantiated as an IList."); + var list = CreateList(returnType, length); //Special case for specific types to avoid boxing return list switch @@ -105,6 +106,31 @@ IList BuildListCommon(IList result, Type targetType) } } + private IList CreateList(Type returnType, ulong length) + { + if (returnType != cachedListType) + { + cachedListType = returnType; + cachedListFactory = returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(List<>) + ? CreateListFactory(returnType) + : null; + } + + if (cachedListFactory != null) + { + return cachedListFactory.Create(checked((int)length)); + } + + return Activator.CreateInstance(returnType) as IList + ?? throw new ArgumentException($"The type '{returnType.Name}' specified in parameter {nameof(returnType)} cannot be instantiated as an IList."); + } + + private static IListFactory CreateListFactory(Type returnType) + { + var factoryType = typeof(ListFactory<>).MakeGenericType(returnType.GetGenericArguments()[0]); + return (IListFactory)Activator.CreateInstance(factoryType)!; + } + internal override void Reset(IntPtr vector) { base.Reset(vector); @@ -119,4 +145,14 @@ public override void Dispose() listDataReader.Dispose(); base.Dispose(); } -} \ No newline at end of file + + private interface IListFactory + { + IList Create(int capacity); + } + + private sealed class ListFactory : IListFactory + { + public IList Create(int capacity) => new List(capacity); + } +} diff --git a/DuckDB.NET.Data/DataChunk/Reader/MapVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/MapVectorDataReader.cs index 640ee839..9e688737 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/MapVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/MapVectorDataReader.cs @@ -1,9 +1,15 @@ -namespace DuckDB.NET.Data.DataChunk.Reader; +using System.Collections.Concurrent; + +namespace DuckDB.NET.Data.DataChunk.Reader; internal sealed class MapVectorDataReader : VectorDataReaderBase { + private static readonly ConcurrentDictionary Materializers = new(); + private readonly VectorDataReaderBase keyReader; private readonly VectorDataReaderBase valueReader; + private Type? cachedTargetType; + private IMapMaterializer? cachedMaterializer; internal unsafe MapVectorDataReader(IntPtr vector, void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, DuckDBLogicalType logicalColumnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName) @@ -37,17 +43,22 @@ internal override unsafe object GetValue(ulong offset, Type targetType) return base.GetValue(offset, targetType); } + var listData = (DuckDBListEntry*)DataPointer + offset; + var materializer = GetMaterializer(targetType); + + if (materializer != null) + { + return materializer.Materialize(keyReader, valueReader, listData->Offset, listData->Length, ColumnName); + } + if (Activator.CreateInstance(targetType) is not IDictionary instance) { throw new InvalidOperationException($"Cannot read Map column {ColumnName} in a non-dictionary type"); } var arguments = targetType.GetGenericArguments(); - var allowsNullValues = arguments.Length == 2 && arguments[1].AllowsNullValue(out _, out _); - var listData = (DuckDBListEntry*)DataPointer + offset; - for (ulong i = 0; i < listData->Length; i++) { var childOffset = i + listData->Offset; @@ -68,6 +79,43 @@ internal override unsafe object GetValue(ulong offset, Type targetType) return instance; } + private IMapMaterializer? GetMaterializer(Type targetType) + { + if (targetType == cachedTargetType) + { + return cachedMaterializer; + } + + cachedTargetType = targetType; + cachedMaterializer = null; + + if (!targetType.IsGenericType || targetType.GetGenericTypeDefinition() != typeof(Dictionary<,>)) + { + return null; + } + + var arguments = targetType.GetGenericArguments(); + if (!CanReadDirectly(keyReader, arguments[0]) || !CanReadDirectly(valueReader, arguments[1])) + { + return null; + } + + cachedMaterializer = Materializers.GetOrAdd(targetType, static type => CreateMaterializer(type)); + return cachedMaterializer; + } + + private static bool CanReadDirectly(VectorDataReaderBase reader, Type targetType) + { + var valueType = Nullable.GetUnderlyingType(targetType) ?? targetType; + return valueType == reader.ClrType || valueType == reader.ProviderSpecificClrType; + } + + private static IMapMaterializer CreateMaterializer(Type targetType) + { + var materializerType = typeof(MapMaterializer<,>).MakeGenericType(targetType.GetGenericArguments()); + return (IMapMaterializer)Activator.CreateInstance(materializerType)!; + } + internal override void Reset(IntPtr vector) { base.Reset(vector); @@ -84,4 +132,49 @@ public override void Dispose() valueReader.Dispose(); base.Dispose(); } -} \ No newline at end of file + + private interface IMapMaterializer + { + object Materialize(VectorDataReaderBase keys, VectorDataReaderBase values, ulong offset, ulong length, string columnName); + } + + private sealed class MapMaterializer : IMapMaterializer where TKey : notnull + { + private static readonly bool AllowsNullValues = typeof(TValue).AllowsNullValue(out _, out _); + + public object Materialize(VectorDataReaderBase keys, VectorDataReaderBase values, ulong offset, ulong length, string columnName) + { + var result = new Dictionary(checked((int)length)); + + for (ulong i = 0; i < length; i++) + { + var childOffset = offset + i; + var key = Read(keys, childOffset); + + if (values.IsValid(childOffset)) + { + result.Add(key, Read(values, childOffset)); + } + else if (AllowsNullValues) + { + result.Add(key, default!); + } + else + { + throw new InvalidCastException($"The Map in column {columnName} contains null value but dictionary does not allow null values"); + } + } + + return result; + } + + private static T Read(VectorDataReaderBase reader, ulong offset) + { + // Object-valued readers need their natural non-generic path. Concrete value types use + // the generic path so they can flow into Dictionary without boxing. + return typeof(T) == typeof(object) + ? (T)reader.GetValue(offset) + : reader.GetValueStrict(offset); + } + } +} diff --git a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs index 9da4bea6..caefd9bc 100644 --- a/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs +++ b/DuckDB.NET.Data/DataChunk/Reader/NumericVectorDataReader.cs @@ -71,6 +71,15 @@ internal override object GetValue(ulong offset, Type targetType) _ => base.GetValue(offset, targetType) }; + // Fast path: when the boxed value already has the requested type (the common case for + // GetValue(ordinal)/this[ordinal], which read using the column's natural ClrType), skip the + // Convert.ChangeType machinery entirely. ChangeType does not early-out on same-type for + // IConvertible values — it re-boxes via ToXxx() — so this also removes a redundant box. + if (value.GetType() == targetType) + { + return value; + } + if (targetType.IsNumeric()) { try diff --git a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs index c56654dc..a51269c0 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/ListVectorDataWriter.cs @@ -1,13 +1,27 @@ using DuckDB.NET.Data.Common; +using System.Reflection; +using System.Runtime.CompilerServices; namespace DuckDB.NET.Data.DataChunk.Writer; internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase { + private delegate void CollectionWriter(ListVectorDataWriter writer, ICollection collection, ulong startIndex); + + private static readonly ConditionalWeakTable CollectionWriterCache = new(); + private static readonly MethodInfo WriteArrayMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteArray), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteListMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteList), BindingFlags.Static | BindingFlags.NonPublic)!; + private static readonly MethodInfo WriteEnumerableMethod = + typeof(ListVectorDataWriter).GetMethod(nameof(WriteEnumerable), BindingFlags.Static | BindingFlags.NonPublic)!; + private ulong offset = 0; private readonly ulong arraySize; private readonly DuckDBLogicalType childType; private readonly VectorDataWriterBase listItemWriter; + private Type? cachedCollectionType; + private CollectionWriterPlan? cachedCollectionWriterPlan; private bool IsList => ColumnType == DuckDBType.List; private ulong vectorReservedSize = DuckDBGlobalData.VectorSize; @@ -27,59 +41,17 @@ internal override bool AppendCollection(ICollection value, ulong rowIndex) ResizeVector(rowIndex % DuckDBGlobalData.VectorSize, count); - _ = value switch - { - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - IEnumerable items => WriteItems(items), - - _ => WriteItemsFallback(value), - }; + ValidateArraySize(count); + + var collectionWriter = GetCollectionWriter(value.GetType()); + if (collectionWriter is not null) + { + collectionWriter(this, value, offset); + } + else + { + WriteItemsFallback(value); + } var duckDBListEntry = new DuckDBListEntry(offset, count); var result = !IsList || AppendValueInternal(duckDBListEntry, rowIndex); @@ -92,42 +64,132 @@ internal override bool AppendCollection(ICollection value, ulong rowIndex) } return result; + } - int WriteItems(IEnumerable items) + private CollectionWriter? GetCollectionWriter(Type collectionType) + { + if (collectionType == cachedCollectionType) { - if (IsList == false && count != arraySize) - { - throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}"); - } + return cachedCollectionWriterPlan!.Writer; + } - var index = 0ul; + cachedCollectionType = collectionType; + cachedCollectionWriterPlan = CollectionWriterCache.GetValue(collectionType, CreateCollectionWriter); + return cachedCollectionWriterPlan.Writer; + } - foreach (var item in items) - { - listItemWriter.WriteValue(item, offset + (index++)); - } + private void ValidateArraySize(ulong count) + { + if (!IsList && count != arraySize) + { + throw new InvalidOperationException( + $"Column has Array size of {arraySize} but the specified value has size of {count}"); + } + } - return 0; + private void WriteItemsFallback(IEnumerable items) + { + var index = 0ul; + + foreach (var item in items) + { + listItemWriter.WriteValue(item, offset + (index++)); } + } + + private static CollectionWriterPlan CreateCollectionWriter(Type collectionType) + { + MethodInfo? openMethod = null; + Type? elementType = null; - int WriteItemsFallback(IEnumerable items) + if (collectionType.IsSZArray) + { + openMethod = WriteArrayMethod; + elementType = collectionType.GetElementType(); + } + else if (collectionType.IsGenericType && + collectionType.GetGenericTypeDefinition() == typeof(List<>)) + { + openMethod = WriteListMethod; + elementType = collectionType.GetGenericArguments()[0]; + } + else { - if (IsList == false && count != arraySize) + elementType = GetEnumerableElementType(collectionType); + openMethod = elementType is null ? null : WriteEnumerableMethod; + } + + return new CollectionWriterPlan( + openMethod is null || elementType is null + ? null + : openMethod.MakeGenericMethod(elementType).CreateDelegate()); + } + + private static Type? GetEnumerableElementType(Type collectionType) + { + Type? elementType = null; + + foreach (var interfaceType in collectionType.GetInterfaces()) + { + if (!interfaceType.IsGenericType || + interfaceType.GetGenericTypeDefinition() != typeof(IEnumerable<>)) { - throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}"); + continue; } - var index = 0ul; - - foreach (var item in items) + var candidateType = interfaceType.GetGenericArguments()[0]; + if (elementType is not null && elementType != candidateType) { - listItemWriter.WriteValue(item, offset + (index++)); + return null; } - return 0; + elementType = candidateType; + } + + return elementType; + } + + private static void WriteArray( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (T[])collection; + + for (var index = 0; index < values.Length; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); + } + } + + private static void WriteList( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var values = (List)collection; + + for (var index = 0; index < values.Count; index++) + { + writer.listItemWriter.WriteValue(values[index], startIndex + (ulong)index); } } + private static void WriteEnumerable( + ListVectorDataWriter writer, + ICollection collection, + ulong startIndex) + { + var index = 0ul; + + foreach (var value in (IEnumerable)collection) + { + writer.listItemWriter.WriteValue(value, startIndex + index++); + } + } + + private sealed record CollectionWriterPlan(CollectionWriter? Writer); + private void ResizeVector(ulong rowIndex, ulong count) { //If writing to a list column we need to make sure that enough space is allocated. Not needed for Arrays as DuckDB does it for us. @@ -163,7 +225,9 @@ private void ResizeVector(ulong rowIndex, ulong count) public override void Dispose() { + cachedCollectionType = null; + cachedCollectionWriterPlan = null; listItemWriter.Dispose(); childType.Dispose(); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs b/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs index 03995710..293c7dd2 100644 --- a/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs +++ b/DuckDB.NET.Data/DataChunk/Writer/VectorDataWriterBase.cs @@ -20,7 +20,7 @@ public void WriteNull(ulong rowIndex) NativeMethods.ValidityMask.DuckDBValiditySetRowValidity(validity, rowIndex, false); } - public void WriteValue(T value, ulong rowIndex) + private void WriteValueFallback(T value, ulong rowIndex) { if (value == null) { @@ -64,6 +64,292 @@ public void WriteValue(T value, ulong rowIndex) }; } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void WriteValue(T value, ulong rowIndex) + { + if (typeof(T) == typeof(bool)) + { + _ = AppendBool(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(bool?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendBool(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(sbyte)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(sbyte?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(short)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(short?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(int)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(int?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(long)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(long?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(byte)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(byte?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(ushort)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(ushort?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(uint)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(uint?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(ulong)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(ulong?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(float)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(float?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(double)) + { + _ = AppendNumeric(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(double?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendNumeric(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(decimal)) + { + _ = AppendDecimal(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(decimal?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDecimal(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(BigInteger)) + { + _ = AppendBigInteger(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(BigInteger?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendBigInteger(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(string)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue is not null) _ = AppendString(typedValue, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(Guid)) + { + _ = AppendGuid(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(Guid?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendGuid(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateTime)) + { + _ = AppendDateTime(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateTime?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateTime(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(TimeSpan)) + { + _ = AppendTimeSpan(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(TimeSpan?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeSpan(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBDateOnly)) + { + _ = AppendDateOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBDateOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBTimeOnly)) + { + _ = AppendTimeOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DuckDBTimeOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateOnly)) + { + _ = AppendDateOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(TimeOnly)) + { + _ = AppendTimeOnly(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(TimeOnly?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendTimeOnly(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + if (typeof(T) == typeof(DateTimeOffset)) + { + _ = AppendDateTimeOffset(Unsafe.As(ref value), rowIndex); + return; + } + + if (typeof(T) == typeof(DateTimeOffset?)) + { + var typedValue = Unsafe.As(ref value); + if (typedValue.HasValue) _ = AppendDateTimeOffset(typedValue.Value, rowIndex); else WriteNull(rowIndex); + return; + } + + WriteValueFallback(value, rowIndex); + } + internal virtual bool AppendBool(bool value, ulong rowIndex) => ThrowException(); internal virtual bool AppendDecimal(decimal value, ulong rowIndex) => ThrowException(); diff --git a/DuckDB.NET.Data/DuckDBAppender.cs b/DuckDB.NET.Data/DuckDBAppender.cs index 5cd6e888..7ef58fdf 100644 --- a/DuckDB.NET.Data/DuckDBAppender.cs +++ b/DuckDB.NET.Data/DuckDBAppender.cs @@ -1,13 +1,21 @@ using DuckDB.NET.Data.Common; using DuckDB.NET.Data.DataChunk.Writer; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; - +using DuckDB.NET.Data.Extensions; namespace DuckDB.NET.Data; +/// +/// Appends rows to a DuckDB table. +/// +/// +/// Instances are not thread-safe. Do not call other methods on the same appender from an +/// or +/// callback. +/// public class DuckDBAppender : IDisposable { private bool closed; + private bool isAppendingRow; + private bool isFaulted; private readonly Native.DuckDBAppender nativeAppender; private readonly string qualifiedTableName; @@ -16,6 +24,7 @@ public class DuckDBAppender : IDisposable private readonly DuckDBLogicalType[] logicalTypes; private readonly DuckDBDataChunk dataChunk; private readonly VectorDataWriterBase[] vectorWriters; + private DuckDBAppenderRow? reusableRow; internal DuckDBAppender(Native.DuckDBAppender appender, string qualifiedTableName) { @@ -42,44 +51,173 @@ internal DuckDBAppender(Native.DuckDBAppender appender, string qualifiedTableNam /// internal IReadOnlyList LogicalTypes => logicalTypes; + /// + /// Creates an independent row. The caller must append every column and call + /// . + /// public IDuckDBAppenderRow CreateRow() { - if (closed) + EnsureUsable(); + return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, PrepareRow(), dataChunk, nativeAppender); + } + + /// + /// Appends a complete row using a reusable row instance. + /// + /// A callback that appends every column value. This method calls + /// after the callback returns. + /// + /// The row is valid only during the callback and must not be retained. The callback must not + /// call other methods on this appender. If the callback or automatic + /// fails, all previously completed rows are flushed, + /// the failed row is discarded, and the appender cannot be reused. + /// + public void AppendRow(Action writeRow) + { + ArgumentNullException.ThrowIfNull(writeRow); + + // Pass the callback as state so the adapter remains static and allocation-free. + AppendRow(writeRow, static (row, callback) => callback(row)); + } + + /// + /// Appends a complete row using a reusable row instance without exposing that instance as a + /// return value. The row passed to is only valid for the duration of + /// the callback and must not be retained or used after the callback returns. + /// + /// The type of value used to populate the row. + /// The value used to populate the row. + /// A callback that appends every column value. This method calls + /// after the callback returns. + /// + /// The callback must not call other methods on this appender. If the callback or automatic + /// fails, all previously completed rows are flushed, + /// the failed row is discarded, and the appender cannot be reused. + /// + public void AppendRow(TState state, Action writeRow) + { + ArgumentNullException.ThrowIfNull(writeRow); + EnsureUsable(); + + DuckDBAppenderRow? row = null; + isAppendingRow = true; + + try { - throw new InvalidOperationException("Appender is already closed"); + row = CreateReusableRow(); + writeRow(row, state); + row.EndRow(); + } + catch (Exception appendException) + { + FinalizeFailedAppendRowOrThrow(row, appendException); + throw; + } + finally + { + isAppendingRow = false; + } + } + + /// + /// Appends a complete row through a stack-only writer that cannot escape the callback. + /// + /// The type of value used to populate the row. + /// The value used to populate the row. + /// A callback that appends every column value. This method validates + /// and completes the row after the callback returns. + /// + /// The callback must not call other methods on this appender. If the callback or automatic + /// row completion fails, all previously completed rows are flushed, the failed row is + /// discarded, and the appender cannot be reused. Use a static or cached callback for an + /// allocation-free per-row path; a capturing callback can allocate. + /// + public void AppendRowScoped(TState state, DuckDBAppenderRowWriterAction writeRow) + { + ArgumentNullException.ThrowIfNull(writeRow); + EnsureUsable(); + + DuckDBAppenderRow? row = null; + isAppendingRow = true; + + try + { + row = CreateReusableRow(); + var writer = new DuckDBAppenderRowWriter(row); + writeRow(ref writer, state); + row.EndRow(); + } + catch (Exception appendException) + { + FinalizeFailedAppendRowOrThrow(row, appendException); + throw; + } + finally + { + isAppendingRow = false; + } + } + + /// + /// Creates a row whose instance may be reused by the next call. This is only safe for internal + /// callers that create, populate and end each row without exposing the row reference. + /// + internal DuckDBAppenderRow CreateReusableRow() + { + var rowIndex = PrepareRow(); + + if (reusableRow is null) + { + reusableRow = new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowIndex, dataChunk, nativeAppender); + } + else + { + reusableRow.Reset(rowIndex); } + return reusableRow; + } + + private ulong PrepareRow() + { if (rowCount % DuckDBGlobalData.VectorSize == 0) { AppendDataChunk(); - InitVectorWriters(); - + // AppendDataChunk resets the chunk. Update the managed count before recreating the + // writers so a writer-initialization failure cannot make CloseCore append the already + // completed chunk a second time. rowCount = 0; + + InitVectorWriters(); } rowCount++; - return new DuckDBAppenderRow(qualifiedTableName, vectorWriters, rowCount - 1, dataChunk, nativeAppender); + return rowCount - 1; } public void Clear() { - if (closed) - { - throw new InvalidOperationException("Appender is already closed"); - } - + EnsureUsable(); + var state = NativeMethods.Appender.DuckDBAppenderClear(nativeAppender); if (!state.IsSuccess()) { - ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } rowCount = 0; NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); + InitVectorWriters(); } public void Close() + { + EnsureUsable(); + CloseCore(); + } + + private void CloseCore() { closed = true; @@ -87,28 +225,38 @@ public void Close() { AppendDataChunk(); - foreach (var logicalType in logicalTypes) + var state = NativeMethods.Appender.DuckDBAppenderClose(nativeAppender); + if (!state.IsSuccess()) { - logicalType.Dispose(); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } - - foreach (var writer in vectorWriters) + } + finally + { + try { - writer?.Dispose(); + DisposeManagedResources(); } - - dataChunk.Dispose(); - - var state = NativeMethods.Appender.DuckDBAppenderClose(nativeAppender); - if (!state.IsSuccess()) + finally { - ThrowLastError(nativeAppender); + nativeAppender.Close(); } } - finally + } + + private void DisposeManagedResources() + { + foreach (var logicalType in logicalTypes) + { + logicalType.Dispose(); + } + + foreach (var writer in vectorWriters) { - nativeAppender.Close(); + writer?.Dispose(); } + + dataChunk.Dispose(); } public void Dispose() @@ -137,18 +285,68 @@ private void AppendDataChunk() if (!state.IsSuccess()) { - ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } NativeMethods.DataChunks.DuckDBDataChunkReset(dataChunk); } - [DoesNotReturn] - [StackTraceHidden] - internal static void ThrowLastError(Native.DuckDBAppender appender) + private void FinalizeFailedAppendRow(DuckDBAppenderRow? row) + { + isFaulted = true; + + if (row is null) + { + // Row preparation failed while flushing or recreating the current chunk. Discard any + // uncommitted managed rows by closing the chunk at size zero so the appender cannot be + // reused in a partially transitioned state. + rowCount = 0; + } + else + { + // The row index is also the number of completed rows before the failed row in this chunk. + rowCount = row.ChunkRowIndex; + row.Invalidate(); + } + + CloseCore(); + } + + private void FinalizeFailedAppendRowOrThrow(DuckDBAppenderRow? row, Exception appendException) + { + try + { + FinalizeFailedAppendRow(row); + } + catch (Exception finalizationException) + { + throw new AggregateException( + "Appending the row failed and the previously completed rows could not be finalized", + appendException, + finalizationException); + } + } + + private void EnsureNotAppendingRow() + { + if (isAppendingRow) + { + throw new InvalidOperationException("The appender cannot be used from inside an AppendRow callback"); + } + } + + private void EnsureUsable() { - var errorMessage = NativeMethods.Appender.DuckDBAppenderError(appender); + EnsureNotAppendingRow(); - throw new DuckDBException(errorMessage); + if (isFaulted) + { + throw new InvalidOperationException("The appender cannot be reused after an AppendRow callback failed"); + } + + if (closed) + { + throw new InvalidOperationException("Appender is already closed"); + } } } diff --git a/DuckDB.NET.Data/DuckDBAppenderRow.cs b/DuckDB.NET.Data/DuckDBAppenderRow.cs index 489d03f3..cd697b2a 100644 --- a/DuckDB.NET.Data/DuckDBAppenderRow.cs +++ b/DuckDB.NET.Data/DuckDBAppenderRow.cs @@ -7,10 +7,12 @@ public class DuckDBAppenderRow : IDuckDBAppenderRow private int columnIndex = 0; private readonly string qualifiedTableName; private readonly VectorDataWriterBase[] vectorWriters; - private readonly ulong rowIndex; + private ulong rowIndex; private readonly DuckDBDataChunk dataChunk; private readonly Native.DuckDBAppender nativeAppender; + internal ulong ChunkRowIndex => rowIndex; + internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vectorWriters, ulong rowIndex, DuckDBDataChunk dataChunk, Native.DuckDBAppender nativeAppender) { @@ -21,6 +23,23 @@ internal DuckDBAppenderRow(string qualifiedTableName, VectorDataWriterBase[] vec this.nativeAppender = nativeAppender; } + /// + /// Re-targets this row instance at a new row index so the appender can reuse a single + /// instead of allocating one per row. The table name, vector + /// writers, data chunk and native appender are stable for the lifetime of the appender, so only + /// the row index and column cursor need to be reset. + /// + internal void Reset(ulong rowIndex) + { + this.rowIndex = rowIndex; + columnIndex = 0; + } + + internal void Invalidate() + { + columnIndex = vectorWriters.Length; + } + public void EndRow() { if (columnIndex < vectorWriters.Length) @@ -135,7 +154,7 @@ public IDuckDBAppenderRow AppendDefault() if (state == DuckDBState.Error) { - DuckDBAppender.ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } columnIndex++; diff --git a/DuckDB.NET.Data/DuckDBAppenderRowWriter.cs b/DuckDB.NET.Data/DuckDBAppenderRowWriter.cs new file mode 100644 index 00000000..9b095ad8 --- /dev/null +++ b/DuckDB.NET.Data/DuckDBAppenderRowWriter.cs @@ -0,0 +1,111 @@ +using System.Runtime.CompilerServices; + +namespace DuckDB.NET.Data; + +/// +/// A stack-only writer for a single appender row. +/// +/// +/// Instances are valid only for the duration of a +/// +/// callback. The stack-only type cannot be boxed, captured, or stored on the managed heap. +/// Row completion is performed by the appender after the callback returns. +/// +public ref struct DuckDBAppenderRowWriter +{ + private readonly DuckDBAppenderRow row; + + internal DuckDBAppenderRowWriter(DuckDBAppenderRow row) + { + this.row = row; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendNullValue() => row.AppendNullValue(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(bool? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(byte[]? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(Span value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(string? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(decimal? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(Guid? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(BigInteger? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(sbyte? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(short? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(int? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(long? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(byte? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(ushort? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(uint? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(ulong? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(TEnum? value) where TEnum : Enum => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(float? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(double? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(DateOnly? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(TimeOnly? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(DuckDBDateOnly? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(DuckDBTimeOnly? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(DateTime? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(DateTimeOffset? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(TimeSpan? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendValue(IEnumerable? value) => row.AppendValue(value); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void AppendDefault() => row.AppendDefault(); +} + +/// +/// Writes one complete row through a stack-only . +/// +public delegate void DuckDBAppenderRowWriterAction(ref DuckDBAppenderRowWriter row, TState state); diff --git a/DuckDB.NET.Data/DuckDBCommand.cs b/DuckDB.NET.Data/DuckDBCommand.cs index a0b67b3c..f21bdaa0 100644 --- a/DuckDB.NET.Data/DuckDBCommand.cs +++ b/DuckDB.NET.Data/DuckDBCommand.cs @@ -1,6 +1,13 @@ using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using System.Threading; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using DuckDB.NET.Data.Arrow; +using DuckDB.NET.Data.DataChunk.Reader; +using PreparedStatementBase = DuckDB.NET.Data.PreparedStatement.PreparedStatement; +using ReusablePreparedStatement = DuckDB.NET.Data.PreparedStatement.ReusablePreparedStatement; namespace DuckDB.NET.Data; @@ -8,6 +15,12 @@ public class DuckDBCommand : DbCommand { private DuckDBConnection? connection; private readonly DuckDBParameterCollection parameters = new(); + private ReusablePreparedStatement? preparedStatement; + private DuckDBConnection? preparedConnection; + private List<(DuckDBConnection Connection, ReusablePreparedStatement Statement)>? deferredPreparedStatements; + private HashSet? registeredConnections; + private int activeExecutions; + private bool disposed; protected override DbTransaction? DbTransaction { get; set; } protected override DbParameterCollection DbParameterCollection => parameters; @@ -35,15 +48,35 @@ public override string CommandText get; set { - // TODO: We shouldn't be able to change the CommandText when the command is in execution (requires CommandState implementation) - field = value ?? string.Empty; + EnsureNotDisposed(); + + var newValue = value ?? string.Empty; + if (string.Equals(field, newValue, StringComparison.Ordinal)) + { + return; + } + + InvalidatePreparedStatements(); + field = newValue; } } = string.Empty; protected override DbConnection? DbConnection { get => connection; - set => connection = (DuckDBConnection?)value; + set + { + EnsureNotDisposed(); + + var newConnection = (DuckDBConnection?)value; + if (ReferenceEquals(connection, newConnection)) + { + return; + } + + InvalidatePreparedStatements(); + connection = newConnection; + } } public DuckDBCommand() @@ -66,7 +99,12 @@ public override int ExecuteNonQuery() { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); + if (preparedStatement is { } reusableStatement) + { + return ExecutePreparedNonQuery(reusableStatement, connection!.NativeConnection); + } + + var results = ExecuteStatements(); var count = 0; @@ -84,6 +122,11 @@ public override int ExecuteNonQuery() { EnsureConnectionOpen(); + if (preparedStatement is { } reusableStatement) + { + return ExecutePreparedScalar(reusableStatement, connection!.NativeConnection); + } + using var reader = ExecuteReader(); return reader.Read() ? reader.GetValue(0) : null; } @@ -102,24 +145,414 @@ protected override DbDataReader ExecuteDbDataReader(CommandBehavior behavior) { EnsureConnectionOpen(); - var results = PreparedStatement.PreparedStatement.PrepareMultiple(connection!.NativeConnection, CommandText, parameters, UseStreamingMode); + var results = ExecuteStatements(); var reader = new DuckDBDataReader(this, results, behavior); return reader; } - public override void Prepare() { } + /// + /// Executes the command and returns the first result set as an Apache Arrow + /// . Each DuckDB data chunk is converted to an Arrow record batch + /// using DuckDB's Arrow C Data Interface, with no row-by-row marshaling. + /// When is enabled, record batches are produced lazily from a + /// streaming result (bounded memory), otherwise the result is materialized first. + /// The caller owns the returned stream and must dispose it. + /// + public IArrowArrayStream ExecuteArrowStream() + { + EnsureConnectionOpen(); + + var results = ExecuteStatements(); + + foreach (var result in results) + { + var current = result; + + if (NativeMethods.Query.DuckDBResultReturnType(current) == DuckDBResultType.QueryResult) + { + return new DuckDBArrowArrayStream(current); + } + + current.Close(); + } + + throw new InvalidOperationException("The command did not return a result set."); + } + + /// + /// Executes the command and asynchronously streams the first result set as Apache Arrow + /// values. The batches are produced lazily, one per DuckDB data chunk. + /// Set to stream from a streaming result with bounded memory. + /// + public async IAsyncEnumerable ExecuteArrowBatchesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var stream = ExecuteArrowStream(); + + try + { + while (await stream.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false) is { } batch) + { + yield return batch; + } + } + finally + { + stream.Dispose(); + } + } + + public override void Prepare() + { + EnsureNotDisposed(); + EnsureConnectionOpen(); + + if (preparedStatement is not null) + { + return; + } + + var statement = PreparedStatementBase.TryPrepareReusable(connection!.NativeConnection, CommandText); + if (statement is null) + { + return; + } + + preparedStatement = statement; + preparedConnection = connection; + RefreshPreparedCommandRegistrations(); + } protected override DbParameter CreateDbParameter() => new DuckDBParameter(); internal void CloseConnection() => Connection!.Close(); + protected override void Dispose(bool disposing) + { + if (disposing && !disposed) + { + disposed = true; + InvalidatePreparedStatements(); + + if (activeExecutions == 0) + { + UnregisterFromConnections(); + } + } + + base.Dispose(disposing); + } + + private IEnumerable ExecuteStatements() + { + EnsureNotDisposed(); + + var nativeConnection = connection!.NativeConnection; + var reusableStatement = preparedStatement; + + return reusableStatement is null + ? PreparedStatementBase.PrepareMultiple(nativeConnection, CommandText, parameters, UseStreamingMode) + : ExecutePreparedStatement(reusableStatement, nativeConnection); + } + + private IEnumerable ExecutePreparedStatement( + ReusablePreparedStatement reusableStatement, + DuckDBNativeConnection nativeConnection) + { + BeginPreparedExecution(); + + try + { + yield return reusableStatement.Execute(parameters, UseStreamingMode, nativeConnection); + } + finally + { + CompletePreparedExecution(); + } + } + + private int ExecutePreparedNonQuery( + ReusablePreparedStatement reusableStatement, + DuckDBNativeConnection nativeConnection) + { + BeginPreparedExecution(); + + try + { + var result = reusableStatement.Execute(parameters, UseStreamingMode, nativeConnection); + + try + { + var current = result; + return (int)NativeMethods.Query.DuckDBRowsChanged(ref current); + } + finally + { + result.Close(); + } + } + finally + { + CompletePreparedExecution(); + } + } + + private object? ExecutePreparedScalar( + ReusablePreparedStatement reusableStatement, + DuckDBNativeConnection nativeConnection) + { + BeginPreparedExecution(); + + try + { + var result = reusableStatement.Execute(parameters, UseStreamingMode, nativeConnection); + + try + { + if (NativeMethods.Query.DuckDBResultReturnType(result) != DuckDBResultType.QueryResult || + NativeMethods.Query.DuckDBColumnCount(ref result) == 0) + { + return null; + } + + return ReadFirstValue(ref result); + } + finally + { + result.Close(); + } + } + finally + { + CompletePreparedExecution(); + } + } + + private static object? ReadFirstValue(ref DuckDBResult result) + { + var streamingResult = NativeMethods.Types.DuckDBResultIsStreaming(result) > 0; + long chunkIndex = 0; + + while (true) + { + using var chunk = streamingResult + ? NativeMethods.StreamingResult.DuckDBStreamFetchChunk(result) + : NativeMethods.Types.DuckDBResultGetChunk(result, chunkIndex++); + + if (chunk is null || chunk.IsInvalid) + { + return null; + } + + if (NativeMethods.DataChunks.DuckDBDataChunkGetSize(chunk) == 0) + { + continue; + } + + var vector = NativeMethods.DataChunks.DuckDBDataChunkGetVector(chunk, 0); + using var logicalType = NativeMethods.Query.DuckDBColumnLogicalType(ref result, 0); + using var reader = VectorDataReaderFactory.CreateReader(vector, logicalType); + + return reader.IsValid(0) ? reader.GetValue(0) : DBNull.Value; + } + } + + private void BeginPreparedExecution() + { + activeExecutions++; + } + + private void CompletePreparedExecution() + { + activeExecutions--; + + if (activeExecutions == 0) + { + DisposeDeferredPreparedStatements(); + } + + if (disposed && activeExecutions == 0) + { + UnregisterFromConnections(); + } + } + + private void InvalidatePreparedStatements() + { + var statement = preparedStatement; + var statementConnection = preparedConnection; + preparedStatement = null; + preparedConnection = null; + + if (statement is null) + { + return; + } + + if (activeExecutions > 0) + { + deferredPreparedStatements ??= []; + deferredPreparedStatements.Add((statementConnection!, statement)); + RefreshPreparedCommandRegistrations(); + return; + } + + statement.Dispose(); + RefreshPreparedCommandRegistrations(); + } + + private void DisposeDeferredPreparedStatements() + { + var deferredStatements = deferredPreparedStatements; + if (deferredStatements is null) + { + return; + } + + foreach (var deferredStatement in deferredStatements) + { + deferredStatement.Statement.Dispose(); + } + + deferredPreparedStatements = null; + RefreshPreparedCommandRegistrations(); + } + + internal void OnConnectionClosing(DuckDBConnection closingConnection) + { + if (ReferenceEquals(preparedConnection, closingConnection)) + { + preparedStatement?.Dispose(); + preparedStatement = null; + preparedConnection = null; + } + + for (var index = deferredPreparedStatements?.Count - 1 ?? -1; index >= 0; index--) + { + var deferredStatement = deferredPreparedStatements![index]; + if (!ReferenceEquals(deferredStatement.Connection, closingConnection)) + { + continue; + } + + deferredStatement.Statement.Dispose(); + deferredPreparedStatements.RemoveAt(index); + } + + if (deferredPreparedStatements?.Count == 0) + { + deferredPreparedStatements = null; + } + + RefreshPreparedCommandRegistrations(); + } + + private void RefreshPreparedCommandRegistrations() + { + List? connectionsToRemove = null; + + if (registeredConnections is not null) + { + foreach (var registeredConnection in registeredConnections) + { + if (IsConnectionRequired(registeredConnection)) + { + continue; + } + + connectionsToRemove ??= []; + connectionsToRemove.Add(registeredConnection); + } + } + + if (connectionsToRemove is not null) + { + foreach (var registeredConnection in connectionsToRemove) + { + registeredConnection.UnregisterPreparedCommand(this); + registeredConnections!.Remove(registeredConnection); + } + } + + if (registeredConnections?.Count == 0) + { + registeredConnections = null; + } + + if (preparedConnection is not null) + { + RegisterWithConnection(preparedConnection); + } + + if (deferredPreparedStatements is not null) + { + foreach (var deferredStatement in deferredPreparedStatements) + { + RegisterWithConnection(deferredStatement.Connection); + } + } + } + + private bool IsConnectionRequired(DuckDBConnection candidate) + { + if (ReferenceEquals(preparedConnection, candidate)) + { + return true; + } + + if (deferredPreparedStatements is null) + { + return false; + } + + foreach (var deferredStatement in deferredPreparedStatements) + { + if (ReferenceEquals(deferredStatement.Connection, candidate)) + { + return true; + } + } + + return false; + } + + private void RegisterWithConnection(DuckDBConnection requiredConnection) + { + registeredConnections ??= []; + if (registeredConnections.Add(requiredConnection)) + { + requiredConnection.RegisterPreparedCommand(this); + } + } + + private void UnregisterFromConnections() + { + if (registeredConnections is null) + { + return; + } + + foreach (var registeredConnection in registeredConnections) + { + registeredConnection.UnregisterPreparedCommand(this); + } + + registeredConnections = null; + } + + private void EnsureNotDisposed() + { + ObjectDisposedException.ThrowIf(disposed, this); + } + private void EnsureConnectionOpen([CallerMemberName] string operation = "") { + EnsureNotDisposed(); + if (Connection is null || Connection.State != ConnectionState.Open) { throw new InvalidOperationException($"{operation} requires an open connection"); } } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/DuckDBConnection.cs b/DuckDB.NET.Data/DuckDBConnection.cs index 54bfbbe6..2dd35758 100644 --- a/DuckDB.NET.Data/DuckDBConnection.cs +++ b/DuckDB.NET.Data/DuckDBConnection.cs @@ -12,7 +12,8 @@ public partial class DuckDBConnection : DbConnection private DuckDBConnectionString? parsedConnection; private ConnectionReference? connectionReference; private bool inMemoryDuplication = false; - + private List>? preparedCommands; + private static readonly StateChangeEventArgs FromClosedToOpenEventArgs = new(ConnectionState.Closed, ConnectionState.Open); private static readonly StateChangeEventArgs FromOpenToClosedEventArgs = new(ConnectionState.Open, ConnectionState.Closed); @@ -88,6 +89,8 @@ public override void Close() throw new InvalidOperationException("Connection is already closed."); } + InvalidatePreparedCommands(); + if (connectionReference is not null) //Should always be the case { connectionManager.ReturnConnectionReference(connectionReference); @@ -97,6 +100,68 @@ public override void Close() OnStateChange(FromOpenToClosedEventArgs); } + internal void RegisterPreparedCommand(DuckDBCommand command) + { + preparedCommands ??= []; + + for (var index = preparedCommands.Count - 1; index >= 0; index--) + { + if (!preparedCommands[index].TryGetTarget(out var preparedCommand)) + { + preparedCommands.RemoveAt(index); + continue; + } + + if (ReferenceEquals(preparedCommand, command)) + { + return; + } + } + + preparedCommands.Add(new WeakReference(command)); + } + + internal void UnregisterPreparedCommand(DuckDBCommand command) + { + if (preparedCommands is null) + { + return; + } + + for (var index = preparedCommands.Count - 1; index >= 0; index--) + { + if (!preparedCommands[index].TryGetTarget(out var preparedCommand) || ReferenceEquals(preparedCommand, command)) + { + preparedCommands.RemoveAt(index); + } + } + + if (preparedCommands.Count == 0) + { + preparedCommands = null; + } + } + + private void InvalidatePreparedCommands() + { + if (preparedCommands is not { Count: > 0 }) + { + return; + } + + var commands = new WeakReference[preparedCommands.Count]; + preparedCommands.CopyTo(commands); + preparedCommands = null; + + foreach (var commandReference in commands) + { + if (commandReference.TryGetTarget(out var command)) + { + command.OnConnectionClosing(this); + } + } + } + public override void Open() { if (connectionState == ConnectionState.Open) @@ -161,7 +226,7 @@ public DuckDBAppender CreateAppender(string? catalog, string? schema, string tab { try { - DuckDBAppender.ThrowLastError(nativeAppender); + NativeMethods.Appender.DuckDBAppenderErrorData(nativeAppender).ThrowOnError(); } finally { diff --git a/DuckDB.NET.Data/DuckDBDataReader.cs b/DuckDB.NET.Data/DuckDBDataReader.cs index 3d74fb26..00e76d62 100644 --- a/DuckDB.NET.Data/DuckDBDataReader.cs +++ b/DuckDB.NET.Data/DuckDBDataReader.cs @@ -1,5 +1,4 @@ -using DuckDB.NET.Data.Common; -using DuckDB.NET.Data.DataChunk.Reader; +using DuckDB.NET.Data.DataChunk.Reader; using System.IO; using System.Runtime.CompilerServices; diff --git a/DuckDB.NET.Data/DuckDBMappedAppender.cs b/DuckDB.NET.Data/DuckDBMappedAppender.cs index ee87f303..8b65eba0 100644 --- a/DuckDB.NET.Data/DuckDBMappedAppender.cs +++ b/DuckDB.NET.Data/DuckDBMappedAppender.cs @@ -1,4 +1,5 @@ using DuckDB.NET.Data.Mapping; +using System.Threading; namespace DuckDB.NET.Data; @@ -9,16 +10,18 @@ namespace DuckDB.NET.Data; /// The AppenderMap type defining the mappings public class DuckDBMappedAppender : IDisposable where TMap : DuckDBAppenderMap, new() { + private static readonly Lazy CompiledMap = + new(CreateCompiledMap, LazyThreadSafetyMode.ExecutionAndPublication); + private readonly DuckDBAppender appender; - private readonly List> mappings; + private readonly Action writeRecord; internal DuckDBMappedAppender(DuckDBAppender appender) { this.appender = appender; - var classMap = new TMap(); - - // Get mappings as List to avoid interface enumerator boxing - mappings = classMap.PropertyMappings; + var compiledMap = CompiledMap.Value; + var mappings = compiledMap.Mappings; + writeRecord = compiledMap.WriteRecord; // Validate mappings match the table structure if (mappings.Count == 0) @@ -50,6 +53,7 @@ internal DuckDBMappedAppender(DuckDBAppender appender) $"Type mismatch at column index {index}: Mapped type is {mapping.PropertyType.Name} (expected DuckDB type: {expectedType}) but actual column type is {columnType}"); } } + } /// @@ -76,14 +80,7 @@ private void AppendRecord(T record) throw new ArgumentNullException(nameof(record)); } - var row = appender.CreateRow(); - - foreach (var mapping in mappings) - { - row = mapping.AppendToRow(row, record); - } - - row.EndRow(); + appender.AppendRow(record, writeRecord); } private static DuckDBType GetExpectedDuckDBType(Type type) @@ -97,6 +94,16 @@ private static DuckDBType GetExpectedDuckDBType(Type type) }; } + private static CompiledAppenderMap CreateCompiledMap() + { + var mappings = new TMap().PropertyMappings; + return new CompiledAppenderMap(mappings, DuckDBAppenderMapCompiler.Compile(mappings)); + } + + private sealed record CompiledAppenderMap( + IReadOnlyList> Mappings, + Action WriteRecord); + /// /// Closes the appender and flushes any remaining data. /// diff --git a/DuckDB.NET.Data/DuckDBParameter.cs b/DuckDB.NET.Data/DuckDBParameter.cs index 7ace2d50..0cf12bf3 100644 --- a/DuckDB.NET.Data/DuckDBParameter.cs +++ b/DuckDB.NET.Data/DuckDBParameter.cs @@ -9,12 +9,28 @@ public class DuckDBParameter : DbParameter private const DbType DefaultDbType = DbType.String; private object? value; + private string parameterName = string.Empty; + private int bindingMetadataVersion; public override DbType DbType { get; set; } [AllowNull] [DefaultValue("")] - public override string ParameterName { get; set; } + public override string ParameterName + { + get => parameterName; + set + { + var newValue = value ?? string.Empty; + if (string.Equals(parameterName, newValue, StringComparison.Ordinal)) + { + return; + } + + parameterName = newValue; + bindingMetadataVersion++; + } + } public override object? Value { @@ -38,6 +54,8 @@ public override object? Value public override bool SourceColumnNullMapping { get; set; } public override int Size { get; set; } + internal int BindingMetadataVersion => bindingMetadataVersion; + public DuckDBParameter() : this (string.Empty, DefaultDbType, null) { } @@ -66,4 +84,86 @@ public DuckDBParameter(string name, DbType type, object? value) } public override void ResetDbType() => DbType = DefaultDbType; -} \ No newline at end of file + + internal virtual bool TryBindScalarValue( + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + => value.TryBindScalarValue(statement, index, duckDBType, DbType, out result); + + internal virtual DuckDBValue ToDuckDBValue(DuckDBLogicalType logicalType, DuckDBType duckDBType) + => value.ToDuckDBValue(logicalType, duckDBType, DbType); +} + +/// +/// A DuckDB parameter that keeps its value in typed storage so repeated prepared executions do +/// not need to box common value types. +/// +/// The CLR type stored by the parameter. +public sealed class DuckDBParameter : DuckDBParameter +{ + private static readonly DbType DefaultTypedDbType = DuckDBTypeMap.GetDbTypeForType(typeof(T)); + private T typedValue; + + /// + /// Gets or sets the strongly typed parameter value. + /// + public T TypedValue + { + get => typedValue; + set => typedValue = value; + } + + public override object? Value + { + get => typedValue; + set + { + if (value is T typed) + { + typedValue = typed; + return; + } + + if (value is null && default(T) is null) + { + typedValue = default!; + return; + } + + throw new InvalidCastException( + $"Parameter '{ParameterName}' requires a value assignable to {typeof(T).Name}."); + } + } + + public DuckDBParameter(T value) + : this(string.Empty, value) + { + } + + public DuckDBParameter(string name, T value) + : base(name, DefaultTypedDbType, null) + { + typedValue = value; + } + + public override void ResetDbType() => DbType = DefaultTypedDbType; + + internal override bool TryBindScalarValue( + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + { + if (ClrToDuckDBConverter.TryBindTypedScalarValue(typedValue, statement, index, duckDBType, out result)) + { + return true; + } + + return ((object?)typedValue).TryBindScalarValue(statement, index, duckDBType, DbType, out result); + } + + internal override DuckDBValue ToDuckDBValue(DuckDBLogicalType logicalType, DuckDBType duckDBType) + => ((object?)typedValue).ToDuckDBValue(logicalType, duckDBType, DbType); +} diff --git a/DuckDB.NET.Data/DuckDBParameterCollection.cs b/DuckDB.NET.Data/DuckDBParameterCollection.cs index 86204812..661370e6 100644 --- a/DuckDB.NET.Data/DuckDBParameterCollection.cs +++ b/DuckDB.NET.Data/DuckDBParameterCollection.cs @@ -5,11 +5,18 @@ namespace DuckDB.NET.Data; public class DuckDBParameterCollection : DbParameterCollection { private readonly List parameters = new(); + private int version; + + internal int Version => version; public new DuckDBParameter this[int index] { get => parameters[index]; - set => parameters[index] = value; + set + { + parameters[index] = value; + version++; + } } public new DuckDBParameter this[string parameterName] @@ -24,22 +31,43 @@ public class DuckDBParameterCollection : DbParameterCollection public override int Add(object value) { parameters.Add((DuckDBParameter)value); + version++; return parameters.Count - 1; } - public override void Clear() => parameters.Clear(); + public override void Clear() + { + if (parameters.Count == 0) + { + return; + } + + parameters.Clear(); + version++; + } public override bool Contains(object value) => parameters.Contains((DuckDBParameter) value); public override int IndexOf(object value) => parameters.IndexOf((DuckDBParameter) value); - public override void Insert(int index, object value) => parameters.Insert(index, (DuckDBParameter) value); + public override void Insert(int index, object value) + { + parameters.Insert(index, (DuckDBParameter)value); + version++; + } - public override void Remove(object value) => parameters.Remove((DuckDBParameter) value); + public override void Remove(object value) + { + if (parameters.Remove((DuckDBParameter)value)) + { + version++; + } + } public int Add(DuckDBParameter value) { parameters.Add(value); + version++; return parameters.Count - 1; } @@ -47,26 +75,41 @@ public int Add(DuckDBParameter value) public int IndexOf(DuckDBParameter value) => parameters.IndexOf(value); - public void Insert(int index, DuckDBParameter value) => parameters.Insert(index, value); + public void Insert(int index, DuckDBParameter value) + { + parameters.Insert(index, value); + version++; + } - public void Remove(DuckDBParameter value) => parameters.Remove(value); + public void Remove(DuckDBParameter value) + { + if (parameters.Remove(value)) + { + version++; + } + } - public override void RemoveAt(int index) => parameters.RemoveAt(index); + public override void RemoveAt(int index) + { + parameters.RemoveAt(index); + version++; + } public override void RemoveAt(string parameterName) { var index = IndexOfSafe(parameterName); parameters.RemoveAt(index); + version++; } protected override void SetParameter(int index, DbParameter value) - => parameters[index] = (DuckDBParameter)value; + => this[index] = (DuckDBParameter)value; protected override void SetParameter(string parameterName, DbParameter value) { var index = IndexOfSafe(parameterName); - parameters[index] = (DuckDBParameter)value; + this[index] = (DuckDBParameter)value; } public override int IndexOf(string parameterName) @@ -95,7 +138,14 @@ public override void AddRange(Array values) => AddRange(values.Cast()); public void AddRange(IEnumerable values) - => parameters.AddRange(values); + { + var oldCount = parameters.Count; + parameters.AddRange(values); + if (parameters.Count != oldCount) + { + version++; + } + } private int IndexOfSafe(string parameterName) { @@ -104,4 +154,4 @@ private int IndexOfSafe(string parameterName) throw new IndexOutOfRangeException($"Parameter '{parameterName}' not found"); return index; } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs b/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs index f737fdee..2d37ffcd 100644 --- a/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs +++ b/DuckDB.NET.Data/Extensions/DateTimeExtensions.cs @@ -16,14 +16,22 @@ public static DuckDBTimeTzStruct ToTimeTzStruct(this DateTimeOffset value) public static DuckDBTimestampStruct ToTimestampStruct(this DateTimeOffset value) { - var timestamp = DuckDBTimestamp.FromDateTime(value.UtcDateTime).ToDuckDBTimestampStruct(); - - return timestamp; + return value.UtcDateTime.ToTimestampStruct(DuckDBType.Timestamp); } public static DuckDBTimestampStruct ToTimestampStruct(this DateTime value, DuckDBType duckDBType) { - var timestamp = DuckDBTimestamp.FromDateTime(value).ToDuckDBTimestampStruct(); + var ticksSinceEpoch = value.Ticks - DateTime.UnixEpoch.Ticks; + var microseconds = ticksSinceEpoch / TicksPerMicrosecond; + + // duckdb_to_timestamp truncates the time-of-day to microseconds after resolving the date, + // which is floor division for pre-epoch values with sub-microsecond ticks. + if (ticksSinceEpoch < 0 && ticksSinceEpoch % TicksPerMicrosecond != 0) + { + microseconds--; + } + + var timestamp = new DuckDBTimestampStruct { Micros = microseconds }; if (duckDBType == DuckDBType.TimestampNs) { diff --git a/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs new file mode 100644 index 00000000..1bbd788b --- /dev/null +++ b/DuckDB.NET.Data/Extensions/DuckDBErrorDataExtensions.cs @@ -0,0 +1,16 @@ +namespace DuckDB.NET.Data.Extensions; + +internal static class DuckDBErrorDataExtensions +{ + public static void ThrowOnError(this DuckDBErrorData errorData, string message = "") + { + using (errorData) + { + if (errorData.HasError) + { + var errorType = NativeMethods.ErrorData.DuckDBErrorDataErrorType(errorData); + throw new DuckDBException($"{message} {errorData.Message}".Trim(), errorType); + } + } + } +} diff --git a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs index 0f8d576f..8583fcde 100644 --- a/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs +++ b/DuckDB.NET.Data/Mapping/DuckDBAppenderMap.cs @@ -1,3 +1,5 @@ +using System.Linq.Expressions; + namespace DuckDB.NET.Data.Mapping; /// @@ -68,7 +70,8 @@ internal interface IPropertyMapping { Type PropertyType { get; } PropertyMappingType MappingType { get; } - IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record); + void AppendToRow(IDuckDBAppenderRow row, T record); + Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record); } internal sealed class PropertyMapping : IPropertyMapping @@ -77,16 +80,17 @@ internal sealed class PropertyMapping : IPropertyMapping public Func Getter { get; set; } = _ => default!; public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) { var value = Getter(record); if (value is null) { - return row.AppendNullValue(); + row.AppendNullValue(); + return; } - return value switch + _ = value switch { // Reference types string v => row.AppendValue(v), @@ -118,6 +122,12 @@ public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) _ => throw new NotSupportedException($"Type {typeof(TProperty).Name} is not supported for appending") }; } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + { + var value = Expression.Invoke(Expression.Constant(Getter), record); + return DuckDBAppenderMapCompiler.CreateAppendExpression(row, value, typeof(TProperty)); + } } internal sealed class DefaultValueMapping : IPropertyMapping @@ -125,10 +135,13 @@ internal sealed class DefaultValueMapping : IPropertyMapping public Type PropertyType { get; set; } = typeof(object); public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) { - return row.AppendDefault(); + row.AppendDefault(); } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + => Expression.Call(row, nameof(IDuckDBAppenderRow.AppendDefault), Type.EmptyTypes); } internal sealed class NullValueMapping : IPropertyMapping @@ -136,8 +149,95 @@ internal sealed class NullValueMapping : IPropertyMapping public Type PropertyType { get; set; } = typeof(object); public PropertyMappingType MappingType { get; set; } - public IDuckDBAppenderRow AppendToRow(IDuckDBAppenderRow row, T record) + public void AppendToRow(IDuckDBAppenderRow row, T record) + { + row.AppendNullValue(); + } + + public Expression BuildWriteExpression(ParameterExpression row, ParameterExpression record) + => Expression.Call(row, nameof(IDuckDBAppenderRow.AppendNullValue), Type.EmptyTypes); +} + +internal static class DuckDBAppenderMapCompiler +{ + private static readonly HashSet SupportedValueTypes = + [ + typeof(bool), + typeof(sbyte), + typeof(short), + typeof(int), + typeof(long), + typeof(byte), + typeof(ushort), + typeof(uint), + typeof(ulong), + typeof(float), + typeof(double), + typeof(decimal), + typeof(DateTime), + typeof(DateTimeOffset), + typeof(TimeSpan), + typeof(Guid), + typeof(BigInteger), + typeof(DuckDBDateOnly), + typeof(DuckDBTimeOnly), + typeof(DateOnly), + typeof(TimeOnly) + ]; + + public static Action Compile(IReadOnlyList> mappings) + { + var row = Expression.Parameter(typeof(IDuckDBAppenderRow), "row"); + var record = Expression.Parameter(typeof(T), "record"); + var writes = new List(mappings.Count + 1); + + for (var index = 0; index < mappings.Count; index++) + { + writes.Add(mappings[index].BuildWriteExpression(row, record)); + } + + // IDuckDBAppenderRow methods return the row to support fluent callers. The mapped + // callback is an Action, so force the compiled block's result type to void. + writes.Add(Expression.Empty()); + + return Expression.Lambda>( + Expression.Block(writes), + row, + record).Compile(); + } + + internal static Expression CreateAppendExpression( + ParameterExpression row, + Expression value, + Type propertyType) { - return row.AppendNullValue(); + if (propertyType == typeof(string) || propertyType == typeof(byte[])) + { + return Expression.Call( + row, + nameof(IDuckDBAppenderRow.AppendValue), + Type.EmptyTypes, + value); + } + + var underlyingType = Nullable.GetUnderlyingType(propertyType); + var valueType = underlyingType ?? propertyType; + + if (!SupportedValueTypes.Contains(valueType) && !valueType.IsEnum) + { + throw new NotSupportedException($"Type {propertyType.Name} is not supported for appending"); + } + + var nullableType = typeof(Nullable<>).MakeGenericType(valueType); + var nullableValue = propertyType == nullableType + ? value + : Expression.Convert(value, nullableType); + var genericArguments = valueType.IsEnum ? [valueType] : Type.EmptyTypes; + + return Expression.Call( + row, + nameof(IDuckDBAppenderRow.AppendValue), + genericArguments, + nullableValue); } } diff --git a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs index 321fdd2a..de55d955 100644 --- a/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs +++ b/DuckDB.NET.Data/PreparedStatement/ClrToDuckDBConverter.cs @@ -1,3 +1,5 @@ +using System.Runtime.CompilerServices; + namespace DuckDB.NET.Data.PreparedStatement; internal static class ClrToDuckDBConverter @@ -57,15 +59,15 @@ public static DuckDBValue ToDuckDBValue(this object? item, DuckDBLogicalType log { (DuckDBType.Boolean, bool value) => NativeMethods.Value.DuckDBCreateBool(value), - (DuckDBType.TinyInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateInt8(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.SmallInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateInt16(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.Integer, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateInt32(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.BigInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateInt64(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.TinyInt, _) => TryConvertTo(item, out sbyte result) ? NativeMethods.Value.DuckDBCreateInt8(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.SmallInt, _) => TryConvertTo(item, out short result) ? NativeMethods.Value.DuckDBCreateInt16(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.Integer, _) => TryConvertTo(item, out int result) ? NativeMethods.Value.DuckDBCreateInt32(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.BigInt, _) => TryConvertTo(item, out long result) ? NativeMethods.Value.DuckDBCreateInt64(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.UnsignedTinyInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateUInt8(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.UnsignedSmallInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateUInt16(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.UnsignedInteger, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateUInt32(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), - (DuckDBType.UnsignedBigInt, _) => TryConvertTo(out var result) ? NativeMethods.Value.DuckDBCreateUInt64(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.UnsignedTinyInt, _) => TryConvertTo(item, out byte result) ? NativeMethods.Value.DuckDBCreateUInt8(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.UnsignedSmallInt, _) => TryConvertTo(item, out ushort result) ? NativeMethods.Value.DuckDBCreateUInt16(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.UnsignedInteger, _) => TryConvertTo(item, out uint result) ? NativeMethods.Value.DuckDBCreateUInt32(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), + (DuckDBType.UnsignedBigInt, _) => TryConvertTo(item, out ulong result) ? NativeMethods.Value.DuckDBCreateUInt64(result) : NativeMethods.Value.DuckDBCreateVarchar(item.ToString()), (DuckDBType.Float, float value) => NativeMethods.Value.DuckDBCreateFloat(value), (DuckDBType.Double, double value) => NativeMethods.Value.DuckDBCreateDouble(value), @@ -97,25 +99,403 @@ public static DuckDBValue ToDuckDBValue(this object? item, DuckDBLogicalType log _ when ValueCreators.TryGetValue(dbType, out var converter) => converter(item), _ => NativeMethods.Value.DuckDBCreateVarchar(item.ToString()) }; + } - bool TryConvertTo(out T result) where T : struct + public static bool TryBindScalarValue(this object? item, DuckDBPreparedStatement statement, long index, DuckDBType duckDBType, DbType dbType, out DuckDBState result) + { + if (item.IsNull()) { - try - { - if (item is T parsable) - { - result = parsable; - return true; - } + result = NativeMethods.PreparedStatements.DuckDBBindNull(statement, index); + return true; + } - result = (T)Convert.ChangeType(item, typeof(T)); + switch (duckDBType, item) + { + case (DuckDBType.Boolean, bool booleanValue): + result = NativeMethods.PreparedStatements.DuckDBBindBoolean(statement, index, booleanValue); return true; - } - catch (Exception) - { + + case (DuckDBType.TinyInt, _): + result = TryConvertTo(item, out sbyte int8Value) + ? NativeMethods.PreparedStatements.DuckDBBindInt8(statement, index, int8Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.SmallInt, _): + result = TryConvertTo(item, out short int16Value) + ? NativeMethods.PreparedStatements.DuckDBBindInt16(statement, index, int16Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.Integer, _): + result = TryConvertTo(item, out int int32Value) + ? NativeMethods.PreparedStatements.DuckDBBindInt32(statement, index, int32Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.BigInt, _): + result = TryConvertTo(item, out long int64Value) + ? NativeMethods.PreparedStatements.DuckDBBindInt64(statement, index, int64Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + + case (DuckDBType.UnsignedTinyInt, _): + result = TryConvertTo(item, out byte uint8Value) + ? NativeMethods.PreparedStatements.DuckDBBindUInt8(statement, index, uint8Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.UnsignedSmallInt, _): + result = TryConvertTo(item, out ushort uint16Value) + ? NativeMethods.PreparedStatements.DuckDBBindUInt16(statement, index, uint16Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.UnsignedInteger, _): + result = TryConvertTo(item, out uint uint32Value) + ? NativeMethods.PreparedStatements.DuckDBBindUInt32(statement, index, uint32Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + case (DuckDBType.UnsignedBigInt, _): + result = TryConvertTo(item, out ulong uint64Value) + ? NativeMethods.PreparedStatements.DuckDBBindUInt64(statement, index, uint64Value) + : NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, item.ToString()!); + return true; + + case (DuckDBType.Float, float floatValue): + result = NativeMethods.PreparedStatements.DuckDBBindFloat(statement, index, floatValue); + return true; + case (DuckDBType.Double, double doubleValue): + result = NativeMethods.PreparedStatements.DuckDBBindDouble(statement, index, doubleValue); + return true; + case (DuckDBType.Decimal, decimal decimalValue): + result = NativeMethods.PreparedStatements.DuckDBBindDecimal(statement, index, ToDuckDBDecimal(decimalValue)); + return true; + case (DuckDBType.HugeInt, BigInteger hugeIntValue): + result = NativeMethods.PreparedStatements.DuckDBBindHugeInt(statement, index, new DuckDBHugeInt(hugeIntValue)); + return true; + + case (DuckDBType.Varchar, string stringValue): + result = NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, stringValue); + return true; + case (DuckDBType.Timestamp, DateTime timestampValue): + result = NativeMethods.PreparedStatements.DuckDBBindTimestamp(statement, index, timestampValue.ToTimestampStruct(duckDBType)); + return true; + case (DuckDBType.TimestampTz, DateTime timestampTzValue): + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz(statement, index, timestampTzValue.ToTimestampStruct(duckDBType)); + return true; + case (DuckDBType.TimestampTz, DateTimeOffset timestampOffsetValue): + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz(statement, index, timestampOffsetValue.ToTimestampStruct()); + return true; + case (DuckDBType.Interval, TimeSpan intervalValue): + result = NativeMethods.PreparedStatements.DuckDBBindInterval(statement, index, intervalValue); + return true; + case (DuckDBType.Date, DateTime dateTimeValue): + result = NativeMethods.PreparedStatements.DuckDBBindDate(statement, index, ((DuckDBDateOnly)dateTimeValue).ToDuckDBDate()); + return true; + case (DuckDBType.Date, DuckDBDateOnly duckDBDateValue): + result = NativeMethods.PreparedStatements.DuckDBBindDate(statement, index, duckDBDateValue.ToDuckDBDate()); + return true; + case (DuckDBType.Date, DateOnly dateOnlyValue): + result = NativeMethods.PreparedStatements.DuckDBBindDate(statement, index, ((DuckDBDateOnly)dateOnlyValue).ToDuckDBDate()); + return true; + case (DuckDBType.Time, DateTime timeDateTimeValue): + result = NativeMethods.PreparedStatements.DuckDBBindTime(statement, index, NativeMethods.DateTimeHelpers.DuckDBToTime((DuckDBTimeOnly)timeDateTimeValue)); + return true; + case (DuckDBType.Time, DuckDBTimeOnly duckDBTimeValue): + result = NativeMethods.PreparedStatements.DuckDBBindTime(statement, index, NativeMethods.DateTimeHelpers.DuckDBToTime(duckDBTimeValue)); + return true; + case (DuckDBType.Time, TimeOnly timeOnlyValue): + result = NativeMethods.PreparedStatements.DuckDBBindTime(statement, index, NativeMethods.DateTimeHelpers.DuckDBToTime(timeOnlyValue)); + return true; + case (DuckDBType.Blob, byte[] blobValue): + result = NativeMethods.PreparedStatements.DuckDBBindBlob(statement, index, blobValue, blobValue.LongLength); + return true; + } + + // These logical types need the exact duckdb_value representation because the scalar C API + // has no equivalent binder, or because its timestamp unit differs from duckdb_bind_timestamp. + if (item is Guid || + item is DateTime && duckDBType is DuckDBType.TimestampS or DuckDBType.TimestampMs or DuckDBType.TimestampNs || + item is DateTimeOffset && duckDBType == DuckDBType.TimeTz || + item is ICollection && (item is not byte[] || duckDBType is DuckDBType.List or DuckDBType.Array)) + { + result = default; + return false; + } + + switch (dbType) + { + case DbType.Currency: + result = NativeMethods.PreparedStatements.DuckDBBindDecimal(statement, index, ToDuckDBDecimal((decimal)item)); + return true; + case DbType.Boolean: + result = NativeMethods.PreparedStatements.DuckDBBindBoolean(statement, index, (bool)item); + return true; + case DbType.SByte: + result = NativeMethods.PreparedStatements.DuckDBBindInt8(statement, index, (sbyte)item); + return true; + case DbType.Int16: + result = NativeMethods.PreparedStatements.DuckDBBindInt16(statement, index, (short)item); + return true; + case DbType.Int32: + result = NativeMethods.PreparedStatements.DuckDBBindInt32(statement, index, (int)item); + return true; + case DbType.Int64: + result = NativeMethods.PreparedStatements.DuckDBBindInt64(statement, index, (long)item); + return true; + case DbType.Byte: + result = NativeMethods.PreparedStatements.DuckDBBindUInt8(statement, index, (byte)item); + return true; + case DbType.UInt16: + result = NativeMethods.PreparedStatements.DuckDBBindUInt16(statement, index, (ushort)item); + return true; + case DbType.UInt32: + result = NativeMethods.PreparedStatements.DuckDBBindUInt32(statement, index, (uint)item); + return true; + case DbType.UInt64: + result = NativeMethods.PreparedStatements.DuckDBBindUInt64(statement, index, (ulong)item); + return true; + case DbType.Single: + result = NativeMethods.PreparedStatements.DuckDBBindFloat(statement, index, (float)item); + return true; + case DbType.Double: + result = NativeMethods.PreparedStatements.DuckDBBindDouble(statement, index, (double)item); + return true; + case DbType.String: + result = NativeMethods.PreparedStatements.DuckDBBindVarchar(statement, index, (string)item); + return true; + case DbType.VarNumeric: + result = NativeMethods.PreparedStatements.DuckDBBindHugeInt(statement, index, new DuckDBHugeInt((BigInteger)item)); + return true; + case DbType.Binary: + var bytes = (byte[])item; + result = NativeMethods.PreparedStatements.DuckDBBindBlob(statement, index, bytes, bytes.LongLength); + return true; + case DbType.Date: + var date = (item is DateOnly dateOnly ? (DuckDBDateOnly)dateOnly : (DuckDBDateOnly)item).ToDuckDBDate(); + result = NativeMethods.PreparedStatements.DuckDBBindDate(statement, index, date); + return true; + case DbType.Time: + var time = NativeMethods.DateTimeHelpers.DuckDBToTime(item is TimeOnly timeOnly ? (DuckDBTimeOnly)timeOnly : (DuckDBTimeOnly)item); + result = NativeMethods.PreparedStatements.DuckDBBindTime(statement, index, time); + return true; + case DbType.DateTime: + var timestamp = (item is DateTime dateTime ? (DuckDBTimestamp)dateTime : (DuckDBTimestamp)item).ToDuckDBTimestampStruct(); + result = NativeMethods.PreparedStatements.DuckDBBindTimestamp(statement, index, timestamp); + return true; + case DbType.DateTimeOffset: + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz(statement, index, ((DateTimeOffset)item).ToTimestampStruct()); + return true; + default: result = default; return false; + } + } + + internal static bool TryBindTypedScalarValue( + T item, + DuckDBPreparedStatement statement, + long index, + DuckDBType duckDBType, + out DuckDBState result) + { + if (item is null) + { + result = NativeMethods.PreparedStatements.DuckDBBindNull(statement, index); + return true; + } + + if (typeof(T) == typeof(bool) && duckDBType == DuckDBType.Boolean) + { + result = NativeMethods.PreparedStatements.DuckDBBindBoolean( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(sbyte) && duckDBType == DuckDBType.TinyInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt8( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(short) && duckDBType == DuckDBType.SmallInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt16( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(int) && duckDBType == DuckDBType.Integer) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt32( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(long) && duckDBType == DuckDBType.BigInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindInt64( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(byte) && duckDBType == DuckDBType.UnsignedTinyInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt8( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(ushort) && duckDBType == DuckDBType.UnsignedSmallInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt16( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(uint) && duckDBType == DuckDBType.UnsignedInteger) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt32( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(ulong) && duckDBType == DuckDBType.UnsignedBigInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindUInt64( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(float) && duckDBType == DuckDBType.Float) + { + result = NativeMethods.PreparedStatements.DuckDBBindFloat( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(double) && duckDBType == DuckDBType.Double) + { + result = NativeMethods.PreparedStatements.DuckDBBindDouble( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(decimal) && duckDBType == DuckDBType.Decimal) + { + result = NativeMethods.PreparedStatements.DuckDBBindDecimal( + statement, index, ToDuckDBDecimal(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(BigInteger) && duckDBType == DuckDBType.HugeInt) + { + result = NativeMethods.PreparedStatements.DuckDBBindHugeInt( + statement, index, new DuckDBHugeInt(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(string) && duckDBType == DuckDBType.Varchar) + { + result = NativeMethods.PreparedStatements.DuckDBBindVarchar( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(DateTime)) + { + var value = Unsafe.As(ref item); + switch (duckDBType) + { + case DuckDBType.Timestamp: + result = NativeMethods.PreparedStatements.DuckDBBindTimestamp( + statement, index, value.ToTimestampStruct(duckDBType)); + return true; + case DuckDBType.TimestampTz: + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz( + statement, index, value.ToTimestampStruct(duckDBType)); + return true; + case DuckDBType.Date: + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, ((DuckDBDateOnly)value).ToDuckDBDate()); + return true; + case DuckDBType.Time: + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, NativeMethods.DateTimeHelpers.DuckDBToTime((DuckDBTimeOnly)value)); + return true; + } + } + + if (typeof(T) == typeof(DateTimeOffset) && duckDBType == DuckDBType.TimestampTz) + { + result = NativeMethods.PreparedStatements.DuckDBBindTimestampTz( + statement, index, Unsafe.As(ref item).ToTimestampStruct()); + return true; + } + + if (typeof(T) == typeof(TimeSpan) && duckDBType == DuckDBType.Interval) + { + result = NativeMethods.PreparedStatements.DuckDBBindInterval( + statement, index, Unsafe.As(ref item)); + return true; + } + + if (typeof(T) == typeof(DateOnly) && duckDBType == DuckDBType.Date) + { + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, ((DuckDBDateOnly)Unsafe.As(ref item)).ToDuckDBDate()); + return true; + } + + if (typeof(T) == typeof(TimeOnly) && duckDBType == DuckDBType.Time) + { + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, + NativeMethods.DateTimeHelpers.DuckDBToTime(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(DuckDBDateOnly) && duckDBType == DuckDBType.Date) + { + result = NativeMethods.PreparedStatements.DuckDBBindDate( + statement, index, Unsafe.As(ref item).ToDuckDBDate()); + return true; + } + + if (typeof(T) == typeof(DuckDBTimeOnly) && duckDBType == DuckDBType.Time) + { + result = NativeMethods.PreparedStatements.DuckDBBindTime( + statement, index, + NativeMethods.DateTimeHelpers.DuckDBToTime(Unsafe.As(ref item))); + return true; + } + + if (typeof(T) == typeof(byte[]) && duckDBType == DuckDBType.Blob) + { + var value = Unsafe.As(ref item); + result = NativeMethods.PreparedStatements.DuckDBBindBlob( + statement, index, value, value.LongLength); + return true; + } + + result = default; + return false; + } + + private static bool TryConvertTo(object item, out T result) where T : struct + { + try + { + if (item is T parsable) + { + result = parsable; + return true; } + + result = (T)Convert.ChangeType(item, typeof(T)); + return true; + } + catch (Exception) + { + result = default; + return false; } } @@ -160,6 +540,9 @@ private static DuckDBValue[] BuildValues(DuckDBLogicalType childType, ICollectio } private static DuckDBValue DecimalToDuckDBValue(decimal value) + => NativeMethods.Value.DuckDBCreateDecimal(ToDuckDBDecimal(value)); + + private static DuckDBDecimal ToDuckDBDecimal(decimal value) { var mantissa = value.GetMantissa(); @@ -167,6 +550,6 @@ private static DuckDBValue DecimalToDuckDBValue(decimal value) ? value.Scale + 1 : Math.Max((int)BigInteger.Log10(BigInteger.Abs(mantissa)) + 1, value.Scale + 1); - return NativeMethods.Value.DuckDBCreateDecimal(new DuckDBDecimal((byte)width, value.Scale, new DuckDBHugeInt(mantissa))); + return new DuckDBDecimal((byte)width, value.Scale, new DuckDBHugeInt(mantissa)); } } diff --git a/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs b/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs index 9469356c..19b9d57d 100644 --- a/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs +++ b/DuckDB.NET.Data/PreparedStatement/DuckDBTypeMap.cs @@ -40,4 +40,10 @@ public static DbType GetDbTypeForValue(object? value) return ClrToDbTypeMap.GetValueOrDefault(type, DbType.Object); } + + internal static DbType GetDbTypeForType(Type type) + { + var underlyingType = Nullable.GetUnderlyingType(type) ?? type; + return ClrToDbTypeMap.GetValueOrDefault(underlyingType, DbType.Object); + } } diff --git a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs index d074baab..8f5b6f39 100644 --- a/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs +++ b/DuckDB.NET.Data/PreparedStatement/PreparedStatement.cs @@ -1,17 +1,20 @@ -using System.Linq; using DuckDB.NET.Data.Connection; namespace DuckDB.NET.Data.PreparedStatement; -internal sealed class PreparedStatement : IDisposable +internal class PreparedStatement : IDisposable { - private readonly DuckDBPreparedStatement statement; + protected readonly DuckDBPreparedStatement Statement; - private PreparedStatement(DuckDBPreparedStatement statement) + internal PreparedStatement(DuckDBPreparedStatement statement) { - this.statement = statement; + Statement = statement; } + protected virtual bool RequiresClearBindings => false; + + protected virtual long ParameterCount => NativeMethods.PreparedStatements.DuckDBParams(Statement); + public static IEnumerable PrepareMultiple(DuckDBNativeConnection connection, string query, DuckDBParameterCollection parameters, bool useStreamingMode) { var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection, query, out var extractedStatements); @@ -36,6 +39,7 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c else { var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(statement); + statement.Dispose(); if (string.IsNullOrEmpty(errorMessage)) { @@ -48,13 +52,61 @@ public static IEnumerable PrepareMultiple(DuckDBNativeConnection c } } - private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBNativeConnection connection) + public static ReusablePreparedStatement? TryPrepareReusable(DuckDBNativeConnection connection, string query) { - BindParameters(statement, parameterCollection); + var statementCount = NativeMethods.ExtractStatements.DuckDBExtractStatements(connection, query, out var extractedStatements); + + using (extractedStatements) + { + if (statementCount <= 0) + { + var error = NativeMethods.ExtractStatements.DuckDBExtractStatementsError(extractedStatements); + throw new DuckDBException(error); + } + + // DuckDB can expand one logical command into dependent statements. Dynamic PIVOT, + // IMPORT and some PRAGMAs require earlier statements to execute before later ones can + // be prepared. Keep those commands on the existing per-execution path. + if (statementCount != 1) + { + return null; + } + + var status = NativeMethods.ExtractStatements.DuckDBPrepareExtractedStatement(connection, extractedStatements, 0, out var statement); + if (status.IsSuccess()) + { + return new ReusablePreparedStatement(statement); + } + + var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(statement); + statement.Dispose(); + + if (string.IsNullOrEmpty(errorMessage)) + { + errorMessage = "DuckDBQuery failed"; + } + + throw new DuckDBException(errorMessage, UdfExceptionStore.Retrieve(connection)); + } + } + + internal DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool useStreamingMode, DuckDBNativeConnection connection) + { + if (RequiresClearBindings) + { + var clearState = NativeMethods.PreparedStatements.DuckDBClearBindings(Statement); + if (!clearState.IsSuccess()) + { + var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(Statement); + throw new InvalidOperationException($"Unable to clear prepared statement bindings: {errorMessage}"); + } + } + + BindParameters(parameterCollection); var status = useStreamingMode - ? NativeMethods.PreparedStatements.DuckDBExecutePreparedStreaming(statement, out var queryResult) - : NativeMethods.PreparedStatements.DuckDBExecutePrepared(statement, out queryResult); + ? NativeMethods.PreparedStatements.DuckDBExecutePreparedStreaming(Statement, out var queryResult) + : NativeMethods.PreparedStatements.DuckDBExecutePrepared(Statement, out queryResult); if (!status.IsSuccess()) { @@ -81,22 +133,25 @@ private DuckDBResult Execute(DuckDBParameterCollection parameterCollection, bool return queryResult; } - private static void BindParameters(DuckDBPreparedStatement preparedStatement, DuckDBParameterCollection parameterCollection) + protected virtual void BindParameters(DuckDBParameterCollection parameterCollection) { - var expectedParameters = NativeMethods.PreparedStatements.DuckDBParams(preparedStatement); - if (parameterCollection.Count < expectedParameters) - { - throw new InvalidOperationException($"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); - } + var expectedParameters = ValidateParameterCount(parameterCollection); - if (parameterCollection.OfType().Any(p => !string.IsNullOrEmpty(p.ParameterName))) + // Index-based iteration over the typed collection avoids the per-execution allocations of + // OfType<>().Any(...) and of the boxed List enumerator that `foreach (DuckDBParameter ...)` + // over the non-generic collection would produce. BindParameters runs on every execution. + var count = parameterCollection.Count; + + var hasNamedParameters = HasNamedParameters(parameterCollection); + + if (hasNamedParameters) { - foreach (DuckDBParameter param in parameterCollection) + for (var i = 0; i < count; i++) { - var state = NativeMethods.PreparedStatements.DuckDBBindParameterIndex(preparedStatement, out var index, param.ParameterName); - if (state.IsSuccess()) + var param = parameterCollection[i]; + if (TryGetParameterIndex(param.ParameterName, out var index)) { - BindParameter(preparedStatement, index, param); + BindParameter(index, param); } } } @@ -105,29 +160,226 @@ private static void BindParameters(DuckDBPreparedStatement preparedStatement, Du for (var i = 0; i < expectedParameters; ++i) { var param = parameterCollection[i]; - BindParameter(preparedStatement, i + 1, param); + BindParameter(i + 1, param); } } } - private static void BindParameter(DuckDBPreparedStatement preparedStatement, long index, DuckDBParameter parameter) + protected virtual bool TryGetParameterIndex(string parameterName, out long index) { - using var parameterLogicalType = NativeMethods.PreparedStatements.DuckDBParamLogicalType(preparedStatement, index); - var duckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(parameterLogicalType); + if (string.IsNullOrEmpty(parameterName)) + { + index = 0; + return false; + } + + var state = NativeMethods.PreparedStatements.DuckDBBindParameterIndex(Statement, out var nativeIndex, parameterName); + index = state.IsSuccess() ? nativeIndex : 0; + + return index > 0; + } - using var duckDBValue = parameter.Value.ToDuckDBValue(parameterLogicalType, duckDBType, parameter.DbType); + protected virtual void BindParameter(long index, DuckDBParameter parameter) + { + using var parameterLogicalType = NativeMethods.PreparedStatements.DuckDBParamLogicalType(Statement, index); + BindParameter(index, parameter, parameterLogicalType); + } - var result = NativeMethods.PreparedStatements.DuckDBBindValue(preparedStatement, index, duckDBValue); + protected void BindParameter(long index, DuckDBParameter parameter, DuckDBLogicalType parameterLogicalType) + { + var duckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(parameterLogicalType); + + DuckDBState result; + if (!parameter.TryBindScalarValue(Statement, index, duckDBType, out result)) + { + using var duckDBValue = parameter.ToDuckDBValue(parameterLogicalType, duckDBType); + result = NativeMethods.PreparedStatements.DuckDBBindValue(Statement, index, duckDBValue); + } if (!result.IsSuccess()) { - var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(preparedStatement); + var errorMessage = NativeMethods.PreparedStatements.DuckDBPrepareError(Statement); throw new InvalidOperationException($"Unable to bind parameter {index}: {errorMessage}"); } } - public void Dispose() + protected long ValidateParameterCount(DuckDBParameterCollection parameterCollection) { - statement.Dispose(); + var expectedParameters = ParameterCount; + if (parameterCollection.Count < expectedParameters) + { + throw new InvalidOperationException( + $"Invalid number of parameters. Expected {expectedParameters}, got {parameterCollection.Count}"); + } + + return expectedParameters; } -} \ No newline at end of file + + protected static bool HasNamedParameters(DuckDBParameterCollection parameterCollection) + { + for (var index = 0; index < parameterCollection.Count; index++) + { + if (!string.IsNullOrEmpty(parameterCollection[index].ParameterName)) + { + return true; + } + } + + return false; + } + + public virtual void Dispose() + { + Statement.Dispose(); + } +} + +internal sealed class ReusablePreparedStatement : PreparedStatement +{ + private readonly long cachedParameterCount; + private readonly DuckDBLogicalType[] cachedParameterTypes; + private readonly Dictionary cachedParameterIndices = new(StringComparer.Ordinal); + private DuckDBParameterCollection? cachedParameterCollection; + private int cachedParameterCollectionVersion = -1; + private int[]? cachedParameterMetadataVersions; + private ParameterBinding[]? cachedBindingPlan; + + public ReusablePreparedStatement(DuckDBPreparedStatement statement) + : base(statement) + { + cachedParameterCount = NativeMethods.PreparedStatements.DuckDBParams(statement); + cachedParameterTypes = new DuckDBLogicalType[cachedParameterCount]; + + try + { + for (var index = 0; index < cachedParameterTypes.Length; index++) + { + cachedParameterTypes[index] = NativeMethods.PreparedStatements.DuckDBParamLogicalType(statement, index + 1); + } + } + catch + { + Dispose(); + throw; + } + } + + protected override bool RequiresClearBindings => true; + + protected override long ParameterCount => cachedParameterCount; + + protected override bool TryGetParameterIndex(string parameterName, out long index) + { + if (cachedParameterIndices.TryGetValue(parameterName, out index)) + { + return index > 0; + } + + var found = base.TryGetParameterIndex(parameterName, out index); + cachedParameterIndices.Add(parameterName, index); + + return found; + } + + protected override void BindParameters(DuckDBParameterCollection parameterCollection) + { + var bindingPlan = GetOrCreateBindingPlan(parameterCollection); + + for (var index = 0; index < bindingPlan.Length; index++) + { + var binding = bindingPlan[index]; + BindParameter(binding.Index, binding.Parameter); + } + } + + protected override void BindParameter(long index, DuckDBParameter parameter) + { + BindParameter(index, parameter, cachedParameterTypes[index - 1]); + } + + private ParameterBinding[] GetOrCreateBindingPlan(DuckDBParameterCollection parameterCollection) + { + if (IsBindingPlanCurrent(parameterCollection)) + { + return cachedBindingPlan!; + } + + var expectedParameters = ValidateParameterCount(parameterCollection); + var hasNamedParameters = HasNamedParameters(parameterCollection); + var plan = new ParameterBinding[hasNamedParameters + ? parameterCollection.Count + : checked((int)expectedParameters)]; + var bindingCount = 0; + + if (hasNamedParameters) + { + for (var parameterIndex = 0; parameterIndex < parameterCollection.Count; parameterIndex++) + { + var parameter = parameterCollection[parameterIndex]; + if (TryGetParameterIndex(parameter.ParameterName, out var nativeIndex)) + { + plan[bindingCount++] = new ParameterBinding(nativeIndex, parameter); + } + } + + if (bindingCount != plan.Length) + { + Array.Resize(ref plan, bindingCount); + } + } + else + { + for (var parameterIndex = 0; parameterIndex < expectedParameters; parameterIndex++) + { + plan[bindingCount++] = new ParameterBinding( + parameterIndex + 1, + parameterCollection[parameterIndex]); + } + } + + cachedParameterCollection = parameterCollection; + cachedParameterCollectionVersion = parameterCollection.Version; + cachedParameterMetadataVersions = new int[parameterCollection.Count]; + for (var index = 0; index < cachedParameterMetadataVersions.Length; index++) + { + cachedParameterMetadataVersions[index] = parameterCollection[index].BindingMetadataVersion; + } + + cachedBindingPlan = plan; + return plan; + } + + private bool IsBindingPlanCurrent(DuckDBParameterCollection parameterCollection) + { + if (!ReferenceEquals(cachedParameterCollection, parameterCollection) || + cachedParameterCollectionVersion != parameterCollection.Version || + cachedBindingPlan is null || + cachedParameterMetadataVersions is null || + cachedParameterMetadataVersions.Length != parameterCollection.Count) + { + return false; + } + + for (var index = 0; index < cachedParameterMetadataVersions.Length; index++) + { + if (cachedParameterMetadataVersions[index] != parameterCollection[index].BindingMetadataVersion) + { + return false; + } + } + + return true; + } + + public override void Dispose() + { + foreach (var parameterType in cachedParameterTypes) + { + parameterType?.Dispose(); + } + + base.Dispose(); + } + + private readonly record struct ParameterBinding(long Index, DuckDBParameter Parameter); +} diff --git a/DuckDB.NET.Test/Arrow/ArrowResultTests.cs b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs new file mode 100644 index 00000000..18e4182e --- /dev/null +++ b/DuckDB.NET.Test/Arrow/ArrowResultTests.cs @@ -0,0 +1,176 @@ +using System; +using System.Threading; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace DuckDB.NET.Test.Arrow; + +public class ArrowResultTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_ReturnsSchemaAndScalarValues(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "select 42 as answer, 'duckdb' as name, cast(3.5 as double) as ratio"; + + var batches = await ReadAllAsync(); + + batches.Should().ContainSingle(); + + var batch = batches[0]; + batch.Schema.FieldsList.Select(f => f.Name).Should().Equal("answer", "name", "ratio"); + batch.Length.Should().Be(1); + + ((Int32Array)batch.Column("answer")).GetValue(0).Should().Be(42); + ((StringArray)batch.Column("name")).GetString(0).Should().Be("duckdb"); + ((DoubleArray)batch.Column("ratio")).GetValue(0).Should().Be(3.5); + + DisposeBatches(batches); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_HandlesNullValues(bool useStreamingMode) + { + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = "select unnest([1, null, 3]) as value"; + + var batches = await ReadAllAsync(); + + var column = (Int32Array)batches.Single().Column("value"); + column.Length.Should().Be(3); + column.GetValue(0).Should().Be(1); + column.GetValue(1).Should().BeNull(); + column.GetValue(2).Should().Be(3); + + DisposeBatches(batches); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteArrowBatches_StreamsMultipleChunks(bool useStreamingMode) + { + const int rowCount = 5000; + Command.UseStreamingMode = useStreamingMode; + Command.CommandText = $"select i from range({rowCount}) t(i)"; + + var batches = await ReadAllAsync(); + + batches.Count.Should().BeGreaterThan(1); + batches.Sum(b => b.Length).Should().Be(rowCount); + + var total = 0L; + foreach (var batch in batches) + { + var column = (Int64Array)batch.Column("i"); + for (var row = 0; row < column.Length; row++) + { + total += column.GetValue(row)!.Value; + } + } + + total.Should().Be((long)rowCount * (rowCount - 1) / 2); + + DisposeBatches(batches); + } + + [Fact] + public void ExecuteArrowStream_ExposesSchemaWithoutReading() + { + Command.CommandText = "select 1 as a, 'x' as b"; + + using var stream = Command.ExecuteArrowStream(); + + stream.Schema.FieldsList.Select(f => f.Name).Should().Equal("a", "b"); + stream.Schema.GetFieldByName("a").DataType.TypeId.Should().Be(ArrowTypeId.Int32); + stream.Schema.GetFieldByName("b").DataType.TypeId.Should().Be(ArrowTypeId.String); + } + + [Fact] + public async Task ExecuteArrowStream_ReadsBatchesUntilNull() + { + Command.CommandText = "select i from range(10) t(i)"; + + using var stream = Command.ExecuteArrowStream(); + + var rows = 0; + while (await stream.ReadNextRecordBatchAsync(CancellationToken.None) is { } batch) + { + rows += batch.Length; + batch.Dispose(); + } + + rows.Should().Be(10); + } + + [Fact] + public void ExecuteArrowStream_ThrowsWhenNoResultSet() + { + Command.CommandText = "create table t_no_result (id integer)"; + + var act = () => Command.ExecuteArrowStream(); + + act.Should().Throw(); + } + + [Fact] + public async Task ReadNextRecordBatchAsync_WithCanceledToken_ReturnsCanceledTask() + { + Command.CommandText = "select i from range(10) t(i)"; + + using var stream = Command.ExecuteArrowStream(); + + var act = async () => await stream.ReadNextRecordBatchAsync(new CancellationToken(canceled: true)); + + await act.Should().ThrowAsync(); + } + + [Fact] + public async Task ReadNextRecordBatchAsync_AfterDispose_Throws() + { + Command.CommandText = "select i from range(10) t(i)"; + + var stream = Command.ExecuteArrowStream(); + stream.Dispose(); + + var act = async () => await stream.ReadNextRecordBatchAsync(CancellationToken.None); + + await act.Should().ThrowAsync(); + } + + [Fact] + public void Dispose_CalledTwice_IsNoOp() + { + Command.CommandText = "select i from range(10) t(i)"; + + var stream = Command.ExecuteArrowStream(); + + stream.Dispose(); + var act = () => stream.Dispose(); + + act.Should().NotThrow(); + } + + private async Task> ReadAllAsync() + { + var batches = new List(); + await foreach (var batch in Command.ExecuteArrowBatchesAsync()) + { + batches.Add(batch); + } + + return batches; + } + + private static void DisposeBatches(List batches) + { + foreach (var batch in batches) + { + batch.Dispose(); + } + } +} diff --git a/DuckDB.NET.Test/DateTimeConversionTests.cs b/DuckDB.NET.Test/DateTimeConversionTests.cs new file mode 100644 index 00000000..15ab6f94 --- /dev/null +++ b/DuckDB.NET.Test/DateTimeConversionTests.cs @@ -0,0 +1,221 @@ +using DuckDB.NET.Data.Extensions; + +namespace DuckDB.NET.Test; + +public class DateTimeConversionTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) +{ + public static IEnumerable TimestampConversionCases() + { + var epoch = DateTime.UnixEpoch; + var values = new[] + { + DateTime.MinValue, + new DateTime(1600, 2, 29, 23, 59, 59, DateTimeKind.Unspecified).AddTicks(9_999_999), + epoch.AddTicks(-11), + epoch.AddTicks(-10), + epoch.AddTicks(-9), + epoch.AddTicks(-1), + epoch, + epoch.AddTicks(1), + epoch.AddTicks(9), + epoch.AddTicks(10), + epoch.AddTicks(11), + new DateTime(2000, 2, 29, 12, 34, 56, DateTimeKind.Utc).AddTicks(7_654_321), + new DateTime(2262, 4, 11, 23, 47, 16, DateTimeKind.Unspecified).AddTicks(8_000_000), + DateTime.MaxValue, + }; + + var types = new[] + { + DuckDBType.Timestamp, + DuckDBType.TimestampS, + DuckDBType.TimestampMs, + DuckDBType.TimestampNs, + DuckDBType.TimestampTz, + }; + + foreach (var value in values) + { + foreach (var type in types) + { + yield return new object[] { value, type }; + } + } + } + + public static IEnumerable TimestampOffsetConversionCases() + { + yield return new object[] { new DateTimeOffset(1970, 1, 1, 0, 30, 0, TimeSpan.FromHours(1)) }; + yield return new object[] { new DateTimeOffset(1969, 12, 31, 23, 30, 0, TimeSpan.FromHours(-1)) }; + yield return new object[] { new DateTimeOffset(2000, 2, 29, 23, 45, 12, 345, TimeSpan.FromHours(5.5)).AddTicks(6_789) }; + yield return new object[] { DateTimeOffset.MinValue }; + yield return new object[] { DateTimeOffset.MaxValue }; + } + + [Theory] + [MemberData(nameof(TimestampConversionCases))] + public void ManagedTimestampConversionMatchesPreviousNativeConversion(DateTime value, DuckDBType type) + { + var expected = ConvertUsingNativeTimestampHelper(value, type); + var actual = value.ToTimestampStruct(type); + + actual.Micros.Should().Be(expected.Micros); + } + + [Theory] + [MemberData(nameof(TimestampOffsetConversionCases))] + public void ManagedTimestampOffsetConversionMatchesPreviousNativeConversion(DateTimeOffset value) + { + var expected = DuckDBTimestamp.FromDateTime(value.UtcDateTime).ToDuckDBTimestampStruct(); + var actual = value.ToTimestampStruct(); + + actual.Micros.Should().Be(expected.Micros); + } + + [Fact] + public void TimestampEdgeCasesRoundTripThroughScopedAppenderAndPreparedStatement() + { + var rows = new[] + { + new TimestampEdgeRow( + 1, + new DateTime(DateTime.UnixEpoch.Ticks - 11, DateTimeKind.Unspecified), + new DateTimeOffset(1970, 1, 1, 0, 30, 0, TimeSpan.FromHours(1))), + new TimestampEdgeRow( + 2, + new DateTime(DateTime.UnixEpoch.Ticks - 1, DateTimeKind.Unspecified), + new DateTimeOffset(1969, 12, 31, 23, 30, 0, TimeSpan.FromHours(-1))), + new TimestampEdgeRow( + 3, + new DateTime(DateTime.UnixEpoch.Ticks + 11, DateTimeKind.Unspecified), + new DateTimeOffset(2000, 2, 29, 23, 45, 12, 345, TimeSpan.FromHours(5.5)).AddTicks(6_789)), + }; + + CreateTimestampTable("managedTimestampScopedAppender"); + using (var appender = Connection.CreateAppender("managedTimestampScopedAppender")) + { + foreach (var row in rows) + { + appender.AppendRowScoped(row, + static (ref DuckDBAppenderRowWriter writer, TimestampEdgeRow value) => + { + writer.AppendValue((int?)value.Id); + writer.AppendValue((DateTime?)value.Timestamp); + writer.AppendValue((DateTime?)value.Timestamp); + writer.AppendValue((DateTime?)value.Timestamp); + writer.AppendValue((DateTime?)value.Timestamp); + writer.AppendValue((DateTimeOffset?)value.TimestampOffset); + }); + } + } + + CreateTimestampTable("managedTimestampPreparedStatement"); + InsertPreparedTimestampRows("managedTimestampPreparedStatement", rows); + + AssertTimestampRows("managedTimestampScopedAppender", rows); + AssertTimestampRows("managedTimestampPreparedStatement", rows); + } + + private void CreateTimestampTable(string tableName) + { + Command.CommandText = $$""" + CREATE TABLE {{tableName}}( + id INTEGER, + timestamp_value TIMESTAMP, + timestamp_s_value TIMESTAMP_S, + timestamp_ms_value TIMESTAMP_MS, + timestamp_ns_value TIMESTAMP_NS, + timestamp_tz_value TIMESTAMPTZ) + """; + Command.ExecuteNonQuery(); + } + + private void InsertPreparedTimestampRows(string tableName, IReadOnlyList rows) + { + using var command = Connection.CreateCommand(); + command.CommandText = $"INSERT INTO {tableName} VALUES (?, ?, ?, ?, ?, ?)"; + + var id = new DuckDBParameter(rows[0].Id); + var timestamp = new DuckDBParameter(rows[0].Timestamp); + var timestampS = new DuckDBParameter(rows[0].Timestamp); + var timestampMs = new DuckDBParameter(rows[0].Timestamp); + var timestampNs = new DuckDBParameter(rows[0].Timestamp); + var timestampTz = new DuckDBParameter(rows[0].TimestampOffset); + command.Parameters.Add(id); + command.Parameters.Add(timestamp); + command.Parameters.Add(timestampS); + command.Parameters.Add(timestampMs); + command.Parameters.Add(timestampNs); + command.Parameters.Add(timestampTz); + command.Prepare(); + + foreach (var row in rows) + { + id.Value = row.Id; + timestamp.Value = row.Timestamp; + timestampS.Value = row.Timestamp; + timestampMs.Value = row.Timestamp; + timestampNs.Value = row.Timestamp; + timestampTz.Value = row.TimestampOffset; + command.ExecuteNonQuery(); + } + } + + private void AssertTimestampRows(string tableName, IReadOnlyList rows) + { + Command.CommandText = $"SELECT * FROM {tableName} ORDER BY id"; + using var reader = Command.ExecuteReader(); + + foreach (var row in rows) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(row.Id); + reader.GetDateTime(1).Ticks.Should().Be(ExpectedDateTime(row.Timestamp, DuckDBType.Timestamp).Ticks); + reader.GetDateTime(2).Ticks.Should().Be(ExpectedDateTime(row.Timestamp, DuckDBType.TimestampS).Ticks); + reader.GetDateTime(3).Ticks.Should().Be(ExpectedDateTime(row.Timestamp, DuckDBType.TimestampMs).Ticks); + reader.GetDateTime(4).Ticks.Should().Be(ExpectedDateTime(row.Timestamp, DuckDBType.TimestampNs).Ticks); + reader.GetFieldValue(5).UtcDateTime.Ticks.Should() + .Be(ExpectedDateTime(row.TimestampOffset.UtcDateTime, DuckDBType.TimestampTz).Ticks); + } + + reader.Read().Should().BeFalse(); + } + + private static DateTime ExpectedDateTime(DateTime value, DuckDBType type) + { + var timestamp = ConvertUsingNativeTimestampHelper(value, type); + var (duckDBTimestamp, additionalTicks) = timestamp.ToDuckDBTimestamp(type); + return duckDBTimestamp.ToDateTime().AddTicks(additionalTicks); + } + + private static DuckDBTimestampStruct ConvertUsingNativeTimestampHelper(DateTime value, DuckDBType type) + { + var timestamp = DuckDBTimestamp.FromDateTime(value).ToDuckDBTimestampStruct(); + + unchecked + { + if (type == DuckDBType.TimestampNs) + { + timestamp.Micros *= 1000; + timestamp.Micros += value.Nanosecond; + } + + if (type == DuckDBType.TimestampMs) + { + timestamp.Micros /= 1000; + } + + if (type == DuckDBType.TimestampS) + { + timestamp.Micros /= 1_000_000; + } + } + + return timestamp; + } + + private readonly record struct TimestampEdgeRow( + int Id, + DateTime Timestamp, + DateTimeOffset TimestampOffset); +} diff --git a/DuckDB.NET.Test/DuckDBCommandTests.cs b/DuckDB.NET.Test/DuckDBCommandTests.cs index b807f2b6..a2cdb0ad 100644 --- a/DuckDB.NET.Test/DuckDBCommandTests.cs +++ b/DuckDB.NET.Test/DuckDBCommandTests.cs @@ -17,4 +17,391 @@ public void SetCommandTextAndConnection() cmd.CommandText.Should().Be("Select 1"); cmd.Connection.Should().Be(Connection); } -} \ No newline at end of file + + [Fact] + public void PreparedCommandCanBeExecutedRepeatedlyWithNewParameterValues() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $left::INTEGER + $right::INTEGER"; + command.Parameters.Add(new DuckDBParameter("left", 10)); + command.Parameters.Add(new DuckDBParameter("right", 1)); + + command.Prepare(); + + command.ExecuteScalar().Should().Be(11); + + command.Parameters["left"].Value = 20; + command.Parameters["right"].Value = 2; + command.ExecuteScalar().Should().Be(22); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarHandlesValuesNullsAndEmptyResults(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + command.Parameters.Add(new DuckDBParameter("value", 42)); + command.UseStreamingMode = useStreamingMode; + command.Prepare(); + + command.ExecuteScalar().Should().Be(42); + + command.Parameters["value"].Value = DBNull.Value; + command.ExecuteScalar().Should().Be(DBNull.Value); + + command.CommandText = "SELECT 42 WHERE FALSE"; + command.Parameters.Clear(); + command.Prepare(); + command.ExecuteScalar().Should().BeNull(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarHandlesRepresentativeReaderTypes(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.UseStreamingMode = useStreamingMode; + + command.CommandText = "SELECT 'duckdb'::VARCHAR"; + command.Prepare(); + command.ExecuteScalar().Should().Be("duckdb"); + + command.CommandText = "SELECT DATE '2026-07-21'"; + command.Prepare(); + command.ExecuteScalar().Should().Be(new DateOnly(2026, 7, 21)); + + command.CommandText = "SELECT [1, 2, 3]::INTEGER[]"; + command.Prepare(); + command.ExecuteScalar().Should().BeEquivalentTo(new[] { 1, 2, 3 }); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void PreparedExecuteScalarCanBeReusedAfterMaterializationFailure(bool useStreamingMode) + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT CASE WHEN $infinite THEN DATE 'infinity' ELSE DATE '2026-07-21' END"; + command.Parameters.Add(new DuckDBParameter("infinite", true)); + command.UseStreamingMode = useStreamingMode; + command.Prepare(); + + command.Invoking(value => value.ExecuteScalar()) + .Should().Throw() + .WithMessage("Cannot convert infinite date value*"); + + command.Parameters["infinite"].Value = false; + command.ExecuteScalar().Should().Be(new DateOnly(2026, 7, 21)); + } + + [Fact] + public void PreparedExecuteScalarReturnsNullForNonQueryStatement() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = "CREATE TABLE prepared_scalar(value INTEGER)"; + command.Prepare(); + + command.ExecuteScalar().Should().BeNull(); + } + + [Fact] + public void TypedPreparedParameterCanBeExecutedRepeatedly() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + var parameter = new DuckDBParameter("value", 10); + command.Parameters.Add(parameter); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + parameter.TypedValue = 20; + command.ExecuteScalar().Should().Be(20); + + parameter.Value = 30; + command.ExecuteScalar().Should().Be(30); + } + + [Fact] + public void TypedParameterRejectsValuesOfAnotherType() + { + DuckDBParameter parameter = new DuckDBParameter("value", 10); + + parameter.Invoking(value => value.Value = "wrong") + .Should().Throw(); + } + + [Fact] + public void PreparedBindingPlanInvalidatesWhenParameterNameChanges() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + var parameter = new DuckDBParameter("value", 10); + command.Parameters.Add(parameter); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + parameter.ParameterName = "unused"; + command.Invoking(value => value.ExecuteScalar()).Should().Throw(); + + parameter.ParameterName = "value"; + parameter.TypedValue = 20; + command.ExecuteScalar().Should().Be(20); + } + + [Fact] + public void PreparedCommandClearsBindingsBeforeReuse() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $used::INTEGER"; + command.Parameters.Add(new DuckDBParameter("used", 10)); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + command.Parameters.Clear(); + command.Parameters.Add(new DuckDBParameter("unused", 20)); + command.Invoking(value => value.ExecuteScalar()).Should().Throw(); + + command.Parameters.Clear(); + command.Parameters.Add(new DuckDBParameter("used", 30)); + command.ExecuteScalar().Should().Be(30); + } + + [Fact] + public void PreparedExecuteNonQueryReturnsAffectedRowsAndCanBeReused() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + ExecuteNonQuery(connection, "CREATE TABLE prepared_values(value INTEGER)"); + + using var command = connection.CreateCommand(); + command.CommandText = "INSERT INTO prepared_values VALUES ($value)"; + command.Parameters.Add(new DuckDBParameter("value", 10)); + command.Prepare(); + + command.ExecuteNonQuery().Should().Be(1); + + command.Parameters["value"].Value = 20; + command.ExecuteNonQuery().Should().Be(1); + + using var query = connection.CreateCommand(); + query.CommandText = "SELECT value FROM prepared_values ORDER BY value"; + using var reader = query.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(10); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(20); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void PreparedExecuteNonQueryCanBeReusedAfterExecutionFailure() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + ExecuteNonQuery(connection, "CREATE TABLE unique_prepared_values(value INTEGER PRIMARY KEY)"); + + using var command = connection.CreateCommand(); + command.CommandText = "INSERT INTO unique_prepared_values VALUES ($value)"; + command.Parameters.Add(new DuckDBParameter("value", 10)); + command.Prepare(); + + command.ExecuteNonQuery().Should().Be(1); + command.Invoking(value => value.ExecuteNonQuery()).Should().Throw(); + + command.Parameters["value"].Value = 20; + command.ExecuteNonQuery().Should().Be(1); + } + + [Fact] + public void PreparePreservesMultipleResultSetsAcrossExecutions() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER; SELECT $value::INTEGER + 1"; + command.Parameters.Add(new DuckDBParameter("value", 10)); + command.Prepare(); + + AssertResults(command, 10, 11); + + command.Parameters["value"].Value = 20; + AssertResults(command, 20, 21); + + static void AssertResults(DuckDBCommand preparedCommand, int first, int second) + { + using var reader = preparedCommand.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(first); + reader.NextResult().Should().BeTrue(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(second); + reader.NextResult().Should().BeFalse(); + } + } + + [Fact] + public void PreparePreservesDependentMultipleStatements() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = """ + CREATE TABLE IF NOT EXISTS prepared_dependency(value INTEGER); + DELETE FROM prepared_dependency; + INSERT INTO prepared_dependency VALUES ($value); + SELECT value FROM prepared_dependency; + """; + command.Parameters.Add(new DuckDBParameter("value", 42)); + command.Prepare(); + + command.ExecuteScalar().Should().Be(42); + + command.Parameters["value"].Value = 84; + command.ExecuteScalar().Should().Be(84); + } + + [Fact] + public void StaticPivotCanBePreparedAndReused() + { + using var connection = CreatePivotConnection(); + + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT * + FROM Cities + PIVOT ( + SUM(Population) + FOR Year IN (2022, 2023) + GROUP BY Country, Name + ); + """; + command.Prepare(); + + AssertPivotValue(command, "2022", 3_688_647L); + + ExecuteNonQuery(connection, "INSERT INTO Cities VALUES ('Georgia', 'Tbilisi', 2022, 100)"); + + AssertPivotValue(command, "2022", 3_688_747L); + } + + [Fact] + public void DynamicPivotIsPreparedPerExecutionSoNewColumnsAreVisible() + { + using var connection = CreatePivotConnection(); + + using var command = connection.CreateCommand(); + command.CommandText = "PIVOT Cities ON Year USING SUM(Population);"; + command.Prepare(); + + GetColumnNames(command).Should().Equal("Country", "Name", "2022", "2023"); + + ExecuteNonQuery(connection, "INSERT INTO Cities VALUES ('Georgia', 'Tbilisi', 2024, 3800000)"); + + GetColumnNames(command).Should().Equal("Country", "Name", "2022", "2023", "2024"); + } + + [Fact] + public void ChangingCommandTextInvalidatesPreparedStatements() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + command.Parameters.Add(new DuckDBParameter("value", 10)); + command.Prepare(); + + command.ExecuteScalar().Should().Be(10); + + command.CommandText = "SELECT $value::INTEGER * 2"; + command.ExecuteScalar().Should().Be(20); + + command.Prepare(); + command.Parameters["value"].Value = 15; + command.ExecuteScalar().Should().Be(30); + } + + [Fact] + public void InvalidatedPreparedStatementsRemainAliveForAnActiveReader() + { + using var command = Connection.CreateCommand(); + command.CommandText = "SELECT value FROM range(1, 4) AS values(value)"; + command.Prepare(); + + using var reader = command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(1); + + command.CommandText = "SELECT 4"; + command.ExecuteScalar().Should().Be(4); + + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(2); + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(3); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void ClosingConnectionInvalidatesPreparedStatementsBeforeDisconnect() + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + using var command = connection.CreateCommand(); + command.CommandText = "SELECT $value::INTEGER"; + command.Parameters.Add(new DuckDBParameter("value", 10)); + command.Prepare(); + command.ExecuteScalar().Should().Be(10); + + connection.Close(); + connection.Open(); + + command.Parameters["value"].Value = 20; + command.ExecuteScalar().Should().Be(20); + } + + private static DuckDBConnection CreatePivotConnection() + { + var connection = new DuckDBConnection("DataSource=:memory:"); + connection.Open(); + + ExecuteNonQuery(connection, """ + CREATE TABLE Cities(Country VARCHAR, Name VARCHAR, Year INT, Population INT); + INSERT INTO Cities VALUES + ('Georgia', 'Tbilisi', 2022, 3688647), + ('Georgia', 'Tbilisi', 2023, 3736400); + """); + + return connection; + } + + private static void ExecuteNonQuery(DuckDBConnection connection, string commandText) + { + using var command = connection.CreateCommand(); + command.CommandText = commandText; + command.ExecuteNonQuery(); + } + + private static void AssertPivotValue(DuckDBCommand command, string columnName, long expected) + { + using var reader = command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt64(reader.GetOrdinal(columnName)).Should().Be(expected); + reader.NextResult().Should().BeFalse(); + } + + private static string[] GetColumnNames(DuckDBCommand command) + { + using var reader = command.ExecuteReader(); + return Enumerable.Range(0, reader.FieldCount).Select(reader.GetName).ToArray(); + } +} diff --git a/DuckDB.NET.Test/DuckDBDataReaderListTests.cs b/DuckDB.NET.Test/DuckDBDataReaderListTests.cs index bca7fe27..32686930 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderListTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderListTests.cs @@ -2,6 +2,19 @@ public class DuckDBDataReaderListTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) { + [Fact] + public void PreSizesListResults() + { + Command.CommandText = "SELECT range(17)::INTEGER[];"; + using var reader = Command.ExecuteReader(); + + reader.Read(); + var list = reader.GetFieldValue>(0); + + list.Should().HaveCount(17); + list.Capacity.Should().Be(17); + } + [Fact] public void ReadListOfIntegers() { @@ -251,4 +264,4 @@ class Person public List Ids { get; set; } public DuckDBDataReaderEnumTests.Mood Mood { get; set; } } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/DuckDBDataReaderMapTests.cs b/DuckDB.NET.Test/DuckDBDataReaderMapTests.cs index b1476c73..12f56e58 100644 --- a/DuckDB.NET.Test/DuckDBDataReaderMapTests.cs +++ b/DuckDB.NET.Test/DuckDBDataReaderMapTests.cs @@ -53,6 +53,32 @@ public void ReadMapStronglyTyped() value.Should().BeEquivalentTo(expectation); } + [Fact] + public void ReadMapWithWiderValueTypeUsesCompatibleFallback() + { + Command.CommandText = "SELECT MAP { 'key1': 1, 'key2': 5, 'key3': 7 }"; + var reader = Command.ExecuteReader(); + + reader.Read(); + var value = reader.GetFieldValue>(0); + + var expectation = new Dictionary() { { "key1", 1 }, { "key2", 5 }, { "key3", 7 } }; + value.Should().BeEquivalentTo(expectation); + } + + [Fact] + public void ReadMapIntoDerivedDictionaryUsesCompatibleFallback() + { + Command.CommandText = "SELECT MAP { 'key1': 1, 'key2': 5, 'key3': 7 }"; + var reader = Command.ExecuteReader(); + + reader.Read(); + var value = reader.GetFieldValue>(0); + + var expectation = new Dictionary() { { "key1", 1 }, { "key2", 5 }, { "key3", 7 } }; + value.Should().BeEquivalentTo(expectation); + } + [Fact] public void ReadMapWithNullInNullableDictionary() { @@ -147,4 +173,8 @@ public int GetHashCode(List obj) return string.Join(",", obj).GetHashCode(); } } -} \ No newline at end of file + + private sealed class CustomDictionary : Dictionary where TKey : notnull + { + } +} diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs index e185cc51..ddb19475 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderListTests.cs @@ -1,4 +1,8 @@ -namespace DuckDB.NET.Test; +using DuckDB.NET.Data.DataChunk.Writer; +using System.Collections.ObjectModel; +using System.Reflection; + +namespace DuckDB.NET.Test; public class DuckDBManagedAppenderListTests(DuckDBDatabaseFixture db) : DuckDBTestBase(db) { @@ -199,6 +203,109 @@ public void ArrayValuesInt() ListValuesInternal("Integer", faker => faker.Random.Int(), 5); } + [Fact] + public void IndexedArrayAndListPathsCrossChunkBoundaries() + { + const int rowCount = 3_000; + + Command.CommandText = """ + CREATE TABLE indexed_collection_paths( + id INTEGER, + array_list INTEGER[], + typed_list INTEGER[], + fixed_array INTEGER[3]); + """; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("indexed_collection_paths")) + { + for (var index = 0; index < rowCount; index++) + { + int?[] arrayValues = [index, null, index + 1]; + List listValues = [index + 2, null, index + 3]; + int[] fixedValues = [index + 4, index + 5, index + 6]; + + appender.AppendRow( + (index, arrayValues, listValues, fixedValues), + static (row, values) => row + .AppendValue(values.index) + .AppendValue(values.arrayValues) + .AppendValue(values.listValues) + .AppendValue(values.fixedValues)); + } + } + + Command.CommandText = "SELECT * FROM indexed_collection_paths ORDER BY id"; + using var reader = Command.ExecuteReader(); + + for (var index = 0; index < rowCount; index++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(index); + reader.GetFieldValue>(1).Should().Equal(index, null, index + 1); + reader.GetFieldValue>(2).Should().Equal(index + 2, null, index + 3); + reader.GetFieldValue>(3).Should().Equal(index + 4, index + 5, index + 6); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void TypedCollectionFallbackSupportsReadOnlyAndDerivedLists() + { + Command.CommandText = """ + CREATE TABLE typed_collection_fallback( + id INTEGER, + read_only_values INTEGER[], + derived_values INTEGER[]); + """; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("typed_collection_fallback")) + { + for (var index = 0; index < 3_000; index++) + { + ReadOnlyCollection readOnlyValues = + Array.AsReadOnly(new[] { index, index + 1, index + 2 }); + DerivedIntList derivedValues = [index + 3, index + 4, index + 5]; + + appender.AppendRow( + (index, readOnlyValues, derivedValues), + static (row, values) => row + .AppendValue(values.index) + .AppendValue(values.readOnlyValues) + .AppendValue(values.derivedValues)); + } + } + + Command.CommandText = "SELECT * FROM typed_collection_fallback ORDER BY id"; + using var reader = Command.ExecuteReader(); + + for (var index = 0; index < 3_000; index++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(index); + reader.GetFieldValue>(1).Should().Equal(index, index + 1, index + 2); + reader.GetFieldValue>(2).Should().Equal(index + 3, index + 4, index + 5); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void NonSzArraysUseTheEnumerableFallback() + { + var values = Array.CreateInstance(typeof(int), [3], [1]); + var createWriter = typeof(ListVectorDataWriter).GetMethod( + "CreateCollectionWriter", + BindingFlags.Static | BindingFlags.NonPublic); + + var plan = createWriter!.Invoke(null, [values.GetType()]); + var writer = plan!.GetType().GetProperty("Writer")!.GetValue(plan); + + writer.Should().BeNull(); + } + [Fact] public void ListValuesEnum() { @@ -357,4 +464,6 @@ private enum TestEnum Test2 = 1, Test3 = 2, } -} \ No newline at end of file + + private sealed class DerivedIntList : List; +} diff --git a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs index e81d8618..6558e470 100644 --- a/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBManagedAppenderTests.cs @@ -1,4 +1,5 @@ using System.Globalization; +using DuckDB.NET.Data.Common; using FluentAssertions.Common; namespace DuckDB.NET.Test; @@ -483,23 +484,28 @@ public void WrongTypesThrowException() } [Fact] - public void ClosedAdapterThrowException() + public void ClosedAppenderRejectsFurtherOperations() { var table = "CREATE TABLE managedAppenderClosedAdapterTest(a BOOLEAN, c Date, b TINYINT);"; Command.CommandText = table; Command.ExecuteNonQuery(); - Connection.Invoking(dbConnection => - { - using var appender = dbConnection.CreateAppender("managedAppenderClosedAdapterTest"); - appender.Close(); - var row = appender.CreateRow(); - row - .AppendValue(false) - .AppendValue((byte)1) - .AppendValue((short?)1) - .EndRow(); - }).Should().Throw(); + using var appender = Connection.CreateAppender("managedAppenderClosedAdapterTest"); + appender.Close(); + + appender.Invoking(value => value.CreateRow()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.AppendRow(row => row.AppendValue(false).AppendNullValue().AppendNullValue())) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Clear()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Close()) + .Should().Throw() + .WithMessage("Appender is already closed"); + appender.Invoking(value => value.Dispose()).Should().NotThrow(); } [Fact] @@ -715,6 +721,532 @@ public void AppendDefault() reader.GetInt32(2).Should().Be(30); } + [Fact] + public void CreateRowReturnsIndependentRows() + { + Command.CommandText = "CREATE TABLE managedAppenderRowLifetime(a INTEGER, b INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderRowLifetime")) + { + var completed = appender.CreateRow(); + completed.AppendValue((int?)10).AppendValue((int?)11).EndRow(); + + var current = appender.CreateRow(); + + completed.Should().NotBeSameAs(current); + completed.Invoking(row => row.AppendValue((int?)99)).Should().Throw(); + current.AppendValue((int?)20).AppendValue((int?)21).EndRow(); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderRowLifetime ORDER BY a"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(10); + reader.GetInt32(1).Should().Be(11); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(20); + reader.GetInt32(1).Should().Be(21); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowStateOverloadWritesRows() + { + Command.CommandText = "CREATE TABLE managedAppenderScopedRow(a INTEGER, b VARCHAR)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderScopedRow")) + { + for (var i = 0; i < 3; i++) + { + appender.AppendRow((Id: i, Name: $"row-{i}"), static (row, value) => + { + row.AppendValue(value.Id).AppendValue(value.Name); + }); + } + } + + Command.CommandText = "SELECT a, b FROM managedAppenderScopedRow ORDER BY a"; + using var reader = Command.ExecuteReader(); + for (var i = 0; i < 3; i++) + { + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(i); + reader.GetString(1).Should().Be($"row-{i}"); + } + + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowScopedWritesMultipleMixedRowsIncludingNulls() + { + Command.CommandText = """ + CREATE TABLE managedAppenderStackOnlyRows( + id INTEGER, + event_time TIMESTAMP, + amount DOUBLE, + category VARCHAR, + active BOOLEAN) + """; + Command.ExecuteNonQuery(); + + var rows = new[] + { + new ScopedMixedRow(1, DateTime.UnixEpoch.AddTicks(-1), 12.5, "alpha", true), + new ScopedMixedRow(2, DateTime.UnixEpoch.AddTicks(10), null, null, null), + new ScopedMixedRow(3, DateTime.UnixEpoch.AddDays(1), -3.25, "gamma", false), + }; + + using (var appender = Connection.CreateAppender("managedAppenderStackOnlyRows")) + { + foreach (var row in rows) + { + appender.AppendRowScoped(row, + static (ref DuckDBAppenderRowWriter writer, ScopedMixedRow value) => + { + writer.AppendValue(value.Id); + writer.AppendValue(value.EventTime); + writer.AppendValue(value.Amount); + writer.AppendValue(value.Category); + writer.AppendValue(value.Active); + }); + } + } + + Command.CommandText = """ + SELECT id, event_time, amount, category, active + FROM managedAppenderStackOnlyRows + ORDER BY id + """; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetDateTime(1).Should().Be(DateTime.UnixEpoch.AddTicks(-10)); + reader.GetDouble(2).Should().Be(12.5); + reader.GetString(3).Should().Be("alpha"); + reader.GetBoolean(4).Should().BeTrue(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(2); + reader.GetDateTime(1).Should().Be(DateTime.UnixEpoch.AddTicks(10)); + reader.IsDBNull(2).Should().BeTrue(); + reader.IsDBNull(3).Should().BeTrue(); + reader.IsDBNull(4).Should().BeTrue(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(3); + reader.GetDateTime(1).Should().Be(DateTime.UnixEpoch.AddDays(1)); + reader.GetDouble(2).Should().Be(-3.25); + reader.GetString(3).Should().Be("gamma"); + reader.GetBoolean(4).Should().BeFalse(); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowScopedWritesAcrossChunkBoundary() + { + Command.CommandText = "CREATE TABLE managedAppenderStackOnlyChunks(id INTEGER, doubled BIGINT)"; + Command.ExecuteNonQuery(); + + var rowCount = checked((int)DuckDBGlobalData.VectorSize + 3); + using (var appender = Connection.CreateAppender("managedAppenderStackOnlyChunks")) + { + for (var i = 0; i < rowCount; i++) + { + appender.AppendRowScoped(i, + static (ref DuckDBAppenderRowWriter writer, int value) => + { + writer.AppendValue(value); + writer.AppendValue((long)value * 2); + }); + } + } + + Command.CommandText = """ + SELECT count(*)::BIGINT, min(id), max(id), sum(doubled)::BIGINT + FROM managedAppenderStackOnlyChunks + """; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(rowCount); + reader.GetInt32(1).Should().Be(0); + reader.GetInt32(2).Should().Be(rowCount - 1); + reader.GetInt64(3).Should().Be((long)(rowCount - 1) * rowCount); + } + + [Fact] + public void AppendRowScopedPreparationFailureFaultsAppenderAtChunkBoundary() + { + Command.CommandText = "CREATE TABLE managedAppenderStackOnlyPreparationFailure(id INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderStackOnlyPreparationFailure")) + { + for (var i = 0; i < checked((int)DuckDBGlobalData.VectorSize); i++) + { + appender.AppendRowScoped(i, + static (ref DuckDBAppenderRowWriter writer, int value) => + writer.AppendValue((int?)value)); + } + + // Constraint checks are deferred until duckdb_appender_close, so close the native + // handle directly to deterministically exercise a failure in the next managed + // vector-boundary flush before CreateReusableRow can return a row. + var nativeAppenderField = typeof(DuckDB.NET.Data.DuckDBAppender).GetField( + "nativeAppender", + System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic); + nativeAppenderField.Should().NotBeNull(); + var nativeAppender = nativeAppenderField!.GetValue(appender) + .Should().BeOfType().Subject; + nativeAppender.Close(); + + var failure = appender.Invoking(value => value.AppendRowScoped(2, + static (ref DuckDBAppenderRowWriter writer, int state) => + writer.AppendValue((int?)state))) + .Should().Throw().Which; + + failure.InnerExceptions.Should().HaveCount(2) + .And.OnlyContain(exception => exception is ObjectDisposedException); + + appender.Invoking(value => value.AppendRowScoped(3, + static (ref DuckDBAppenderRowWriter writer, int state) => + writer.AppendValue((int?)state))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + } + + [Fact] + public void IncompleteAppendRowScopedDiscardsFailedRowAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderIncompleteStackOnlyRow(a INTEGER, b INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderIncompleteStackOnlyRow")) + { + appender.Invoking(value => value.AppendRowScoped(1, + static (ref DuckDBAppenderRowWriter writer, int state) => writer.AppendValue(state))) + .Should().Throw() + .WithMessage("*specified only 1 values"); + + appender.Invoking(value => value.AppendRowScoped((2, 3), + static (ref DuckDBAppenderRowWriter writer, (int First, int Second) state) => + { + writer.AppendValue(state.First); + writer.AppendValue(state.Second); + })) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT count(*) FROM managedAppenderIncompleteStackOnlyRow"; + Command.ExecuteScalar().Should().Be(0); + } + + [Fact] + public void AppendRowScopedFailureFlushesCompletedRowsAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderThrownStackOnlyRow(a INTEGER, b VARCHAR)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderThrownStackOnlyRow")) + { + appender.AppendRowScoped((Id: 1, Name: "complete"), + static (ref DuckDBAppenderRowWriter writer, (int Id, string Name) value) => + { + writer.AppendValue(value.Id); + writer.AppendValue(value.Name); + }); + + appender.Invoking(value => value.AppendRowScoped((Id: 2, Name: "discarded"), + static (ref DuckDBAppenderRowWriter writer, (int Id, string Name) state) => + { + writer.AppendValue(state.Id); + writer.AppendValue(state.Name); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + + appender.Invoking(value => value.AppendRowScoped((Id: 3, Name: "rejected"), + static (ref DuckDBAppenderRowWriter writer, (int Id, string Name) state) => + { + writer.AppendValue(state.Id); + writer.AppendValue(state.Name); + })) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderThrownStackOnlyRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetString(1).Should().Be("complete"); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowActionOverloadWritesCompleteRow() + { + Command.CommandText = "CREATE TABLE managedAppenderActionRow(a INTEGER, b VARCHAR)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderActionRow")) + { + appender.AppendRow(row => row.AppendValue((int?)42).AppendValue("answer")); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderActionRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(42); + reader.GetString(1).Should().Be("answer"); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowActionOverloadRejectsNullCallback() + { + Command.CommandText = "CREATE TABLE managedAppenderNullAction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderNullAction"); + appender.Invoking(value => value.AppendRow((Action)null!)) + .Should().Throw() + .WithParameterName("writeRow"); + } + + [Fact] + public void AppendRowStateOverloadRejectsNullCallback() + { + Command.CommandText = "CREATE TABLE managedAppenderNullStateAction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderNullStateAction"); + appender.Invoking(value => value.AppendRow(1, (Action)null!)) + .Should().Throw() + .WithParameterName("writeRow"); + } + + [Fact] + public void IncompleteAppendRowDiscardsFailedRowAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderIncompleteScopedRow(a INTEGER, b INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderIncompleteScopedRow")) + { + appender.Invoking(value => value.AppendRow(1, static (row, state) => row.AppendValue(state))) + .Should().Throw() + .WithMessage("*specified only 1 values"); + + appender.Invoking(value => value.AppendRow((2, 3), static (row, state) => + row.AppendValue(state.Item1).AppendValue(state.Item2))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderIncompleteScopedRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowFailureFlushesCompletedRowsAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderThrownScopedRow(a INTEGER, b INTEGER[])"; + Command.ExecuteNonQuery(); + + IDuckDBAppenderRow failedRow = null!; + + using (var appender = Connection.CreateAppender("managedAppenderThrownScopedRow")) + { + appender.AppendRow(row => row.AppendValue((int?)1).AppendValue(new[] { 1, 2 })); + + appender.Invoking(value => value.AppendRow(row => + { + failedRow = row; + row.AppendValue((int?)2).AppendValue(new[] { 3, 4 }); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + + failedRow.Should().NotBeNull(); + failedRow.Invoking(row => row.AppendValue((int?)99)).Should().Throw(); + + appender.Invoking(value => value.AppendRow(row => row.AppendValue((int?)3).AppendValue(new[] { 5, 6 }))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.CreateRow()) + .Should().Throw() + .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.Clear()) + .Should().Throw() + .WithMessage("*cannot be reused*"); + + appender.Invoking(value => value.Close()) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderThrownScopedRow ORDER BY a"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetFieldValue>(1).Should().Equal(1, 2); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowFailureAggregatesFinalizationFailureAndCanBeDisposed() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedFinalization(a INTEGER UNIQUE)"; + Command.ExecuteNonQuery(); + Command.CommandText = "INSERT INTO managedAppenderFailedFinalization VALUES (1)"; + Command.ExecuteNonQuery(); + + using var appender = Connection.CreateAppender("managedAppenderFailedFinalization"); + appender.AppendRow(row => row.AppendValue((int?)1)); + + var exception = appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)2); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw().Which; + + exception.InnerExceptions.Should().HaveCount(2); + exception.InnerExceptions[0].Should().BeOfType() + .Which.Message.Should().Be("callback failed"); + exception.InnerExceptions[1].Should().BeOfType() + .Which.ErrorType.Should().Be(DuckDBErrorType.Constraint); + + appender.Invoking(value => value.Dispose()).Should().NotThrow(); + appender.Invoking(value => value.Dispose()).Should().NotThrow(); + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedFinalization"; + Command.ExecuteScalar().Should().Be(1); + } + + [Fact] + public void AppendRowFailureFlushesCompletedRowsAcrossDataChunks() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedAcrossChunks(a INTEGER)"; + Command.ExecuteNonQuery(); + + var completedRowCount = checked((int)DuckDBGlobalData.VectorSize + 3); + + using (var appender = Connection.CreateAppender("managedAppenderFailedAcrossChunks")) + { + for (var i = 0; i < completedRowCount; i++) + { + appender.AppendRow(i, static (row, value) => row.AppendValue(value)); + } + + appender.Invoking(value => value.AppendRow(completedRowCount, static (row, failedValue) => + { + row.AppendValue(failedValue); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + } + + Command.CommandText = """ + SELECT count(*)::BIGINT, min(a), max(a), sum(a)::BIGINT + FROM managedAppenderFailedAcrossChunks + """; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt64(0).Should().Be(completedRowCount); + reader.GetInt32(1).Should().Be(0); + reader.GetInt32(2).Should().Be(completedRowCount - 1); + reader.GetInt64(3).Should().Be((long)(completedRowCount - 1) * completedRowCount / 2); + } + + [Fact] + public void AppendRowFailureLeavesCompletedRowsInCallerTransaction() + { + Command.CommandText = "CREATE TABLE managedAppenderFailedTransaction(a INTEGER)"; + Command.ExecuteNonQuery(); + + using (var transaction = Connection.BeginTransaction()) + { + using (var appender = Connection.CreateAppender("managedAppenderFailedTransaction")) + { + appender.AppendRow(row => row.AppendValue((int?)1)); + + appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)2); + throw new InvalidOperationException("callback failed"); + })) + .Should().Throw() + .WithMessage("callback failed"); + } + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedTransaction"; + Command.ExecuteScalar().Should().Be(1); + + transaction.Rollback(); + } + + Command.CommandText = "SELECT count(*) FROM managedAppenderFailedTransaction"; + Command.ExecuteScalar().Should().Be(0); + } + + [Fact] + public void AppendRowWritesListValue() + { + Command.CommandText = "CREATE TABLE managedAppenderScopedListRow(a INTEGER, b INTEGER[])"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderScopedListRow")) + { + appender.AppendRow(row => row.AppendValue((int?)1).AppendValue(new[] { 1, 2 })); + } + + Command.CommandText = "SELECT a, b FROM managedAppenderScopedListRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetFieldValue>(1).Should().Equal(1, 2); + reader.Read().Should().BeFalse(); + } + + [Fact] + public void AppendRowRejectsReentrantAppenderUseAndFaultsAppender() + { + Command.CommandText = "CREATE TABLE managedAppenderReentrantScopedRow(a INTEGER)"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("managedAppenderReentrantScopedRow")) + { + appender.Invoking(value => value.AppendRow(row => + { + row.AppendValue((int?)1); + value.AppendRow(nested => nested.AppendValue((int?)2)); + })) + .Should().Throw() + .WithMessage("*inside an AppendRow callback"); + + appender.Invoking(value => value.AppendRow(row => row.AppendValue((int?)3))) + .Should().Throw() + .WithMessage("*cannot be reused*"); + } + + Command.CommandText = "SELECT a FROM managedAppenderReentrantScopedRow"; + using var reader = Command.ExecuteReader(); + reader.Read().Should().BeFalse(); + } + [Fact] public void ClearAppender() { @@ -828,6 +1360,13 @@ private static string GetQualifiedObjectName(params string[] parts) => Select(p => '"' + p + '"') ); + private readonly record struct ScopedMixedRow( + int Id, + DateTime EventTime, + double? Amount, + string Category, + bool? Active); + private enum TestEnum1 { Test1 = 0, @@ -851,4 +1390,4 @@ private enum EnumNotValidValueTestEnum { NotValid = 12345, } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs index 9186bd58..3497bb6e 100644 --- a/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs +++ b/DuckDB.NET.Test/DuckDBMappedAppenderTests.cs @@ -185,4 +185,46 @@ public void MappedAppender_SupportsDefaultAndNull() reader.GetInt32(2).Should().Be(18); reader.IsDBNull(3).Should().BeTrue(); } + + public class NullableValue + { + public int Id { get; set; } + public int? Value { get; set; } + } + + public class NullableValueMap : DuckDBAppenderMap + { + public NullableValueMap() + { + Map(value => value.Id); + Map(value => value.Value); + } + } + + [Fact] + public void MappedAppender_CompiledWriterSupportsNullableValues() + { + Command.CommandText = "CREATE TABLE nullable_values(id INTEGER, value INTEGER);"; + Command.ExecuteNonQuery(); + + using (var appender = Connection.CreateAppender("nullable_values")) + { + appender.AppendRecords( + [ + new NullableValue { Id = 1, Value = 42 }, + new NullableValue { Id = 2, Value = null } + ]); + } + + Command.CommandText = "SELECT id, value FROM nullable_values ORDER BY id"; + using var reader = Command.ExecuteReader(); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(1); + reader.GetInt32(1).Should().Be(42); + + reader.Read().Should().BeTrue(); + reader.GetInt32(0).Should().Be(2); + reader.IsDBNull(1).Should().BeTrue(); + } } diff --git a/DuckDB.NET.Test/Parameters/ListParameterTests.cs b/DuckDB.NET.Test/Parameters/ListParameterTests.cs index 4fa0f307..011ed0d3 100644 --- a/DuckDB.NET.Test/Parameters/ListParameterTests.cs +++ b/DuckDB.NET.Test/Parameters/ListParameterTests.cs @@ -78,6 +78,22 @@ public void CanBindByteList() TestInsertSelect("UTinyInt", faker => faker.Random.Byte().OrNull(faker)); } + [Theory] + [InlineData("UTINYINT[]")] + [InlineData("UTINYINT[3]")] + public void CanBindByteArrayAsCollection(string duckDbType) + { + byte[] expected = [1, 2, byte.MaxValue]; + + Command.CommandText = $"SELECT ?::{duckDbType};"; + Command.Parameters.Add(new DuckDBParameter(expected)); + + using var reader = Command.ExecuteReader(); + reader.Read(); + + reader.GetFieldValue>(0).Should().Equal(expected); + } + [Fact] public void CanBindUShortList() { @@ -189,4 +205,4 @@ public void CanBindTimeOnlyList() return new TimeOnly(dateTime.TimeOfDay.Ticks - dateTime.TimeOfDay.Ticks % 10); }); } -} \ No newline at end of file +} diff --git a/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs b/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs index c4c2f896..245bcdfa 100644 --- a/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs +++ b/DuckDB.NET.Test/Parameters/ParameterCollectionTests.cs @@ -46,10 +46,10 @@ public void ParameterCountMismatchTest() } [Fact] - public void PrepareCommandNoOperationTest() + public void PrepareInvalidCommandThrows() { Command.CommandText = "SELECT ? FROM nowhere"; - Command.Invoking(dbCommand => dbCommand.Prepare()).Should().NotThrow(); + Command.Invoking(dbCommand => dbCommand.Prepare()).Should().Throw(); } [Fact] diff --git a/DuckDB.NET.Test/Test.csproj b/DuckDB.NET.Test/Test.csproj index 37ced068..25a2753e 100644 --- a/DuckDB.NET.Test/Test.csproj +++ b/DuckDB.NET.Test/Test.csproj @@ -1,7 +1,7 @@  - net8.0 + net8.0;net10.0 false true diff --git a/DuckDB.NET.slnx b/DuckDB.NET.slnx index 715e640c..b40a6d65 100644 --- a/DuckDB.NET.slnx +++ b/DuckDB.NET.slnx @@ -1,4 +1,5 @@ + diff --git a/README-FORK.md b/README-FORK.md new file mode 100644 index 00000000..4a2fa48b --- /dev/null +++ b/README-FORK.md @@ -0,0 +1,33 @@ +# Skuirrels DuckDB.NET fork + +This is an unofficial stable build from the +[`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork. It +packages the consolidated performance work for DuckDB v1.5.4 under distinct +`Skuirrels.DuckDB.NET.*` package IDs. + +Install the bundled provider explicitly: + +```shell +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5 +``` + +NuGet packages: + +- [`Skuirrels.DuckDB.NET.Data.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) +- [`Skuirrels.DuckDB.NET.Bindings.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) + +The package keeps the official `DuckDB.NET.Data` namespaces and assembly names, +so application source code does not need to change. Do not reference this fork +package and the official `DuckDB.NET.Data.Full` package in the same dependency +graph because they contain assemblies with the same identities. + +This release bundles DuckDB v1.5.4 and includes the consolidated appender, +parameter binding, prepared-command, result materialisation, and scoped-writer +optimisations from the fork. When equivalent upstream changes are released, +move back to the official `DuckDB.NET.Data.Full` package. + +The original DuckDB.NET and DuckDB licences and attribution are included in the +package. + +Report fork-specific problems in the +[`skuirrels/DuckDB.NET` issue tracker](https://github.com/skuirrels/DuckDB.NET/issues/new). diff --git a/README-PREVIEW.md b/README-PREVIEW.md new file mode 100644 index 00000000..0e3b2dd2 --- /dev/null +++ b/README-PREVIEW.md @@ -0,0 +1,33 @@ +# DuckDB.NET fork preview + +This is an unofficial preview build from the +[`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) fork. It +packages the consolidated performance work while the corresponding upstream +pull requests are under review. + +Install the bundled provider explicitly: + +```shell +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.4-preview.2 +``` + +NuGet packages: + +- [`Skuirrels.DuckDB.NET.Data.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full/) +- [`Skuirrels.DuckDB.NET.Bindings.Full`](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full/) + +The package keeps the official `DuckDB.NET.Data` namespaces and assembly names, +so application source code does not need to change. Do not reference this fork +package and the official `DuckDB.NET.Data.Full` package in the same dependency +graph because they contain assemblies with the same identities. + +This preview bundles DuckDB v1.5.4 and includes the consolidated appender, +parameter binding, prepared-command, result materialisation, and scoped-writer +optimisations from the fork. When equivalent upstream changes are released, +move back to the official `DuckDB.NET.Data.Full` package. + +The original DuckDB.NET and DuckDB licences and attribution are included in the +package. + +Report fork-specific problems in the +[`skuirrels/DuckDB.NET` issue tracker](https://github.com/skuirrels/DuckDB.NET/issues/new). diff --git a/README.md b/README.md index ea5f8b02..b667f01a 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,25 @@ -# DuckDB.NET +# DuckDB.NET - Performance fork -[DuckDB](https://duckdb.org/) bindings for C# +This is the [`skuirrels/DuckDB.NET`](https://github.com/skuirrels/DuckDB.NET) +fork of the [upstream DuckDB.NET project](https://github.com/Giorgi/DuckDB.NET). +It provides stable fork packages containing the cutting-edge performance work while +the corresponding upstream pull requests are reviewed and released. -[![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Giorgi/DuckDB.NET/ci.yml?branch=main&logo=GitHub&style=for-the-badge)](https://github.com/Giorgi/DuckDB.NET/actions/workflows/ci.yml) -[![Coveralls](https://img.shields.io/coveralls/github/Giorgi/DuckDB.NET?logo=coveralls&style=for-the-badge)](https://coveralls.io/github/Giorgi/DuckDB.NET) -[![License](https://img.shields.io/badge/License-Mit-blue.svg?style=for-the-badge&logo=mit)](LICENSE.md) -[![Ko-Fi](https://img.shields.io/static/v1?style=for-the-badge&message=Support%20the%20Project&color=success&logo=ko-fi&label=$$)](https://ko-fi.com/U6U81LHU8) -[![Discord](https://img.shields.io/badge/DuckDB-.Net-%23FFF000?logo=DuckDB&style=for-the-badge)](https://discord.com/channels/909674491309850675/1051088721996427265) - -[![NuGet DuckDB.NET.Data](https://img.shields.io/nuget/dt/DuckDB.NET.Data.svg?label=DuckDB.NET.Data&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Data/) -[![NuGet DuckDB.NET.Bindings](https://img.shields.io/nuget/dt/DuckDB.NET.Bindings.svg?label=DuckDB.NET.Bindings&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Bindings/) - -[![NuGet DuckDB.NET.Data.Full](https://img.shields.io/nuget/dt/DuckDB.NET.Data.Full.svg?label=DuckDB.NET.Data.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Data.Full/) -[![NuGet DuckDB.NET.Bindings.Full](https://img.shields.io/nuget/dt/DuckDB.NET.Bindings.Full.svg?label=DuckDB.NET.Bindings.Full&style=for-the-badge&logo=NuGet)](https://www.nuget.org/packages/DuckDB.NET.Bindings.Full/) - -![Project Icon](https://raw.githubusercontent.com/Giorgi/DuckDB.NET/main/Logo.jpg "DuckDB.NET Project Icon") +[![NuGet (Data)](https://img.shields.io/nuget/v/Skuirrels.DuckDB.NET.Data.Full.svg?label=NuGet%20%28Data%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Data.Full) +[![NuGet (Bindings)](https://img.shields.io/nuget/v/Skuirrels.DuckDB.NET.Bindings.Full.svg?label=NuGet%20%28Bindings%29)](https://www.nuget.org/packages/Skuirrels.DuckDB.NET.Bindings.Full) +[![.NET 8 and 10](https://img.shields.io/badge/.NET-8%20%7C%2010-512BD4)](https://dotnet.microsoft.com/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md) ## Usage ```sh -dotnet add package DuckDB.NET.Data.Full +dotnet add package Skuirrels.DuckDB.NET.Data.Full --version 1.5.5 ``` +The fork packages retain the official `DuckDB.NET.Data` namespaces and assembly +names. Do not reference this package and the official `DuckDB.NET.Data.Full` +package in the same dependency graph. + ```cs using (var duckDBConnection = new DuckDBConnection("Data Source=file.db")) { @@ -91,9 +89,11 @@ Documentation is available at [https://duckdb.net](https://duckdb.net) ## Support -If you encounter a bug with the library [Create an Issue](https://github.com/Giorgi/DuckDB.NET/issues/new). Join the [DuckDB `dotnet` channel](https://discord.duckdb.org/) for DuckDB.NET-related topics. +For a fork-specific problem, [create an issue in this fork](https://github.com/skuirrels/DuckDB.NET/issues/new). +For upstream DuckDB.NET support, use the [upstream issue tracker](https://github.com/Giorgi/DuckDB.NET/issues/new). +You can also join the [DuckDB `dotnet` channel](https://discord.duckdb.org/) for DuckDB.NET-related topics. -## Contributors +## Upstream contributors [![Contributors](https://contrib.rocks/image?repo=Giorgi/DuckDB.NET)](https://github.com/Giorgi/DuckDB.NET/graphs/contributors) diff --git a/run-driver-comparison.sh b/run-driver-comparison.sh new file mode 100755 index 00000000..5c8b1c7c --- /dev/null +++ b/run-driver-comparison.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PROJECT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)" +ARTIFACTS_ROOT="${DRIVER_BENCHMARK_OUTPUT:-${PROJECT_ROOT}/BenchmarkDotNet.Artifacts/DriverComparison/${RUN_ID}}" +GO_RUN_COUNT="${GO_COUNT:-10}" +GO_RUN_BENCHTIME="${GO_BENCHTIME:-500ms}" +GO_MODULE_CACHE="${DRIVER_BENCHMARK_GOMODCACHE:-${GOMODCACHE:-}}" +GO_BUILD_CACHE="${DRIVER_BENCHMARK_GOCACHE:-${GOCACHE:-}}" +JAVA_RUN_WARMUP_ITERATIONS="${JAVA_WARMUP_ITERATIONS:-5}" +JAVA_RUN_MEASUREMENT_ITERATIONS="${JAVA_MEASUREMENT_ITERATIONS:-10}" +JAVA_RUN_ITERATION_TIME="${JAVA_ITERATION_TIME:-500ms}" +SKIP_RELEASE_BUILD="${SKIP_DOTNET_BUILD:-false}" +SKIP_JAVA_RELEASE_BUILD="${SKIP_JAVA_BUILD:-false}" + +# Keep runner overrides away from MSBuild; environment variables are also +# imported as MSBuild properties and can interfere with the .NET project build. +unset DRIVER_BENCHMARK_OUTPUT GO_COUNT GO_BENCHTIME SKIP_DOTNET_BUILD SKIP_JAVA_BUILD +unset DRIVER_BENCHMARK_GOMODCACHE DRIVER_BENCHMARK_GOCACHE +unset JAVA_WARMUP_ITERATIONS JAVA_MEASUREMENT_ITERATIONS JAVA_ITERATION_TIME +unset GOMODCACHE GOCACHE + +mkdir -p "${ARTIFACTS_ROOT}/dotnet-1.5.3" +mkdir -p "${ARTIFACTS_ROOT}/dotnet-efcore-1.13.0" +mkdir -p "${ARTIFACTS_ROOT}/dotnet-fork-1.5.4" +mkdir -p "${ARTIFACTS_ROOT}/java" + +{ + uname -a + dotnet --version + go version + java -version 2>&1 + mvn -version + echo "DuckDB.NET package baseline: 1.5.3" + echo "DuckDB.EFCoreProvider package: 1.13.0 (depends on DuckDB.NET.Data.Full 1.5.3)" + echo "DuckDB.NET source lane: consolidated fork 1.5.4" + echo "duckdb-go: v2.10504.0 (DuckDB 1.5.4)" + echo "DuckDB JDBC: org.duckdb:duckdb_jdbc:1.5.4.0 (DuckDB 1.5.4)" + echo "DuckDB threads per connection: 1" + echo "Analytical rows: 2000000" + echo "Materialization rows: 100000" + echo "Ingest rows per transaction: 10000" + echo "TPC-H scale factor: 0.1; queries: 1, 6, 12, 14" +} > "${ARTIFACTS_ROOT}/environment.txt" + +if [[ "${SKIP_RELEASE_BUILD}" != "1" && "${SKIP_RELEASE_BUILD}" != "true" ]]; then + dotnet build \ + "${PROJECT_ROOT}/DuckDB.NET.Benchmarks/Benchmarks.csproj" \ + --configuration Release \ + --disable-build-servers \ + --nologo + + dotnet build \ + "${PROJECT_ROOT}/DuckDB.NET.1_5_3.Benchmarks/DuckDB.NET.1_5_3.Benchmarks.csproj" \ + --configuration Release \ + --disable-build-servers \ + --nologo + + dotnet build \ + "${PROJECT_ROOT}/DuckDB.EFCoreProvider.1_13_0.Benchmarks/DuckDB.EFCoreProvider.1_13_0.Benchmarks.csproj" \ + --configuration Release \ + --disable-build-servers \ + --nologo +fi + +if [[ "${SKIP_JAVA_RELEASE_BUILD}" != "1" && "${SKIP_JAVA_RELEASE_BUILD}" != "true" ]]; then + ( + cd "${PROJECT_ROOT}/DuckDB.Java.Benchmarks" + mvn --batch-mode --no-transfer-progress package + ) +fi + +dotnet run \ + --no-build \ + --configuration Release \ + --project "${PROJECT_ROOT}/DuckDB.NET.1_5_3.Benchmarks/DuckDB.NET.1_5_3.Benchmarks.csproj" \ + -- \ + --filter '*' \ + --artifacts "${ARTIFACTS_ROOT}/dotnet-1.5.3" + +dotnet run \ + --no-build \ + --configuration Release \ + --project "${PROJECT_ROOT}/DuckDB.EFCoreProvider.1_13_0.Benchmarks/DuckDB.EFCoreProvider.1_13_0.Benchmarks.csproj" \ + -- \ + --filter '*' \ + --artifacts "${ARTIFACTS_ROOT}/dotnet-efcore-1.13.0" + +dotnet run \ + --no-build \ + --configuration Release \ + --project "${PROJECT_ROOT}/DuckDB.NET.Benchmarks/Benchmarks.csproj" \ + -- \ + --filter '*' \ + --artifacts "${ARTIFACTS_ROOT}/dotnet-fork-1.5.4" + +( + cd "${PROJECT_ROOT}/DuckDB.Go.Benchmarks" + if [[ -n "${GO_MODULE_CACHE}" ]]; then + export GOMODCACHE="${GO_MODULE_CACHE}" + fi + if [[ -n "${GO_BUILD_CACHE}" ]]; then + export GOCACHE="${GO_BUILD_CACHE}" + fi + go test \ + -run '^$' \ + -bench '^Benchmark(PreparedCommand|AnalyticalQuery|ResultMaterialization|BulkIngestion|Tpch)' \ + -benchmem \ + -count "${GO_RUN_COUNT}" \ + -benchtime "${GO_RUN_BENCHTIME}" \ + . +) | tee "${ARTIFACTS_ROOT}/go.txt" + +( + cd "${PROJECT_ROOT}/DuckDB.Java.Benchmarks" + java --enable-native-access=ALL-UNNAMED \ + -jar target/benchmarks-all.jar \ + 'org.duckdb.benchmarks.*' \ + -f 1 \ + -wi "${JAVA_RUN_WARMUP_ITERATIONS}" \ + -i "${JAVA_RUN_MEASUREMENT_ITERATIONS}" \ + -w "${JAVA_RUN_ITERATION_TIME}" \ + -r "${JAVA_RUN_ITERATION_TIME}" \ + -tu ns \ + -foe true \ + -prof gc \ + -rf json \ + -rff "${ARTIFACTS_ROOT}/java/results.json" +) | tee "${ARTIFACTS_ROOT}/java.txt" + +echo "Comparison artifacts: ${ARTIFACTS_ROOT}" diff --git a/scripts/validate-fork-packages.sh b/scripts/validate-fork-packages.sh new file mode 100755 index 00000000..269189cc --- /dev/null +++ b/scripts/validate-fork-packages.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "Usage: $0 " >&2 + exit 2 +fi + +package_directory=$(cd "$1" && pwd) +package_version=$2 +data_package="$package_directory/Skuirrels.DuckDB.NET.Data.Full.$package_version.nupkg" +bindings_package="$package_directory/Skuirrels.DuckDB.NET.Bindings.Full.$package_version.nupkg" + +if [[ "$package_version" == *-preview.* ]]; then + expected_readme=README-PREVIEW.md +else + expected_readme=README-FORK.md +fi + +[[ -f "$data_package" ]] || { echo "Missing $data_package" >&2; exit 1; } +[[ -f "$bindings_package" ]] || { echo "Missing $bindings_package" >&2; exit 1; } + +validation_directory=$(mktemp -d) +trap 'rm -rf "$validation_directory"' EXIT + +unzip -p "$data_package" '*.nuspec' > "$validation_directory/data.nuspec" +unzip -p "$bindings_package" '*.nuspec' > "$validation_directory/bindings.nuspec" +unzip -Z1 "$data_package" > "$validation_directory/data-files.txt" +unzip -Z1 "$bindings_package" > "$validation_directory/bindings-files.txt" + +grep -Fq 'Skuirrels.DuckDB.NET.Data.Full' "$validation_directory/data.nuspec" +grep -Fq "$package_version" "$validation_directory/data.nuspec" +grep -Fq "Skuirrels.DuckDB.NET.Bindings.Full' "$validation_directory/bindings.nuspec" +grep -Fq "$package_version" "$validation_directory/bindings.nuspec" +grep -Fq "$expected_readme" "$validation_directory/data.nuspec" +grep -Fq "$expected_readme" "$validation_directory/bindings.nuspec" +grep -Fxq "$expected_readme" "$validation_directory/data-files.txt" +grep -Fxq "$expected_readme" "$validation_directory/bindings-files.txt" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/data.nuspec" +grep -Fq 'https://github.com/skuirrels/DuckDB.NET' "$validation_directory/bindings.nuspec" + +for native_asset in \ + runtimes/win-x64/native/duckdb.dll \ + runtimes/win-arm64/native/duckdb.dll \ + runtimes/linux-x64/native/libduckdb.so \ + runtimes/linux-arm64/native/libduckdb.so \ + runtimes/osx/native/libduckdb.dylib +do + grep -Fxq "$native_asset" "$validation_directory/bindings-files.txt" +done + +for target_framework in net8.0 net10.0 +do + smoke_directory="$validation_directory/smoke-$target_framework" + mkdir -p "$smoke_directory" + + cat > "$smoke_directory/ForkPackageSmoke.csproj" < + + Exe + $target_framework + enable + enable + + + + + +EOF + + cat > "$smoke_directory/NuGet.config" < + + + + + + + + + + + + + + + +EOF + + cat > "$smoke_directory/Program.cs" <<'EOF' +using DuckDB.NET.Data; + +using var connection = new DuckDBConnection("Data Source=:memory:"); +connection.Open(); + +using (var command = connection.CreateCommand()) +{ + command.CommandText = "CREATE TABLE fork_package_smoke(value INTEGER)"; + command.ExecuteNonQuery(); +} + +using (var appender = connection.CreateAppender("fork_package_smoke")) +{ + appender.AppendRowScoped(42, + static (ref DuckDBAppenderRowWriter writer, int value) => writer.AppendValue(value)); +} + +using var verification = connection.CreateCommand(); +verification.CommandText = "SELECT version(), sum(value) FROM fork_package_smoke"; +using var reader = verification.ExecuteReader(); +if (!reader.Read() || !reader.GetString(0).Contains("v1.5.4", StringComparison.Ordinal) || reader.GetInt64(1) != 42) +{ + throw new InvalidOperationException("Fork package smoke test failed."); +} + +Console.WriteLine($"Fork package smoke passed with {reader.GetString(0)}"); +EOF + + dotnet restore "$smoke_directory/ForkPackageSmoke.csproj" \ + --configfile "$smoke_directory/NuGet.config" \ + --packages "$validation_directory/packages-$target_framework" + dotnet run --project "$smoke_directory/ForkPackageSmoke.csproj" --configuration Release --no-restore +done + +echo "Validated fork packages at version $package_version" diff --git a/scripts/validate-preview-packages.sh b/scripts/validate-preview-packages.sh new file mode 100755 index 00000000..b770b25c --- /dev/null +++ b/scripts/validate-preview-packages.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_directory=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +exec "$script_directory/validate-fork-packages.sh" "$@"