diff --git a/.formatter.exs b/.formatter.exs index d9c81cd4..d33e8cbd 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,5 +1,5 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench}/**/*.{ex,exs}"], + inputs: ["{mix,.formatter}.exs", "{config,lib,test,bench,dev}/**/*.{ex,exs}"], import_deps: [:stream_data] ] diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9481148c..bd12107d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -2,7 +2,6 @@ name: bench on: workflow_dispatch: - pull_request: push: branches: [master] schedule: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52263a0a..b2d0ccf5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: jobs: test: - name: "test @ elixir:${{ matrix.elixir }} otp:${{ matrix.otp }} clickhouse:${{ matrix.clickhouse || 'latest' }} tz:${{ matrix.timezone || 'utc' }}${{ matrix.dialyzer && ' +dialyzer' || '' }}${{ matrix.lint && ' +lint' || '' }}" + name: "test @ elixir:${{ matrix.elixir }} otp:${{ matrix.otp }} clickhouse:${{ matrix.clickhouse }} ${{ matrix.dialyzer && ' +dialyzer' || '' }}${{ matrix.lint && ' +lint' || '' }}" runs-on: ubuntu-latest env: @@ -19,42 +19,26 @@ jobs: strategy: matrix: include: - # some old elixir/erlang version - - elixir: 1.15 - otp: 25 - - # some recent version and non-UTC timezone - - elixir: 1.18 - otp: 27 - timezone: Europe/Berlin - - # the latest elixir/erlang version with all static checks - - elixir: 1.19 - otp: 28 + # some old elixir/clickhouse version with the minimum supported OTP + # see https://hexdocs.pm/elixir/compatibility-and-deprecations.html#between-elixir-and-erlang-otp + - elixir: "1.18" + otp: "28" + clickhouse: "24.5.4.49" + + # the latest versions with all static checks + - elixir: ">0" + otp: ">0" + clickhouse: "latest" dialyzer: true lint: true coverage: true - # Plausible versions - # - https://github.com/plausible/analytics/blob/master/.tool-versions - # - https://github.com/plausible/analytics/blob/master/.github/workflows/elixir.yml - - elixir: 1.19.4 - otp: 27.3.4.6 - clickhouse: 25.11.5.8 - - # some older pre-JSON ClickHouse version - # https://github.com/plausible/ch/issues/273 - - elixir: 1.18 - otp: 28 - clickhouse: 24.5.4.49 - services: clickhouse: - image: clickhouse/clickhouse-server:${{ matrix.clickhouse || 'latest' }} + image: clickhouse/clickhouse-server:${{ matrix.clickhouse }} ports: - 8123:8123 env: - TZ: ${{ matrix.timezone || 'UTC' }} # https://github.com/ClickHouse/ClickHouse/issues/75494 CLICKHOUSE_SKIP_USER_SETUP: 1 options: >- @@ -110,9 +94,9 @@ jobs: uses: actions/cache@v5 with: path: plts - key: plts-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ github.head_ref || github.ref }} + key: plts-${{ steps.beam.outputs.elixir-version }}-${{ github.head_ref || github.ref }} restore-keys: | - plts-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-refs/heads/master + plts-${{ steps.beam.outputs.elixir-version }}-refs/heads/master - run: mix dialyzer --format github if: ${{ matrix.dialyzer }} diff --git a/CHANGELOG.md b/CHANGELOG.md index eff1ad26..96601a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## Unreleased + +- **Breaking:** replace DBConnection with NimblePool. +- **Breaking:** require Elixir 1.18 or later for the built-in `JSON` module and Erlang/OTP 28 or later for `:zstd`. +- **Breaking:** `Ch.start_link/1` no longer accepts DBConnection options or connection-level ClickHouse options such as `:database`, `:username`, `:password`, `:settings`, `:timeout`, `:scheme`, `:hostname`, `:port`, and `:transport_opts`. Use `:url` for the endpoint, pass ClickHouse settings per query with `Ch.query/4`'s `:settings` option, and pass ClickHouse/database/auth headers per query with `:headers`. +- **Breaking:** remove DBConnection compatibility APIs and structs such as `Ch.stream/4`, `Ch.run/3`, `Ch.Query`, `Ch.Stream`, DBConnection transactions/checkouts, and DBConnection streaming/collectable inserts. +- **Breaking:** `Ch.query/4` only accepts named query parameters as a map. Positional and pseudo-positional params such as `[value]` with `{$0:Type}` are no longer supported. +- **Breaking:** remove query command detection and the `:command` query option. +- **Breaking:** remove query options `:format`, `:types`, `:encode`, `:decode`, and `:multipart`. Use an `x-clickhouse-format` header or explicit `FORMAT ...` SQL for formats, and pass already-encoded request bodies for inserts. +- **Breaking:** remove automatic RowBinary insert encoding from `Ch.query/4`. Call `Ch.RowBinary.encode_rows/2` or `Ch.RowBinary.encode_names_and_types/2` explicitly and pass the resulting iodata in the query body. +- **Breaking:** remove multipart query parameter requests for now. `multipart: true` is no longer supported; see https://github.com/plausible/ch/issues/344 for restoring it. +- **Breaking:** `Ch.query/4` now returns `%Ch.Result{names: names, rows: rows, headers: headers, data: data}` for successful responses. Decoded `RowBinaryWithNamesAndTypes` responses populate `:names` and `:rows`; raw formats, inserts, DDL, and other empty responses keep the response body in `:data` and leave `:names` and `:rows` as `nil`. +- **Breaking:** successful inserts, DDL, and other empty responses no longer return `%Ch.Result{command: command, num_rows: num_rows}`. `x-clickhouse-summary` written-row counts are no longer surfaced. +- **Breaking:** `Ch.RowBinary` no longer has a separate `:binary` type. Use `:string` for ClickHouse `String`; it now preserves raw bytes and no longer replaces invalid UTF-8 with the replacement character. +- Remove the `Jason` dependency. JSON encoding/decoding now uses Elixir's built-in `JSON` module. +- Add explicit request and response compression support through HTTP headers. `zstd` and `gzip` response bodies are decompressed automatically for decoded `RowBinaryWithNamesAndTypes` and error responses; raw successful responses are kept as received in `Ch.Result.data`. + ## 0.8.3 (2026-05-12) - use DBConnection v2.10 https://github.com/plausible/ch/pull/339 diff --git a/README.md b/README.md index 8db70a15..610c0a6e 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ HTTP [ClickHouse](https://clickhouse.com) client for Elixir. -Used in [Ecto ClickHouse adapter.](https://github.com/plausible/ecto_ch) +Used in [Ecto ClickHouse adapter](https://github.com/plausible/ecto_ch). ### Key features @@ -14,303 +14,106 @@ Used in [Ecto ClickHouse adapter.](https://github.com/plausible/ecto_ch) - Native query parameters - Per query settings -Your ideas are welcome [here.](https://github.com/plausible/ch/issues/82) - ## Installation ```elixir defp deps do [ - {:ch, "~> 0.8.0"} + {:ch, "~> 0.9.0"} ] end ``` ## Usage -#### Start [DBConnection](https://github.com/elixir-ecto/db_connection) pool - -```elixir -defaults = [ - scheme: "http", - hostname: "localhost", - port: 8123, - database: "default", - settings: [], - pool_size: 1, - timeout: :timer.seconds(15) -] - -# note that starting in ClickHouse 25.1.3.23 `default` user doesn't have -# network access by default in the official Docker images -# see https://github.com/ClickHouse/ClickHouse/pull/75259 -{:ok, pid} = Ch.start_link(defaults) -``` - -#### Select rows - -```elixir -{:ok, pid} = Ch.start_link() - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT 3") - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {$0:UInt8}", [3]) - -{:ok, %Ch.Result{rows: [[0], [1], [2]]}} = - Ch.query(pid, "SELECT * FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 3}) -``` - -Note on datetime encoding in query parameters: - -- `%NaiveDateTime{}` is encoded as text to make it assume the column's or ClickHouse server's timezone -- `%DateTime{}` is encoded as unix timestamp and is treated as UTC timestamp by ClickHouse - -#### Select rows (lots of params, reverse proxy) - -> [!NOTE] -> -> Support for multipart requests was added in `v0.6.2` - -For queries with many parameters the resulting URL can become too long for some reverse proxies, resulting in a `414 Request-URI Too Large` error. - -To avoid this, you can use the `multipart: true` option to send the query and parameters in the request body. - -```elixir -{:ok, pid} = Ch.start_link() - -# Moves parameters from the URL to a multipart/form-data body -%Ch.Result{rows: [[[1, 2, 3 | _rest]]]} = - Ch.query!(pid, "SELECT {ids:Array(UInt64)}", %{"ids" => Enum.to_list(1..10_000)}, multipart: true) -``` - -> [!NOTE] -> -> `multipart: true` is currently required on each individual query. Support for pool-wide configuration is planned for a future release. - -#### Insert rows - -```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES (0), (1)") - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({$0:UInt8}), ({$1:UInt32})", [0, 1]) - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) VALUES ({a:UInt16}), ({b:UInt64})", %{"a" => 0, "b" => 1}) - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) SELECT number FROM system.numbers LIMIT {limit:UInt8}", %{"limit" => 2}) -``` - -#### Insert rows as [RowBinary](https://clickhouse.com/docs/en/interfaces/formats/RowBinary) (efficient) - -```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -types = ["UInt64"] -# or -types = [Ch.Types.u64()] -# or -types = [:u64] - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) FORMAT RowBinary", [[0], [1]], types: types) -``` - -Note that RowBinary format encoding requires `:types` option to be provided. - -Similarly, you can use [RowBinaryWithNamesAndTypes](https://clickhouse.com/docs/en/interfaces/formats/RowBinaryWithNamesAndTypes) which would additionally do something like a type check. - -```elixir -sql = "INSERT INTO ch_demo FORMAT RowBinaryWithNamesAndTypes" -opts = [names: ["id"], types: ["UInt64"]] -rows = [[0], [1]] - -%Ch.Result{num_rows: 2} = Ch.query!(pid, sql, rows, opts) -``` - -#### Insert rows in custom [format](https://clickhouse.com/docs/en/interfaces/formats) - -```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -csv = [0, 1] |> Enum.map(&to_string/1) |> Enum.intersperse(?\n) - -%Ch.Result{num_rows: 2} = - Ch.query!(pid, "INSERT INTO ch_demo(id) FORMAT CSV", csv, encode: false) -``` - -#### Insert rows as chunked RowBinary stream +Start a pool: ```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE IF NOT EXISTS ch_demo(id UInt64) ENGINE Null") - -stream = Stream.repeatedly(fn -> [:rand.uniform(100)] end) -chunked = Stream.chunk_every(stream, 100) -encoded = Stream.map(chunked, fn chunk -> Ch.RowBinary.encode_rows(chunk, _types = ["UInt64"]) end) -ten_encoded_chunks = Stream.take(encoded, 10) - -%Ch.Result{num_rows: 1000} = - Ch.query(pid, "INSERT INTO ch_demo(id) FORMAT RowBinary", ten_encoded_chunks, encode: false) +{:ok, pool} = Ch.start_link(url: "http://localhost:8123") ``` -This query makes a [`transfer-encoding: chunked`](https://en.wikipedia.org/wiki/Chunked_transfer_encoding) HTTP request while unfolding the stream resulting in lower memory usage. - -#### Query with custom [settings](https://clickhouse.com/docs/en/operations/settings/settings) +Run a query with named ClickHouse parameters: ```elixir -{:ok, pid} = Ch.start_link() - -settings = [async_insert: 1] - -%Ch.Result{rows: [["async_insert", "Bool", "0"]]} = - Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'") - -%Ch.Result{rows: [["async_insert", "Bool", "1"]]} = - Ch.query!(pid, "SHOW SETTINGS LIKE 'async_insert'", [], settings: settings) +Ch.query!( + pool, + "SELECT {limit:UInt64}", + %{"limit" => 42} +) ``` -## Caveats - -#### NULL in RowBinary - -It's the same as in [ch-go](https://clickhouse.com/docs/en/integrations/go#nullable) - -> At insert time, Nil can be passed for both the normal and Nullable version of a column. For the former, the default value for the type will be persisted, e.g., an empty string for string. For the nullable version, a NULL value will be stored in ClickHouse. +Positional parameters such as `{$0:UInt64}` are no longer supported. Use named parameters instead: ```elixir -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, """ -CREATE TABLE ch_nulls ( - a UInt8 NULL, - b UInt8 DEFAULT 10, - c UInt8 NOT NULL -) ENGINE Memory -""") - -types = ["Nullable(UInt8)", "UInt8", "UInt8"] -inserted_rows = [[nil, nil, nil]] -selected_rows = [[nil, 0, 0]] +# before +Ch.query!(pool, "SELECT {$0:UInt64}", [42]) -%Ch.Result{num_rows: 1} = - Ch.query!(pid, "INSERT INTO ch_nulls(a, b, c) FORMAT RowBinary", inserted_rows, types: types) - -%Ch.Result{rows: ^selected_rows} = - Ch.query!(pid, "SELECT * FROM ch_nulls") +# now +Ch.query!(pool, "SELECT {value:UInt64}", %{"value" => 42}) ``` -Note that in this example `DEFAULT 10` is ignored and `0` (the default value for `UInt8`) is persisted instead. - -However, [`input()`](https://clickhouse.com/docs/en/sql-reference/table-functions/input) can be used as a workaround: +By default, `Ch.query/4` requests `RowBinaryWithNamesAndTypes` and returns a decoded result: ```elixir -sql = """ -INSERT INTO ch_nulls - SELECT * FROM input('a Nullable(UInt8), b Nullable(UInt8), c UInt8') - FORMAT RowBinary\ -""" - -Ch.query!(pid, sql, inserted_rows, types: ["Nullable(UInt8)", "Nullable(UInt8)", "UInt8"]) - -%Ch.Result{rows: [[0], [10]]} = - Ch.query!(pid, "SELECT b FROM ch_nulls ORDER BY b") +%Ch.Result{ + names: ["number"], + rows: [[42]], + headers: headers, + data: raw_body +} = Ch.query!(pool, "SELECT 42 AS number") ``` -#### UTF-8 in RowBinary - -When decoding [`String`](https://clickhouse.com/docs/en/sql-reference/data-types/string) columns non UTF-8 characters are replaced with `�` (U+FFFD). This behaviour is similar to [`toValidUTF8`](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#tovalidutf8) and [JSON format.](https://clickhouse.com/docs/en/interfaces/formats#json) +To get a raw CSV or JSON response, override the ClickHouse response format and read `data`: ```elixir -{:ok, pid} = Ch.start_link() +%Ch.Result{data: csv} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 3", + %{}, + headers: [{"x-clickhouse-format", "CSV"}] + ) -Ch.query!(pid, "CREATE TABLE ch_utf8(str String) ENGINE Memory") - -bin = "\x61\xF0\x80\x80\x80b" -utf8 = "a�b" - -%Ch.Result{num_rows: 1} = - Ch.query!(pid, "INSERT INTO ch_utf8(str) FORMAT RowBinary", [[bin]], types: ["String"]) - -%Ch.Result{rows: [[^utf8]]} = - Ch.query!(pid, "SELECT * FROM ch_utf8") - -%Ch.Result{rows: %{"data" => [[^utf8]]}} = - pid |> Ch.query!("SELECT * FROM ch_utf8 FORMAT JSONCompact") |> Map.update!(:rows, &Jason.decode!/1) -``` - -To get raw binary from `String` columns use `:binary` type that skips UTF-8 checks. - -```elixir -%Ch.Result{rows: [[^bin]]} = - Ch.query!(pid, "SELECT * FROM ch_utf8", [], types: [:binary]) +%Ch.Result{data: json} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 3", + %{}, + headers: [{"x-clickhouse-format", "JSONEachRow"}] + ) ``` -#### Timezones in RowBinary - -Decoding non-UTC datetimes like `DateTime('Asia/Taipei')` requires a [timezone database.](https://hexdocs.pm/elixir/DateTime.html#module-time-zone-database) +Insert RowBinary data by encoding it explicitly: ```elixir -Mix.install([:ch, :tz]) - -:ok = Calendar.put_time_zone_database(Tz.TimeZoneDatabase) - -{:ok, pid} = Ch.start_link() - -%Ch.Result{rows: [[~N[2023-04-25 17:45:09]]]} = - Ch.query!(pid, "SELECT CAST(now() as DateTime)") +rows = [[1, "one"], [2, "two"]] +types = ["UInt8", "String"] +rowbinary = Ch.RowBinary.encode_rows(rows, types) -%Ch.Result{rows: [[~U[2023-04-25 17:45:11Z]]]} = - Ch.query!(pid, "SELECT CAST(now() as DateTime('UTC'))") - -%Ch.Result{rows: [[%DateTime{time_zone: "Asia/Taipei"} = taipei]]} = - Ch.query!(pid, "SELECT CAST(now() as DateTime('Asia/Taipei'))") - -"2023-04-26 01:45:12+08:00 CST Asia/Taipei" = to_string(taipei) +Ch.query!(pool, [ + "INSERT INTO events FORMAT RowBinary\n", + rowbinary +]) ``` -Encoding non-UTC datetimes works but might be slow due to timezone conversion: +Compressed inserts use the same shape, with the whole request body compressed: ```elixir -Mix.install([:ch, :tz]) - -:ok = Calendar.put_time_zone_database(Tz.TimeZoneDatabase) - -{:ok, pid} = Ch.start_link() - -Ch.query!(pid, "CREATE TABLE ch_datetimes(name String, datetime DateTime) ENGINE Memory") - -naive = NaiveDateTime.utc_now() -utc = DateTime.utc_now() -taipei = DateTime.shift_zone!(utc, "Asia/Taipei") +names = ["id", "name"] +types = ["UInt8", "String"] +rows = [[1, "one"], [2, "two"]] -rows = [["naive", naive], ["utc", utc], ["taipei", taipei]] +payload = + :zstd.compress([ + "INSERT INTO events FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) + ]) -Ch.query!(pid, "INSERT INTO ch_datetimes(name, datetime) FORMAT RowBinary", rows, types: ["String", "DateTime"]) - -%Ch.Result{ - rows: [ - ["naive", ~U[2024-12-21 05:24:40Z]], - ["utc", ~U[2024-12-21 05:24:40Z]], - ["taipei", ~U[2024-12-21 05:24:40Z]] - ] -} = - Ch.query!(pid, "SELECT name, CAST(datetime as DateTime('UTC')) FROM ch_datetimes") +Ch.query!( + pool, + payload, + %{}, + headers: [{"content-encoding", "zstd"}] +) ``` - -## [Benchmarks](./bench) - -See [GitHub Pages](https://plausible.github.io/ch/dev/bench/) for latest results. diff --git a/bench/stream.exs b/bench/stream.exs deleted file mode 100644 index 66bbbe52..00000000 --- a/bench/stream.exs +++ /dev/null @@ -1,64 +0,0 @@ -IO.puts(""" -This benchmark is based on https://github.com/ClickHouse/ch-bench - -It tests how quickly a client can select N rows from the system.numbers_mt table: - - SELECT number FROM system.numbers_mt LIMIT {limit:UInt64} FORMAT RowBinary -""") - -port = String.to_integer(System.get_env("CH_PORT") || "8123") -hostname = System.get_env("CH_HOSTNAME") || "localhost" -scheme = System.get_env("CH_SCHEME") || "http" - -limits = fn limits -> - Map.new(limits, fn limit -> - {"limit=#{limit}", limit} - end) -end - -Benchee.run( - %{ - # "Ch.query" => fn %{pool: pool, limit: limit} -> - # Ch.query!( - # pool, - # "SELECT number FROM system.numbers_mt LIMIT {limit:UInt64}", - # %{"limit" => limit}, - # timeout: :infinity - # ) - # end, - "Ch.stream w/o decoding (i.e. pass-through)" => fn %{pool: pool, limit: limit} -> - DBConnection.run( - pool, - fn conn -> - conn - |> Ch.stream( - "SELECT number FROM system.numbers_mt LIMIT {limit:UInt64}", - %{"limit" => limit}, - decode: false - ) - |> Stream.run() - end, - timeout: :infinity - ) - end, - "Ch.stream with decoding" => fn %{pool: pool, limit: limit} -> - DBConnection.run( - pool, - fn conn -> - conn - |> Ch.stream( - "SELECT number FROM system.numbers_mt LIMIT {limit:UInt64}", - %{"limit" => limit} - ) - |> Stream.run() - end, - timeout: :infinity - ) - end - }, - before_scenario: fn limit -> - {:ok, pool} = Ch.start_link(scheme: scheme, hostname: hostname, port: port, pool_size: 1) - %{pool: pool, limit: limit} - end, - inputs: limits.([500, 500_000, 500_000_000]) -) diff --git a/bench/support/github_action_benchmark_formatter.ex b/dev/support/github_action_benchmark_formatter.ex similarity index 96% rename from bench/support/github_action_benchmark_formatter.ex rename to dev/support/github_action_benchmark_formatter.ex index fb78594a..442325f4 100644 --- a/bench/support/github_action_benchmark_formatter.ex +++ b/dev/support/github_action_benchmark_formatter.ex @@ -27,7 +27,7 @@ defmodule GitHubActionBenchmarkFormatter do |> Path.dirname() |> File.mkdir_p!() - File.write!(path, Jason.encode_to_iodata!(data, pretty: true)) + File.write!(path, JSON.encode_to_iodata!(data)) end defp benchmark_name(suite_name, scenario) do diff --git a/bench/support/row_binary_benchmark_data.ex b/dev/support/row_binary_benchmark_data.ex similarity index 100% rename from bench/support/row_binary_benchmark_data.ex rename to dev/support/row_binary_benchmark_data.ex diff --git a/lib/ch.ex b/lib/ch.ex index 20ed8c8f..92f7e41c 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -1,124 +1,459 @@ defmodule Ch do - @moduledoc "Minimal HTTP ClickHouse client." - alias Ch.{Connection, Query, Result} + @moduledoc """ + Minimal HTTP ClickHouse client. + + `Ch` starts a lazy pool of HTTP/1 connections to ClickHouse. The pool opens + connections on demand and reuses them while they remain healthy. + + By default, queries request `RowBinaryWithNamesAndTypes` and return decoded + rows: + + {:ok, pool} = Ch.start_link(url: "http://localhost:8123") + + {:ok, %Ch.Result{names: ["number"], rows: [[1], [2], [3]]}} = + Ch.query(pool, "SELECT number FROM system.numbers LIMIT {limit:UInt8}", %{ + "limit" => 3 + }) + + For large decoded responses, ask ClickHouse for compressed response bodies: + + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 1_000_000", + %{}, + headers: [{"accept-encoding", "zstd"}] + ) + + `Ch` automatically decompresses successful responses that it decodes itself + (`RowBinaryWithNamesAndTypes`) and error responses. Successful responses in + other formats keep the raw response body in `Ch.Result.data`, including any + `content-encoding`. + """ + @behaviour NimblePool + + @dialyzer :no_improper_lists + + @query_timeout to_timeout(second: 30) + @user_agent "ch/#{Ch.MixProject.version()}" + + @start_options_schema [ + name: [ + type: {:custom, __MODULE__, :validate_name, []}, + doc: """ + The name of the Ch pool instance, used to identify and interact with it. Supported values are atoms and via tuples. + """ + ], + pool_size: [ + type: :pos_integer, + doc: + "Maximum number of concurrent connections. Pool is lazy so it starts out without any connections and they are open on demand.", + default: 20 + ], + worker_idle_timeout: [ + type: :timeout, + doc: """ + Time a connection can stay idle before the pool closes it. + Should be lower than ClickHouse's [`keep_alive_timeout`](https://clickhouse.com/docs/operations/server-configuration-parameters/settings#keep_alive_timeout) + to avoid sending a request over a connection that would be closed by ClickHouse soon-ish. + """, + default: to_timeout(second: 5) + ], + url: [ + type: :string, + doc: "The ClickHouse endpoint URL.", + default: "http://localhost:8123" + ] + ] + + @doc false + def validate_name(name) when is_atom(name), do: {:ok, name} + def validate_name({:via, module, _term} = via) when is_atom(module), do: {:ok, via} + + def validate_name(name) do + {:error, + "expected :name to be an atom or a {:via, module, term} tuple, got: #{inspect(name)}"} + end + + @typedoc """ + The query payload. + + This can be a standard SQL string or SQL appended with data (`[sql, ?\n, rowbinary]`). + If providing compressed payloads, don't forget to pass the appropriate `content-encoding` header. + """ + @type query_statement :: iodata @typedoc """ - Options shared by both connection startup and query execution. + Named query parameters. + + Keys are parameter names without the ClickHouse `param_` prefix. Values are + encoded in ClickHouse's escaped parameter format and passed in the URL query + string. - * `:database` - Database, defaults to `"default"` - * `:username` - Username - * `:password` - User password - * `:settings` - Keyword list of ClickHouse settings - * `:timeout` - HTTP request/receive timeout in milliseconds + Ch.query(pool, "SELECT {name:String}", %{"name" => "Ada"}) """ - @type common_option :: - {:database, String.t()} - | {:username, String.t()} - | {:password, String.t()} - | {:settings, Keyword.t()} - | {:timeout, timeout} + @type query_params :: %{String.t() => term} @typedoc """ - Options for starting the connection pool. + Query execution options. + + * `:timeout` - Request timeout, defaults to 30 seconds. + * `:settings` - An enumerable (usually a map or a keyword list) added to the URL query string. + * `:headers` - Headers passed directly to Mint. + """ + @type query_option :: + {:timeout, timeout} + | {:settings, Enumerable.t()} + | {:headers, Mint.Types.headers()} - Includes all keys from `t:common_option/0` and `t:DBConnection.start_option/0` plus: + @typedoc """ + The parsed query response. - * `:scheme` - HTTP scheme, defaults to `"http"` - * `:hostname` - server hostname, defaults to `"localhost"` - * `:port` - HTTP port, defaults to `8123` - * `:transport_opts` - options to be given to the transport being used. See `Mint.HTTP1.connect/4` for more info + If the response format is `RowBinaryWithNamesAndTypes`, `Ch` returns decoded + column names and rows in `Ch.Result`. Other successful formats keep the raw + response body in `Ch.Result.data`. """ - @type start_option :: - common_option - | {:scheme, String.t()} - | {:hostname, String.t()} - | {:port, :inet.port_number()} - | {:transport_opts, [:gen_tcp.connect_option() | :ssl.tls_client_option()]} - | DBConnection.start_option() + @type query_result :: Ch.Result.t() + + @typedoc """ + A query execution error. + + Returns `Ch.Error` for ClickHouse errors or Mint errors for network/HTTP failures. + """ + @type query_error :: Ch.Error.t() | Mint.Types.error() + + @typedoc """ + The options supported by `start_link/1`. + """ + @type start_option :: unquote(NimbleOptions.option_typespec(@start_options_schema)) @doc """ - Start the connection pool process. + Starts a new Ch pool process. - See `t:start_option/0` for available options. + Supported options: + #{NimbleOptions.docs(@start_options_schema)} """ @spec start_link([start_option]) :: GenServer.on_start() - def start_link(opts \\ []) do - DBConnection.start_link(Connection, opts) + def start_link(options \\ []) do + options = NimbleOptions.validate!(options, @start_options_schema) + + name = Keyword.get(options, :name) + pool_size = Keyword.fetch!(options, :pool_size) + worker_idle_timeout = Keyword.fetch!(options, :worker_idle_timeout) + url = Keyword.fetch!(options, :url) + + %URI{scheme: scheme, host: host, port: port} = URI.parse(url) + + scheme = + case scheme do + "http" -> :http + "https" -> :https + _other -> raise ArgumentError, "unexpected HTTP scheme: #{inspect(scheme)}" + end + + initial_pool_state = %{ + template: {:template, scheme, host, port} + } + + NimblePool.start_link( + worker: {__MODULE__, initial_pool_state}, + pool_size: pool_size, + worker_idle_timeout: worker_idle_timeout, + lazy: true, + name: name + ) end @doc """ - Returns a supervisor child specification for a connection pool. + Returns a child spec to allow Ch pool to be started under a supervisor. + + ## Options - See `t:start_option/0` for supported options. + The options are exactly the same as for `start_link/1`. """ - @spec child_spec([start_option]) :: :supervisor.child_spec() - def child_spec(opts) do - DBConnection.child_spec(Connection, opts) + @spec child_spec([start_option]) :: Supervisor.child_spec() + def child_spec(options) do + %{id: __MODULE__, start: {__MODULE__, :start_link, [options]}} end - @typedoc """ - Options for executing a query. - - Includes all keys from `t:common_option/0` and `t:DBConnection.connection_option/0` plus: + @doc """ + Stops the given `pool`. - * `:command` - Command tag for the query - * `:headers` - Custom HTTP headers for the request - * `:format` - Custom response format for the request - * `:decode` - Whether to automatically decode the response - * `:multipart` - Whether to send the query as multipart/form-data + The pool exits with the given `reason`. The pool has `timeout` milliseconds to stop + before it's unilaterally killed by the runtime. """ - @type query_option :: - common_option - | {:command, Ch.Query.command()} - | {:headers, [{String.t(), String.t()}]} - | {:format, String.t()} - | {:types, [String.t() | atom | tuple]} - # TODO remove - | {:encode, boolean} - | {:decode, boolean} - | {:multipart, boolean} - | DBConnection.connection_option() + @spec stop(NimblePool.pool(), reason :: term, timeout) :: :ok + def stop(pool, reason \\ :normal, timeout \\ :infinity) do + NimblePool.stop(pool, reason, timeout) + end @doc """ - Runs a query and returns the result as `{:ok, %Ch.Result{}}` or - `{:error, Exception.t()}` if there was a database error. + Executes a ClickHouse query. + + `statement` is usually a SQL string. It can also be iodata, which is useful + for `INSERT ... FORMAT RowBinary` requests: + + rowbinary = Ch.RowBinary.encode_rows([[1, "Ada"]], ["UInt8", "String"]) + Ch.query!(pool, ["INSERT INTO users FORMAT RowBinary\n", rowbinary]) + + `params` are named ClickHouse query parameters used by placeholders such as + `{limit:UInt8}`. + + Options: + + * `:timeout` - request timeout, defaults to 30 seconds. + * `:settings` - ClickHouse settings added to the URL query string. + * `:headers` - HTTP headers sent with the request. + + By default, `Ch` adds `x-clickhouse-format: RowBinaryWithNamesAndTypes`, + decodes that response format, and returns `%Ch.Result{names: names, rows: rows}`. + Passing a different `x-clickhouse-format` header disables automatic row + decoding and keeps the response body in `%Ch.Result{data: data}`. - See `t:query_option/0` for available options. + If an error response is compressed with `gzip` or `zstd`, `Ch` decompresses it + before returning `%Ch.Error{}`. """ - @spec query(DBConnection.conn(), iodata, params, [query_option]) :: - {:ok, Result.t()} | {:error, Exception.t()} - when params: map | [term] | [row :: [term]] | iodata | Enumerable.t() - def query(conn, statement, params \\ [], opts \\ []) do - query = Query.build(statement, opts) - - with {:ok, _query, result} <- DBConnection.execute(conn, query, params, opts) do - {:ok, result} + @spec query(NimblePool.pool(), query_statement, query_params, [query_option]) :: + {:ok, query_result} | {:error, query_error} + def query(pool, statement, params \\ %{}, options \\ []) do + timeout = Keyword.get(options, :timeout, @query_timeout) + settings = Keyword.get(options, :settings, []) + + headers = + options + |> Keyword.get(:headers, []) + |> put_new_header("user-agent", @user_agent) + |> put_new_header("x-clickhouse-format", "RowBinaryWithNamesAndTypes") + + deadline = Ch.HTTP.to_deadline(timeout) + path = Ch.HTTP.path(params, settings) + + result = + NimblePool.checkout!( + pool, + :request, + fn {pid, _ref}, conn_or_template -> + with {:ok, conn} <- connect(conn_or_template, pid, deadline), + {:ok, conn, status, headers, data} <- + request(conn, "POST", path, headers, statement, deadline) do + {{:ok, status, headers, data}, checkin(conn)} + else + {:error, reason} = error -> {error, {:remove, reason}} + end + end, + timeout + ) + + with {:ok, status, headers, data} <- result do + decode_query_response(status, headers, data) end end @doc """ - Runs a query and returns the result or raises `Ch.Error` if - there was an error. See `query/4`. + Executes a query on the given pool, raising on error. + + Returns the `query_result` directly. Raises an exception if the query fails. """ - @spec query!(DBConnection.conn(), iodata, params, [query_option]) :: Result.t() - when params: map | [term] | [row :: [term]] | iodata | Enumerable.t() - def query!(conn, statement, params \\ [], opts \\ []) do - query = Query.build(statement, opts) - DBConnection.execute!(conn, query, params, opts) + @spec query!(NimblePool.pool(), query_statement, query_params, [query_option]) :: query_result + def query!(pool, statement, params \\ %{}, options \\ []) do + case query(pool, statement, params, options) do + {:ok, result} -> result + {:error, error} -> raise error + end end - @doc false - @spec stream(DBConnection.t(), iodata, map | [term], [query_option]) :: Ch.Stream.t() - def stream(conn, statement, params \\ [], opts \\ []) do - query = Query.build(statement, opts) - %Ch.Stream{conn: conn, query: query, params: params, opts: opts} + @impl NimblePool + def init_pool(config) do + {:ok, config} end - # TODO drop - @doc false - @spec run(DBConnection.conn(), (DBConnection.t() -> any), Keyword.t()) :: any - def run(conn, f, opts \\ []) when is_function(f, 1) do - DBConnection.run(conn, f, opts) + @impl NimblePool + def init_worker(config) do + {:ok, :template, config} + end + + @impl NimblePool + def handle_checkout(:request, _from, :template = template, config) do + {:ok, config.template, template, config} + end + + def handle_checkout(:request, _from, %Mint.HTTP1{} = conn, config) do + {:ok, {:ok, conn}, conn, config} + end + + @impl NimblePool + def handle_checkin({:ok, conn}, _from, _prev, config) do + {:ok, conn, config} + end + + def handle_checkin({:remove, reason}, _from, _prev, config) do + {:remove, reason, config} + end + + @impl NimblePool + def handle_ping(_conn, _config) do + {:remove, :worker_idle_timeout} + end + + @impl NimblePool + def terminate_worker(_reason, conn_or_template, config) do + case conn_or_template do + :template -> :ok + conn -> Mint.HTTP1.close(conn) + end + + {:ok, config} + end + + defp connect({:template, scheme, host, port}, owner, deadline) do + timeout = Ch.HTTP.to_timeout(deadline) + + case Mint.HTTP1.connect(scheme, host, port, mode: :passive, timeout: timeout) do + {:ok, conn} -> + case Mint.HTTP1.controlling_process(conn, owner) do + {:ok, _conn} = ok -> + ok + + {:error, _reason} = error -> + Mint.HTTP1.close(conn) + error + end + + {:error, _reason} = error -> + error + end + end + + defp connect({:ok, _conn} = ok, _owner, _deadline), do: ok + + defp request(conn, method, path, headers, body, deadline) do + result = + with {:ok, conn, _ref} <- Mint.HTTP1.request(conn, method, path, headers, body) do + recv_all(conn, nil, [], nil, deadline) + end + + with {:error, conn, reason} <- result do + Mint.HTTP1.close(conn) + {:error, reason} + end + end + + defp recv_all(conn, status, headers, data, deadline) do + timeout = Ch.HTTP.to_timeout(deadline) + + case Mint.HTTP1.recv(conn, 0, timeout) do + {:ok, conn, responses} -> + case handle_responses(responses, status, headers, data) do + {:ok, status, headers, data} -> {:ok, conn, status, headers, data} + {:more, status, headers, data} -> recv_all(conn, status, headers, data, deadline) + {:error, reason} -> {:error, conn, reason} + end + + {:error, conn, reason, _responses} -> + {:error, conn, reason} + end + end + + defp handle_responses([{:status, _ref, status} | rest], _prev_status = nil, headers, data) do + handle_responses(rest, status, headers, data) + end + + defp handle_responses([{:headers, _ref, new_headers} | rest], status, prev_headers, data) do + handle_responses(rest, status, prev_headers ++ new_headers, data) + end + + defp handle_responses([{:data, _ref, new_data} | rest], status, headers, prev_data) do + next_data = + case prev_data do + nil -> new_data + _ -> [prev_data | new_data] + end + + handle_responses(rest, status, headers, next_data) + end + + defp handle_responses([{:done, _ref}], status, headers, data) do + {:ok, status, headers, data} + end + + defp handle_responses([{:error, _ref, reason} | _rest], _status, _headers, _data) do + {:error, reason} + end + + defp handle_responses([], status, headers, data) do + {:more, status, headers, data} + end + + defp checkin(conn) do + if Mint.HTTP1.open?(conn) do + {:ok, conn} + else + {:remove, Mint.TransportError.exception(reason: :closed)} + end + end + + defp maybe_decompress(data, headers) do + case get_header(headers, "content-encoding") do + "zstd" when data != nil -> data |> IO.iodata_to_binary() |> :zstd.decompress() + "gzip" when data != nil -> data |> IO.iodata_to_binary() |> :zlib.gunzip() + nil -> data + _ when data == nil -> data + other -> raise "unsupported content encoding: #{inspect(other)}" + end + end + + defp decode_query_response(200, headers, body) do + format = get_header(headers, "x-clickhouse-format") + + if format == "RowBinaryWithNamesAndTypes" do + case body |> maybe_decompress(headers) |> response_body_to_binary() do + "" -> + {:ok, %Ch.Result{headers: headers, data: body}} + + decoded_data -> + [names | rows] = Ch.RowBinary.decode_names_and_rows(decoded_data) + + {:ok, + %Ch.Result{ + names: names, + rows: rows, + headers: headers, + data: body + }} + end + else + {:ok, %Ch.Result{headers: headers, data: body}} + end + end + + defp decode_query_response(_status, headers, body) do + message = + body + |> maybe_decompress(headers) + |> response_body_to_binary() + + code = + if code = get_header(headers, "x-clickhouse-error-code") do + String.to_integer(code) + end + + {:error, %Ch.Error{code: code, message: message}} + end + + defp response_body_to_binary(nil), do: "" + defp response_body_to_binary(body), do: IO.iodata_to_binary(body) + + @compile inline: [get_header: 2] + defp get_header(headers, name) do + with {_, value} <- List.keyfind(headers, name, 0, nil), do: value + end + + @compile inline: [put_new_header: 3] + defp put_new_header(headers, name, value) do + if List.keymember?(headers, name, 0) do + headers + else + [{name, value} | headers] + end end if Code.ensure_loaded?(Ecto.ParameterizedType) do diff --git a/lib/ch/connection.ex b/lib/ch/connection.ex deleted file mode 100644 index b53394ab..00000000 --- a/lib/ch/connection.ex +++ /dev/null @@ -1,527 +0,0 @@ -defmodule Ch.Connection do - @moduledoc false - use DBConnection - require Logger - alias Ch.{Error, Query, Result, RowBinary} - alias Mint.HTTP1, as: HTTP - - @user_agent "ch/" <> Mix.Project.config()[:version] - - @typep conn :: HTTP.t() - - @impl true - @spec connect([Ch.start_option()]) :: {:ok, conn} | {:error, Error.t() | Mint.Types.error()} - def connect(opts) do - scheme = String.to_existing_atom(opts[:scheme] || "http") - address = opts[:hostname] || "localhost" - port = opts[:port] || 8123 - mint_opts = [mode: :passive] ++ Keyword.take(opts, [:hostname, :transport_opts]) - - with {:ok, conn} <- HTTP.connect(scheme, address, port, mint_opts) do - conn = - conn - |> HTTP.put_private(:timeout, opts[:timeout] || :timer.seconds(15)) - |> maybe_put_private(:database, opts[:database]) - |> maybe_put_private(:username, opts[:username]) - |> maybe_put_private(:password, opts[:password]) - |> maybe_put_private(:settings, opts[:settings]) - - handshake = Query.build("select 1, version()") - params = DBConnection.Query.encode(handshake, _params = [], _opts = []) - - case handle_execute(handshake, params, _opts = [], conn) do - {:ok, handshake, responses, conn} -> - case DBConnection.Query.decode(handshake, responses, _opts = []) do - %Result{rows: [[1, version]]} -> - conn = - if parse_version(version) >= parse_version("24.10") do - settings = - HTTP.get_private(conn, :settings, []) - |> Keyword.put_new(:input_format_binary_read_json_as_string, 1) - |> Keyword.put_new(:output_format_binary_write_json_as_string, 1) - - HTTP.put_private(conn, :settings, settings) - else - conn - end - - {:ok, conn} - - result -> - {:ok, _conn} = HTTP.close(conn) - reason = Error.exception("unexpected result for '#{handshake}': #{inspect(result)}") - {:error, reason} - end - - {:error, reason, conn} -> - {:ok, _conn} = HTTP.close(conn) - {:error, reason} - - {disconnect, reason, conn} when disconnect in [:disconnect, :disconnect_and_retry] -> - {:ok, _conn} = HTTP.close(conn) - {:error, reason} - end - end - catch - _kind, reason -> {:error, reason} - end - - defp parse_version(version) do - version - |> String.split(".") - |> Enum.flat_map(fn segment -> - case Integer.parse(segment) do - {int, _rest} -> [int] - :error -> [] - end - end) - end - - @impl true - @spec ping(conn) :: {:ok, conn} | {:disconnect, Mint.Types.error() | Error.t(), conn} - def ping(conn) do - headers = [{"user-agent", @user_agent}] - - case request(conn, "GET", "/ping", headers, _body = "", _opts = []) do - {:ok, conn, _response} -> {:ok, conn} - {:error, error, conn} -> {:disconnect, error, conn} - {:disconnect, _error, _conn} = disconnect -> disconnect - end - end - - @impl true - @spec checkout(conn) :: {:ok, conn} - def checkout(conn), do: {:ok, conn} - - # we "support" these four tx callbacks for Repo.checkout - # even though ClickHouse doesn't support txs - - @impl true - def handle_begin(_opts, conn), do: {:ok, %{}, conn} - @impl true - def handle_commit(_opts, conn), do: {:ok, %{}, conn} - @impl true - def handle_rollback(_opts, conn), do: {:ok, %{}, conn} - @impl true - def handle_status(_opts, conn), do: {:idle, conn} - - @impl true - def handle_prepare(_query, _opts, conn) do - {:error, Error.exception("prepared statements are not supported"), conn} - end - - @impl true - def handle_close(_query, _opts, conn) do - {:error, Error.exception("prepared statements are not supported"), conn} - end - - @impl true - def handle_declare(query, params, opts, conn) do - %Query{command: command, decode: decode} = query - {query_params, extra_headers, body} = params - - path = path(conn, query_params, opts) - headers = headers(conn, extra_headers, opts) - timeout = timeout(conn, opts) - - with {:ok, conn, _ref} <- send_request(conn, "POST", path, headers, body), - {:ok, conn, columns, headers, reader} <- recv_declare(conn, decode, timeout) do - result = %Result{ - command: command, - columns: columns, - rows: [], - num_rows: 0, - headers: headers, - data: [] - } - - {:ok, query, result, {conn, reader}} - else - {:error, _reason, _conn} = client_error -> client_error - {:disconnect, reason, conn} -> {:disconnect_and_retry, reason, conn} - end - end - - defp recv_declare(conn, decode, timeout) do - acc = %{decode: decode, step: :status, buffer: [], headers: []} - recv_declare_continue(conn, acc, timeout) - end - - defp recv_declare_continue(conn, acc, timeout) do - case HTTP.recv(conn, 0, timeout) do - {:ok, conn, responses} -> - case handle_recv_declare(responses, acc) do - {:ok, columns, headers, reader} -> - {:ok, conn, columns, headers, reader} - - {:more, acc} -> - recv_declare_continue(conn, acc, timeout) - - :error -> - all_responses_result = - case handle_all_responses(responses, []) do - {:ok, responses} -> {:ok, conn, responses} - {:more, acc} -> recv_all(conn, acc, timeout) - end - - with {:ok, conn, responses} <- all_responses_result do - [_status, headers | data] = responses - message = IO.iodata_to_binary(data) - - code = - if code = get_header(headers, "x-clickhouse-exception-code") do - String.to_integer(code) - end - - {:error, Error.exception(code: code, message: message), conn} - end - end - - {:error, conn, error, _responses} -> - {:disconnect, error, conn} - end - end - - defp handle_recv_declare([{:status, _ref, status} | responses], %{step: :status} = acc) do - case status do - 200 -> handle_recv_declare(responses, %{acc | step: :headers}) - _other -> :error - end - end - - defp handle_recv_declare([{:headers, _ref, headers} | responses], %{step: :headers} = acc) do - with %{decode: true} <- acc, - "RowBinaryWithNamesAndTypes" <- get_header(headers, "x-clickhouse-format") do - handle_recv_declare(responses, %{acc | headers: headers, step: :columns}) - else - _ -> - reader = %{decode: false, responses: responses} - {:ok, _columns = nil, headers, reader} - end - end - - defp handle_recv_declare([{:data, _ref, data} | responses], %{step: :columns} = acc) do - buffer = maybe_concat_buffer(acc.buffer, data) - - case RowBinary.decode_header(buffer) do - {:ok, names, types, buffer} -> - reader = %{buffer: buffer, types: types, state: nil, responses: responses} - {:ok, names, acc.headers, reader} - - :more -> - handle_recv_declare(responses, %{acc | buffer: buffer}) - end - end - - defp handle_recv_declare([], acc), do: {:more, acc} - - @compile inline: [maybe_concat_buffer: 2] - defp maybe_concat_buffer("", data), do: data - defp maybe_concat_buffer(buffer, data) when is_binary(buffer), do: buffer <> data - defp maybe_concat_buffer([], data), do: data - - @impl true - def handle_fetch(query, %Result{} = result, opts, {conn, reader}) do - case reader do - %{responses: []} -> - handle_fetch_recv(query, result, opts, conn, reader) - - %{decode: false, responses: responses} -> - case responses do - [{:data, _ref, data} | responses] -> - result = %Result{result | data: data} - reader = %{reader | responses: responses} - {:cont, result, {conn, reader}} - - [{:done, _ref}] -> - reader = %{reader | responses: []} - {:halt, result, {conn, reader}} - end - - %{buffer: buffer, types: types, state: state, responses: responses} -> - case responses do - [{:data, _ref, data} | responses] -> - buffer = maybe_concat_buffer(buffer, data) - {rows, buffer, state} = RowBinary.decode_rows_continue(buffer, types, state) - result = %Result{result | data: data, rows: rows, num_rows: length(rows)} - reader = %{reader | buffer: buffer, state: state, responses: responses} - {:cont, result, {conn, reader}} - - [{:done, _ref}] -> - reader = %{reader | responses: []} - {:halt, result, {conn, reader}} - end - end - end - - defp handle_fetch_recv(query, result, opts, conn, reader) do - timeout = timeout(conn, opts) - - case HTTP.recv(conn, 0, timeout) do - {:ok, conn, responses} -> - reader = %{reader | responses: responses} - handle_fetch(query, result, opts, {conn, reader}) - - {:error, conn, reason, _responses} -> - {:disconnect, reason, conn} - end - end - - @impl true - def handle_deallocate(_query, %Result{} = result, _opts, {conn, _reader}) do - case HTTP.open_request_count(conn) do - 0 -> - {:ok, %{result | data: []}, conn} - - 1 -> - error = - Error.exception("stopping stream before receiving full response by closing connection") - - {:disconnect, error, conn} - end - end - - @impl true - def handle_execute(%Query{} = query, {:stream, params}, opts, conn) do - {query_params, extra_headers, body} = params - - path = path(conn, query_params, opts) - headers = headers(conn, extra_headers, opts) - - with {:ok, conn, ref} <- send_request(conn, "POST", path, headers, :stream) do - case HTTP.stream_request_body(conn, ref, body) do - {:ok, conn} -> {:ok, query, ref, conn} - {:error, conn, reason} -> {:disconnect_and_retry, reason, conn} - end - end - end - - def handle_execute(%Query{} = query, {:stream, ref, body}, opts, conn) do - case HTTP.stream_request_body(conn, ref, body) do - {:ok, conn} -> - case body do - :eof -> - with {:ok, conn, responses} <- receive_full_response(conn, timeout(conn, opts)) do - {:ok, query, responses, conn} - end - - _other -> - {:ok, query, ref, conn} - end - - {:error, conn, reason} -> - {:disconnect_and_retry, reason, conn} - end - end - - def handle_execute(%Query{command: :insert} = query, params, opts, conn) do - {query_params, extra_headers, body} = params - - path = path(conn, query_params, opts) - headers = headers(conn, extra_headers, opts) - - result = - if is_function(body, 2) do - request_chunked(conn, "POST", path, headers, body, opts) - else - request(conn, "POST", path, headers, body, opts) - end - - case result do - {:ok, conn, responses} -> {:ok, query, responses, conn} - {:error, _reason, _conn} = client_error -> client_error - {:disconnect, reason, conn} -> {:disconnect_and_retry, reason, conn} - end - end - - def handle_execute(query, params, opts, conn) do - {query_params, extra_headers, body} = params - - path = path(conn, query_params, opts) - headers = headers(conn, extra_headers, opts) - - case request(conn, "POST", path, headers, body, opts) do - {:ok, conn, responses} -> {:ok, query, responses, conn} - {:error, _reason, _conn} = client_error -> client_error - {:disconnect, reason, conn} -> {:disconnect_and_retry, reason, conn} - end - end - - @impl true - def disconnect(error, {conn, _reader}) do - disconnect(error, conn) - end - - def disconnect(_error, conn) do - {:ok = ok, _conn} = HTTP.close(conn) - ok - end - - @typep response :: Mint.Types.status() | Mint.Types.headers() | binary - - @spec request(conn, binary, binary, Mint.Types.headers(), iodata, [Ch.query_option()]) :: - {:ok, conn, [response]} - | {:error, Error.t(), conn} - | {:disconnect, Mint.Types.error(), conn} - defp request(conn, method, path, headers, body, opts) do - with {:ok, conn, _ref} <- send_request(conn, method, path, headers, body) do - receive_full_response(conn, timeout(conn, opts)) - end - end - - @spec request_chunked(conn, binary, binary, Mint.Types.headers(), Enumerable.t(), Keyword.t()) :: - {:ok, conn, [response]} - | {:error, Error.t(), conn} - | {:disconnect, Mint.Types.error(), conn} - def request_chunked(conn, method, path, headers, stream, opts) do - with {:ok, conn, ref} <- send_request(conn, method, path, headers, :stream), - {:ok, conn} <- stream_body(conn, ref, stream), - do: receive_full_response(conn, timeout(conn, opts)) - end - - @spec stream_body(conn, Mint.Types.request_ref(), Enumerable.t()) :: - {:ok, conn} | {:disconnect, Mint.Types.error(), conn} - defp stream_body(conn, ref, stream) do - result = - stream - |> Stream.concat([:eof]) - |> Enum.reduce_while({:ok, conn}, fn - chunk, {:ok, conn} -> {:cont, HTTP.stream_request_body(conn, ref, chunk)} - _chunk, {:error, _conn, _reason} = error -> {:halt, error} - end) - - case result do - {:ok, _conn} = ok -> ok - {:error, conn, reason} -> {:disconnect, reason, conn} - end - end - - # stacktrace is a bit cleaner with this function inlined - @compile inline: [send_request: 5] - defp send_request(conn, method, path, headers, body) do - case HTTP.request(conn, method, path, headers, body) do - {:ok, _conn, _ref} = ok -> ok - {:error, conn, reason} -> {:disconnect, reason, conn} - end - end - - @spec receive_full_response(conn, timeout) :: - {:ok, conn, [response]} - | {:error, Error.t(), conn} - | {:disconnect, Mint.Types.error(), conn} - defp receive_full_response(conn, timeout) do - with {:ok, conn, responses} <- recv_all(conn, [], timeout) do - case responses do - [200, headers | _rest] -> - conn = ensure_same_server(conn, headers) - {:ok, conn, responses} - - [_status, headers | data] -> - message = IO.iodata_to_binary(data) - - code = - if code = get_header(headers, "x-clickhouse-exception-code") do - String.to_integer(code) - end - - {:error, Error.exception(code: code, message: message), conn} - end - end - end - - @spec recv_all(conn, [response], timeout()) :: - {:ok, conn, [response]} | {:disconnect, Mint.Types.error(), conn} - defp recv_all(conn, acc, timeout) do - case HTTP.recv(conn, 0, timeout) do - {:ok, conn, responses} -> - case handle_all_responses(responses, acc) do - {:ok, responses} -> {:ok, conn, responses} - {:more, acc} -> recv_all(conn, acc, timeout) - end - - {:error, conn, reason, _responses} -> - {:disconnect, reason, conn} - end - end - - for tag <- [:data, :status, :headers] do - defp handle_all_responses([{unquote(tag), _ref, data} | rest], acc) do - handle_all_responses(rest, [data | acc]) - end - end - - defp handle_all_responses([{:done, _ref}], acc), do: {:ok, :lists.reverse(acc)} - defp handle_all_responses([], acc), do: {:more, acc} - - defp maybe_put_private(conn, _k, nil), do: conn - defp maybe_put_private(conn, k, v), do: HTTP.put_private(conn, k, v) - - defp timeout(conn), do: HTTP.get_private(conn, :timeout) - defp timeout(conn, opts), do: Keyword.get(opts, :timeout) || timeout(conn) - - defp settings(conn, opts) do - default_settings = HTTP.get_private(conn, :settings, []) - opts_settings = Keyword.get(opts, :settings, []) - Keyword.merge(default_settings, opts_settings) - end - - defp headers(conn, extra_headers, opts) do - extra_headers - |> maybe_put_new_header("x-clickhouse-user", get_opts_or_private(conn, opts, :username)) - |> maybe_put_new_header("x-clickhouse-key", get_opts_or_private(conn, opts, :password)) - |> maybe_put_new_header("x-clickhouse-database", get_opts_or_private(conn, opts, :database)) - |> maybe_put_new_header("user-agent", @user_agent) - end - - defp get_opts_or_private(conn, opts, key) do - Keyword.get(opts, key) || HTTP.get_private(conn, key) - end - - defp maybe_put_new_header(headers, _name, _no_value = nil), do: headers - - defp maybe_put_new_header(headers, name, value) do - if List.keymember?(headers, name, 0) do - headers - else - [{name, value} | headers] - end - end - - defp get_header(headers, key) do - case List.keyfind(headers, key, 0) do - {_, value} -> value - nil = not_found -> not_found - end - end - - defp path(conn, query_params, opts) do - settings = settings(conn, opts) - "/?" <> URI.encode_query(settings ++ query_params) - end - - @server_display_name_key :server_display_name - - @spec ensure_same_server(conn, Mint.Types.headers()) :: conn - defp ensure_same_server(conn, headers) do - expected_name = HTTP.get_private(conn, @server_display_name_key) - actual_name = get_header(headers, "x-clickhouse-server-display-name") - - cond do - expected_name && actual_name -> - unless actual_name == expected_name do - Logger.warning( - "Server mismatch detected. Expected #{inspect(expected_name)} but got #{inspect(actual_name)}!" <> - " Connection pooling might be unstable." - ) - end - - conn - - actual_name -> - HTTP.put_private(conn, @server_display_name_key, actual_name) - - true -> - conn - end - end -end diff --git a/lib/ch/http.ex b/lib/ch/http.ex new file mode 100644 index 00000000..3311b2ac --- /dev/null +++ b/lib/ch/http.ex @@ -0,0 +1,170 @@ +defmodule Ch.HTTP do + @moduledoc """ + `Mint.HTTP1` helpers for ClickHouse. + """ + + import Kernel, except: [to_timeout: 1] + + @typedoc """ + Represents a deadline for an operation. + + Either `:infinity` or `{:deadline, timestamp}` where `timestamp` is an absolute + time in milliseconds from `System.monotonic_time(:millisecond)`. + """ + @type deadline :: {:deadline, integer} | :infinity + + @doc """ + Converts a relative timeout (milliseconds) to a `t:deadline/0`. + """ + @spec to_deadline(timeout | deadline) :: deadline + def to_deadline(:infinity = inf), do: inf + def to_deadline({:deadline, _timestamp} = deadline), do: deadline + + def to_deadline(timeout) when is_integer(timeout) do + {:deadline, System.monotonic_time(:millisecond) + timeout} + end + + @doc """ + Returns the remaining milliseconds until a `t:deadline/0`. + """ + @spec to_timeout(timeout | deadline) :: timeout + def to_timeout(:infinity = inf), do: inf + def to_timeout(timeout) when is_integer(timeout), do: timeout + + def to_timeout({:deadline, timestamp}) do + max(0, timestamp - System.monotonic_time(:millisecond)) + end + + @doc """ + Builds the request path for a ClickHouse HTTP request. + + ### Examples + + iex> Ch.HTTP.path(%{}) + "/" + + iex> Ch.HTTP.path(%{"city" => "Prague"}) + "/?param_city=Prague" + + iex> Ch.HTTP.path(%{}, output_format_binary_write_json_as_string: true) + "/?output_format_binary_write_json_as_string=true" + + iex> Ch.HTTP.path(%{"city" => "Prague"}, %{"query_id" => "550e8400"}) + "/?param_city=Prague&query_id=550e8400" + + """ + @spec path(Ch.query_params(), Enumerable.t()) :: String.t() + def path(params, options \\ []) do + params = params |> encode_params() |> URI.encode_query() + options = URI.encode_query(options) + + case {params, options} do + {"", ""} -> "/" + {"", options} -> "/?" <> options + {params, ""} -> "/?" <> params + {params, options} -> "/?" <> params <> "&" <> options + end + end + + # Encodes query parameters for ClickHouse HTTP URL binding. + # + # ClickHouse uses an "escaped" parameter format identical to its TSV format escaping + # (see https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters): + # tab (\t), newline (\n), and backslash (\) are backslash-escaped. + defp encode_params(params) do + Enum.map(params, fn {k, v} -> {"param_#{k}", encode_param(v)} end) + end + + defp encode_param(n) when is_integer(n), do: Integer.to_string(n) + defp encode_param(f) when is_float(f), do: Float.to_string(f) + + defp encode_param(b) when is_binary(b) do + escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) + end + + defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) + defp encode_param(nil), do: "\\N" + defp encode_param(%Decimal{} = d), do: decimal_to_string!(d) + defp encode_param(%Date{} = date), do: Date.to_iso8601(date) + defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) + defp encode_param(%Time{} = time), do: Time.to_iso8601(time) + + defp encode_param(%DateTime{microsecond: microsecond} = dt) do + dt = DateTime.shift_zone!(dt, "Etc/UTC") + + case microsecond do + {val, precision} when val > 0 and precision > 0 -> + size = round(:math.pow(10, precision)) + unix = DateTime.to_unix(dt, size) + seconds = div(unix, size) + fractional = rem(unix, size) + + IO.iodata_to_binary([ + Integer.to_string(seconds), + ?., + String.pad_leading(Integer.to_string(fractional), precision, "0") + ]) + + _ -> + dt |> DateTime.to_unix(:second) |> Integer.to_string() + end + end + + defp encode_param(tuple) when is_tuple(tuple) do + IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) + end + + defp encode_param(a) when is_list(a) do + IO.iodata_to_binary([?[, encode_array_params(a), ?]]) + end + + defp encode_param(m) when is_map(m) do + IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) + end + + defp encode_array_params([last]), do: encode_array_param(last) + + defp encode_array_params([s | rest]) do + [encode_array_param(s), ?, | encode_array_params(rest)] + end + + defp encode_array_params([] = empty), do: empty + + defp encode_map_params([last]), do: encode_map_param(last) + + defp encode_map_params([kv | rest]) do + [encode_map_param(kv), ?, | encode_map_params(rest)] + end + + defp encode_map_params([] = empty), do: empty + + defp encode_array_param(s) when is_binary(s) do + [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] + end + + defp encode_array_param(nil), do: "null" + + defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do + [?', encode_param(param), ?'] + end + + defp encode_array_param(v), do: encode_param(v) + + defp encode_map_param({k, v}) do + [encode_array_param(k), ?:, encode_array_param(v)] + end + + defp escape_param([{pattern, replacement} | escapes], param) do + param = String.replace(param, pattern, replacement) + escape_param(escapes, param) + end + + defp escape_param([], param), do: param + + @compile inline: [decimal_to_string!: 1] + defp decimal_to_string!(%Decimal{coef: coef}) when coef in [:NaN, :inf] do + raise ArgumentError, "ClickHouse Decimal values must be finite" + end + + defp decimal_to_string!(d), do: Decimal.to_string(d, :scientific) +end diff --git a/lib/ch/query.ex b/lib/ch/query.ex deleted file mode 100644 index 4fad7e89..00000000 --- a/lib/ch/query.ex +++ /dev/null @@ -1,427 +0,0 @@ -defmodule Ch.Query do - @moduledoc "Query struct wrapping the SQL statement." - defstruct [:statement, :command, :encode, :decode, :multipart] - - @typedoc """ - The Query struct. - - ## Fields - - * `:statement` - The SQL statement to be executed (as `t:iodata/0`). - * `:command` - The detected or enforced SQL command type (e.g., `:select`, `:insert`). - * `:encode` - Whether to encode parameters (defaults to `true`). - * `:decode` - Whether to decode the response (defaults to `true`). - * `:multipart` - Whether to use `multipart/form-data` for the request (defaults to `false`). - """ - @type t :: %__MODULE__{ - statement: iodata, - command: command, - encode: boolean, - decode: boolean, - multipart: boolean - } - - @doc false - @spec build(iodata, [Ch.query_option()]) :: t - def build(statement, opts \\ []) do - command = Keyword.get(opts, :command) || extract_command(statement) - encode = Keyword.get(opts, :encode, true) - decode = Keyword.get(opts, :decode, true) - multipart = Keyword.get(opts, :multipart, false) - - %__MODULE__{ - statement: statement, - command: command, - encode: encode, - decode: decode, - multipart: multipart - } - end - - statements = [ - {"SELECT", :select}, - {"INSERT", :insert}, - {"CREATE", :create}, - {"ALTER", :alter}, - {"DELETE", :delete}, - {"SYSTEM", :system}, - {"SHOW", :show}, - # as of ClickHouse 24.11, WITH is only allowed in SELECT - # https://clickhouse.com/docs/en/sql-reference/statements/select/with/ - {"WITH", :select}, - {"GRANT", :grant}, - {"EXPLAIN", :explain}, - {"REVOKE", :revoke}, - {"UPDATE", :update}, - {"ATTACH", :attach}, - {"CHECK", :check}, - {"DESCRIBE", :describe}, - {"DETACH", :detach}, - {"DROP", :drop}, - {"EXISTS", :exists}, - {"KILL", :kill}, - {"OPTIMIZE", :optimize}, - {"RENAME", :rename}, - {"EXCHANGE", :exchange}, - {"SET", :set}, - {"TRUNCATE", :truncate}, - {"USE", :use}, - {"WATCH", :watch}, - {"MOVE", :move}, - {"UNDROP", :undrop} - ] - - command_union = - statements - |> Enum.map(fn {_, command} -> command end) - |> Enum.reduce(&{:|, [], [&1, &2]}) - - @typedoc """ - Atom representing the type of SQL command. - - Derived automatically from the start of the SQL statement (e.g., `"SELECT ..."` -> `:select`), - or provided explicitly via options. - """ - @type command :: unquote(command_union) - - defp extract_command(statement) - - for {statement, command} <- statements do - defp extract_command(unquote(statement) <> _), do: unquote(command) - defp extract_command(unquote(String.downcase(statement)) <> _), do: unquote(command) - end - - defp extract_command(<>) when whitespace in [?\s, ?\t, ?\n] do - extract_command(rest) - end - - defp extract_command([first_segment | _] = statement) do - extract_command(first_segment) || extract_command(IO.iodata_to_binary(statement)) - end - - defp extract_command(_other), do: nil -end - -defimpl DBConnection.Query, for: Ch.Query do - @dialyzer :no_improper_lists - alias Ch.{Query, Result, RowBinary} - - @spec parse(Query.t(), [Ch.query_option()]) :: Query.t() - def parse(query, _opts), do: query - - @spec describe(Query.t(), [Ch.query_option()]) :: Query.t() - def describe(query, _opts), do: query - - # stream: insert init - @spec encode(Query.t(), {:stream, term}, [Ch.query_option()]) :: - {:stream, {[{String.t(), String.t()}], Mint.Types.headers(), iodata}} - def encode(query, {:stream, params}, opts) do - {:stream, encode(query, params, opts)} - end - - # stream: insert data chunk - @spec encode(Query.t(), {:stream, Mint.Types.request_ref(), iodata | :eof}, [Ch.query_option()]) :: - {:stream, Mint.Types.request_ref(), iodata | :eof} - def encode(_query, {:stream, ref, data}, _opts) do - {:stream, ref, data} - end - - @spec encode(Query.t(), params, [Ch.query_option()]) :: - {query_params, Mint.Types.headers(), body} - when params: map | [term] | [row :: [term]] | iodata | Enumerable.t(), - query_params: [{String.t(), String.t()}], - body: iodata | Enumerable.t() - - def encode(%Query{command: :insert, encode: false, statement: statement}, data, opts) do - body = - case data do - _ when is_list(data) or is_binary(data) -> [statement, ?\n | data] - _ -> Stream.concat([[statement, ?\n]], data) - end - - {_query_params = [], headers(opts), body} - end - - def encode(%Query{command: :insert, statement: statement}, params, opts) do - cond do - names = Keyword.get(opts, :names) -> - types = Keyword.fetch!(opts, :types) - header = RowBinary.encode_names_and_types(names, types) - data = RowBinary.encode_rows(params, types) - {_query_params = [], headers(opts), [statement, ?\n, header | data]} - - format_row_binary?(statement) -> - types = Keyword.fetch!(opts, :types) - data = RowBinary.encode_rows(params, types) - {_query_params = [], headers(opts), [statement, ?\n | data]} - - true -> - {query_params(params), headers(opts), statement} - end - end - - def encode(%Query{multipart: true, statement: statement}, params, opts) do - types = Keyword.get(opts, :types) - default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" - format = Keyword.get(opts, :format) || default_format - - boundary = "ChFormBoundary" <> Base.url_encode64(:crypto.strong_rand_bytes(24)) - content_type = "multipart/form-data; boundary=\"#{boundary}\"" - enc_boundary = "--#{boundary}\r\n" - multipart = multipart_params(params, enc_boundary) - multipart = add_multipart_part(multipart, "query", statement, enc_boundary) - multipart = [multipart | "--#{boundary}--\r\n"] - - {_no_query_params = [], - [{"x-clickhouse-format", format}, {"content-type", content_type} | headers(opts)], multipart} - end - - def encode(%Query{statement: statement}, params, opts) do - types = Keyword.get(opts, :types) - default_format = if types, do: "RowBinary", else: "RowBinaryWithNamesAndTypes" - format = Keyword.get(opts, :format) || default_format - {query_params(params), [{"x-clickhouse-format", format} | headers(opts)], statement} - end - - defp multipart_params(params, boundary) when is_map(params) do - multipart_named_params(Map.to_list(params), boundary, []) - end - - defp multipart_params(params, boundary) when is_list(params) do - multipart_positional_params(params, 0, boundary, []) - end - - defp multipart_named_params([{name, value} | params], boundary, acc) do - acc = - add_multipart_part( - acc, - "param_" <> URI.encode_www_form(name), - encode_param(value), - boundary - ) - - multipart_named_params(params, boundary, acc) - end - - defp multipart_named_params([], _boundary, acc), do: acc - - defp multipart_positional_params([value | params], idx, boundary, acc) do - acc = - add_multipart_part( - acc, - "param_$" <> Integer.to_string(idx), - encode_param(value), - boundary - ) - - multipart_positional_params(params, idx + 1, boundary, acc) - end - - defp multipart_positional_params([], _idx, _boundary, acc), do: acc - - @compile inline: [add_multipart_part: 4] - defp add_multipart_part(multipart, name, value, boundary) do - part = [ - boundary, - "content-disposition: form-data; name=\"", - name, - "\"\r\n\r\n", - value, - "\r\n" - ] - - case multipart do - [] -> part - _ -> [multipart | part] - end - end - - defp format_row_binary?(statement) when is_binary(statement) do - statement |> String.trim_trailing() |> String.ends_with?("RowBinary") - end - - defp format_row_binary?(statement) when is_list(statement) do - statement - |> IO.iodata_to_binary() - |> format_row_binary?() - end - - # stream: select result - @spec decode(Query.t(), result, [Ch.query_option()]) :: result when result: Result.t() - def decode(_query, %Result{} = result, _opts), do: result - # stream: insert result - @spec decode(Query.t(), ref, [Ch.query_option()]) :: ref when ref: Mint.Types.request_ref() - def decode(_query, ref, _opts) when is_reference(ref), do: ref - - @spec decode(Query.t(), [response], [Ch.query_option()]) :: Result.t() - when response: Mint.Types.status() | Mint.Types.headers() | binary - def decode(%Query{command: :insert}, responses, _opts) do - [_status, headers | _data] = responses - - num_rows = - if summary = get_header(headers, "x-clickhouse-summary") do - summary = Jason.decode!(summary) - - if written_rows = Map.get(summary, "written_rows") do - String.to_integer(written_rows) - end - end - - %Result{num_rows: num_rows, rows: nil, command: :insert, headers: headers} - end - - def decode(%Query{decode: false, command: command}, responses, _opts) when is_list(responses) do - # TODO potentially fails on x-progress-headers - [_status, headers | data] = responses - %Result{rows: data, data: data, command: command, headers: headers} - end - - def decode(%Query{command: command}, responses, opts) when is_list(responses) do - # TODO potentially fails on x-progress-headers - [_status, headers | data] = responses - - case get_header(headers, "x-clickhouse-format") do - "RowBinary" -> - types = Keyword.fetch!(opts, :types) - rows = data |> IO.iodata_to_binary() |> RowBinary.decode_rows(types) - %Result{num_rows: length(rows), rows: rows, command: command, headers: headers} - - "RowBinaryWithNamesAndTypes" -> - [names | rows] = data |> IO.iodata_to_binary() |> RowBinary.decode_names_and_rows() - - %Result{ - num_rows: length(rows), - columns: names, - rows: rows, - command: command, - headers: headers - } - - _other -> - %Result{rows: data, data: data, command: command, headers: headers} - end - end - - defp get_header(headers, key) do - case List.keyfind(headers, key, 0) do - {_, value} -> value - nil = not_found -> not_found - end - end - - defp query_params(params) when is_map(params) do - Enum.map(params, fn {k, v} -> {"param_#{k}", encode_param(v)} end) - end - - defp query_params(params) when is_list(params) do - params - |> Enum.with_index() - |> Enum.map(fn {v, idx} -> {"param_$#{idx}", encode_param(v)} end) - end - - defp encode_param(n) when is_integer(n), do: Integer.to_string(n) - defp encode_param(f) when is_float(f), do: Float.to_string(f) - - # TODO possibly speed up - # For more info see - # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - defp encode_param(b) when is_binary(b) do - escape_param([{"\\", "\\\\"}, {"\t", "\\\t"}, {"\n", "\\\n"}], b) - end - - defp encode_param(b) when is_boolean(b), do: Atom.to_string(b) - defp encode_param(nil), do: "\\N" - defp encode_param(%Decimal{} = d), do: decimal_to_string!(d) - defp encode_param(%Date{} = date), do: Date.to_iso8601(date) - defp encode_param(%NaiveDateTime{} = naive), do: NaiveDateTime.to_iso8601(naive) - defp encode_param(%Time{} = time), do: Time.to_iso8601(time) - - defp encode_param(%DateTime{microsecond: microsecond} = dt) do - dt = DateTime.shift_zone!(dt, "Etc/UTC") - - case microsecond do - {val, precision} when val > 0 and precision > 0 -> - size = Integer.pow(10, precision) - unix = DateTime.to_unix(dt, size) - seconds = div(unix, size) - fractional = rem(unix, size) - - IO.iodata_to_binary([ - Integer.to_string(seconds), - ?., - String.pad_leading(Integer.to_string(fractional), precision, "0") - ]) - - _ -> - dt |> DateTime.to_unix(:second) |> Integer.to_string() - end - end - - defp encode_param(tuple) when is_tuple(tuple) do - IO.iodata_to_binary([?(, encode_array_params(Tuple.to_list(tuple)), ?)]) - end - - defp encode_param(a) when is_list(a) do - IO.iodata_to_binary([?[, encode_array_params(a), ?]]) - end - - defp encode_param(m) when is_map(m) do - IO.iodata_to_binary([?{, encode_map_params(Map.to_list(m)), ?}]) - end - - defp encode_array_params([last]), do: encode_array_param(last) - - defp encode_array_params([s | rest]) do - [encode_array_param(s), ?, | encode_array_params(rest)] - end - - defp encode_array_params([] = empty), do: empty - - defp encode_map_params([last]), do: encode_map_param(last) - - defp encode_map_params([kv | rest]) do - [encode_map_param(kv), ?, | encode_map_params(rest)] - end - - defp encode_map_params([] = empty), do: empty - - defp encode_array_param(s) when is_binary(s) do - [?', escape_param([{"'", "''"}, {"\\", "\\\\"}], s), ?'] - end - - defp encode_array_param(nil), do: "null" - - defp encode_array_param(%s{} = param) when s in [Date, NaiveDateTime] do - [?', encode_param(param), ?'] - end - - defp encode_array_param(v), do: encode_param(v) - - defp encode_map_param({k, v}) do - [encode_array_param(k), ?:, encode_array_param(v)] - end - - defp escape_param([{pattern, replacement} | escapes], param) do - param = String.replace(param, pattern, replacement) - escape_param(escapes, param) - end - - defp escape_param([], param), do: param - - @compile inline: [decimal_to_string!: 1] - defp decimal_to_string!(%Decimal{coef: coef}) when coef in [:NaN, :inf] do - raise ArgumentError, "ClickHouse Decimal values must be finite" - end - - defp decimal_to_string!(d), do: Decimal.to_string(d, :scientific) - - @spec headers(Keyword.t()) :: Mint.Types.headers() - defp headers(opts), do: Keyword.get(opts, :headers, []) -end - -defimpl String.Chars, for: Ch.Query do - def to_string(%{statement: statement}) do - IO.iodata_to_binary(statement) - end -end diff --git a/lib/ch/result.ex b/lib/ch/result.ex index 8d4f0868..776f8c71 100644 --- a/lib/ch/result.ex +++ b/lib/ch/result.ex @@ -1,28 +1,31 @@ defmodule Ch.Result do @moduledoc """ - Result struct returned from any successful query. + ClickHouse query result. + + `Ch.query/4` returns this struct for successful responses. """ - defstruct [:command, :num_rows, :columns, :rows, :headers, :data] + defstruct [ + :names, + :rows, + :headers, + :data + ] @typedoc """ - The Result struct. + Query result. ## Fields - * `:command` - An atom of the query command, for example: `:select`, `:insert` - * `:columns` - A list of column names - * `:rows` - A list of lists (each inner list corresponding to a row, each element in the inner list corresponds to a column) - * `:num_rows` - The number of fetched or affected rows - * `:headers` - The HTTP response headers - * `:data` - The raw iodata from the response + * `:names` - Column names returned by ClickHouse, or `nil` when Ch did not decode rows. + * `:rows` - Decoded rows, or `nil` when Ch did not decode rows. + * `:headers` - HTTP response headers. + * `:data` - Raw response body iodata as received from ClickHouse. """ @type t :: %__MODULE__{ - command: Ch.Query.command() | nil, - num_rows: non_neg_integer | nil, - columns: [String.t()] | nil, - rows: [[term]] | iodata | nil, + names: [String.t()] | nil, + rows: [[term]] | nil, headers: Mint.Types.headers(), - data: iodata + data: iodata | nil } end diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index a531be92..2d6d5d41 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -91,7 +91,6 @@ defmodule Ch.RowBinary do defp encoding_type(t) when t in [ :string, - :binary, :json, :dynamic, :boolean, @@ -184,7 +183,7 @@ defmodule Ch.RowBinary do def encode(:varint, i) when is_integer(i) and i < 128, do: i def encode(:varint, i) when is_integer(i), do: encode_varint_cont(i) - def encode(type, str) when type in [:string, :binary] do + def encode(:string, str) do case str do _ when is_binary(str) -> [encode(:varint, byte_size(str)) | str] _ when is_list(str) -> [encode(:varint, IO.iodata_length(str)) | str] @@ -196,7 +195,7 @@ defmodule Ch.RowBinary do # assuming it can be sent as text and not "native" binary JSON # i.e. assumes `settings: [input_format_binary_read_json_as_string: 1]` # TODO - encode(:string, Jason.encode_to_iodata!(json)) + encode(:string, JSON.encode_to_iodata!(json)) end def encode({:fixed_string, size}, str) when byte_size(str) == size do @@ -700,7 +699,6 @@ defmodule Ch.RowBinary do defp decoding_type(t) when t in [ :string, - :binary, :json, :dynamic, :boolean, @@ -821,7 +819,7 @@ defmodule Ch.RowBinary do rows, types ) do - decode_rows(types_rest, bin, [to_utf8(s) | row], rows, types) + decode_rows(types_rest, bin, [s | row], rows, types) end end @@ -829,47 +827,6 @@ defmodule Ch.RowBinary do to_be_continued(rows, bin, [:string | types_rest], row) end - @doc false - def to_utf8(str) do - utf8 = to_utf8(str, 0, 0, str, []) - IO.iodata_to_binary(utf8) - end - - @dialyzer {:no_improper_lists, to_utf8: 5, to_utf8_escape: 5} - - defp to_utf8(<>, from, len, original, acc) do - to_utf8(rest, from, len + utf8_size(valid), original, acc) - end - - defp to_utf8(<<_invalid, rest::bytes>>, from, len, original, acc) do - acc = [acc | binary_part(original, from, len)] - to_utf8_escape(rest, from + len, 1, original, acc) - end - - defp to_utf8(<<>>, from, len, original, acc) do - [acc | binary_part(original, from, len)] - end - - defp to_utf8_escape(<>, from, len, original, acc) do - acc = [acc | "�"] - to_utf8(rest, from + len, utf8_size(valid), original, acc) - end - - defp to_utf8_escape(<<_invalid, rest::bytes>>, from, len, original, acc) do - to_utf8_escape(rest, from, len + 1, original, acc) - end - - defp to_utf8_escape(<<>>, _from, _len, _original, acc) do - [acc | "�"] - end - - # UTF-8 encodes code points in one to four bytes - @compile inline: [utf8_size: 1] - defp utf8_size(codepoint) when codepoint <= 0x7F, do: 1 - defp utf8_size(codepoint) when codepoint <= 0x7FF, do: 2 - defp utf8_size(codepoint) when codepoint <= 0xFFFF, do: 3 - defp utf8_size(codepoint) when codepoint <= 0x10FFFF, do: 4 - @compile inline: [decode_string_json_decode_rows: 5] for {pattern, size} <- varints do @@ -880,7 +837,7 @@ defmodule Ch.RowBinary do rows, types ) do - decode_rows(types_rest, bin, [Jason.decode!(s) | row], rows, types) + decode_rows(types_rest, bin, [JSON.decode!(s) | row], rows, types) end end @@ -888,24 +845,6 @@ defmodule Ch.RowBinary do to_be_continued(rows, bin, [:json | types_rest], row) end - @compile inline: [decode_binary_decode_rows: 5] - - for {pattern, size} <- varints do - defp decode_binary_decode_rows( - <>, - types_rest, - row, - rows, - types - ) do - decode_rows(types_rest, bin, [s | row], rows, types) - end - end - - defp decode_binary_decode_rows(<>, types_rest, row, rows, _types) do - to_be_continued(rows, bin, [:binary | types_rest], row) - end - @compile inline: [decode_array_decode_rows: 6] defp decode_array_decode_rows(<<0, bin::bytes>>, _type, types_rest, row, rows, types) do decode_rows(types_rest, bin, [[] | row], rows, types) @@ -1322,9 +1261,6 @@ defmodule Ch.RowBinary do :string -> decode_string_decode_rows(bin, types_rest, row, rows, types) - :binary -> - decode_binary_decode_rows(bin, types_rest, row, rows, types) - :json -> # assuming it arrives as text and not "native" binary JSON # i.e. assumes `settings: [output_format_binary_write_json_as_string: 1]` @@ -1337,7 +1273,6 @@ defmodule Ch.RowBinary do {:dynamic, dynamic} -> decode_dynamic(bin, dynamic, types_rest, row, rows, types) - # TODO utf8? {:fixed_string, size} -> case bin do <> -> diff --git a/lib/ch/stream.ex b/lib/ch/stream.ex deleted file mode 100644 index 9ec8b5fd..00000000 --- a/lib/ch/stream.ex +++ /dev/null @@ -1,43 +0,0 @@ -defmodule Ch.Stream do - @moduledoc false - - @derive {Inspect, only: []} - defstruct [:conn, :ref, :query, :params, :opts] - - @type t :: %__MODULE__{ - conn: DBConnection.conn(), - ref: Mint.Types.request_ref() | nil, - query: Ch.Query.t(), - params: term, - opts: [Ch.query_option()] - } - - defimpl Enumerable do - def reduce(stream, acc, fun) do - %Ch.Stream{conn: conn, query: query, params: params, opts: opts} = stream - stream = %DBConnection.Stream{conn: conn, query: query, params: params, opts: opts} - DBConnection.reduce(stream, acc, fun) - end - - def member?(_, _), do: {:error, __MODULE__} - def count(_), do: {:error, __MODULE__} - def slice(_), do: {:error, __MODULE__} - end - - defimpl Collectable do - def into(stream) do - %Ch.Stream{conn: conn, query: query, params: params, opts: opts} = stream - ref = DBConnection.execute!(conn, query, {:stream, params}, opts) - {%{stream | ref: ref}, &collect/2} - end - - defp collect(%{conn: conn, query: query, ref: ref} = stream, {:cont, data}) do - ^ref = DBConnection.execute!(conn, query, {:stream, ref, data}) - stream - end - - defp collect(%{conn: conn, query: query, ref: ref}, eof) when eof in [:halt, :done] do - DBConnection.execute!(conn, query, {:stream, ref, :eof}) - end - end -end diff --git a/lib/ch/types.ex b/lib/ch/types.ex index 0c0c2503..fc280321 100644 --- a/lib/ch/types.ex +++ b/lib/ch/types.ex @@ -579,9 +579,9 @@ defmodule Ch.Types do end def encode(:datetime), do: "DateTime" - def encode({:time64, p}), do: ["Time64(", String.Chars.Integer.to_string(p), ?)] + def encode({:time64, p}), do: ["Time64(", Integer.to_string(p), ?)] def encode({:nullable, type}), do: ["Nullable(", encode(type), ?)] - def encode({:fixed_string, n}), do: ["FixedString(", String.Chars.Integer.to_string(n), ?)] + def encode({:fixed_string, n}), do: ["FixedString(", Integer.to_string(n), ?)] def encode({:array, type}), do: ["Array(", encode(type), ?)] def encode({:tuple, types}), do: ["Tuple(", encode_intersperse(types, ", "), ?)] def encode({:variant, types}), do: ["Variant(", encode_intersperse(types, ", "), ?)] @@ -609,13 +609,7 @@ defmodule Ch.Types do end def encode({:decimal, precision, scale}) do - [ - "Decimal(", - String.Chars.Integer.to_string(precision), - ", ", - String.Chars.Integer.to_string(scale), - ?) - ] + ["Decimal(", Integer.to_string(precision), ", ", Integer.to_string(scale), ?)] end def encode({:datetime, timezone}) when is_binary(timezone) do @@ -623,11 +617,11 @@ defmodule Ch.Types do end def encode({:datetime64, precision}) do - ["DateTime64(", String.Chars.Integer.to_string(precision), ?)] + ["DateTime64(", Integer.to_string(precision), ?)] end def encode({:datetime64, precision, timezone}) when is_binary(timezone) do - ["DateTime64(", String.Chars.Integer.to_string(precision), ", '", timezone, "')"] + ["DateTime64(", Integer.to_string(precision), ", '", timezone, "')"] end def encode({:enum8, mapping}) do @@ -653,11 +647,11 @@ defmodule Ch.Types do defp encode_intersperse([] = empty, _separator), do: empty defp encode_mapping([{k, v}]) when is_binary(k) do - [k, "' = ", String.Chars.Integer.to_string(v)] + [k, "' = ", Integer.to_string(v)] end defp encode_mapping([{k, v} | mapping]) when is_binary(k) do - [k, "' = ", String.Chars.Integer.to_string(v), ", '" | encode_mapping(mapping)] + [k, "' = ", Integer.to_string(v), ", '" | encode_mapping(mapping)] end defp encode_mapping([] = empty), do: empty diff --git a/mix.exs b/mix.exs index 14c6495d..c730fe15 100644 --- a/mix.exs +++ b/mix.exs @@ -2,22 +2,24 @@ defmodule Ch.MixProject do use Mix.Project @source_url "https://github.com/plausible/ch" - @version "0.8.3" + @version "0.9.0" + + def version, do: @version def project do [ app: :ch, - version: @version, - elixir: "~> 1.15", + version: version(), + elixir: "~> 1.18", elixirc_paths: elixirc_paths(Mix.env()), deps: deps(), name: "Ch", - description: "HTTP ClickHouse driver for Elixir", + description: "HTTP ClickHouse client for Elixir", docs: docs(), package: package(), source_url: @source_url, dialyzer: [plt_local_path: "plts", plt_core_path: "plts", plt_ignore_apps: [:xmerl]], - test_coverage: [tool: ExCoveralls, ignore_modules: [Ch.Test]] + test_coverage: [tool: ExCoveralls, ignore_modules: [Help]] ] end @@ -42,25 +44,25 @@ defmodule Ch.MixProject do # Specifies which paths to compile per environment. defp elixirc_paths(:test), do: ["lib", "test/support"] - defp elixirc_paths(:bench), do: ["lib", "bench/support"] + defp elixirc_paths(:dev), do: ["lib", "dev/support"] defp elixirc_paths(_env), do: ["lib"] - defp extra_applications(:test), do: [:inets, :tools] + defp extra_applications(:test), do: [:tools] defp extra_applications(:dev), do: [:tools] defp extra_applications(_env), do: [] # Run "mix help deps" to learn about dependencies. defp deps do [ - {:mint, "~> 1.0"}, - {:db_connection, "~> 2.10.1"}, - {:jason, "~> 1.0"}, + {:mint, "~> 1.8"}, + {:nimble_pool, "~> 1.1"}, + {:nimble_options, "~> 1.1"}, {:decimal, "~> 2.0 or ~> 3.0"}, {:ecto, "~> 3.13.0", optional: true}, - {:benchee, "~> 1.0", only: [:bench]}, + {:benchee, "~> 1.0", only: :dev}, {:dialyxir, "~> 1.0", only: [:dev, :test], runtime: false}, - {:ex_doc, ">= 0.0.0", only: :docs}, - {:tz, "~> 0.28.1", only: [:dev, :test, :bench]}, + {:ex_doc, ">= 0.0.0", only: :dev}, + {:tz, "~> 0.28.1", only: [:dev, :test]}, {:excoveralls, "~> 0.18.5", only: :test}, {:stream_data, "~> 1.3", only: :test} ] @@ -71,7 +73,13 @@ defmodule Ch.MixProject do source_url: @source_url, source_ref: "v#{@version}", main: "readme", - extras: ["README.md", "CHANGELOG.md"], + extras: [ + "README.md", + "CHANGELOG.md", + "pages/query.md", + "pages/datetime-timezones.md", + "pages/compression.md" + ], skip_undefined_reference_warnings_on: ["CHANGELOG.md"] ] end diff --git a/mix.lock b/mix.lock index 5eab42dc..77041b3d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,21 +1,22 @@ %{ "benchee": {:hex, :benchee, "1.5.0", "4d812c31d54b0ec0167e91278e7de3f596324a78a096fd3d0bea68bb0c513b10", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.1", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "5b075393aea81b8ae74eadd1c28b1d87e8a63696c649d8293db7c4df3eb67535"}, - "db_connection": {:hex, :db_connection, "2.10.1", "d5465f6bcc125c1b8981c1dbf23c193ca16f446ec0b25832dc174f74f18be510", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "18ed94c6e627b4bf452dbd4df61b69a35a1e768525140bc1917b7a685026a6a3"}, "decimal": {:hex, :decimal, "3.1.0", "9ede268cff827e6f0c4fb1b34747c82630dce5d7b877dfb22ec8f0cb25855fce", [:mix], [], "hexpm", "e8b3efb3bb3a13cb5e4268ffe128569067b1972e9dee013537c71a5b073168f9"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ecto": {:hex, :ecto, "3.13.6", "352135b474f91d1ab99a1b502171d207e9db60421c9e3d0ecab4c7ab96b24d14", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8afa059bc16cd2c94739ec0a11e3e5df69d828125119109bef35f20a21a76af2"}, - "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, - "ex_doc": {:hex, :ex_doc, "0.40.1", "67542e4b6dde74811cfd580e2c0149b78010fd13001fda7cfeb2b2c2ffb1344d", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "bcef0e2d360d93ac19f01a85d58f91752d930c0a30e2681145feea6bd3516e00"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, + "ex_doc": {:hex, :ex_doc, "0.40.2", "f50edec428c4b0a457a167de42414c461122a3585a99515a69d09fff19e5597e", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "4fa426e2beb47854a162e2c488727fdec51cd4692e319b23810c2804cb1a40fe"}, "excoveralls": {:hex, :excoveralls, "0.18.5", "e229d0a65982613332ec30f07940038fe451a2e5b29bce2a5022165f0c9b157e", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "523fe8a15603f86d64852aab2abe8ddbd78e68579c8525ae765facc5eae01562"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, - "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, + "mint": {:hex, :mint, "1.8.0", "b964eaf4416f2dee2ba88968d52239fca5621b0402b9c95f55a08eb9d74803e9", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "f3c572c11355eccf00f22275e9b42463bc17bd28db13be1e28f8e0bb4adbc849"}, + "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, + "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, diff --git a/pages/compression.md b/pages/compression.md new file mode 100644 index 00000000..5554d3d7 --- /dev/null +++ b/pages/compression.md @@ -0,0 +1,100 @@ +# Compression + +ClickHouse supports compressing HTTP response bodies and accepting compressed request bodies. `Ch` keeps compression explicit: callers choose when to ask for compressed data by passing HTTP headers. + +## Response Compression + +Ask ClickHouse to compress a response with `accept-encoding`: + +```elixir +Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 1_000_000", + %{}, + headers: [{"accept-encoding", "zstd"}] +) +``` + +Supported automatic decompression: + +| Response kind | `content-encoding: zstd` | `content-encoding: gzip` | Other encodings | +| --- | --- | --- | --- | +| decoded `RowBinaryWithNamesAndTypes` success | decompressed automatically | decompressed automatically | raises | +| error response | decompressed automatically | decompressed automatically | raises | +| raw successful response | stored as received in `Ch.Result.data` | stored as received in `Ch.Result.data` | stored as received in `Ch.Result.data` | + +By default, `Ch.query/4` requests `RowBinaryWithNamesAndTypes`, decodes it, and returns `%Ch.Result{names: names, rows: rows, headers: headers, data: data}`. If you add `accept-encoding: zstd` or `accept-encoding: gzip`, `Ch` decompresses before decoding. + +If you override the response format, `Ch` returns `%Ch.Result{}` with the successful body as received in `data`: + +```elixir +%Ch.Result{data: csv_gz} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 1_000_000", + %{}, + headers: [ + {"x-clickhouse-format", "CSV"}, + {"accept-encoding", "gzip"} + ] + ) +``` + +In that example, `csv_gz` is still gzip-compressed. This is intentional so callers can write compressed exports directly. + +## Request Compression + +To send a compressed request body, compress the body and set `content-encoding`: + +```elixir +names = ["id", "name"] +types = ["UInt8", "String"] +rows = [[1, "one"], [2, "two"]] + +payload = + :zstd.compress([ + "INSERT INTO users FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) + ]) + +Ch.query!( + pool, + payload, + %{}, + headers: [{"content-encoding", "zstd"}] +) +``` + +ClickHouse decompresses the request body before parsing the SQL and input format. + +The uncompressed form is the same explicit RowBinary insert body: + +```elixir +Ch.query!(pool, [ + "INSERT INTO users FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) +]) +``` + +## Why ZSTD Is Not Default + +`Ch` does not add `accept-encoding: zstd` automatically. + +Compression is useful for large responses, but making it the default would also affect small queries and raw export workflows. Keeping it explicit means: + +- small queries avoid compression overhead; +- raw responses can be stored exactly as ClickHouse sent them; +- compressed CSV/RowBinary exports can be written directly; +- callers choose the tradeoff per query. + +For large decoded query results, prefer: + +```elixir +headers: [{"accept-encoding", "zstd"}] +``` + +## Errors + +Error bodies are always treated as part of Ch's API, not as raw payloads. If ClickHouse returns an error with `content-encoding: zstd` or `content-encoding: gzip`, `Ch` decompresses it before building `%Ch.Error{}`. diff --git a/pages/datetime-timezones.md b/pages/datetime-timezones.md new file mode 100644 index 00000000..60d1039a --- /dev/null +++ b/pages/datetime-timezones.md @@ -0,0 +1,105 @@ +# DateTime and Time Zones + +ClickHouse `DateTime` and `DateTime64` values are stored as Unix timestamps. A time zone affects how a value is parsed from text and how it is displayed, not the stored instant. + +`Ch` has two relevant encoding paths: + +- Query parameters are sent as text in the HTTP URL. +- RowBinary values are sent as binary Unix timestamps. + +## Query Parameters + +For query parameters, `Ch` encodes: + +- `NaiveDateTime` as an ISO8601 datetime without a time zone, for example `2022-12-12T12:00:00`. +- `DateTime` as a Unix timestamp shifted to UTC, for example `1670846400`. + +ClickHouse then interprets the parameter according to the query parameter type. + +| Elixir value | ClickHouse parameter type | Server/session time zone | Meaning | +| --- | --- | --- | --- | +| `~N[2022-12-12 12:00:00]` | `DateTime` | server UTC | parsed as noon UTC | +| `~N[2022-12-12 12:00:00]` | `DateTime` | server `Europe/Berlin` | parsed as noon Berlin time | +| `~N[2022-12-12 12:00:00]` | `DateTime` | `session_timezone: "Asia/Bangkok"` | parsed as noon Bangkok time | +| `~N[2022-12-12 12:00:00]` | `DateTime('UTC')` | any | parsed as noon UTC | +| `~N[2022-12-12 12:00:00]` | `DateTime('Asia/Bangkok')` | any | parsed as noon Bangkok time | +| `~U[2022-12-12 12:00:00Z]` | `DateTime` | any | sent as a UTC Unix timestamp | +| `DateTime` in any zone | `DateTime('Asia/Bangkok')` | any | sent as a UTC Unix timestamp, displayed in Bangkok time | + +The same rules apply to `DateTime64`, except fractional precision is preserved. + +## Session Time Zone + +ClickHouse's `session_timezone` setting controls the implicit time zone for `DateTime` and `DateTime64` types that do not specify one. + +```elixir +Ch.query!( + pool, + "SELECT {dt:DateTime} AS d, toString(d), timeZone()", + %{"dt" => ~N[2022-12-12 12:00:00]}, + settings: [session_timezone: "Asia/Bangkok"] +) +``` + +returns the stored UTC instant for noon in Bangkok, while `toString(d)` displays the session-local value: + +```elixir +%{rows: [[~N[2022-12-12 05:00:00], "2022-12-12 12:00:00", "Asia/Bangkok"]]} +``` + +An explicit ClickHouse type time zone ignores `session_timezone`: + +```elixir +Ch.query!( + pool, + "SELECT {dt:DateTime('Asia/Bangkok')} AS d, toString(d)", + %{"dt" => ~N[2022-12-12 12:00:00]}, + settings: [session_timezone: "UTC"] +) +``` + +returns: + +```elixir +%{rows: [[#DateTime<2022-12-12 12:00:00+07:00 +07 Asia/Bangkok>, "2022-12-12 12:00:00"]]} +``` + +## Decoding Results + +When Ch decodes `RowBinaryWithNamesAndTypes`: + +| ClickHouse result type | Elixir value | +| --- | --- | +| `DateTime` | `NaiveDateTime` in UTC | +| `DateTime64(P)` | `NaiveDateTime` in UTC | +| `DateTime('UTC')` | `DateTime` in UTC | +| `DateTime64(P, 'UTC')` | `DateTime` in UTC | +| `DateTime('Europe/Berlin')` | `DateTime` in `Europe/Berlin` | +| `DateTime64(P, 'Europe/Berlin')` | `DateTime` in `Europe/Berlin` | + +For implicit time zone result types, ClickHouse sends only the stored Unix timestamp and the type name `DateTime` or `DateTime64(P)`. The response does not include the server or session time zone, so Ch decodes those values as UTC `NaiveDateTime`. + +For explicit time zone result types, the time zone is part of the type name, so Ch decodes to `DateTime` in that zone. + +## RowBinary Inserts + +RowBinary does not send text for `DateTime` values. It sends Unix timestamps directly. + +| Elixir value | RowBinary type | Encoding | +| --- | --- | --- | +| `NaiveDateTime` | `DateTime` | treated as a UTC naive value and encoded as Unix seconds | +| `NaiveDateTime` | `DateTime64(P)` | treated as a UTC naive value and encoded as Unix ticks | +| `DateTime` | `DateTime` | encoded as Unix seconds for the instant | +| `DateTime` | `DateTime64(P)` | encoded as Unix ticks for the instant | +| any | `DateTime('UTC')` | same as `DateTime` | +| any | `DateTime64(P, 'UTC')` | same as `DateTime64(P)` | +| any | non-UTC `DateTime(...)` / `DateTime64(...)` | not supported by `Ch.RowBinary.encode_rows/2` | + +Use query parameters for non-UTC textual interpretation, or normalize values to UTC before RowBinary insertion. + +## Practical Guidance + +- Prefer `DateTime` values when the Elixir value represents a real instant. +- Use `NaiveDateTime` only when you intentionally want ClickHouse to interpret the wall time using an implicit or explicit ClickHouse time zone. +- Prefer explicit ClickHouse column types like `DateTime('UTC')` or `DateTime64(6, 'UTC')` for unambiguous schemas. +- Use `session_timezone` in tests when you need deterministic behavior for implicit `DateTime` or `DateTime64` types without changing the ClickHouse server time zone. diff --git a/pages/query.md b/pages/query.md new file mode 100644 index 00000000..d713071c --- /dev/null +++ b/pages/query.md @@ -0,0 +1,131 @@ +# Querying + +`Ch.query/4` sends SQL to ClickHouse over HTTP. + +By default, Ch asks ClickHouse for `RowBinaryWithNamesAndTypes`, decodes the response, and returns `%Ch.Result{}`: + +```elixir +%Ch.Result{ + names: ["number"], + rows: [[42]], + headers: headers, + data: raw_body +} = Ch.query!(pool, "SELECT 42 AS number") +``` + +## Named Parameters + +Query parameters are named. The map keys do not include ClickHouse's `param_` prefix: + +```elixir +Ch.query!( + pool, + "SELECT {value:UInt64}", + %{"value" => 42} +) +``` + +Positional parameters are not supported: + +```elixir +# before +Ch.query!(pool, "SELECT {$0:UInt64}", [42]) + +# now +Ch.query!(pool, "SELECT {value:UInt64}", %{"value" => 42}) +``` + +Use the same naming style for multiple parameters: + +```elixir +Ch.query!( + pool, + "SELECT {name:String}, {age:UInt8}", + %{"name" => "Ada", "age" => 37} +) +``` + +## Raw Formats + +The default response format is decoded. To receive raw CSV, JSON, TSV, or another ClickHouse format, override the `x-clickhouse-format` header: + +```elixir +%Ch.Result{data: csv} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 3", + %{}, + headers: [{"x-clickhouse-format", "CSV"}] + ) +``` + +```elixir +%Ch.Result{data: json_each_row} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 3", + %{}, + headers: [{"x-clickhouse-format", "JSONEachRow"}] + ) +``` + +For raw successful responses, Ch returns `%Ch.Result{}` with the body as received in `data`. It does not decode rows or decompress compressed raw responses. + +```elixir +%Ch.Result{names: nil, rows: nil, data: csv} = + Ch.query!( + pool, + "SELECT number FROM system.numbers LIMIT 3", + %{}, + headers: [{"x-clickhouse-format", "CSV"}] + ) +``` + +## RowBinary Inserts + +RowBinary inserts are explicit. Encode rows with `Ch.RowBinary` and pass the SQL plus encoded data as the request body: + +```elixir +rows = [[1, "one"], [2, "two"]] +types = ["UInt8", "String"] +rowbinary = Ch.RowBinary.encode_rows(rows, types) + +Ch.query!(pool, [ + "INSERT INTO events FORMAT RowBinary\n", + rowbinary +]) +``` + +For `RowBinaryWithNamesAndTypes`, include the encoded names and types header: + +```elixir +names = ["id", "name"] +types = ["UInt8", "String"] +rows = [[1, "one"], [2, "two"]] + +Ch.query!(pool, [ + "INSERT INTO events FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) +]) +``` + +## Compressed Inserts + +ClickHouse accepts compressed request bodies when the `content-encoding` header is set. Compress the entire SQL plus data body: + +```elixir +payload = + :zstd.compress([ + "INSERT INTO events FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) + ]) + +Ch.query!( + pool, + payload, + %{}, + headers: [{"content-encoding", "zstd"}] +) +``` diff --git a/test/ch/aggregation_test.exs b/test/ch/aggregation_test.exs index 651622c1..e1aad349 100644 --- a/test/ch/aggregation_test.exs +++ b/test/ch/aggregation_test.exs @@ -2,12 +2,11 @@ defmodule Ch.AggregationTest do use ExUnit.Case, async: true setup do - conn = start_supervised!({Ch, database: Ch.Test.database()}) - {:ok, conn: conn} + {:ok, pool: start_supervised!(Ch)} end - test "select SimpleAggregateFunction types", %{conn: conn} do - Ch.query!(conn, """ + test "select SimpleAggregateFunction types", %{pool: pool} do + Help.query!(""" CREATE TABLE candle_fragments ( ticker LowCardinality(String), time DateTime('UTC') CODEC(Delta, Default), @@ -19,7 +18,9 @@ defmodule Ch.AggregationTest do ORDER BY (ticker, time) """) - Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop table candle_fragments") end) + + Help.query!(""" CREATE MATERIALIZED VIEW candles_one_hour_amt ( ticker LowCardinality(String), @@ -43,7 +44,9 @@ defmodule Ch.AggregationTest do GROUP BY ticker, time """) - Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop view candles_one_hour_amt") end) + + Ch.query!(pool, """ INSERT INTO candle_fragments(ticker, time, high, open, close, low) VALUES ('INTC', '2023-04-13 20:33:00', 32, 32, 32, 32), ('INTC', '2023-04-13 20:34:00', 33, 33, 33, 33), @@ -51,7 +54,8 @@ defmodule Ch.AggregationTest do ('INTC', '2023-04-13 20:36:00', 32, 27, 27, 27) """) - assert Ch.query!(conn, """ + assert pool + |> Ch.query!(""" SELECT t.ticker AS ticker, toStartOfHour(t.time) AS start_time, @@ -63,23 +67,25 @@ defmodule Ch.AggregationTest do min(t.low) AS low FROM candles_one_hour_amt t GROUP BY ticker, time - """).rows == [ + """) + |> Help.to_maps() == [ - "INTC", - ~U[2023-04-13 20:00:00Z], - ~U[2023-04-13 21:00:00Z], - ~D[2023-04-13], - 33.0, - 32.0, - 27.0, - 26.0 + %{ + "ticker" => "INTC", + "start_time" => ~U[2023-04-13 20:00:00Z], + "end_time" => ~U[2023-04-13 21:00:00Z], + "date" => ~D[2023-04-13], + "high" => 33.0, + "open" => 32.0, + "close" => 27.0, + "low" => 26.0 + } ] - ] end # based on https://github.com/ClickHouse/clickhouse-java/issues/1232 - test "insert AggregateFunction via input()", %{conn: conn} do - Ch.query!(conn, """ + test "insert AggregateFunction via input()", %{pool: pool} do + Help.query!(""" CREATE TABLE test_insert_aggregate_function ( uid Int16, updated SimpleAggregateFunction(max, DateTime), @@ -87,35 +93,47 @@ defmodule Ch.AggregationTest do ) ENGINE AggregatingMergeTree ORDER BY uid """) + on_exit(fn -> Help.query!("drop table test_insert_aggregate_function") end) + rows = [ [1, ~N[2020-01-02 00:00:00], "b"], [1, ~N[2020-01-01 00:00:00], "a"] ] - assert %{num_rows: 2} = - Ch.query!( - conn, - """ - INSERT INTO test_insert_aggregate_function - SELECT uid, updated, arrayReduce('argMaxState', [name], [updated]) - FROM input('uid Int16, updated DateTime, name String') - FORMAT RowBinary\ - """, - rows, - types: ["Int16", "DateTime", "String"] - ) - - assert Ch.query!(conn, """ + rowbinary = Ch.RowBinary.encode_rows(rows, _types = ["Int16", "DateTime", "String"]) + + insert = """ + INSERT INTO test_insert_aggregate_function + SELECT uid, updated, arrayReduce('argMaxState', [name], [updated]) + FROM input('uid Int16, updated DateTime, name String') + FORMAT RowBinary + """ + + Ch.query!(pool, [insert | rowbinary]) + + assert Ch.query!(pool, """ SELECT uid, max(updated) AS updated, argMaxMerge(name) FROM test_insert_aggregate_function GROUP BY uid - """).rows == [[1, ~N[2020-01-02 00:00:00], "b"]] + """).rows == [ + [1, ~N[2020-01-02 00:00:00], "b"] + ] end # https://kb.altinity.com/altinity-kb-schema-design/ingestion-aggregate-function/ describe "altinity examples" do - test "ephemeral column", %{conn: conn} do - Ch.query!(conn, """ + setup do + rows = [ + [1231, ~N[2020-01-02 00:00:00], "Jane"], + [1231, ~N[2020-01-01 00:00:00], "John"] + ] + + rowbinary = Ch.RowBinary.encode_rows(rows, ["Int16", "DateTime", "String"]) + {:ok, rowbinary: rowbinary} + end + + test "ephemeral column", %{pool: pool, rowbinary: rowbinary} do + Help.query!(""" CREATE TABLE test_users_ephemeral_column ( uid Int16, updated SimpleAggregateFunction(max, DateTime), @@ -124,25 +142,24 @@ defmodule Ch.AggregationTest do ) ENGINE AggregatingMergeTree ORDER BY uid """) - Ch.query!( - conn, - "INSERT INTO test_users_ephemeral_column(uid, updated, name_stub) FORMAT RowBinary", - _rows = [ - [1231, ~N[2020-01-02 00:00:00], "Jane"], - [1231, ~N[2020-01-01 00:00:00], "John"] - ], - types: ["Int16", "DateTime", "String"] - ) - - assert Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop table test_users_ephemeral_column") end) + + Ch.query!(pool, [ + "INSERT INTO test_users_ephemeral_column(uid, updated, name_stub) FORMAT RowBinary\n" + | rowbinary + ]) + + assert Ch.query!(pool, """ SELECT uid, max(updated) AS updated, argMaxMerge(name) FROM test_users_ephemeral_column GROUP BY uid - """).rows == [[1231, ~N[2020-01-02 00:00:00], "Jane"]] + """).rows == [ + [1231, ~N[2020-01-02 00:00:00], "Jane"] + ] end - test "input function", %{conn: conn} do - Ch.query!(conn, """ + test "input function", %{pool: pool, rowbinary: rowbinary} do + Help.query!(""" CREATE TABLE test_users_input_function ( uid Int16, updated SimpleAggregateFunction(max, DateTime), @@ -150,29 +167,28 @@ defmodule Ch.AggregationTest do ) ENGINE AggregatingMergeTree ORDER BY uid """) - Ch.query!( - conn, + on_exit(fn -> Help.query!("drop table test_users_input_function") end) + + Ch.query!(pool, [ """ INSERT INTO test_users_input_function SELECT uid, updated, arrayReduce('argMaxState', [name], [updated]) - FROM input('uid Int16, updated DateTime, name String') FORMAT RowBinary\ - """, - _rows = [ - [1231, ~N[2020-01-02 00:00:00], "Jane"], - [1231, ~N[2020-01-01 00:00:00], "John"] - ], - types: ["Int16", "DateTime", "String"] - ) - - assert Ch.query!(conn, """ + FROM input('uid Int16, updated DateTime, name String') FORMAT RowBinary + """ + | rowbinary + ]) + + assert Ch.query!(pool, """ SELECT uid, max(updated) AS updated, argMaxMerge(name) FROM test_users_input_function GROUP BY uid - """).rows == [[1231, ~N[2020-01-02 00:00:00], "Jane"]] + """).rows == [ + [1231, ~N[2020-01-02 00:00:00], "Jane"] + ] end - test "materialized view and null engine", %{conn: conn} do - Ch.query!(conn, """ + test "materialized view and null engine", %{pool: pool, rowbinary: rowbinary} do + Help.query!(""" CREATE TABLE test_users_mv_ne ( uid Int16, updated SimpleAggregateFunction(max, DateTime), @@ -180,7 +196,9 @@ defmodule Ch.AggregationTest do ) ENGINE AggregatingMergeTree ORDER BY uid """) - Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop table test_users_mv_ne") end) + + Help.query!(""" CREATE TABLE test_users_ne ( uid Int16, updated DateTime, @@ -188,27 +206,25 @@ defmodule Ch.AggregationTest do ) ENGINE Null """) - Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop table test_users_ne") end) + + Help.query!(""" CREATE MATERIALIZED VIEW test_users_mv TO test_users_mv_ne AS SELECT uid, updated, arrayReduce('argMaxState', [name], [updated]) name FROM test_users_ne """) - Ch.query!( - conn, - "INSERT INTO test_users_ne FORMAT RowBinary", - _rows = [ - [1231, ~N[2020-01-02 00:00:00], "Jane"], - [1231, ~N[2020-01-01 00:00:00], "John"] - ], - types: ["Int16", "DateTime", "String"] - ) - - assert Ch.query!(conn, """ + on_exit(fn -> Help.query!("drop view test_users_mv") end) + + Ch.query!(pool, ["INSERT INTO test_users_ne FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, """ SELECT uid, max(updated) AS updated, argMaxMerge(name) FROM test_users_mv_ne GROUP BY uid - """).rows == [[1231, ~N[2020-01-02 00:00:00], "Jane"]] + """).rows == [ + [1231, ~N[2020-01-02 00:00:00], "Jane"] + ] end end end diff --git a/test/ch/compression_test.exs b/test/ch/compression_test.exs new file mode 100644 index 00000000..e2c8aecc --- /dev/null +++ b/test/ch/compression_test.exs @@ -0,0 +1,150 @@ +defmodule Ch.CompressionTest do + use ExUnit.Case, async: true + + setup do + {:ok, pool: start_supervised!(Ch)} + end + + test "can request GZIP response through headers", %{pool: pool} do + assert <<0x1F, 0x8B, _rest::bytes>> = + pool + |> Ch.query!( + "select number from system.numbers limit {limit:UInt32}", + %{"limit" => 1_000_000}, + headers: [{"accept-encoding", "gzip"}, {"x-clickhouse-format", "RowBinary"}], + settings: %{"enable_http_compression" => 1} + ) + |> Map.fetch!(:data) + |> IO.iodata_to_binary() + end + + test "can request LZ4 response through headers", %{pool: pool} do + assert <<0x04, 0x22, 0x4D, 0x18, _rest::bytes>> = + pool + |> Ch.query!( + "select number from system.numbers limit {limit:UInt32}", + %{"limit" => 1_000_000}, + headers: [{"accept-encoding", "lz4"}, {"x-clickhouse-format", "RowBinary"}], + settings: %{"enable_http_compression" => 1} + ) + |> Map.fetch!(:data) + |> IO.iodata_to_binary() + end + + test "can request ZSTD response through headers", %{pool: pool} do + assert <<0x28, 0xB5, 0x2F, 0xFD, _rest::bytes>> = + pool + |> Ch.query!( + "select number from system.numbers limit {limit:UInt32}", + %{"limit" => 1_000_000}, + headers: [{"accept-encoding", "zstd"}, {"x-clickhouse-format", "RowBinary"}], + settings: %{"enable_http_compression" => 1} + ) + |> Map.fetch!(:data) + |> IO.iodata_to_binary() + end + + test "automatically decompresses and decodes ZSTD RowBinaryWithNamesAndTypes", %{pool: pool} do + assert %{names: ["number"], rows: rows} = + Ch.query!( + pool, + "select number from system.numbers limit {limit:UInt32}", + %{"limit" => 1_000_000}, + headers: [{"accept-encoding", "zstd"}], + settings: %{"enable_http_compression" => 1} + ) + + assert length(rows) == 1_000_000 + end + + test "automatically decompresses and decodes GZIP RowBinaryWithNamesAndTypes", %{pool: pool} do + assert %{names: ["number"], rows: rows} = + Ch.query!( + pool, + "select number from system.numbers limit {limit:UInt32}", + %{"limit" => 1_000_000}, + headers: [{"accept-encoding", "gzip"}], + settings: %{"enable_http_compression" => 1} + ) + + assert length(rows) == 1_000_000 + end + + test "automatically handles empty ZSTD RowBinaryWithNamesAndTypes responses", %{pool: pool} do + on_exit(fn -> Help.query!("DROP TABLE compression_test_zstd_empty_response") end) + + assert %Ch.Result{names: nil, rows: nil, data: nil, headers: headers} = + Ch.query!( + pool, + "CREATE TABLE compression_test_zstd_empty_response(a UInt8) ENGINE Memory", + %{}, + headers: [{"accept-encoding", "zstd"}], + settings: %{"enable_http_compression" => 1} + ) + + assert is_list(headers) + end + + test "automatically handles empty GZIP RowBinaryWithNamesAndTypes responses", %{pool: pool} do + on_exit(fn -> Help.query!("DROP TABLE compression_test_gzip_empty_response") end) + + assert %Ch.Result{names: nil, rows: nil, data: nil, headers: headers} = + Ch.query!( + pool, + "CREATE TABLE compression_test_gzip_empty_response(a UInt8) ENGINE Memory", + %{}, + headers: [{"accept-encoding", "gzip"}], + settings: %{"enable_http_compression" => 1} + ) + + assert is_list(headers) + end + + test "automatically decompresses ZSTD error responses", %{pool: pool} do + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, "SELECT missing_column", %{}, + headers: [{"accept-encoding", "zstd"}], + settings: %{"enable_http_compression" => 1} + ) + + assert message =~ "UNKNOWN_IDENTIFIER" + refute message =~ <<0x28, 0xB5, 0x2F, 0xFD>> + end + + test "automatically decompresses GZIP error responses", %{pool: pool} do + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, "SELECT missing_column", %{}, + headers: [{"accept-encoding", "gzip"}], + settings: %{"enable_http_compression" => 1} + ) + + assert message =~ "UNKNOWN_IDENTIFIER" + refute message =~ <<0x1F, 0x8B>> + end + + test "can send ZSTD compressed RowBinaryWithNamesAndTypes payloads", %{pool: pool} do + Help.query!("CREATE TABLE compression_test_zstd_payload(id UInt8, name String) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE compression_test_zstd_payload") end) + + names = ["id", "name"] + types = ["UInt8", "String"] + rows = [[1, "one"], [2, "two"]] + + payload = + :zstd.compress([ + "INSERT INTO compression_test_zstd_payload FORMAT RowBinaryWithNamesAndTypes\n", + Ch.RowBinary.encode_names_and_types(names, types), + Ch.RowBinary.encode_rows(rows, types) + ]) + + assert %Ch.Result{names: nil, rows: nil, data: nil} = + Ch.query!(pool, payload, %{}, headers: [{"content-encoding", "zstd"}]) + + assert Ch.query!(pool, "SELECT * FROM compression_test_zstd_payload ORDER BY id").rows == rows + + assert Ch.query!(pool, "SELECT * FROM compression_test_zstd_payload ORDER BY id", %{}, + headers: [{"accept-encoding", "zstd"}], + settings: %{"enable_http_compression" => 1} + ).rows == rows + end +end diff --git a/test/ch/connect_test.exs b/test/ch/connect_test.exs deleted file mode 100644 index d21edcb8..00000000 --- a/test/ch/connect_test.exs +++ /dev/null @@ -1,22 +0,0 @@ -defmodule Ch.ConnectTest do - use ExUnit.Case - import ExUnit.CaptureLog - - @tag :slow - test "retries to connect even with exceptions / exits / throws" do - # See https://github.com/plausible/ch/issues/208 - bad_transport_opts = [sndbuf: nil] - - logs = - capture_log(fn -> - {:ok, conn} = - Ch.start_link(database: Ch.Test.database(), transport_opts: bad_transport_opts) - - :timer.sleep(100) - - assert Process.alive?(conn) - end) - - assert logs =~ "failed to connect: ** (ArgumentError) argument error" - end -end diff --git a/test/ch/connection_property_test.exs b/test/ch/connection_property_test.exs new file mode 100644 index 00000000..5c945e59 --- /dev/null +++ b/test/ch/connection_property_test.exs @@ -0,0 +1,209 @@ +defmodule Ch.ConnectionPropertyTest do + use ExUnit.Case, async: true + use ExUnitProperties + + setup do + {:ok, pool: start_supervised!(Ch)} + end + + describe "query/4" do + test "selects rows and column names", %{pool: pool} do + assert %{names: ["one", "two"], rows: [[1, 2]]} = + Ch.query!(pool, "SELECT 1 AS one, 2 AS two") + end + + test "accepts iodata statements", %{pool: pool} do + assert Ch.query!(pool, ["S", ?E, ["LEC" | "T"], " ", ~c"123"]).rows == [[123]] + end + + test "returns ClickHouse errors", %{pool: pool} do + assert {:error, %Ch.Error{message: message}} = Ch.query(pool, "wat") + assert message =~ "Code: 62" + assert message =~ "SYNTAX_ERROR" + end + + test "reuses the pool after a query error", %{pool: pool} do + assert {:error, %Ch.Error{}} = Ch.query(pool, "SELECT 123 + 'a'") + assert Ch.query!(pool, "SELECT 42").rows == [[42]] + end + + test "runs concurrent queries", %{pool: pool} do + parent = self() + + for _ <- 1..10 do + spawn_link(fn -> send(parent, Ch.query!(pool, "SELECT sleep(0.05)").rows) end) + end + + assert Ch.query!(pool, "SELECT 42").rows == [[42]] + + for _ <- 1..10 do + assert_receive [[0]] + end + end + end + + describe "query params" do + property "scalar params round-trip through ClickHouse", %{pool: pool} do + check all {type, value, expected} <- scalar_param() do + assert Ch.query!(pool, "SELECT {value:#{type}}", %{"value" => value}).rows == [[expected]] + end + end + + property "array params round-trip through ClickHouse", %{pool: pool} do + check all {type, values, expected} <- array_param() do + assert Ch.query!(pool, "SELECT {value:Array(#{type})}", %{"value" => values}).rows == [ + [expected] + ] + end + end + + test "identifier params can address tables", %{pool: pool} do + Help.query!("CREATE TABLE connection_property_identifier_params (a UInt8) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE connection_property_identifier_params") end) + + Ch.query!(pool, "INSERT INTO {table:Identifier} VALUES (1), (2)", %{ + "table" => "connection_property_identifier_params" + }) + + assert Ch.query!(pool, "SELECT sum(a) FROM {table:Identifier}", %{ + "table" => "connection_property_identifier_params" + }).rows == [[3]] + end + end + + describe "RowBinary inserts" do + property "rows encoded as RowBinary can be inserted and selected", %{pool: pool} do + Help.query!(""" + CREATE TABLE connection_property_rowbinary ( + id UInt8, + name String, + active Bool + ) ENGINE Memory + """) + + on_exit(fn -> Help.query!("DROP TABLE connection_property_rowbinary") end) + + check all rows <- rowbinary_rows() do + Ch.query!(pool, "TRUNCATE TABLE connection_property_rowbinary") + + rowbinary = Ch.RowBinary.encode_rows(rows, ["UInt8", "String", "Bool"]) + + Ch.query!(pool, [ + "INSERT INTO connection_property_rowbinary FORMAT RowBinary\n" | rowbinary + ]) + + assert Ch.query!(pool, "SELECT * FROM connection_property_rowbinary ORDER BY id").rows == + Enum.sort_by(rows, &List.first/1) + end + end + + test "supports RowBinaryWithNamesAndTypes payloads", %{pool: pool} do + Help.query!(""" + CREATE TABLE connection_property_rowbinary_names_types ( + country_code FixedString(2), + rare_string LowCardinality(String), + maybe_int32 Nullable(Int32) + ) ENGINE Memory + """) + + on_exit(fn -> + Help.query!("DROP TABLE connection_property_rowbinary_names_types") + end) + + names = ["country_code", "rare_string", "maybe_int32"] + types = ["FixedString(2)", "LowCardinality(String)", "Nullable(Int32)"] + rows = [["AB", "rare", -42], ["CD", "another", nil]] + + rowbinary = [ + Ch.RowBinary.encode_names_and_types(names, types) + | Ch.RowBinary.encode_rows(rows, types) + ] + + Ch.query!(pool, [ + "INSERT INTO connection_property_rowbinary_names_types FORMAT RowBinaryWithNamesAndTypes\n" + | rowbinary + ]) + + assert Ch.query!( + pool, + "SELECT * FROM connection_property_rowbinary_names_types ORDER BY country_code" + ).rows == + rows + end + end + + defp scalar_param do + one_of([ + gen_constant("UInt8", integer(0..255)), + gen_constant("Int16", integer(-32_768..32_767)), + gen_constant("Bool", boolean()), + gen_constant("String", safe_string()), + gen_constant("Date", date_gen()), + gen_constant("Date32", date32_gen()), + gen_constant("Decimal(18, 4)", decimal_gen()) + ]) + end + + defp gen_constant(type, generator) do + gen all value <- generator do + expected = + case type do + "Decimal(18, 4)" -> Decimal.round(value, 4) + _ -> value + end + + {type, value, expected} + end + end + + defp array_param do + one_of([ + gen_array("UInt8", integer(0..255)), + gen_array("Int16", integer(-32_768..32_767)), + gen_array("Bool", boolean()), + gen_array("String", safe_string()), + gen_array("Date", date_gen()) + ]) + end + + defp gen_array(type, generator) do + gen all values <- list_of(generator, max_length: 8) do + {type, values, values} + end + end + + defp rowbinary_rows do + uniq_list_of( + fixed_list([ + integer(0..255), + safe_string(), + boolean() + ]), + max_length: 12 + ) + end + + defp safe_string do + string(:printable, max_length: 32) + end + + defp date_gen do + gen all days <- integer(0..20_000) do + Date.add(~D[1970-01-01], days) + end + end + + defp date32_gen do + gen all days <- integer(-25_567..120_529) do + Date.add(~D[1970-01-01], days) + end + end + + defp decimal_gen do + gen all sign <- member_of([1, -1]), + coef <- integer(0..999_999_999), + exp <- integer(-4..4) do + Decimal.new(sign, coef, exp) + end + end +end diff --git a/test/ch/connection_test.exs b/test/ch/connection_test.exs index 57c48d31..2cb0af9c 100644 --- a/test/ch/connection_test.exs +++ b/test/ch/connection_test.exs @@ -1,1830 +1,497 @@ defmodule Ch.ConnectionTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - - import Ch.Test, - only: [ - parameterize_query: 2, - parameterize_query: 3, - parameterize_query: 4, - parameterize_query!: 2, - parameterize_query!: 3, - parameterize_query!: 4 - ] + use ExUnit.Case, async: true alias Ch.RowBinary setup do - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} - end - - test "select without params", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select 1") + {:ok, pool: start_supervised!(Ch)} end - test "select with types", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select 1", [], types: ["UInt8"]) + test "selects without params", %{pool: pool} do + assert Ch.query!(pool, "select 1").rows == [[1]] end - test "select with params", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt8}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[true]]}} = - parameterize_query(ctx, "select {b:Bool}", %{"b" => true}) - - assert {:ok, %{num_rows: 1, rows: [[false]]}} = - parameterize_query(ctx, "select {b:Bool}", %{"b" => false}) - - assert {:ok, %{num_rows: 1, rows: [[nil]]}} = - parameterize_query(ctx, "select {n:Nullable(Nothing)}", %{"n" => nil}) - - assert {:ok, %{num_rows: 1, rows: [[1.0]]}} = - parameterize_query(ctx, "select {a:Float32}", %{"a" => 1.0}) - - assert {:ok, %{num_rows: 1, rows: [["a&b=c"]]}} = - parameterize_query(ctx, "select {a:String}", %{"a" => "a&b=c"}) - - assert {:ok, %{num_rows: 1, rows: [["a\n"]]}} = - parameterize_query(ctx, "select {a:String}", %{"a" => "a\n"}) - - assert {:ok, %{num_rows: 1, rows: [["a\t"]]}} = - parameterize_query(ctx, "select {a:String}", %{"a" => "a\t"}) - - assert {:ok, %{num_rows: 1, rows: [[["a\tb"]]]}} = - parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a\tb"]}) - - assert {:ok, %{num_rows: 1, rows: [[[true, false]]]}} = - parameterize_query(ctx, "select {a:Array(Bool)}", %{"a" => [true, false]}) - - assert {:ok, %{num_rows: 1, rows: [[["a", nil, "b"]]]}} = - parameterize_query(ctx, "select {a:Array(Nullable(String))}", %{ - "a" => ["a", nil, "b"] - }) - - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query(ctx, "select {a:Decimal(9,4)}", %{"a" => Decimal.new("2000.333")}) - - assert row == [Decimal.new("2000.3330")] - - assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - parameterize_query(ctx, "select {a:Date}", %{"a" => ~D[2022-01-01]}) - - assert {:ok, %{num_rows: 1, rows: [[~D[2022-01-01]]]}} = - parameterize_query(ctx, "select {a:Date32}", %{"a" => ~D[2022-01-01]}) - - naive_noon = ~N[2022-01-01 12:00:00] - - # datetimes in params are sent in text and ClickHouse translates them to UTC from server timezone by default - # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime - # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ - assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - parameterize_query(ctx, "select {naive:DateTime}", %{"naive" => naive_noon}) - - # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse - # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result - {_, timezone} = List.keyfind!(headers, "x-clickhouse-timezone", 0) - - assert naive_datetime == - naive_noon - |> DateTime.from_naive!(timezone) - |> DateTime.shift_zone!("Etc/UTC") - |> DateTime.to_naive() - - # when the timezone information is provided in the type, we don't need to rely on server timezone - assert {:ok, %{num_rows: 1, rows: [[bkk_datetime]]}} = - parameterize_query(ctx, "select {$0:DateTime('Asia/Bangkok')}", [naive_noon]) - - assert bkk_datetime == DateTime.from_naive!(naive_noon, "Asia/Bangkok") - - assert {:ok, %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00Z]]]}} = - parameterize_query(ctx, "select {$0:DateTime('UTC')}", [naive_noon]) + test "selects with named params", %{pool: pool} do + assert Ch.query!(pool, "select {a:UInt8}", %{"a" => 1}).rows == [[1]] + assert Ch.query!(pool, "select {b:Bool}", %{"b" => true}).rows == [[true]] + assert Ch.query!(pool, "select {b:Bool}", %{"b" => false}).rows == [[false]] + assert Ch.query!(pool, "select {n:Nullable(Nothing)}", %{"n" => nil}).rows == [[nil]] + assert Ch.query!(pool, "select {a:Float32}", %{"a" => 1.0}).rows == [[1.0]] + assert Ch.query!(pool, "select {a:String}", %{"a" => "a&b=c"}).rows == [["a&b=c"]] + assert Ch.query!(pool, "select {a:String}", %{"a" => "a\n"}).rows == [["a\n"]] + assert Ch.query!(pool, "select {a:String}", %{"a" => "a\t"}).rows == [["a\t"]] - naive_noon_ms = ~N[2022-01-01 12:00:00.123] + assert Ch.query!(pool, "select {a:Array(String)}", %{"a" => ["a\tb"]}).rows == [ + [["a\tb"]] + ] - assert {:ok, %{num_rows: 1, rows: [[naive_datetime]]}} = - parameterize_query(ctx, "select {$0:DateTime64(3)}", [naive_noon_ms]) + assert Ch.query!(pool, "select {a:Array(Bool)}", %{"a" => [true, false]}).rows == [ + [[true, false]] + ] - assert NaiveDateTime.compare( - naive_datetime, - naive_noon_ms - |> DateTime.from_naive!(timezone) - |> DateTime.shift_zone!("Etc/UTC") - |> DateTime.to_naive() - ) == :eq + assert Ch.query!(pool, "select {a:Array(Nullable(String))}", %{ + "a" => ["a", nil, "b"] + }).rows == [[["a", nil, "b"]]] - assert {:ok, %{num_rows: 1, rows: [[["a", "b'", "\\'c"]]]}} = - parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a", "b'", "\\'c"]}) + assert Ch.query!(pool, "select {a:Decimal(9,4)}", %{ + "a" => Decimal.new("2000.333") + }).rows == [[Decimal.new("2000.3330")]] - assert {:ok, %{num_rows: 1, rows: [[["a\n", "b\tc"]]]}} = - parameterize_query(ctx, "select {a:Array(String)}", %{"a" => ["a\n", "b\tc"]}) + assert Ch.query!(pool, "select {a:Date}", %{"a" => ~D[2022-01-01]}).rows == [ + [~D[2022-01-01]] + ] - assert {:ok, %{num_rows: 1, rows: [[[1, 2, 3]]]}} = - parameterize_query(ctx, "select {a:Array(UInt8)}", %{"a" => [1, 2, 3]}) - - assert {:ok, %{num_rows: 1, rows: [[[[1], [2, 3], []]]]}} = - parameterize_query(ctx, "select {a:Array(Array(UInt8))}", %{"a" => [[1], [2, 3], []]}) + assert Ch.query!(pool, "select {a:Date32}", %{"a" => ~D[2022-01-01]}).rows == [ + [~D[2022-01-01]] + ] uuid = "9B29BD20-924C-4DE5-BDB3-8C2AA1FCE1FC" uuid_bin = uuid |> String.replace("-", "") |> Base.decode16!() - assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - parameterize_query(ctx, "select {a:UUID}", %{"a" => uuid}) - - # TODO - # assert {:ok, %{num_rows: 1, rows: [[^uuid_bin]]}} = - # parameterize_query(ctx, "select {a:UUID}", %{"a" => uuid_bin}) - - # pseudo-positional bind - assert {:ok, %{num_rows: 1, rows: [[1]]}} = parameterize_query(ctx, "select {$0:UInt8}", [1]) - end - - test "utc datetime query param encoding", ctx do - utc = ~U[2021-01-01 12:00:00Z] - msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00], "Europe/Moscow") - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - - assert parameterize_query!(ctx, "select {$0:DateTime} as d, toString(d)", [utc]).rows == - [[~N[2021-01-01 12:00:00], to_string(naive)]] - - assert parameterize_query!(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [utc]).rows == - [[utc, "2021-01-01 12:00:00"]] - - assert parameterize_query!(ctx, "select {$0:DateTime('Europe/Moscow')} as d, toString(d)", [ - utc - ]).rows == - [[msk, "2021-01-01 15:00:00"]] - end - - test "non-utc datetime query param encoding", ctx do - jp = DateTime.shift_zone!(~U[2021-01-01 12:34:56Z], "Asia/Tokyo") - assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" - - assert [[utc, jp]] = - parameterize_query!( - ctx, - "select {$0:DateTime('UTC')}, {$0:DateTime('Asia/Tokyo')}", - [jp] - ).rows - - assert inspect(utc) == "~U[2021-01-01 12:34:56Z]" - assert inspect(jp) == "#DateTime<2021-01-01 21:34:56+09:00 JST Asia/Tokyo>" - end - - test "non-utc datetime rowbinary encoding", ctx do - parameterize_query!( - ctx, - "create table ch_non_utc_datetimes(name String, datetime DateTime) engine Memory" - ) - - on_exit(fn -> Ch.Test.query("drop table ch_non_utc_datetimes") end) - - utc = ~U[2024-12-21 05:35:19.886393Z] - - taipei = DateTime.shift_zone!(utc, "Asia/Taipei") - tokyo = DateTime.shift_zone!(utc, "Asia/Tokyo") - vienna = DateTime.shift_zone!(utc, "Europe/Vienna") - - rows = [["taipei", taipei], ["tokyo", tokyo], ["vienna", vienna]] - - parameterize_query!( - ctx, - "insert into ch_non_utc_datetimes(name, datetime) format RowBinary", - rows, - types: ["String", "DateTime"] - ) - - result = - parameterize_query!( - ctx, - "select name, cast(datetime as DateTime('UTC')) from ch_non_utc_datetimes" - ) - |> Map.fetch!(:rows) - |> Map.new(fn [name, datetime] -> {name, datetime} end) - - assert result["taipei"] == ~U[2024-12-21 05:35:19Z] - assert result["tokyo"] == ~U[2024-12-21 05:35:19Z] - assert result["vienna"] == ~U[2024-12-21 05:35:19Z] + assert Ch.query!(pool, "select {a:UUID}", %{"a" => uuid}).rows == [[uuid_bin]] end - test "utc datetime64 query param encoding", ctx do - utc = ~U[2021-01-01 12:00:00.123456Z] - msk = DateTime.new!(~D[2021-01-01], ~T[15:00:00.123456], "Europe/Moscow") - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() - - assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == - [[~N[2021-01-01 12:00:00.123456], to_string(naive)]] - - assert parameterize_query!(ctx, "select {$0:DateTime64(6, 'UTC')} as d, toString(d)", [utc]).rows == - [[utc, "2021-01-01 12:00:00.123456"]] + test "accepts query settings", %{pool: pool} do + assert Ch.query!(pool, "show settings like 'async_insert'", %{}, settings: [async_insert: 1]).rows == + [["async_insert", "Bool", "1"]] - assert parameterize_query!( - ctx, - "select {$0:DateTime64(6,'Europe/Moscow')} as d, toString(d)", - [utc] - ).rows == - [[msk, "2021-01-01 15:00:00.123456"]] + assert Ch.query!(pool, "show settings like 'async_insert'", %{}, settings: [async_insert: 0]).rows == + [["async_insert", "Bool", "0"]] end - test "utc datetime64 zero microseconds query param encoding", ctx do - # this test case guards against a previous bug where DateTimes with a microsecond value of 0 and precision > 0 would - # get encoded as a val like "1.6095024e9" which ClickHouse would be unable to parse to a DateTime. - utc = ~U[2021-01-01 12:00:00.000000Z] - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() + test "creates and drops a table", %{pool: pool} do + Ch.query!(pool, "CREATE TABLE connection_test_create(a UInt8) ENGINE Memory") - assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == - [[~N[2021-01-01 12:00:00.000000], to_string(naive)]] - end - - test "utc datetime64 microseconds with more precision than digits", ctx do - # this test case guards against a previous bug where DateTimes with a microsecond value of with N digits - # and a precision > N would be encoded with a space like `234235234. 234123` - utc = ~U[2024-05-26 20:00:46.099856Z] - naive = utc |> DateTime.shift_zone!(Ch.Test.clickhouse_tz(ctx.conn)) |> DateTime.to_naive() + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_create") + Ch.stop(cleanup) + end) - assert parameterize_query!(ctx, "select {$0:DateTime64(6)} as d, toString(d)", [utc]).rows == - [[~N[2024-05-26 20:00:46.099856Z], to_string(naive)]] + assert Ch.query!(pool, "SHOW TABLES LIKE 'connection_test_create'").rows == [ + ["connection_test_create"] + ] end - test "select with options", ctx do - assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "1"]]}} = - parameterize_query(ctx, "show settings like 'async_insert'", [], - settings: [async_insert: 1] + test "returns readonly errors for create", %{pool: pool} do + assert {:error, %Ch.Error{message: message}} = + Ch.query( + pool, + "CREATE TABLE connection_test_create_readonly(a UInt8) ENGINE Memory", + %{}, + settings: [readonly: 1] ) - assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - parameterize_query(ctx, "show settings like 'async_insert'", [], - settings: [async_insert: 0] - ) + assert message =~ "Cannot execute query in readonly mode" end - test "create", ctx do - assert {:ok, %{command: :create, num_rows: nil, rows: [], data: []}} = - parameterize_query(ctx, "create table create_example(a UInt8) engine = Memory") - - on_exit(fn -> Ch.Test.query("drop table create_example") end) - end + test "inserts values and insert-selects rows", %{pool: pool} do + Ch.query!( + pool, + "CREATE TABLE connection_test_insert(a UInt8 DEFAULT 1, b String) ENGINE Memory" + ) - test "create with options", ctx do - assert {:error, %Ch.Error{code: 164, message: message}} = - parameterize_query(ctx, "create table create_example(a UInt8) engine = Memory", [], - settings: [readonly: 1] + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_insert") + Ch.stop(cleanup) + end) + + assert %Ch.Result{names: nil, rows: nil, data: nil} = + Ch.query!(pool, """ + INSERT INTO connection_test_insert VALUES + (1, 'a'), (2, 'b'), (NULL, NULL) + """) + + assert Ch.query!(pool, "SELECT * FROM connection_test_insert ORDER BY a, b").rows == [ + [1, ""], + [1, "a"], + [2, "b"] + ] + + assert %Ch.Result{names: nil, rows: nil, data: nil} = + Ch.query!( + pool, + """ + INSERT INTO connection_test_insert(a, b) + SELECT a, b FROM connection_test_insert WHERE a > {min:UInt8} + """, + %{"min" => 1} ) - assert message =~ ~r/Cannot execute query in readonly mode/ + assert Ch.query!(pool, "SELECT * FROM connection_test_insert WHERE a > 1").rows == [ + [2, "b"], + [2, "b"] + ] end - describe "insert" do - setup ctx do - table = "insert_t_#{System.unique_integer([:positive])}" - - parameterize_query!( - ctx, - "create table #{table}(a UInt8 default 1, b String) engine = Memory" - ) - - {:ok, table: table} - end - - test "values", %{table: table} = ctx do - parameterize_query( - ctx, - "insert into {table:Identifier} values (1, 'a'),(2,'b'), (null, null)", - %{"table" => table} - ) - - assert {:ok, %{rows: rows}} = - parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert rows == [[1, "a"], [2, "b"], [1, ""]] - - parameterize_query( - ctx, - "insert into {$0:Identifier}(a, b) values ({$1:UInt8},{$2:String}),({$3:UInt8},{$4:String})", - [table, 4, "d", 5, "e"] - ) - - assert {:ok, %{rows: rows}} = - parameterize_query(ctx, "select * from {table:Identifier} where a >= 4", %{ - "table" => table - }) - - assert rows == [[4, "d"], [5, "e"]] - end - - test "when readonly", %{table: table} = ctx do - settings = [readonly: 1] - - assert {:error, %Ch.Error{code: 164, message: message}} = - parameterize_query( - ctx, - "insert into {table:Identifier} values (1, 'a'), (2, 'b')", - %{"table" => table}, - settings: settings - ) + test "inserts RowBinary data", %{pool: pool} do + Ch.query!(pool, "CREATE TABLE connection_test_rowbinary(a UInt8, b String) ENGINE Memory") - assert message =~ "Cannot execute query in readonly mode." - end + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_rowbinary") + Ch.stop(cleanup) + end) - test "automatic RowBinary", %{table: table} = ctx do - stmt = "insert into #{table}(a, b) format RowBinary" - types = ["UInt8", "String"] - rows = [[1, "a"], [2, "b"]] + rows = [[1, "a"], [2, "b"], [3, "c"]] + rowbinary = RowBinary.encode_rows(rows, ["UInt8", "String"]) - parameterize_query!(ctx, stmt, rows, types: types) + assert %Ch.Result{names: nil, rows: nil, data: nil} = + Ch.query!(pool, [ + "INSERT INTO connection_test_rowbinary FORMAT RowBinary\n" | rowbinary + ]) - assert %{rows: rows} = - parameterize_query!(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert rows == [[1, "a"], [2, "b"]] - end - - test "manual RowBinary", %{table: table} = ctx do - stmt = "insert into #{table}(a, b) format RowBinary" - - types = ["UInt8", "String"] - rows = [[1, "a"], [2, "b"]] - data = RowBinary.encode_rows(rows, types) - - parameterize_query!(ctx, stmt, data, encode: false) - - assert %{rows: rows} = - parameterize_query!(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert rows == [[1, "a"], [2, "b"]] - end - - test "chunked", %{table: table} = ctx do - types = ["UInt8", "String"] - rows = [[1, "a"], [2, "b"], [3, "c"]] - - stream = - rows - |> Stream.chunk_every(2) - |> Stream.map(fn chunk -> RowBinary.encode_rows(chunk, types) end) - - parameterize_query( - ctx, - "insert into #{table}(a, b) format RowBinary", - stream, - encode: false - ) - - assert {:ok, %{rows: rows}} = - parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert rows == [[1, "a"], [2, "b"], [3, "c"]] - end - - test "select", %{table: table} = ctx do - parameterize_query( - ctx, - "insert into {table:Identifier} values (1, 'a'), (2, 'b'), (null, null)", - %{"table" => table} - ) - - parameterize_query( - ctx, - "insert into {table:Identifier}(a, b) select a, b from {table:Identifier}", - %{"table" => table} - ) - - assert {:ok, %{rows: rows}} = - parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert rows == [[1, "a"], [2, "b"], [1, ""], [1, "a"], [2, "b"], [1, ""]] - - assert {:ok, %{num_rows: 2}} = - parameterize_query( - ctx, - "insert into {$0:Identifier}(a, b) select a, b from {$0:Identifier} where a > {$1:UInt8}", - [table, 1] - ) - - assert {:ok, %{rows: new_rows}} = - parameterize_query(ctx, "select * from {table:Identifier}", %{"table" => table}) - - assert new_rows -- rows == [[2, "b"], [2, "b"]] - end + assert Ch.query!(pool, "SELECT * FROM connection_test_rowbinary ORDER BY a").rows == rows end - test "delete", ctx do - parameterize_query!( - ctx, - "create table delete_t(a UInt8, b String) engine = MergeTree order by tuple()" - ) - - on_exit(fn -> Ch.Test.query("drop table delete_t") end) + test "returns readonly errors", %{pool: pool} do + Ch.query!(pool, "CREATE TABLE connection_test_readonly(a UInt8) ENGINE Memory") - parameterize_query(ctx, "insert into delete_t values (1,'a'), (2,'b')") + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_readonly") + Ch.stop(cleanup) + end) - settings = [allow_experimental_lightweight_delete: 1] - - assert {:ok, %{rows: [], data: [], command: :delete}} = - parameterize_query(ctx, "delete from delete_t where 1", [], settings: settings) - end + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, "INSERT INTO connection_test_readonly VALUES (1)", %{}, + settings: [readonly: 1] + ) - test "query!", ctx do - assert %{num_rows: 1, rows: [[1]]} = parameterize_query!(ctx, "select 1") + assert message =~ "Cannot execute query in readonly mode" end - describe "types" do - test "multiple types", ctx do - assert {:ok, %{num_rows: 1, rows: [[1, "a"]]}} = - parameterize_query(ctx, "select {a:Int8}, {b:String}", %{"a" => 1, "b" => "a"}) - end - - test "ints", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:Int8}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[-1000]]}} = - parameterize_query(ctx, "select {a:Int16}", %{"a" => -1000}) - - assert {:ok, %{num_rows: 1, rows: [[100_000]]}} = - parameterize_query(ctx, "select {a:Int32}", %{"a" => 100_000}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:Int64}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:Int128}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:Int256}", %{"a" => 1}) - end - - test "uints", ctx do - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt8}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt16}", %{"a" => 1}) + test "deletes rows", %{pool: pool} do + Ch.query!(pool, """ + CREATE TABLE connection_test_delete(a UInt8, b String) + ENGINE MergeTree + ORDER BY tuple() + """) - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt32}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt64}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt128}", %{"a" => 1}) - - assert {:ok, %{num_rows: 1, rows: [[1]]}} = - parameterize_query(ctx, "select {a:UInt256}", %{"a" => 1}) - end - - test "fixed string", ctx do - assert {:ok, %{num_rows: 1, rows: [[<<0, 0>>]]}} = - parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => ""}) - - assert {:ok, %{num_rows: 1, rows: [["a" <> <<0>>]]}} = - parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => "a"}) - - assert {:ok, %{num_rows: 1, rows: [["aa"]]}} = - parameterize_query(ctx, "select {a:FixedString(2)}", %{"a" => "aa"}) - - assert {:ok, %{num_rows: 1, rows: [["aaaaa"]]}} = - parameterize_query(ctx, "select {a:FixedString(5)}", %{"a" => "aaaaa"}) - - parameterize_query!(ctx, "create table fixed_string_t(a FixedString(3)) engine = Memory") - on_exit(fn -> Ch.Test.query("drop table fixed_string_t") end) - - parameterize_query( - ctx, - "insert into fixed_string_t(a) format RowBinary", - [ - [""], - ["a"], - ["aa"], - ["aaa"] - ], - types: ["FixedString(3)"] - ) - - assert parameterize_query!(ctx, "select * from fixed_string_t").rows == [ - [<<0, 0, 0>>], - ["a" <> <<0, 0>>], - ["aa" <> <<0>>], - ["aaa"] - ] - end - - test "decimal", ctx do - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query(ctx, "SELECT toDecimal32(2, 4) AS x, x / 3, toTypeName(x)") - - assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(9, 4)"] - - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query(ctx, "SELECT toDecimal64(2, 4) AS x, x / 3, toTypeName(x)") - - assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(18, 4)"] - - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query(ctx, "SELECT toDecimal128(2, 4) AS x, x / 3, toTypeName(x)") - - assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(38, 4)"] - - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query(ctx, "SELECT toDecimal256(2, 4) AS x, x / 3, toTypeName(x)") - - assert row == [Decimal.new("2.0000"), Decimal.new("0.6666"), "Decimal(76, 4)"] - - parameterize_query!(ctx, "create table decimal_t(d Decimal32(4)) engine = Memory") - on_exit(fn -> Ch.Test.query("drop table decimal_t") end) - - parameterize_query!( - ctx, - "insert into decimal_t(d) format RowBinary", - _rows = [ - [Decimal.new("2.66")], - [Decimal.new("2.6666")], - [Decimal.new("2.66666")] - ], - types: ["Decimal32(4)"] - ) - - assert parameterize_query!(ctx, "select * from decimal_t").rows == [ - [Decimal.new("2.6600")], - [Decimal.new("2.6666")], - [Decimal.new("2.6667")] - ] - end - - test "boolean", ctx do - assert {:ok, %{num_rows: 1, rows: [[true, "Bool"]]}} = - parameterize_query(ctx, "select true as col, toTypeName(col)") - - assert {:ok, %{num_rows: 1, rows: [[1, "UInt8"]]}} = - parameterize_query(ctx, "select true == 1 as col, toTypeName(col)") - - assert {:ok, %{num_rows: 1, rows: [[true, false]]}} = - parameterize_query(ctx, "select true, false") - - parameterize_query!(ctx, "create table test_bool(A Int64, B Bool) engine = Memory") - on_exit(fn -> Ch.Test.query("drop table test_bool") end) - - parameterize_query!(ctx, "INSERT INTO test_bool VALUES (1, true),(2,0)") - - parameterize_query!( - ctx, - "insert into test_bool(A, B) format RowBinary", - _rows = [[3, true], [4, false]], - types: ["Int64", "Bool"] - ) - - # anything > 0 is `true`, here `2` is `true` - parameterize_query!(ctx, "insert into test_bool(A, B) values (5, 2)") - - assert %{ - rows: [ - [1, true, 1], - [2, false, 0], - [3, true, 3], - [4, false, 0], - [5, true, 5] - ] - } = parameterize_query!(ctx, "SELECT *, A * B FROM test_bool ORDER BY A") - end + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_delete") + Ch.stop(cleanup) + end) - test "uuid", ctx do - assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>]]}} = - parameterize_query(ctx, "select generateUUIDv4()") + Ch.query!(pool, "INSERT INTO connection_test_delete VALUES (1, 'a'), (2, 'b')") - assert {:ok, %{num_rows: 1, rows: [[uuid, "417ddc5d-e556-4d27-95dd-a34d84e46a50"]]}} = - parameterize_query(ctx, "select {uuid:UUID} as u, toString(u)", %{ - "uuid" => "417ddc5d-e556-4d27-95dd-a34d84e46a50" - }) - - assert uuid == - "417ddc5d-e556-4d27-95dd-a34d84e46a50" - |> String.replace("-", "") - |> Base.decode16!(case: :lower) - - parameterize_query!(ctx, " CREATE TABLE t_uuid (x UUID, y String) ENGINE Memory") - on_exit(fn -> Ch.Test.query("drop table t_uuid") end) - - parameterize_query!(ctx, "INSERT INTO t_uuid SELECT generateUUIDv4(), 'Example 1'") - - assert {:ok, %{num_rows: 1, rows: [[<<_::16-bytes>>, "Example 1"]]}} = - parameterize_query(ctx, "SELECT * FROM t_uuid") - - parameterize_query!(ctx, "INSERT INTO t_uuid (y) VALUES ('Example 2')") - - parameterize_query!( - ctx, - "insert into t_uuid(x,y) format RowBinary", - _rows = [[uuid, "Example 3"]], - types: ["UUID", "String"] - ) - - assert {:ok, - %{ - num_rows: 3, - rows: [ - [<<_::16-bytes>>, "Example 1"], - [<<0::128>>, "Example 2"], - [^uuid, "Example 3"] - ] - }} = parameterize_query(ctx, "SELECT * FROM t_uuid ORDER BY y") - end - - @tag :skip - test "json", ctx do - settings = [allow_experimental_object_type: 1] - - parameterize_query!(ctx, "CREATE TABLE json(o JSON) ENGINE = Memory", [], - settings: settings - ) - - parameterize_query!( - ctx, - ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')| - ) - - assert parameterize_query!(ctx, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]] - - # named tuples are not supported yet - assert_raise ArgumentError, fn -> parameterize_query!(ctx, "SELECT o FROM json") end - end - - @tag :json - test "json as string", ctx do - # after v25 ClickHouse started rendering numbers in JSON as strings - [[version]] = parameterize_query!(ctx, "select version()").rows - - parse_version = fn version -> - version |> String.split(".") |> Enum.map(&String.to_integer/1) - end - - version = parse_version.(version) - numbers_as_strings? = version >= [25] and version <= [25, 8] - - [expected1, expected2] = - if numbers_as_strings? do - [ - [[~s|{"answer":"42"}|]], - [[~s|{"a":"42"}|], [~s|{"b":"10"}|]] - ] - else - [ - [[~s|{"answer":42}|]], - [[~s|{"a":42}|], [~s|{"b":10}|]] - ] - end - - assert parameterize_query!(ctx, ~s|select '{"answer":42}'::JSON::String|, [], - settings: [enable_json_type: 1] - ).rows == expected1 - - parameterize_query!(ctx, "CREATE TABLE test_json_as_string(json JSON) ENGINE = Memory", [], - settings: [enable_json_type: 1] - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE test_json_as_string") end) - - parameterize_query!( - ctx, - "INSERT INTO test_json_as_string(json) FORMAT RowBinary", - _rows = [[Jason.encode_to_iodata!(%{"a" => 42})], [Jason.encode_to_iodata!(%{"b" => 10})]], - types: [:string], - settings: [ - enable_json_type: 1, - input_format_binary_read_json_as_string: 1 - ] - ) - - assert parameterize_query!(ctx, "select json::String from test_json_as_string", [], - settings: [enable_json_type: 1] - ).rows == expected2 - end - - # TODO enum16 - - test "enum8", ctx do - assert {:ok, %{num_rows: 1, rows: [["Enum8('a' = 1, 'b' = 2)"]]}} = - parameterize_query( - ctx, - "SELECT toTypeName(CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)'))" - ) - - assert {:ok, %{num_rows: 1, rows: [["a"]]}} = - parameterize_query(ctx, "SELECT CAST('a', 'Enum(\\'a\\' = 1, \\'b\\' = 2)')") - - assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - parameterize_query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => "b"}) - - assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - parameterize_query(ctx, "select {enum:Enum('a' = 1, 'b' = 2)}", %{"enum" => 2}) - - assert {:ok, %{num_rows: 1, rows: [["b"]]}} = - parameterize_query(ctx, "select {enum:Enum16('a' = 1, 'b' = 2)}", %{"enum" => 2}) - - parameterize_query!( - ctx, - "CREATE TABLE t_enum(i UInt8, x Enum('hello' = 1, 'world' = 2)) ENGINE Memory" - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE t_enum") end) - - parameterize_query!( - ctx, - "INSERT INTO t_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')" - ) - - assert parameterize_query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == - [ - [0, "hello", 1], - [1, "world", 2], - [2, "hello", 1] - ] - - parameterize_query!( - ctx, - "INSERT INTO t_enum(i, x) FORMAT RowBinary", - _rows = [[3, "hello"], [4, "world"], [5, 1], [6, 2]], - types: ["UInt8", "Enum8('hello' = 1, 'world' = 2)"] - ) - - assert parameterize_query!(ctx, "SELECT *, CAST(x, 'Int8') FROM t_enum ORDER BY i").rows == - [ - [0, "hello", 1], - [1, "world", 2], - [2, "hello", 1], - [3, "hello", 1], - [4, "world", 2], - [5, "hello", 1], - [6, "world", 2] - ] - - # TODO nil enum - end - - test "map", ctx do - assert parameterize_query!( - ctx, - "SELECT CAST(([1, 2, 3], ['Ready', 'Steady', 'Go']), 'Map(UInt8, String)') AS map" - ).rows == [[%{1 => "Ready", 2 => "Steady", 3 => "Go"}]] - - assert parameterize_query!(ctx, "select {map:Map(String, UInt8)}", %{ - "map" => %{"pg" => 13, "hello" => 100} - }).rows == [[%{"hello" => 100, "pg" => 13}]] - - parameterize_query!(ctx, "CREATE TABLE table_map (a Map(String, UInt64)) ENGINE=Memory") - on_exit(fn -> Ch.Test.query("DROP TABLE table_map") end) - - parameterize_query!( - ctx, - "INSERT INTO table_map VALUES ({'key1':1, 'key2':10}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30})" - ) - - assert parameterize_query!(ctx, "SELECT a['key2'] FROM table_map").rows == [ - [10], - [20], - [30] - ] - - assert parameterize_query!(ctx, "INSERT INTO table_map VALUES ({'key3':100}), ({})") - - assert parameterize_query!(ctx, "SELECT a['key3'] FROM table_map ORDER BY 1 DESC").rows == [ - [100], - [0], - [0], - [0], - [0] - ] - - assert parameterize_query!( - ctx, - "INSERT INTO table_map FORMAT RowBinary", - _rows = [ - [%{"key10" => 20, "key20" => 40}], - # empty map - [%{}], - # null map - [nil], - # empty proplist map - [[]], - [[{"key50", 100}]] - ], - types: ["Map(String, UInt64)"] + assert %Ch.Result{names: nil, rows: nil, data: nil} = + Ch.query!(pool, "DELETE FROM connection_test_delete WHERE 1", %{}, + settings: [mutations_sync: 1] ) - assert parameterize_query!(ctx, "SELECT * FROM table_map ORDER BY a ASC").rows == [ - [%{}], - [%{}], - [%{}], - [%{}], - [%{"key1" => 1, "key2" => 10}], - [%{"key1" => 2, "key2" => 20}], - [%{"key1" => 3, "key2" => 30}], - [%{"key10" => 20, "key20" => 40}], - [%{"key3" => 100}], - [%{"key50" => 100}] - ] - end - - test "tuple", ctx do - assert parameterize_query!(ctx, "SELECT tuple(1,'a') AS x, toTypeName(x)").rows == [ - [{1, "a"}, "Tuple(UInt8, String)"] - ] - - assert parameterize_query!(ctx, "SELECT {$0:Tuple(Int8, String)}", [{-1, "abs"}]).rows == [ - [{-1, "abs"}] - ] - - assert parameterize_query!(ctx, "SELECT tuple('a') AS x").rows == [[{"a"}]] - - assert parameterize_query!(ctx, "SELECT tuple(1, NULL) AS x, toTypeName(x)").rows == [ - [{1, nil}, "Tuple(UInt8, Nullable(Nothing))"] - ] - - # TODO named tuples - parameterize_query!(ctx, "CREATE TABLE tuples_t (`a` Tuple(String, Int64)) ENGINE = Memory") - on_exit(fn -> Ch.Test.query("DROP TABLE tuples_t") end) - - parameterize_query!(ctx, "INSERT INTO tuples_t VALUES (('y', 10)), (('x',-10))") - - parameterize_query!( - ctx, - "INSERT INTO tuples_t FORMAT RowBinary", - _rows = [[{"a", 20}], [{"b", 30}]], - types: ["Tuple(String, Int64)"] - ) - - assert parameterize_query!(ctx, "SELECT a FROM tuples_t ORDER BY a.1 ASC").rows == [ - [{"a", 20}], - [{"b", 30}], - [{"x", -10}], - [{"y", 10}] - ] - end - - test "datetime", ctx do - parameterize_query!( - ctx, - "CREATE TABLE dt(`timestamp` DateTime('Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE dt") end) - - parameterize_query!( - ctx, - "INSERT INTO dt Values (1546300800, 1), ('2019-01-01 00:00:00', 2)" - ) - - assert {:ok, %{num_rows: 2, rows: rows}} = - parameterize_query(ctx, "SELECT *, toString(timestamp) FROM dt") - - assert rows == [ - [ - DateTime.new!(~D[2019-01-01], ~T[03:00:00], "Asia/Istanbul"), - 1, - "2019-01-01 03:00:00" - ], - [ - DateTime.new!(~D[2019-01-01], ~T[00:00:00], "Asia/Istanbul"), - 2, - "2019-01-01 00:00:00" - ] - ] - - naive_noon = ~N[2022-12-12 12:00:00] - - # datetimes in params are sent in text and ClickHouse translates them to UTC from server timezone by default - # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime - # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ - assert {:ok, - %{num_rows: 1, rows: [[naive_datetime, "2022-12-12 12:00:00"]], headers: headers}} = - parameterize_query(ctx, "select {$0:DateTime} as d, toString(d)", [naive_noon]) - - # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse - # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result - {_, timezone} = List.keyfind!(headers, "x-clickhouse-timezone", 0) - - assert naive_datetime == - naive_noon - |> DateTime.from_naive!(timezone) - |> DateTime.shift_zone!("Etc/UTC") - |> DateTime.to_naive() - - assert {:ok, %{num_rows: 1, rows: [[~U[2022-12-12 12:00:00Z], "2022-12-12 12:00:00"]]}} = - parameterize_query(ctx, "select {$0:DateTime('UTC')} as d, toString(d)", [ - naive_noon - ]) - - assert {:ok, %{num_rows: 1, rows: rows}} = - parameterize_query(ctx, "select {$0:DateTime('Asia/Bangkok')} as d, toString(d)", [ - naive_noon - ]) - - assert rows == [ - [ - DateTime.new!(~D[2022-12-12], ~T[12:00:00], "Asia/Bangkok"), - "2022-12-12 12:00:00" - ] - ] - - # simulate unknown timezone - prev_tz_db = Calendar.get_time_zone_database() - Calendar.put_time_zone_database(Calendar.UTCOnlyTimeZoneDatabase) - on_exit(fn -> Calendar.put_time_zone_database(prev_tz_db) end) - - assert_raise ArgumentError, ~r/:utc_only_time_zone_database/, fn -> - parameterize_query(ctx, "select {$0:DateTime('Asia/Tokyo')}", [naive_noon]) - end - end - - # TODO are negatives correct? what's the range? - test "date32", ctx do - parameterize_query!( - ctx, - "CREATE TABLE new(`timestamp` Date32, `event_id` UInt8) ENGINE = Memory;" - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE new") end) - - parameterize_query!(ctx, "INSERT INTO new VALUES (4102444800, 1), ('2100-01-01', 2)") - - assert {:ok, - %{ - num_rows: 2, - rows: [first_event, [~D[2100-01-01], 2, "2100-01-01"]] - }} = parameterize_query(ctx, "SELECT *, toString(timestamp) FROM new") - - # TODO use timezone info to be more exact - assert first_event in [ - [~D[2099-12-31], 1, "2099-12-31"], - [~D[2100-01-01], 1, "2100-01-01"] - ] - - assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) - - # max - assert {:ok, %{num_rows: 1, rows: [[~D[2299-12-31], "2299-12-31"]]}} = - parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[2299-12-31]]) - - # min - assert {:ok, %{num_rows: 1, rows: [[~D[1900-01-01], "1900-01-01"]]}} = - parameterize_query(ctx, "select {$0:Date32} as d, toString(d)", [~D[1900-01-01]]) - - parameterize_query!( - ctx, - "insert into new(timestamp, event_id) format RowBinary", - _rows = [[~D[1960-01-01], 3]], - types: ["Date32", "UInt8"] - ) - - assert %{ - num_rows: 3, - rows: [ - first_event, - [~D[2100-01-01], 2, "2100-01-01"], - [~D[1960-01-01], 3, "1960-01-01"] - ] - } = - parameterize_query!( - ctx, - "SELECT *, toString(timestamp) FROM new ORDER BY event_id" - ) - - # TODO use timezone info to be more exact - assert first_event in [ - [~D[2099-12-31], 1, "2099-12-31"], - [~D[2100-01-01], 1, "2100-01-01"] - ] - - assert %{num_rows: 1, rows: [[3]]} = - parameterize_query!(ctx, "SELECT event_id FROM new WHERE timestamp = '1960-01-01'") - end - - # https://clickhouse.com/docs/sql-reference/data-types/time - @tag :time - test "time", ctx do - settings = [enable_time_time64_type: 1] - - parameterize_query!( - ctx, - "CREATE TABLE time_t(`time` Time, `event_id` UInt8) ENGINE = Memory", - [], - settings: settings - ) - - on_exit(fn -> - Ch.Test.query("DROP TABLE time_t", [], settings: settings) - end) - - parameterize_query!(ctx, "INSERT INTO time_t VALUES ('100:00:00', 1), (12453, 2)", [], - settings: settings - ) - - # ClickHouse supports Time values of [-999:59:59, 999:59:59] - # and Elixir's Time supports values of [00:00:00, 23:59:59] - # so we raise an error when ClickHouse's Time value is out of Elixir's Time range - - assert_raise ArgumentError, - "ClickHouse Time value 3.6e5 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> - parameterize_query!(ctx, "select * from time_t", [], settings: settings) - end - - parameterize_query!( - ctx, - "INSERT INTO time_t(time, event_id) FORMAT RowBinary", - _rows = [ - [~T[00:00:00], 3], - [~T[12:34:56], 4], - [~T[23:59:59], 5] - ], - settings: settings, - types: ["Time", "UInt8"] - ) - - assert parameterize_query!( - ctx, - "select * from time_t where event_id > 1 order by event_id", - [], - settings: settings - ).rows == - [[~T[03:27:33], 2], [~T[00:00:00], 3], [~T[12:34:56], 4], [~T[23:59:59], 5]] - end - - # https://clickhouse.com/docs/sql-reference/data-types/time64 - @tag :time - test "Time64(3)", ctx do - settings = [enable_time_time64_type: 1] - - parameterize_query!( - ctx, - "CREATE TABLE time64_3_t(`time` Time64(3), `event_id` UInt8) ENGINE = Memory", - [], - settings: settings - ) - - on_exit(fn -> - Ch.Test.query("DROP TABLE time64_3_t", [], settings: settings) - end) - - parameterize_query!( - ctx, - "INSERT INTO time64_3_t VALUES (15463123, 1), (154600.123, 2), ('100:00:00', 3);", - [], - settings: settings - ) - - # ClickHouse supports Time64 values of [-999:59:59.999999999, 999:59:59.999999999] - # and Elixir's Time supports values of [00:00:00.000000, 23:59:59.999999] - # so we raise an error when ClickHouse's Time64 value is out of Elixir's Time range - - assert_raise ArgumentError, - "ClickHouse Time value 154600.123 (seconds) is out of Elixir's Time range (00:00:00.000000 - 23:59:59.999999)", - fn -> - parameterize_query!(ctx, "select * from time64_3_t", [], settings: settings) - end - - parameterize_query!( - ctx, - "INSERT INTO time64_3_t(time, event_id) FORMAT RowBinary", - _rows = [ - [~T[00:00:00.000000], 4], - [~T[12:34:56.012300], 5], - [~T[12:34:56.123456], 6], - [~T[12:34:56.120000], 7], - [~T[23:59:59.999999], 8] - ], - settings: settings, - types: ["Time64(3)", "UInt8"] - ) - - assert parameterize_query!( - ctx, - "select * from time64_3_t where time < {max_elixir_time:Time64(6)} order by event_id", - %{"max_elixir_time" => ~T[23:59:59.999999]}, - settings: settings - ).rows == - [ - [~T[04:17:43.123], 1], - [~T[00:00:00.000], 4], - [~T[12:34:56.012], 5], - [~T[12:34:56.123], 6], - [~T[12:34:56.120], 7], - [~T[23:59:59.999], 8] - ] - end - - @tag :time - test "Time64(6)", ctx do - settings = [enable_time_time64_type: 1] - - parameterize_query!( - ctx, - "CREATE TABLE time64_6_t(`time` Time64(6), `event_id` UInt8) ENGINE = Memory", - [], - settings: settings - ) - - on_exit(fn -> - Ch.Test.query("DROP TABLE time64_6_t", [], settings: settings) - end) - - parameterize_query!( - ctx, - "INSERT INTO time64_6_t(time, event_id) FORMAT RowBinary", - _rows = [ - [~T[00:00:00.000000], 1], - [~T[12:34:56.123456], 2], - [~T[12:34:56.123000], 3], - [~T[12:34:56.000123], 4], - [~T[23:59:59.999999], 5] - ], - settings: settings, - types: ["Time64(6)", "UInt8"] - ) - - assert parameterize_query!( - ctx, - "select * from time64_6_t order by event_id", - [], - settings: settings - ).rows == - [ - [~T[00:00:00.000000], 1], - [~T[12:34:56.123456], 2], - [~T[12:34:56.123000], 3], - [~T[12:34:56.000123], 4], - [~T[23:59:59.999999], 5] - ] - end - - @tag :time - test "Time64(9)", ctx do - settings = [enable_time_time64_type: 1] - - parameterize_query!( - ctx, - "CREATE TABLE time64_9_t(`time` Time64(9), `event_id` UInt8) ENGINE = Memory", - [], - settings: settings - ) - - on_exit(fn -> - Ch.Test.query("DROP TABLE time64_9_t", [], settings: settings) - end) - - parameterize_query!( - ctx, - "INSERT INTO time64_9_t(time, event_id) FORMAT RowBinary", - _rows = [ - [~T[00:00:00.000000], 1], - [~T[12:34:56.123456], 2], - [~T[12:34:56.123000], 3], - [~T[12:34:56.000123], 4], - [~T[23:59:59.999999], 5] - ], - settings: settings, - types: ["Time64(9)", "UInt8"] - ) - - assert parameterize_query!( - ctx, - "select * from time64_9_t order by event_id", - [], - settings: settings - ).rows == - [ - [~T[00:00:00.000000], 1], - [~T[12:34:56.123456], 2], - [~T[12:34:56.123000], 3], - [~T[12:34:56.000123], 4], - [~T[23:59:59.999999], 5] - ] - end - - test "datetime64", ctx do - parameterize_query!( - ctx, - "CREATE TABLE datetime64_t(`timestamp` DateTime64(3, 'Asia/Istanbul'), `event_id` UInt8) ENGINE = Memory" - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE datetime64_t") end) - - parameterize_query!( - ctx, - "INSERT INTO datetime64_t Values (1546300800123, 1), (1546300800.123, 2), ('2019-01-01 00:00:00', 3)" - ) - - assert {:ok, %{num_rows: 3, rows: rows}} = - parameterize_query(ctx, "SELECT *, toString(timestamp) FROM datetime64_t") - - assert rows == [ - [ - DateTime.new!(~D[2019-01-01], ~T[03:00:00.123], "Asia/Istanbul"), - 1, - "2019-01-01 03:00:00.123" - ], - [ - DateTime.new!(~D[2019-01-01], ~T[03:00:00.123], "Asia/Istanbul"), - 2, - "2019-01-01 03:00:00.123" - ], - [ - DateTime.new!(~D[2019-01-01], ~T[00:00:00.000], "Asia/Istanbul"), - 3, - "2019-01-01 00:00:00.000" - ] - ] - - parameterize_query!( - ctx, - "insert into datetime64_t(event_id, timestamp) format RowBinary", - _rows = [ - [4, ~N[2021-01-01 12:00:00.123456]], - [5, ~N[2021-01-01 12:00:00]] - ], - types: ["UInt8", "DateTime64(3)"] - ) - - assert {:ok, %{num_rows: 2, rows: rows}} = - parameterize_query( - ctx, - "SELECT *, toString(timestamp) FROM datetime64_t WHERE timestamp > '2020-01-01'" - ) + assert Ch.query!(pool, "SELECT * FROM connection_test_delete").rows == [] + end - assert rows == [ - [ - DateTime.new!(~D[2021-01-01], ~T[15:00:00.123], "Asia/Istanbul"), - 4, - "2021-01-01 15:00:00.123" - ], - [ - DateTime.new!(~D[2021-01-01], ~T[15:00:00.000], "Asia/Istanbul"), - 5, - "2021-01-01 15:00:00.000" - ] + test "decodes representative scalar types", %{pool: pool} do + assert Ch.query!(pool, """ + SELECT + -1::Int8, + 1::UInt8, + true, + 'abc'::String, + toDecimal32(2, 4), + '417ddc5d-e556-4d27-95dd-a34d84e46a50'::UUID, + '2022-01-01'::Date, + '1900-01-01'::Date32 + """).rows == [ + [ + -1, + 1, + true, + "abc", + Decimal.new("2.0000"), + Base.decode16!("417ddc5de5564d2795dda34d84e46a50", case: :lower), + ~D[2022-01-01], + ~D[1900-01-01] ] + ] + end - for precision <- 0..9 do - naive_noon = ~N[2022-01-01 12:00:00] - - # datetimes in params are sent in text and ClickHouse translates them to UTC from server timezone by default - # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime - # https://kb.altinity.com/altinity-kb-queries-and-syntax/time-zones/ - assert {:ok, %{num_rows: 1, rows: [[naive_datetime]], headers: headers}} = - parameterize_query(ctx, "select {$0:DateTime64(#{precision})}", [naive_noon]) - - # to make this test pass for contributors with non UTC timezone we perform the same steps as ClickHouse - # i.e. we give server timezone to the naive datetime and shift it to UTC before comparing with the result - {_, timezone} = List.keyfind!(headers, "x-clickhouse-timezone", 0) - - expected = - naive_noon - |> DateTime.from_naive!(timezone) - |> DateTime.shift_zone!("Etc/UTC") - |> DateTime.to_naive() - - assert NaiveDateTime.compare(naive_datetime, expected) == :eq - end - - assert {:ok, - %{num_rows: 1, rows: [[~U[2022-01-01 12:00:00.123Z], "2022-01-01 12:00:00.123"]]}} = - parameterize_query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ - "dt" => ~N[2022-01-01 12:00:00.123] - }) - - assert {:ok, - %{num_rows: 1, rows: [[~U[1900-01-01 12:00:00.123Z], "1900-01-01 12:00:00.123"]]}} = - parameterize_query(ctx, "select {dt:DateTime64(3,'UTC')} as d, toString(d)", %{ - "dt" => ~N[1900-01-01 12:00:00.123] - }) - - assert {:ok, %{num_rows: 1, rows: [row]}} = - parameterize_query( - ctx, - "select {dt:DateTime64(3,'Asia/Bangkok')} as d, toString(d)", - %{ - "dt" => ~N[2022-01-01 12:00:00.123] - } - ) - - assert row == [ - DateTime.new!(~D[2022-01-01], ~T[12:00:00.123], "Asia/Bangkok"), - "2022-01-01 12:00:00.123" + test "decodes compound types", %{pool: pool} do + assert Ch.query!(pool, """ + SELECT + map('hello', 100::UInt64, 'pg', 13::UInt64), + tuple('a', 1), + [1, 2, 3], + CAST((10, 20), 'Point') + """).rows == [ + [ + %{"hello" => 100, "pg" => 13}, + {"a", 1}, + [1, 2, 3], + {10.0, 20.0} ] - end + ] + end - test "nullable", ctx do - parameterize_query!( - ctx, - "CREATE TABLE nullable (`n` Nullable(UInt32)) ENGINE = MergeTree ORDER BY tuple()" + test "inserts and selects nullable/default values", %{pool: pool} do + Ch.query!(pool, """ + CREATE TABLE connection_test_nulls ( + a UInt8, + b Nullable(UInt8), + c UInt8 DEFAULT 10, + d Nullable(UInt8) DEFAULT 10 + ) ENGINE Memory + """) + + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_nulls") + Ch.stop(cleanup) + end) + + rowbinary = + RowBinary.encode_rows( + [[nil, nil, nil, nil]], + ["UInt8", "Nullable(UInt8)", "UInt8", "Nullable(UInt8)"] ) - on_exit(fn -> Ch.Test.query("DROP TABLE nullable") end) - - parameterize_query!(ctx, "INSERT INTO nullable VALUES (1) (NULL) (2) (NULL)") + Ch.query!(pool, [ + "INSERT INTO connection_test_nulls(a, b, c, d) FORMAT RowBinary\n" | rowbinary + ]) - assert {:ok, %{num_rows: 4, rows: [[0], [1], [0], [1]]}} = - parameterize_query(ctx, "SELECT n.null FROM nullable") - - assert {:ok, %{num_rows: 4, rows: [[1], [nil], [2], [nil]]}} = - parameterize_query(ctx, "SELECT n FROM nullable") - - # weird thing about nullables is that, similar to bool, in binary format, any byte larger than 0 is `null` - parameterize_query( - ctx, - "insert into nullable format RowBinary", - <<1, 2, 3, 4, 5>>, - encode: false - ) + assert Ch.query!(pool, "SELECT * FROM connection_test_nulls").rows == [[0, nil, 0, nil]] + end - assert %{num_rows: 1, rows: [[count]]} = - parameterize_query!(ctx, "select count(*) from nullable where n is null") + test "inserts nullable input rows and applies defaults", %{pool: pool} do + Ch.query!(pool, """ + CREATE TABLE connection_test_input_default( + n Int32, + s String DEFAULT 'secret' + ) ENGINE Memory + """) + + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_input_default") + Ch.stop(cleanup) + end) + + rows = [[1, nil], [-1, nil]] + rowbinary = RowBinary.encode_rows(rows, ["UInt32", "Nullable(String)"]) + + Ch.query!(pool, [ + """ + INSERT INTO connection_test_input_default + SELECT id, name + FROM input('id UInt32, name Nullable(String)') + FORMAT RowBinary + """ + | rowbinary + ]) + + assert Ch.query!(pool, "SELECT * FROM connection_test_input_default ORDER BY n").rows == [ + [-1, "secret"], + [1, "secret"] + ] + end - assert count == 2 + 5 - end + test "inserts RowBinaryWithNamesAndTypes", %{pool: pool} do + Ch.query!(pool, """ + CREATE TABLE connection_test_names_types ( + country_code FixedString(2), + rare_string LowCardinality(String), + maybe_int32 Nullable(Int32) + ) ENGINE Memory + """) + + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_names_types") + Ch.stop(cleanup) + end) + + names = ["country_code", "rare_string", "maybe_int32"] + types = ["FixedString(2)", "LowCardinality(String)", "Nullable(Int32)"] + rows = [["AB", "rare", -42], ["CD", "other", nil]] + + rowbinary = [ + RowBinary.encode_names_and_types(names, types) + | RowBinary.encode_rows(rows, types) + ] - test "nullable + default", ctx do - parameterize_query!(ctx, """ - CREATE TABLE ch_nulls ( - a UInt8, - b UInt8 NULL, - c UInt8 DEFAULT 10, - d Nullable(UInt8) DEFAULT 10, - ) ENGINE Memory - """) + Ch.query!(pool, [ + "INSERT INTO connection_test_names_types FORMAT RowBinaryWithNamesAndTypes\n" + | rowbinary + ]) - on_exit(fn -> Ch.Test.query("DROP TABLE ch_nulls") end) + assert Ch.query!(pool, "SELECT * FROM connection_test_names_types ORDER BY country_code").rows == + rows + end - parameterize_query!( - ctx, - "INSERT INTO ch_nulls(a, b, c, d) FORMAT RowBinary", - [[nil, nil, nil, nil]], - types: ["UInt8", "Nullable(UInt8)", "UInt8", "Nullable(UInt8)"] - ) + test "returns RowBinaryWithNamesAndTypes type mismatch errors", %{pool: pool} do + Ch.query!(pool, """ + CREATE TABLE connection_test_names_types_mismatch ( + country_code FixedString(2), + rare_string LowCardinality(String), + maybe_int32 Nullable(Int32) + ) ENGINE Memory + """) + + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_names_types_mismatch") + Ch.stop(cleanup) + end) + + names = ["country_code", "rare_string", "maybe_int32"] + rows = [["AB", "rare", -42]] + + rowbinary = [ + RowBinary.encode_names_and_types(names, ["FixedString(2)", "String", "Nullable(Int32)"]) + | RowBinary.encode_rows(rows, ["FixedString(2)", "String", "Nullable(Int32)"]) + ] - # default is ignored... - assert parameterize_query!(ctx, "SELECT * FROM ch_nulls").rows == [[0, nil, 0, nil]] - end - - # based on https://github.com/ClickHouse/clickhouse-java/pull/1345/files - test "nullable + input() + default", ctx do - parameterize_query!(ctx, """ - CREATE TABLE test_insert_default_value( - n Int32, - s String DEFAULT 'secret' - ) ENGINE Memory - """) - - on_exit(fn -> Ch.Test.query("DROP TABLE test_insert_default_value") end) - - parameterize_query!( - ctx, - """ - INSERT INTO test_insert_default_value - SELECT id, name - FROM input('id UInt32, name Nullable(String)') - FORMAT RowBinary\ - """, - [[1, nil], [-1, nil]], - types: ["UInt32", "Nullable(String)"] - ) + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, [ + "INSERT INTO connection_test_names_types_mismatch FORMAT RowBinaryWithNamesAndTypes\n" + | rowbinary + ]) - assert parameterize_query!(ctx, "SELECT * FROM test_insert_default_value ORDER BY n").rows == - [ - [-1, "secret"], - [1, "secret"] - ] - end + assert message =~ "Type of 'rare_string' must be LowCardinality(String), not String" + end - test "can decode casted Point", ctx do - assert parameterize_query!(ctx, "select cast((0, 1) as Point)").rows == [ - _row = [_point = {0.0, 1.0}] - ] - end + test "inserts and selects geo types", %{pool: pool} do + Ch.query!(pool, "CREATE TABLE connection_test_geo_point(p Point) ENGINE Memory") + Ch.query!(pool, "CREATE TABLE connection_test_geo_ring(r Ring) ENGINE Memory") + Ch.query!(pool, "CREATE TABLE connection_test_geo_polygon(pg Polygon) ENGINE Memory") - test "can encode and then decode Point in query params", ctx do - assert parameterize_query!(ctx, "select {$0:Point}", [{10, 10}]).rows == [ - _row = [_point = {10.0, 10.0}] - ] - end + Ch.query!( + pool, + "CREATE TABLE connection_test_geo_multipolygon(mp MultiPolygon) ENGINE Memory" + ) - test "can insert and select Point", ctx do - parameterize_query!(ctx, "CREATE TABLE geo_point (p Point) ENGINE = Memory()") - on_exit(fn -> Ch.Test.query("DROP TABLE geo_point") end) + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP TABLE connection_test_geo_point") + Ch.query!(cleanup, "DROP TABLE connection_test_geo_ring") + Ch.query!(cleanup, "DROP TABLE connection_test_geo_polygon") + Ch.query!(cleanup, "DROP TABLE connection_test_geo_multipolygon") + Ch.stop(cleanup) + end) - parameterize_query!(ctx, "INSERT INTO geo_point VALUES((10, 10))") + Ch.query!(pool, "INSERT INTO connection_test_geo_point VALUES((10, 10))") - parameterize_query!(ctx, "INSERT INTO geo_point FORMAT RowBinary", [[{20, 20}]], - types: ["Point"] - ) + Ch.query!(pool, [ + "INSERT INTO connection_test_geo_point FORMAT RowBinary\n", + RowBinary.encode_rows([[{20, 20}]], ["Point"]) + ]) - assert parameterize_query!(ctx, "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC").rows == - [ - [{10.0, 10.0}, "Point"], - [{20.0, 20.0}, "Point"] - ] + assert Ch.query!(pool, "SELECT p FROM connection_test_geo_point ORDER BY p").rows == [ + [{10.0, 10.0}], + [{20.0, 20.0}] + ] - # to make our RowBinary is not garbage in garbage out we also test a text format response - assert parameterize_query!( - ctx, - "SELECT p, toTypeName(p) FROM geo_point ORDER BY p ASC FORMAT JSONCompact" - ).rows - |> Jason.decode!() - |> Map.fetch!("data") == [ - [[10, 10], "Point"], - [[20, 20], "Point"] - ] - end + ring = [{20, 20}, {0, 0}, {0, 20}] - test "can decode casted Ring", ctx do - ring = [{0.0, 1.0}, {10.0, 3.0}] + Ch.query!( + pool, + "INSERT INTO connection_test_geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])" + ) - assert parameterize_query!(ctx, "select cast([(0,1),(10,3)] as Ring)").rows == [ - _row = [ring] - ] - end + Ch.query!(pool, [ + "INSERT INTO connection_test_geo_ring FORMAT RowBinary\n", + RowBinary.encode_rows([[ring]], ["Ring"]) + ]) - test "can encode and then decode Ring in query params", ctx do - ring = [{0.0, 1.0}, {10.0, 3.0}] - assert parameterize_query!(ctx, "select {$0:Ring}", [ring]).rows == [_row = [ring]] - end + assert Ch.query!(pool, "SELECT r FROM connection_test_geo_ring ORDER BY r").rows == [ + [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}]], + [[{20.0, 20.0}, {0.0, 0.0}, {0.0, 20.0}]] + ] - test "can insert and select Ring", ctx do - parameterize_query!(ctx, "CREATE TABLE geo_ring (r Ring) ENGINE = Memory()") - on_exit(fn -> Ch.Test.query("DROP TABLE geo_ring") end) + polygon = [[{0, 1.0}, {10, 3.2}], [], [{2, 2}]] - parameterize_query!( - ctx, - "INSERT INTO geo_ring VALUES([(0, 0), (10, 0), (10, 10), (0, 10)])" - ) + Ch.query!( + pool, + "INSERT INTO connection_test_geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]])" + ) - ring = [{20, 20}, {0, 0}, {0, 20}] - parameterize_query!(ctx, "INSERT INTO geo_ring FORMAT RowBinary", [[ring]], types: ["Ring"]) + Ch.query!(pool, [ + "INSERT INTO connection_test_geo_polygon FORMAT RowBinary\n", + RowBinary.encode_rows([[polygon]], ["Polygon"]) + ]) - assert parameterize_query!(ctx, "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC").rows == + assert Ch.query!(pool, "SELECT pg FROM connection_test_geo_polygon ORDER BY pg").rows == [ + [[[{0.0, 1.0}, {10.0, 3.2}], [], [{2.0, 2.0}]]], + [ [ - [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}], "Ring"], - [[{20.0, 20.0}, {0.0, 0.0}, {0.0, 20.0}], "Ring"] + [{20.0, 20.0}, {50.0, 20.0}, {50.0, 50.0}, {20.0, 50.0}], + [{30.0, 30.0}, {50.0, 50.0}, {50.0, 30.0}] ] - - # to make our RowBinary is not garbage in garbage out we also test a text format response - assert parameterize_query!( - ctx, - "SELECT r, toTypeName(r) FROM geo_ring ORDER BY r ASC FORMAT JSONCompact" - ).rows - |> Jason.decode!() - |> Map.fetch!("data") == [ - [[[0, 0], [10, 0], [10, 10], [0, 10]], "Ring"], - [[[20, 20], [0, 0], [0, 20]], "Ring"] ] - end + ] - test "can decode casted Polygon", ctx do - polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] + multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - assert parameterize_query!(ctx, "select cast([[(0,1),(10,3)],[],[(2,2)]] as Polygon)").rows == - [ - _row = [polygon] - ] - end - - test "can encode and then decode Polygon in query params", ctx do - polygon = [[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]] - assert parameterize_query!(ctx, "select {$0:Polygon}", [polygon]).rows == [_row = [polygon]] - end - - test "can insert and select Polygon", ctx do - parameterize_query!(ctx, "CREATE TABLE geo_polygon (pg Polygon) ENGINE = Memory()") - on_exit(fn -> Ch.Test.query("DROP TABLE geo_polygon") end) - - parameterize_query!( - ctx, - "INSERT INTO geo_polygon VALUES([[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]])" - ) - - polygon = [[{0, 1.0}, {10, 3.2}], [], [{2, 2}]] + Ch.query!( + pool, + "INSERT INTO connection_test_geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)], [(30, 30), (50, 50), (50, 30)]]])" + ) - parameterize_query!(ctx, "INSERT INTO geo_polygon FORMAT RowBinary", [[polygon]], - types: ["Polygon"] - ) + Ch.query!(pool, [ + "INSERT INTO connection_test_geo_multipolygon FORMAT RowBinary\n", + RowBinary.encode_rows([[multipolygon]], ["MultiPolygon"]) + ]) - assert parameterize_query!( - ctx, - "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC" - ).rows == + assert Ch.query!(pool, "SELECT mp FROM connection_test_geo_multipolygon ORDER BY mp").rows == + [ [ - [[[{0.0, 1.0}, {10.0, 3.2}], [], [{2.0, 2.0}]], "Polygon"], [ + [[{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}]], [ [{20.0, 20.0}, {50.0, 20.0}, {50.0, 50.0}, {20.0, 50.0}], [{30.0, 30.0}, {50.0, 50.0}, {50.0, 30.0}] - ], - "Polygon" - ] - ] - - # to make our RowBinary is not garbage in garbage out we also test a text format response - assert parameterize_query!( - ctx, - "SELECT pg, toTypeName(pg) FROM geo_polygon ORDER BY pg ASC FORMAT JSONCompact" - ).rows - |> Jason.decode!() - |> Map.fetch!("data") == [ - [[[[0, 1], [10, 3.2]], [], [[2, 2]]], "Polygon"], - [ - [[[20, 20], [50, 20], [50, 50], [20, 50]], [[30, 30], [50, 50], [50, 30]]], - "Polygon" - ] - ] - end - - test "can decode casted MultiPolygon", ctx do - multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - - assert parameterize_query!( - ctx, - "select cast([[[(0,1),(10,3)],[],[(2,2)]],[],[[(3, 3)]]] as MultiPolygon)" - ).rows == [ - _row = [multipolygon] - ] - end - - test "can encode and then decode MultiPolygon in query params", ctx do - multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - - assert parameterize_query!(ctx, "select {$0:MultiPolygon}", [multipolygon]).rows == [ - _row = [multipolygon] - ] - end - - test "can insert and select MultiPolygon", ctx do - parameterize_query!( - ctx, - "CREATE TABLE geo_multipolygon (mpg MultiPolygon) ENGINE = Memory()" - ) - - on_exit(fn -> Ch.Test.query("DROP TABLE geo_multipolygon") end) - - parameterize_query!( - ctx, - "INSERT INTO geo_multipolygon VALUES([[[(0, 0), (10, 0), (10, 10), (0, 10)]], [[(20, 20), (50, 20), (50, 50), (20, 50)],[(30, 30), (50, 50), (50, 30)]]])" - ) - - multipolygon = [[[{0.0, 1.0}, {10.0, 3.0}], [], [{2, 2}]], [], [[{3, 3}]]] - - parameterize_query!(ctx, "INSERT INTO geo_multipolygon FORMAT RowBinary", [[multipolygon]], - types: ["MultiPolygon"] - ) - - assert parameterize_query!( - ctx, - "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC" - ).rows == - [ - _row = [ - _multipolygon = [ - _polygon = [ - _ring = [{0.0, 0.0}, {10.0, 0.0}, {10.0, 10.0}, {0.0, 10.0}] - ], - [ - [{20.0, 20.0}, {50.0, 20.0}, {50.0, 50.0}, {20.0, 50.0}], - [{30.0, 30.0}, {50.0, 50.0}, {50.0, 30.0}] - ] - ], - "MultiPolygon" - ], - [ - [ - [ - [{0.0, 1.0}, {10.0, 3.0}], - [], - [{2.0, 2.0}] - ], - [], - [ - [{3.0, 3.0}] - ] - ], - "MultiPolygon" + ] ] - ] - - # to make our RowBinary is not garbage in garbage out we also test a text format response - assert parameterize_query!( - ctx, - "SELECT mpg, toTypeName(mpg) FROM geo_multipolygon ORDER BY mpg ASC FORMAT JSONCompact" - ).rows - |> Jason.decode!() - |> Map.fetch!("data") == [ - [ - [ - [[[0, 0], [10, 0], [10, 10], [0, 10]]], - [[[20, 20], [50, 20], [50, 50], [20, 50]], [[30, 30], [50, 50], [50, 30]]] - ], - "MultiPolygon" ], - [[[[[0, 1], [10, 3]], [], [[2, 2]]], [], [[[3, 3]]]], "MultiPolygon"] + [[[[{0.0, 1.0}, {10.0, 3.0}], [], [{2.0, 2.0}]], [], [[{3.0, 3.0}]]]] ] - end end - describe "options" do - # this test is flaky, sometimes it raises due to ownership timeout - @tag capture_log: true, skip: true - test "can provide custom timeout", ctx do - assert {:error, %Mint.TransportError{reason: :timeout} = error} = - parameterize_query(ctx, "select sleep(1)", _params = [], timeout: 100) - - assert Exception.message(error) == "timeout" - end - - test "errors on invalid creds", ctx do - assert {:error, %Ch.Error{code: 516} = error} = - parameterize_query(ctx, "select 1 + 1", _params = [], - username: "no-exists", - password: "wrong" - ) - - assert Exception.message(error) =~ - "Code: 516. DB::Exception: no-exists: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED)" - end - - test "errors on invalid database", ctx do - assert {:error, %Ch.Error{code: 81} = error} = - parameterize_query(ctx, "select 1 + 1", _params = [], database: "no-db") - - assert Exception.message(error) =~ "`no-db`" - assert Exception.message(error) =~ "UNKNOWN_DATABASE" - end - - test "can provide custom database", ctx do - assert {:ok, %{num_rows: 1, rows: [[2]]}} = - parameterize_query(ctx, "select 1 + 1", [], database: "default") - end - end + test "accepts database and auth through headers", %{pool: pool} do + Ch.query!(pool, "CREATE DATABASE connection_test_database_header") - describe "transactions" do - test "commit", ctx do - DBConnection.transaction(ctx.conn, fn conn -> - ctx = Map.put(ctx, :conn, conn) - parameterize_query!(ctx, "select 1 + 1") - end) - end - - test "rollback", ctx do - DBConnection.transaction(ctx.conn, fn conn -> - DBConnection.rollback(conn, :some_reason) - end) - end - - test "status", ctx do - assert DBConnection.status(ctx.conn) == :idle - end - end + on_exit(fn -> + {:ok, cleanup} = Ch.start_link() + Ch.query!(cleanup, "DROP DATABASE connection_test_database_header") + Ch.stop(cleanup) + end) - describe "stream" do - test "emits result structs containing raw data", ctx do - results = - DBConnection.run(ctx.conn, fn conn -> - conn - |> Ch.stream( - "select number from system.numbers limit {limit:UInt64}", - %{"limit" => 10_000}, - decode: false - ) - |> Enum.into([]) - end) - - assert length(results) >= 2 - - assert results - |> Enum.map(& &1.data) - |> IO.iodata_to_binary() - |> RowBinary.decode_rows() == Enum.map(0..9999, &[&1]) - end - - test "disconnects on early halt", ctx do - logs = - ExUnit.CaptureLog.capture_log(fn -> - Ch.run(ctx.conn, fn conn -> - conn |> Ch.stream("select number from system.numbers") |> Enum.take(1) - end) - - assert parameterize_query!(ctx, "select 1 + 1").rows == [[2]] - end) - - assert logs =~ - "disconnected: ** (Ch.Error) stopping stream before receiving full response by closing connection" - end - end + Ch.query!( + pool, + "CREATE TABLE connection_test_database_header.example(a UInt8) ENGINE Memory" + ) - describe "prepare" do - test "no-op", ctx do - query = Ch.Query.build("select 1 + 1") + assert Ch.query!(pool, "SHOW TABLES", %{}, + headers: [{"x-clickhouse-database", "connection_test_database_header"}] + ).rows == [["example"]] - assert {:error, %Ch.Error{message: "prepared statements are not supported"}} = - DBConnection.prepare(ctx.conn, query) - end - end + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, "SELECT 1", %{}, + headers: [{"x-clickhouse-user", "no-exists"}, {"x-clickhouse-key", "wrong"}] + ) - describe "start_link/1" do - test "can pass options to start_link/1", ctx do - db = "#{Ch.Test.database()}_#{System.unique_integer([:positive])}" - Ch.Test.query("CREATE DATABASE {db:Identifier}", %{"db" => db}) - on_exit(fn -> Ch.Test.query("DROP DATABASE {db:Identifier}", %{"db" => db}) end) - - {:ok, conn} = Ch.start_link(database: db) - ctx = Map.put(ctx, :conn, conn) - parameterize_query!(ctx, "create table example(a UInt8) engine=Memory") - assert {:ok, %{rows: [["example"]]}} = parameterize_query(ctx, "show tables") - end - - test "can start without options", ctx do - {:ok, conn} = Ch.start_link() - ctx = Map.put(ctx, :conn, conn) - assert {:ok, %{num_rows: 1, rows: [[2]]}} = parameterize_query(ctx, "select 1 + 1") - end - end + assert message =~ "AUTHENTICATION_FAILED" - describe "RowBinaryWithNamesAndTypes" do - setup ctx do - parameterize_query!(ctx, """ - create table if not exists row_binary_names_and_types_t ( - country_code FixedString(2), - rare_string LowCardinality(String), - maybe_int32 Nullable(Int32) - ) engine Memory - """) - - on_exit(fn -> Ch.Test.query("truncate row_binary_names_and_types_t") end) - end - - test "error on type mismatch", ctx do - stmt = "insert into row_binary_names_and_types_t format RowBinaryWithNamesAndTypes" - rows = [["AB", "rare", -42]] - names = ["country_code", "rare_string", "maybe_int32"] - - opts = [ - names: names, - types: [Ch.Types.fixed_string(2), Ch.Types.string(), Ch.Types.nullable(Ch.Types.u32())] - ] - - assert {:error, %Ch.Error{code: 117, message: message}} = - parameterize_query(ctx, stmt, rows, opts) - - assert message =~ "Type of 'rare_string' must be LowCardinality(String), not String" - - opts = [ - names: names, - types: [ - Ch.Types.fixed_string(2), - Ch.Types.low_cardinality(Ch.Types.string()), - Ch.Types.nullable(Ch.Types.u32()) - ] - ] - - assert {:error, %Ch.Error{code: 117, message: message}} = - parameterize_query(ctx, stmt, rows, opts) - - assert message =~ "Type of 'maybe_int32' must be Nullable(Int32), not Nullable(UInt32)" - end - - test "ok on valid types", ctx do - stmt = "insert into row_binary_names_and_types_t format RowBinaryWithNamesAndTypes" - rows = [["AB", "rare", -42]] - names = ["country_code", "rare_string", "maybe_int32"] - - opts = [ - names: names, - types: [ - Ch.Types.fixed_string(2), - Ch.Types.low_cardinality(Ch.Types.string()), - Ch.Types.nullable(Ch.Types.i32()) - ] - ] - - parameterize_query(ctx, stmt, rows, opts) - - assert parameterize_query!(ctx, "select * from row_binary_names_and_types_t").rows == [ - ["AB", "rare", -42] - ] - end + assert {:error, %Ch.Error{message: message}} = + Ch.query(pool, "SELECT 1", %{}, headers: [{"x-clickhouse-database", "no-db"}]) - test "select with lots of columns", ctx do - select = Enum.map_join(1..1000, ", ", fn i -> "#{i} as col_#{i}" end) - stmt = "select #{select} format RowBinaryWithNamesAndTypes" + assert message =~ "UNKNOWN_DATABASE" + end - assert %Ch.Result{columns: columns, rows: [row]} = parameterize_query!(ctx, stmt) + test "selects many columns in RowBinaryWithNamesAndTypes", %{pool: pool} do + select = Enum.map_join(1..1000, ", ", fn i -> "#{i} AS col_#{i}" end) - assert length(columns) == 1000 - assert List.first(columns) == "col_1" - assert List.last(columns) == "col_1000" + assert %{names: columns, rows: [row]} = Ch.query!(pool, "SELECT #{select}") - assert length(row) == 1000 - assert List.first(row) == 1 - assert List.last(row) == 1000 - end + assert length(columns) == 1000 + assert List.first(columns) == "col_1" + assert List.last(columns) == "col_1000" + assert length(row) == 1000 + assert List.first(row) == 1 + assert List.last(row) == 1000 end end diff --git a/test/ch/decimal_param_test.exs b/test/ch/decimal_param_test.exs index da996b21..0e4d5e6e 100644 --- a/test/ch/decimal_param_test.exs +++ b/test/ch/decimal_param_test.exs @@ -1,15 +1,9 @@ defmodule Ch.DecimalParamTest do - use ExUnit.Case, - parameterize: [%{query_options: []}, %{query_options: [multipart: true]}], - async: true - + use ExUnit.Case, async: true use ExUnitProperties - import Ch.Test, only: [parameterize_query: 4, parameterize_query!: 4] - - setup ctx do - {:ok, conn} = Ch.start_link() - {:ok, conn: conn, query_options: ctx[:query_options] || []} + setup do + {:ok, pool: start_supervised!(Ch), query_options: []} end test "decimal parameter boundaries", ctx do @@ -92,9 +86,9 @@ defmodule Ch.DecimalParamTest do end defp assert_decimal_param(ctx, decimal, type, expected) do - assert %Ch.Result{rows: [[actual, ^type]]} = - parameterize_query!( - ctx, + assert %{rows: [[actual, ^type]]} = + Ch.query!( + ctx.pool, "select {d:#{type}} as x, toTypeName(x)", %{"d" => decimal}, ctx.query_options @@ -105,8 +99,8 @@ defmodule Ch.DecimalParamTest do defp decimal_error(ctx, decimal, type) do assert {:error, error} = - parameterize_query( - ctx, + Ch.query( + ctx.pool, "select {d:#{type}}", %{"d" => decimal}, ctx.query_options @@ -132,14 +126,12 @@ defmodule Ch.DecimalParamTest do end defp encoded_decimal_param(query_options, decimal) do - query = Ch.Query.build("select {d:Decimal(76, 0)}", query_options) - - {query_params, _headers, body} = - DBConnection.Query.encode(query, %{"d" => decimal}, []) - - case query_params do - [{"param_d", value}] -> value - [] -> IO.iodata_to_binary(body) - end + query_options + |> Keyword.get(:settings, []) + |> then(&Ch.HTTP.path(%{"d" => decimal}, &1)) + |> URI.parse() + |> Map.fetch!(:query) + |> URI.decode_query() + |> Map.fetch!("param_d") end end diff --git a/test/ch/dynamic_test.exs b/test/ch/dynamic_test.exs index 7a916d69..9fbec93e 100644 --- a/test/ch/dynamic_test.exs +++ b/test/ch/dynamic_test.exs @@ -1,31 +1,28 @@ defmodule Ch.DynamicTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - import Ch.Test, only: [parameterize_query!: 2, parameterize_query!: 3, parameterize_query!: 4] + use ExUnit.Case, async: true @moduletag :dynamic setup do - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} + {:ok, pool: start_supervised!(Ch)} end - test "it works", ctx do + test "it works", %{pool: pool} do select = fn literal -> - [row] = parameterize_query!(ctx, "select #{literal}::Dynamic as d, dynamicType(d)").rows + [row] = Ch.query!(pool, "select #{literal}::Dynamic as d, dynamicType(d)").rows row end - parameterize_query!(ctx, "CREATE TABLE test (d Dynamic, id String) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) + Help.query!("CREATE TABLE dynamic_test (d Dynamic, id String) ENGINE = Memory;") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) insert = fn value -> id = inspect(value) + rowbinary = Ch.RowBinary.encode_row([value, id], ["Dynamic", "String"]) + Ch.query!(pool, ["insert into dynamic_test(d, id) format RowBinary\n" | rowbinary]) - parameterize_query!(ctx, "insert into test(d, id) format RowBinary", [[value, id]], - types: ["Dynamic", "String"] - ).rows - - [[inserted]] = - parameterize_query!(ctx, "select d from test where id = {id:String}", %{"id" => id}).rows + %{rows: [[inserted]]} = + Ch.query!(pool, "select d from dynamic_test where id = {id:String}", %{"id" => id}) inserted end @@ -96,10 +93,7 @@ defmodule Ch.DynamicTest do assert select.("'2020-01-01'::Date32") == [~D[2020-01-01], "Date32"] # DateTime 0x11 - assert select.("'2020-01-01 12:34:56'::DateTime") == [ - Ch.Test.to_clickhouse_naive(ctx.conn, ~N[2020-01-01 12:34:56]), - "DateTime" - ] + assert select.("'2020-01-01 12:34:56'::DateTime") == [~N[2020-01-01 12:34:56], "DateTime"] assert insert.(~N[2020-01-01 12:34:56]) == ~N[2020-01-01 12:34:56] @@ -111,10 +105,7 @@ defmodule Ch.DynamicTest do # DateTime64(P) 0x13 assert select.("'2020-01-01 12:34:56.123456'::DateTime64(6)") == - [ - Ch.Test.to_clickhouse_naive(ctx.conn, ~N[2020-01-01 12:34:56.123456]), - "DateTime64(6)" - ] + [~N[2020-01-01 12:34:56.123456], "DateTime64(6)"] # DateTime64(P, time_zone) 0x14 assert [dt64, "DateTime64(6, 'Europe/Prague')"] = @@ -270,17 +261,17 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#creating-dynamic - test "creating dynamic", ctx do + test "creating dynamic", %{pool: pool} do # Using Dynamic type in table column definition: - parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) + Help.query!("CREATE TABLE dynamic_test (d Dynamic) ENGINE = Memory;") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) - parameterize_query!( - ctx, - "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" + Ch.query!( + pool, + "INSERT INTO dynamic_test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" ) - assert parameterize_query!(ctx, "SELECT d, dynamicType(d) FROM test;").rows == [ + assert Ch.query!(pool, "SELECT d, dynamicType(d) FROM dynamic_test;").rows == [ [nil, "None"], [42, "Int64"], ["Hello, World!", "String"], @@ -288,20 +279,20 @@ defmodule Ch.DynamicTest do ] # Using CAST from ordinary column: - assert parameterize_query!(ctx, "SELECT 'Hello, World!'::Dynamic AS d, dynamicType(d);").rows == + assert Ch.query!(pool, "SELECT 'Hello, World!'::Dynamic AS d, dynamicType(d);").rows == [ ["Hello, World!", "String"] ] # Using CAST from Variant column: - assert parameterize_query!( - ctx, + assert Ch.query!( + pool, "SELECT multiIf((number % 3) = 0, number, (number % 3) = 1, range(number + 1), NULL)::Dynamic AS d, dynamicType(d) FROM numbers(3)", - [], - settings: [ - enable_variant_type: 1, - use_variant_as_common_type: 1 - ] + _params = %{}, + settings: %{ + "enable_variant_type" => 1, + "use_variant_as_common_type" => 1 + } ).rows == [ [0, "UInt64"], [[0, 1], "Array(UInt64)"], @@ -310,18 +301,18 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#reading-dynamic-nested-types-as-subcolumns - test "reading dynamic nested types as subcolumns", ctx do - parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) + test "reading dynamic nested types as subcolumns", %{pool: pool} do + Help.query!("CREATE TABLE dynamic_test (d Dynamic) ENGINE = Memory;") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) - parameterize_query!( - ctx, - "INSERT INTO test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" + Ch.query!( + pool, + "INSERT INTO dynamic_test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" ) - assert parameterize_query!( - ctx, - "SELECT d, dynamicType(d), d.String, d.Int64, d.`Array(Int64)`, d.Date, d.`Array(String)` FROM test;" + assert Ch.query!( + pool, + "SELECT d, dynamicType(d), d.String, d.Int64, d.`Array(Int64)`, d.Date, d.`Array(String)` FROM dynamic_test;" ).rows == [ [nil, "None", nil, nil, [], nil, []], [42, "Int64", nil, 42, [], nil, []], @@ -329,9 +320,9 @@ defmodule Ch.DynamicTest do [[1, 2, 3], "Array(Int64)", nil, nil, [1, 2, 3], nil, []] ] - assert parameterize_query!( - ctx, - "SELECT toTypeName(d.String), toTypeName(d.Int64), toTypeName(d.`Array(Int64)`), toTypeName(d.Date), toTypeName(d.`Array(String)`) FROM test LIMIT 1;" + assert Ch.query!( + pool, + "SELECT toTypeName(d.String), toTypeName(d.Int64), toTypeName(d.`Array(Int64)`), toTypeName(d.Date), toTypeName(d.`Array(String)`) FROM dynamic_test LIMIT 1;" ).rows == [ [ "Nullable(String)", @@ -342,9 +333,9 @@ defmodule Ch.DynamicTest do ] ] - assert parameterize_query!( - ctx, - "SELECT d, dynamicType(d), dynamicElement(d, 'String'), dynamicElement(d, 'Int64'), dynamicElement(d, 'Array(Int64)'), dynamicElement(d, 'Date'), dynamicElement(d, 'Array(String)') FROM test;" + assert Ch.query!( + pool, + "SELECT d, dynamicType(d), dynamicElement(d, 'String'), dynamicElement(d, 'Int64'), dynamicElement(d, 'Array(Int64)'), dynamicElement(d, 'Date'), dynamicElement(d, 'Array(String)') FROM dynamic_test;" ).rows == [ [nil, "None", nil, nil, [], nil, []], [42, "Int64", nil, 42, [], nil, []], @@ -354,12 +345,12 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-string-column-to-a-dynamic-column-through-parsing - test "converting a string column to a dynamic column through parsing", ctx do - assert parameterize_query!( - ctx, + test "converting a string column to a dynamic column through parsing", %{pool: pool} do + assert Ch.query!( + pool, "SELECT CAST(materialize(map('key1', '42', 'key2', 'true', 'key3', '2020-01-01')), 'Map(String, Dynamic)') as map_of_dynamic, mapApply((k, v) -> (k, dynamicType(v)), map_of_dynamic) as map_of_dynamic_types;", - [], - settings: [cast_string_to_dynamic_use_inference: 1] + _params = %{}, + settings: %{"cast_string_to_dynamic_use_inference" => 1} ).rows == [ [ %{"key1" => 42, "key2" => true, "key3" => ~D[2020-01-01]}, @@ -369,12 +360,12 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamic-column-to-an-ordinary-column - test "converting a dynamic column to an ordinary column", ctx do - parameterize_query!(ctx, "CREATE TABLE test (d Dynamic) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - parameterize_query!(ctx, "INSERT INTO test VALUES (NULL), (42), ('42.42'), (true), ('e10');") + test "converting a dynamic column to an ordinary column", %{pool: pool} do + Help.query!("CREATE TABLE dynamic_test (d Dynamic) ENGINE = Memory;") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) + Ch.query!(pool, "INSERT INTO dynamic_test VALUES (NULL), (42), ('42.42'), (true), ('e10');") - assert parameterize_query!(ctx, "SELECT d::Nullable(Float64) FROM test;").rows == [ + assert Ch.query!(pool, "SELECT d::Nullable(Float64) FROM dynamic_test;").rows == [ [nil], [42.0], [42.42], @@ -384,16 +375,15 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-variant-column-to-dynamic-column - test "converting a variant column to dynamic column", ctx do - parameterize_query!( - ctx, - "CREATE TABLE test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory;" + test "converting a variant column to dynamic column", %{pool: pool} do + Help.query!( + "CREATE TABLE dynamic_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory;" ) - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) - parameterize_query!(ctx, "INSERT INTO test VALUES (NULL), (42), ('String'), ([1, 2, 3]);") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) + Ch.query!(pool, "INSERT INTO dynamic_test VALUES (NULL), (42), ('String'), ([1, 2, 3]);") - assert parameterize_query!(ctx, "SELECT v::Dynamic AS d, dynamicType(d) FROM test;").rows == [ + assert Ch.query!(pool, "SELECT v::Dynamic AS d, dynamicType(d) FROM dynamic_test;").rows == [ [nil, "None"], [42, "UInt64"], ["String", "String"], @@ -402,18 +392,18 @@ defmodule Ch.DynamicTest do end # https://clickhouse.com/docs/sql-reference/data-types/dynamic#converting-a-dynamicmax_typesn-column-to-another-dynamicmax_typesk - test "converting a Dynamic(max_types=N) column to another Dynamic(max_types=K)", ctx do - parameterize_query!(ctx, "CREATE TABLE test (d Dynamic(max_types=4)) ENGINE = Memory;") - on_exit(fn -> Ch.Test.query("DROP TABLE test") end) + test "converting a Dynamic(max_types=N) column to another Dynamic(max_types=K)", %{pool: pool} do + Help.query!("CREATE TABLE dynamic_test (d Dynamic(max_types=4)) ENGINE = Memory;") + on_exit(fn -> Help.query!("DROP TABLE dynamic_test") end) - parameterize_query!( - ctx, - "INSERT INTO test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);" + Ch.query!( + pool, + "INSERT INTO dynamic_test VALUES (NULL), (42), (43), ('42.42'), (true), ([1, 2, 3]);" ) - assert parameterize_query!( - ctx, - "SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM test;" + assert Ch.query!( + pool, + "SELECT d::Dynamic(max_types=5) as d2, dynamicType(d2) FROM dynamic_test;" ).rows == [ [nil, "None"], @@ -424,9 +414,9 @@ defmodule Ch.DynamicTest do [[1, 2, 3], "Array(Int64)"] ] - assert parameterize_query!( - ctx, - "SELECT d, dynamicType(d), d::Dynamic(max_types=2) as d2, dynamicType(d2), isDynamicElementInSharedData(d2) FROM test;" + assert Ch.query!( + pool, + "SELECT d, dynamicType(d), d::Dynamic(max_types=2) as d2, dynamicType(d2), isDynamicElementInSharedData(d2) FROM dynamic_test;" ).rows == [ [nil, "None", nil, "None", false], [42, "Int64", 42, "Int64", false], diff --git a/test/ch/faults_test.exs b/test/ch/faults_test.exs index bcc7457a..c1e60fb1 100644 --- a/test/ch/faults_test.exs +++ b/test/ch/faults_test.exs @@ -1,551 +1,97 @@ defmodule Ch.FaultsTest do - alias Ch.Result - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - import Ch.Test, only: [intercept_packets: 1] - - defp capture_async_log(f) do - ExUnit.CaptureLog.capture_log([async: true], f) - end + use ExUnit.Case, async: true @socket_opts [:binary, {:active, true}, {:packet, :raw}] setup do - # this setup makes the test act as MITM for clickhouse and ch's http conn (mint) - # allowing the test to intercept, slow down, and modify packets to cause failures {:ok, clickhouse} = :gen_tcp.connect({127, 0, 0, 1}, 8123, @socket_opts) {:ok, listen} = :gen_tcp.listen(0, @socket_opts) {:ok, port} = :inet.port(listen) {:ok, clickhouse: clickhouse, listen: listen, port: port} end - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - describe "connect/1" do - test "reconnects to eventually reachable server", ctx do - %{listen: listen, port: port, clickhouse: clickhouse, query_options: query_options} = ctx - - # make the server unreachable - :ok = :gen_tcp.close(listen) - test = self() - - {:ok, conn} = Ch.start_link(port: port, queue_interval: 100, backoff_min: 0) - - log = - capture_async_log(fn -> - assert {:error, %DBConnection.ConnectionError{reason: :queue_timeout}} = - Ch.query(conn, "select 1 + 1", [], query_options) - - # make the server reachable - {:ok, listen} = :gen_tcp.listen(port, @socket_opts) - {:ok, mint} = :gen_tcp.accept(listen) + test "returns transport errors when ClickHouse is unreachable", %{listen: listen, port: port} do + :ok = :gen_tcp.close(listen) + {:ok, pool} = Ch.start_link(url: "http://localhost:#{port}") - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) + assert {:error, %Mint.TransportError{reason: reason}} = + Ch.query(pool, "select 1", %{}, timeout: 100) - spawn_link(fn -> - assert {:ok, %{num_rows: 1, rows: [[2]]}} = - Ch.query(conn, "select 1 + 1", [], query_options) - - send(test, :done) - end) - - # select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert_receive :done - refute_receive _anything - end) - - assert log =~ "failed to connect: ** (Mint.TransportError) connection refused" - end + assert reason in [:econnrefused, :closed] end - describe "connect/1 handshake" do - test "reconnects after timeout", %{port: port, listen: listen, clickhouse: clickhouse} do - log = - capture_async_log(fn -> - Ch.start_link(port: port, timeout: 100, backoff_min: 0) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # failed handshake - handshake = intercept_packets(mint) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - handshake = intercept_packets(mint) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - end) - - assert log =~ "failed to connect: ** (Mint.TransportError) timeout" - end - - test "reconnects after closed", %{port: port, listen: listen, clickhouse: clickhouse} do - log = - capture_async_log(fn -> - Ch.start_link(port: port, backoff_min: 0) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # failed handshake - handshake = intercept_packets(mint) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - :gen_tcp.close(mint) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - handshake = intercept_packets(mint) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - end) - - assert log =~ "failed to connect: ** (Mint.TransportError) socket closed" - end - - test "reconnects after unexpected status code", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx - - log = - capture_async_log(fn -> - Ch.start_link(port: port, backoff_min: 0) - - # connect - {:ok, mint1} = :gen_tcp.accept(listen) - - # failed handshake - handshake = intercept_packets(mint1) - assert handshake =~ "select 1, version()" - altered_handshake = String.replace(handshake, "select 1", "select x") - :ok = :gen_tcp.send(clickhouse, altered_handshake) - :ok = :gen_tcp.send(mint1, intercept_packets(clickhouse)) - - # reconnect - {:ok, mint2} = :gen_tcp.accept(listen) - - # handshake - handshake = intercept_packets(mint2) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint2, intercept_packets(clickhouse)) + test "removes a timed out connection and reconnects on the next query", ctx do + %{port: port, listen: listen, clickhouse: clickhouse} = ctx + {:ok, pool} = Ch.start_link(url: "http://localhost:#{port}") - # no socket leak - refute Port.info(mint1) - assert Port.info(mint2) - end) + select = + Task.async(fn -> + Ch.query(pool, "select 1 + 1", %{}, timeout: 100) + end) - assert log =~ "UNKNOWN_IDENTIFIER" - end - - test "reconnects after incorrect query result", ctx do - %{port: port, listen: listen, clickhouse: clickhouse} = ctx - - log = - capture_async_log(fn -> - Ch.start_link(port: port, backoff_min: 0) - - # connect - {:ok, mint1} = :gen_tcp.accept(listen) + {:ok, mint} = :gen_tcp.accept(listen) + :ok = :gen_tcp.send(clickhouse, read_packets(mint)) + :ok = :gen_tcp.send(mint, first_byte(read_packets(clickhouse))) - # failed handshake - handshake = intercept_packets(mint1) - assert handshake =~ "select 1, version()" + assert {:error, %Mint.TransportError{reason: :timeout}} = Task.await(select) - altered_handshake = - String.replace(handshake, "select 1, version()", "select 2, version()") + select = + Task.async(fn -> + Ch.query(pool, "select 1 + 1", %{}, timeout: 1_000) + end) - :ok = :gen_tcp.send(clickhouse, altered_handshake) - :ok = :gen_tcp.send(mint1, intercept_packets(clickhouse)) + {:ok, mint} = :gen_tcp.accept(listen) + :ok = :gen_tcp.send(clickhouse, read_packets(mint)) + :ok = :gen_tcp.send(mint, read_packets(clickhouse)) - # reconnect - {:ok, mint2} = :gen_tcp.accept(listen) - - # handshake - handshake = intercept_packets(mint2) - assert handshake =~ "select 1, version()" - :ok = :gen_tcp.send(clickhouse, handshake) - :ok = :gen_tcp.send(mint2, intercept_packets(clickhouse)) - - # no socket leak - refute Port.info(mint1) - assert Port.info(mint2) - end) - - assert log =~ "failed to connect: ** (Ch.Error) unexpected result for 'select 1, version()'" - end + assert {:ok, %{rows: [[2]]}} = Task.await(select) end - describe "ping/1" do - test "reconnects after timeout", %{port: port, listen: listen, clickhouse: clickhouse} do - log = - capture_async_log(fn -> - Ch.start_link(port: port, timeout: 100, idle_interval: 20) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # failed ping - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # ping - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - end) - - assert log =~ "disconnected: ** (Mint.TransportError) timeout" - end - - test "reconnects after close", %{port: port, listen: listen, clickhouse: clickhouse} do - log = - capture_async_log(fn -> - Ch.start_link(port: port, idle_interval: 40) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) + test "removes a closed connection and reconnects on the next query", ctx do + %{port: port, listen: listen, clickhouse: clickhouse} = ctx + {:ok, pool} = Ch.start_link(url: "http://localhost:#{port}") - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) + select = + Task.async(fn -> + Ch.query(pool, "select 1 + 1", %{}, timeout: 1_000) + end) - # failed ping - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - :ok = :gen_tcp.close(mint) + {:ok, mint} = :gen_tcp.accept(listen) + :ok = :gen_tcp.send(clickhouse, read_packets(mint)) + :ok = :gen_tcp.send(mint, first_byte(read_packets(clickhouse))) + :ok = :gen_tcp.close(mint) - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) + assert {:error, %Mint.TransportError{reason: :closed}} = Task.await(select) - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) + select = + Task.async(fn -> + Ch.query(pool, "select 1 + 1", %{}, timeout: 1_000) + end) - # ping - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - end) + {:ok, mint} = :gen_tcp.accept(listen) + :ok = :gen_tcp.send(clickhouse, read_packets(mint)) + :ok = :gen_tcp.send(mint, read_packets(clickhouse)) - assert log =~ "disconnected: ** (Mint.TransportError) socket closed" - end + assert {:ok, %{rows: [[2]]}} = Task.await(select) end - describe "query" do - test "reconnects after timeout", %{ - port: port, - listen: listen, - clickhouse: clickhouse, - query_options: query_options - } do - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(port: port, timeout: 100) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - select = - Task.async(fn -> - Ch.query(conn, "select 1 + 1", [], query_options) - end) - - # failed select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert {:ok, %Ch.Result{rows: [[2]]}} = Task.await(select) - end) - - assert log =~ "disconnected: ** (Mint.TransportError) timeout" + defp read_packets(socket) do + receive do + {:tcp, ^socket, packet} -> read_packets(socket, packet) + {:tcp_closed, ^socket} -> "" end + end - test "reconnects after closed on response", ctx do - %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx - - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(port: port) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - select = - Task.async(fn -> - Ch.query(conn, "select 1 + 1", [], query_options) - end) - - # failed select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, first_byte(intercept_packets(clickhouse))) - :ok = :gen_tcp.close(mint) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert {:ok, %{rows: [[2]]}} = Task.await(select) - end) - - assert log =~ "disconnected: ** (Mint.TransportError) socket closed" - end - - test "reconnects after Connection: close response from server", ctx do - %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx - - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(port: port) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - select = - Task.async(fn -> - Ch.query(conn, "select 1 + 1", [], query_options) - end) - - # first select 1 + 1 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - - response = - String.replace( - intercept_packets(clickhouse), - "Connection: Keep-Alive", - "Connection: Close" - ) - - assert response =~ "Connection: Close" - - :ok = :gen_tcp.send(mint, response) - :ok = :gen_tcp.close(mint) - - assert {:ok, %Ch.Result{rows: [[2]]}} = Task.await(select) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - select = - Task.async(fn -> - Ch.query(conn, "select 2 + 2", [], query_options) - end) - - # select 2 + 2 - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert {:ok, %Ch.Result{rows: [[4]]}} = Task.await(select) - end) - - assert log =~ "disconnected: ** (Mint.HTTPError) the connection is closed" - end - - # TODO non-chunked request - - test "reconnects after closed before streaming request", ctx do - %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx - - rows = [[1, 2], [3, 4]] - stream = Stream.map(rows, fn row -> Ch.RowBinary.encode_row(row, [:u8, :u8]) end) - - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(database: Ch.Test.database(), port: port) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # disconnect before insert - :ok = :gen_tcp.close(mint) - - insert = - Task.async(fn -> - Ch.query( - conn, - "insert into unknown_table(a,b) format RowBinary", - stream, - Keyword.merge(query_options, encode: false) - ) - end) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # insert - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert {:error, %Ch.Error{code: 60, message: message}} = Task.await(insert) - assert message =~ ~r/UNKNOWN_TABLE/ - end) - - assert log =~ "disconnected: ** (Mint.TransportError) socket closed" - end - - test "reconnects after closed while streaming request", ctx do - %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx - - rows = [[1, 2], [3, 4]] - stream = Stream.map(rows, fn row -> Ch.RowBinary.encode_row(row, [:u8, :u8]) end) - - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(database: Ch.Test.database(), port: port) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - insert = - Task.async(fn -> - Ch.query( - conn, - "insert into unknown_table(a,b) format RowBinary", - stream, - Keyword.merge(query_options, encode: false) - ) - end) - - # close after first packet from mint arrives - assert_receive {:tcp, ^mint, _packet} - :ok = :gen_tcp.close(mint) - - # reconnect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - # insert - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - assert {:error, %Ch.Error{code: 60, message: message}} = Task.await(insert) - assert message =~ ~r/UNKNOWN_TABLE/ - end) - - assert log =~ "disconnected: ** (Mint.TransportError) socket closed" - end - - test "warns on different server name", ctx do - %{port: port, listen: listen, clickhouse: clickhouse, query_options: query_options} = ctx - test = self() - - header = "X-ClickHouse-Server-Display-Name" - %Result{headers: headers} = Ch.Test.query("select 1") - {_, expected_name} = List.keyfind!(headers, String.downcase(header), 0) - - log = - capture_async_log(fn -> - {:ok, conn} = Ch.start_link(port: port) - - # connect - {:ok, mint} = :gen_tcp.accept(listen) - - # handshake - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - :ok = :gen_tcp.send(mint, intercept_packets(clickhouse)) - - spawn_link(fn -> - assert {:ok, %Result{rows: [[1]]}} = Ch.query(conn, "select 1", [], query_options) - send(test, :done) - end) - - # query - :ok = :gen_tcp.send(clickhouse, intercept_packets(mint)) - - response = - String.replace( - intercept_packets(clickhouse), - "#{header}: #{expected_name}", - "#{header}: not-#{expected_name}" - ) - - :ok = :gen_tcp.send(mint, response) - - assert_receive :done - end) - - assert log =~ - "[warning] Server mismatch detected." <> - " Expected \"#{expected_name}\" but got \"not-#{expected_name}\"!" <> - " Connection pooling might be unstable." + defp read_packets(socket, acc) do + receive do + {:tcp, ^socket, packet} -> read_packets(socket, [acc | packet]) + {:tcp_closed, ^socket} -> acc + after + 10 -> acc end end defp first_byte(binary) do - :binary.part(binary, 0, 1) + :binary.part(IO.iodata_to_binary(binary), 0, 1) end end diff --git a/test/ch/headers_test.exs b/test/ch/headers_test.exs deleted file mode 100644 index 2d3da43b..00000000 --- a/test/ch/headers_test.exs +++ /dev/null @@ -1,77 +0,0 @@ -defmodule Ch.HeadersTest do - use ExUnit.Case, - async: true, - parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - - setup do - {:ok, conn} = Ch.start_link() - {:ok, conn: conn} - end - - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - test "can request gzipped response through headers", %{conn: conn, query_options: query_options} do - assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query( - conn, - "select number from system.numbers limit 100", - [], - Keyword.merge(query_options, - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "gzip"}] - ) - ) - - assert :proplists.get_value("content-type", headers) == "application/octet-stream" - assert :proplists.get_value("content-encoding", headers) == "gzip" - assert :proplists.get_value("x-clickhouse-format", headers) == "RowBinaryWithNamesAndTypes" - - # https://en.wikipedia.org/wiki/Gzip - assert <<0x1F, 0x8B, _rest::bytes>> = IO.iodata_to_binary(data) - end - - test "can request lz4 response through headers", %{conn: conn, query_options: query_options} do - assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query( - conn, - "select number from system.numbers limit 100", - [], - Keyword.merge(query_options, - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "lz4"}] - ) - ) - - assert :proplists.get_value("content-type", headers) == "application/octet-stream" - assert :proplists.get_value("content-encoding", headers) == "lz4" - assert :proplists.get_value("x-clickhouse-format", headers) == "RowBinaryWithNamesAndTypes" - - # https://en.wikipedia.org/wiki/LZ4_(compression_algorithm) - assert <<0x04, 0x22, 0x4D, 0x18, _rest::bytes>> = IO.iodata_to_binary(data) - end - - test "can request zstd response through headers", %{conn: conn, query_options: query_options} do - assert {:ok, %{rows: data, data: data, headers: headers}} = - Ch.query( - conn, - "select number from system.numbers limit 100", - [], - Keyword.merge(query_options, - decode: false, - settings: [enable_http_compression: 1], - headers: [{"accept-encoding", "zstd"}] - ) - ) - - assert :proplists.get_value("content-type", headers) == "application/octet-stream" - assert :proplists.get_value("content-encoding", headers) == "zstd" - assert :proplists.get_value("x-clickhouse-format", headers) == "RowBinaryWithNamesAndTypes" - - # https://en.wikipedia.org/wiki/LZ4_(compression_algorithm) - assert <<0x28, 0xB5, 0x2F, 0xFD, _rest::bytes>> = IO.iodata_to_binary(data) - end -end diff --git a/test/ch/http_test.exs b/test/ch/http_test.exs deleted file mode 100644 index c0f802b4..00000000 --- a/test/ch/http_test.exs +++ /dev/null @@ -1,66 +0,0 @@ -defmodule Ch.HTTPTest do - use ExUnit.Case, - async: true, - parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - - @moduletag :slow - - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - describe "user-agent" do - setup do - {:ok, ch: start_supervised!(Ch)} - end - - test "sets user-agent to ch/ by default", %{ch: ch, query_options: query_options} do - %Ch.Result{rows: [[123]], headers: resp_header} = - Ch.query!(ch, "select 123", [], query_options) - - {"x-clickhouse-query-id", query_id} = List.keyfind!(resp_header, "x-clickhouse-query-id", 0) - - assert query_http_user_agent(ch, query_id, query_options) == - "ch/" <> Mix.Project.config()[:version] - end - - test "uses the provided user-agent", %{ch: ch, query_options: query_options} do - req_headers = [{"user-agent", "plausible/0.1.0"}] - - %Ch.Result{rows: [[123]], headers: resp_header} = - Ch.query!( - ch, - "select 123", - _params = [], - Keyword.merge(query_options, headers: req_headers) - ) - - {"x-clickhouse-query-id", query_id} = List.keyfind!(resp_header, "x-clickhouse-query-id", 0) - assert query_http_user_agent(ch, query_id, query_options) == "plausible/0.1.0" - end - end - - defp query_http_user_agent(ch, query_id, query_options) do - retry(fn -> - %Ch.Result{rows: [[user_agent]]} = - Ch.query!( - ch, - "select http_user_agent from system.query_log where query_id = {query_id:String} limit 1", - %{"query_id" => query_id}, - query_options - ) - - user_agent - end) - end - - defp retry(f) do - try do - f.() - catch - _, _ -> - :timer.sleep(100) - retry(f) - end - end -end diff --git a/test/ch/json_test.exs b/test/ch/json_test.exs index b41a82d0..a08e533f 100644 --- a/test/ch/json_test.exs +++ b/test/ch/json_test.exs @@ -1,112 +1,87 @@ defmodule Ch.JSONTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + use ExUnit.Case, async: true @moduletag :json - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - setup do - on_exit(fn -> Ch.Test.query("DROP TABLE IF EXISTS json_test") end) - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} + {:ok, pool: start_supervised!(Ch)} end - test "simple json", %{conn: conn, query_options: query_options} do + test "select literal json", %{pool: pool} do select = fn literal -> - [[value]] = Ch.query!(conn, "select '#{literal}'::json", [], query_options).rows + assert %{rows: [[value]]} = + Ch.query!(pool, "select '#{literal}'::json", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ) + value end + assert select.(~s|{}|) == %{} assert select.(~s|{"a":"b","c":"d"}|) == %{"a" => "b", "c" => "d"} - - # note that 42 was a string in pre-25.0 and post-25.8 ClickHouse versions - assert select.(~s|{"a":42}|) == %{"a" => 42} - - assert select.(~s|{}|) == %{} - - # null fields are removed? - assert select.(~s|{"a":null}|) == %{} - assert select.(~s|{"a":3.14}|) == %{"a" => 3.14} - assert select.(~s|{"a":true}|) == %{"a" => true} - assert select.(~s|{"a":false}|) == %{"a" => false} - assert select.(~s|{"a":{"b":"c"}}|) == %{"a" => %{"b" => "c"}} - - # numbers in arrays become strings assert select.(~s|{"a":[1,2,3]}|) == %{"a" => [1, 2, 3]} - - # this is weird, fields with dots are treated as nested objects - assert select.(~s|{"a.b":"c"}|) == %{"a" => %{"b" => "c"}} - assert select.(~s|{"a":[]}|) == %{"a" => []} - assert select.(~s|{"a":[null]}|) == %{"a" => [nil]} - - # everything in an array gets converted to "lcd" type, aka string assert select.(~s|{"a":[1,3.14,"hello",null]}|) == %{"a" => [1, 3.14, "hello", nil]} - - # but not if the array has nested objects, then the array becomes a tuple and can support mixed types assert select.(~s|{"a":[1,2.13,"s",{"a":"b"}]}|) == %{"a" => [1, 2.13, "s", %{"a" => "b"}]} + + # now the weird bits: + # - null fields are removed + assert select.(~s|{"a":null}|) == %{} + # - fields with dots are treated as nested objects + assert select.(~s|{"a.b":"c"}|) == %{"a" => %{"b" => "c"}} end # https://clickhouse.com/docs/sql-reference/data-types/newjson#using-json-in-a-table-column-definition - test "basic", %{conn: conn, query_options: query_options} do - Ch.query!( - conn, - "CREATE TABLE json_test (json JSON, id UInt8) ENGINE = Memory", - [], - query_options - ) - - Ch.query!( - conn, - """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}', 0), - ('{"f" : "Hello, World!"}', 1), - ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}', 2) - """, - [], - query_options - ) - - assert Ch.query!( - conn, - "SELECT json FROM json_test ORDER BY id", - [], - query_options + test "basic", %{pool: pool} do + Help.query!("CREATE TABLE json_test (json JSON, id UInt8) ENGINE = Memory") + on_exit(fn -> Help.query!("drop table json_test") end) + + Ch.query!(pool, """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}', 0), + ('{"f" : "Hello, World!"}', 1), + ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}', 2) + """) + + assert Ch.query!(pool, "SELECT json FROM json_test ORDER BY id", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} ).rows == [ [%{"a" => %{"b" => 42}, "c" => [1, 2, 3]}], [%{"f" => "Hello, World!"}], [%{"a" => %{"b" => 43, "e" => 10}, "c" => [4, 5, 6]}] ] + rows = [[%{"a" => %{"b" => 999}, "some other" => "json value", "from" => "rowbinary"}, 3]] + rowbinary = Ch.RowBinary.encode_rows(rows, ["JSON", "UInt8"]) + Ch.query!( - conn, - "INSERT INTO json_test(json, id) FORMAT RowBinary", - [[%{"a" => %{"b" => 999}, "some other" => "json value", "from" => "rowbinary"}, 3]], - Keyword.merge(query_options, types: ["JSON", "UInt8"]) + pool, + ["INSERT INTO json_test(json, id) FORMAT RowBinary\n" | rowbinary], + _params = %{}, + settings: %{"input_format_binary_read_json_as_string" => 1} ) assert Ch.query!( - conn, + pool, "SELECT json FROM json_test where json.from = 'rowbinary'", - [], - query_options - ).rows == [ - [%{"from" => "rowbinary", "some other" => "json value", "a" => %{"b" => 999}}] - ] + _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == + [ + [%{"from" => "rowbinary", "some other" => "json value", "a" => %{"b" => 999}}] + ] assert Ch.query!( - conn, + pool, "select json.a.b, json.a.g, json.c, json.d from json_test order by id", - [], - query_options + _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} ).rows == [ [42, nil, [1, 2, 3], nil], @@ -117,31 +92,19 @@ defmodule Ch.JSONTest do end # https://clickhouse.com/docs/sql-reference/data-types/newjson#using-json-in-a-table-column-definition - test "with skip (i.e. extra type options)", %{conn: conn, query_options: query_options} do - Ch.query!( - conn, - "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory;", - [], - query_options - ) - - Ch.query!( - conn, - """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), - ('{"f" : "Hello, World!"}'), - ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}'); - """, - [], - query_options - ) - - assert Ch.query!( - conn, - "SELECT json FROM json_test", - [], - query_options + test "with skip (i.e. extra type options)", %{pool: pool} do + Help.query!("CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory;") + on_exit(fn -> Help.query!("drop table json_test") end) + + Ch.query!(pool, """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), + ('{"f" : "Hello, World!"}'), + ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}'); + """) + + assert Ch.query!(pool, "SELECT json FROM json_test", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} ).rows == [ [%{"a" => %{"b" => 42}, "c" => [1, 2, 3]}], [%{"a" => %{"b" => 0}, "f" => "Hello, World!"}], @@ -150,31 +113,19 @@ defmodule Ch.JSONTest do end # https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns - test "reading json paths as subcolumns", %{conn: conn, query_options: query_options} do - Ch.query!( - conn, - "CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory", - [], - query_options - ) - - Ch.query!( - conn, - """ - INSERT INTO json_test VALUES - ('{"a" : {"b" : 42, "g" : 42.42}, "c" : [1, 2, 3], "d" : "2020-01-01"}'), - ('{"f" : "Hello, World!", "d" : "2020-01-02"}'), - ('{"a" : {"b" : 43, "e" : 10, "g" : 43.43}, "c" : [4, 5, 6]}'); - """, - [], - query_options - ) - - assert Ch.query!( - conn, - "SELECT json FROM json_test", - [], - query_options + test "reading json paths as subcolumns", %{pool: pool} do + Help.query!("CREATE TABLE json_test (json JSON(a.b UInt32, SKIP a.e)) ENGINE = Memory") + on_exit(fn -> Help.query!("drop table json_test") end) + + Ch.query!(pool, """ + INSERT INTO json_test VALUES + ('{"a" : {"b" : 42, "g" : 42.42}, "c" : [1, 2, 3], "d" : "2020-01-01"}'), + ('{"f" : "Hello, World!", "d" : "2020-01-02"}'), + ('{"a" : {"b" : 43, "e" : 10, "g" : 43.43}, "c" : [4, 5, 6]}'); + """) + + assert Ch.query!(pool, "SELECT json FROM json_test", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} ).rows == [ [%{"a" => %{"b" => 42, "g" => 42.42}, "c" => [1, 2, 3], "d" => "2020-01-01"}], [%{"a" => %{"b" => 0}, "d" => "2020-01-02", "f" => "Hello, World!"}], @@ -182,17 +133,20 @@ defmodule Ch.JSONTest do ] assert Ch.query!( - conn, + pool, "SELECT json.a.b, json.a.g, json.c, json.d FROM json_test", - [], - query_options - ).rows == [ - [42, 42.42, [1, 2, 3], ~D[2020-01-01]], - [0, nil, nil, ~D[2020-01-02]], - [43, 43.43, [4, 5, 6], nil] - ] + _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == + [ + [42, 42.42, [1, 2, 3], ~D[2020-01-01]], + [0, nil, nil, ~D[2020-01-02]], + [43, 43.43, [4, 5, 6], nil] + ] - assert Ch.query!(conn, "SELECT json.non.existing.path FROM json_test", [], query_options).rows == + assert Ch.query!(pool, "SELECT json.non.existing.path FROM json_test", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == [ [nil], [nil], @@ -200,70 +154,57 @@ defmodule Ch.JSONTest do ] assert Ch.query!( - conn, - "SELECT toTypeName(json.a.b), toTypeName(json.a.g), toTypeName(json.c), toTypeName(json.d) FROM json_test;", - [], - query_options - ).rows == [ - ["UInt32", "Dynamic", "Dynamic", "Dynamic"], - ["UInt32", "Dynamic", "Dynamic", "Dynamic"], - ["UInt32", "Dynamic", "Dynamic", "Dynamic"] - ] + pool, + "SELECT toTypeName(json.a.b), toTypeName(json.a.g), toTypeName(json.c), toTypeName(json.d) FROM json_test;" + ).rows == + [ + ["UInt32", "Dynamic", "Dynamic", "Dynamic"], + ["UInt32", "Dynamic", "Dynamic", "Dynamic"], + ["UInt32", "Dynamic", "Dynamic", "Dynamic"] + ] - assert Ch.query!( - conn, - """ - SELECT - json.a.g.:Float64, - dynamicType(json.a.g), - json.d.:Date, - dynamicType(json.d) - FROM json_test - """, - [], - query_options - ).rows == [ + assert Ch.query!(pool, """ + SELECT + json.a.g.:Float64, + dynamicType(json.a.g), + json.d.:Date, + dynamicType(json.d) + FROM json_test + """).rows == [ [42.42, "Float64", ~D[2020-01-01], "Date"], [nil, "None", ~D[2020-01-02], "Date"], [43.43, "Float64", nil, "None"] ] - assert Ch.query!( - conn, - """ - SELECT json.a.g::UInt64 AS uint - FROM json_test; - """, - [], - query_options - ).rows == [ + assert Ch.query!(pool, "SELECT json.a.g::UInt64 AS uint FROM json_test").rows == [ [42], [0], [43] ] assert_raise Ch.Error, ~r/Conversion between numeric types and UUID is not supported/, fn -> - Ch.query!(conn, "SELECT json.a.g::UUID AS float FROM json_test;", [], query_options) + Ch.query!(pool, "SELECT json.a.g::UUID AS float FROM json_test;") end end # https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-sub-objects-as-sub-columns - test "reading json subobjects as subcolumns", %{conn: conn, query_options: query_options} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;", [], query_options) + test "reading json subobjects as subcolumns", %{pool: pool} do + Help.query!("CREATE TABLE json_test (json JSON) ENGINE = Memory;") + on_exit(fn -> Help.query!("drop table json_test") end) Ch.query!( - conn, + pool, """ INSERT INTO json_test VALUES ('{"a" : {"b" : {"c" : 42, "g" : 42.42}}, "c" : [1, 2, 3], "d" : {"e" : {"f" : {"g" : "Hello, World", "h" : [1, 2, 3]}}}}'), ('{"f" : "Hello, World!", "d" : {"e" : {"f" : {"h" : [4, 5, 6]}}}}'), ('{"a" : {"b" : {"c" : 43, "e" : 10, "g" : 43.43}}, "c" : [4, 5, 6]}'); - """, - [], - query_options + """ ) - assert Ch.query!(conn, "SELECT json FROM json_test;", [], query_options).rows == [ + assert Ch.query!(pool, "SELECT json FROM json_test", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == [ [ %{ "a" => %{"b" => %{"c" => 42, "g" => 42.42}}, @@ -280,7 +221,9 @@ defmodule Ch.JSONTest do ] ] - assert Ch.query!(conn, "SELECT json.^a.b, json.^d.e.f FROM json_test;", [], query_options).rows == + assert Ch.query!(pool, "SELECT json.^a.b, json.^d.e.f FROM json_test;", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == [ [%{"c" => 42, "g" => 42.42}, %{"g" => "Hello, World", "h" => [1, 2, 3]}], [%{}, %{"h" => [4, 5, 6]}], @@ -288,24 +231,24 @@ defmodule Ch.JSONTest do ] end - # TODO # https://clickhouse.com/docs/sql-reference/data-types/newjson#handling-arrays-of-json-objects - test "handling arrays of json objects", %{conn: conn, query_options: query_options} do - Ch.query!(conn, "CREATE TABLE json_test (json JSON) ENGINE = Memory;", [], query_options) + test "handling arrays of json objects", %{pool: pool} do + Help.query!("CREATE TABLE json_test (json JSON) ENGINE = Memory;") + on_exit(fn -> Help.query!("drop table json_test") end) Ch.query!( - conn, + pool, """ INSERT INTO json_test VALUES ('{"a" : {"b" : [{"c" : 42, "d" : "Hello", "f" : [[{"g" : 42.42}]], "k" : {"j" : 1000}}, {"c" : 43}, {"e" : [1, 2, 3], "d" : "My", "f" : [[{"g" : 43.43, "h" : "2020-01-01"}]], "k" : {"j" : 2000}}]}}'), ('{"a" : {"b" : [1, 2, 3]}}'), ('{"a" : {"b" : [{"c" : 44, "f" : [[{"h" : "2020-01-02"}]]}, {"e" : [4, 5, 6], "d" : "World", "f" : [[{"g" : 44.44}]], "k" : {"j" : 3000}}]}}'); - """, - [], - query_options + """ ) - assert Ch.query!(conn, "SELECT json FROM json_test;", [], query_options).rows == [ + assert Ch.query!(pool, "SELECT json FROM json_test;", _params = %{}, + settings: %{"output_format_binary_write_json_as_string" => 1} + ).rows == [ [ %{ "a" => %{ @@ -347,15 +290,13 @@ defmodule Ch.JSONTest do # TODO assert_raise ArgumentError, "unsupported dynamic type JSON", fn -> - Ch.query!(conn, "SELECT json.a.b, dynamicType(json.a.b) FROM json_test;", [], query_options) + Ch.query!(pool, "SELECT json.a.b, dynamicType(json.a.b) FROM json_test;") end assert_raise ArgumentError, "unsupported dynamic type JSON", fn -> Ch.query!( - conn, - "SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f, json.a.b.:`Array(JSON)`.d FROM json_test;", - [], - query_options + pool, + "SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f, json.a.b.:`Array(JSON)`.d FROM json_test;" ) end end diff --git a/test/ch/naive_datetime_timezone_test.exs b/test/ch/naive_datetime_timezone_test.exs new file mode 100644 index 00000000..dd5852c8 --- /dev/null +++ b/test/ch/naive_datetime_timezone_test.exs @@ -0,0 +1,75 @@ +defmodule Ch.NaiveDateTimeTimezoneTest do + use ExUnit.Case, async: true + + setup do + {:ok, pool: start_supervised!(Ch)} + end + + test "naive DateTime params use session_timezone for implicit timezone types", %{pool: pool} do + naive = ~N[2022-12-12 12:00:00] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime} AS d, toString(d), timeZone()", + %{"dt" => naive}, + settings: [session_timezone: "Asia/Bangkok"] + ).rows == [[~N[2022-12-12 05:00:00], "2022-12-12 12:00:00", "Asia/Bangkok"]] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime} AS d, toString(d), timeZone()", + %{"dt" => naive}, + settings: [session_timezone: "Europe/Berlin"] + ).rows == [[~N[2022-12-12 11:00:00], "2022-12-12 12:00:00", "Europe/Berlin"]] + end + + test "naive DateTime64 params use session_timezone for implicit timezone types", %{pool: pool} do + naive = ~N[2022-12-12 12:00:00.123] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime64(3)} AS d, toString(d), timeZone()", + %{"dt" => naive}, + settings: [session_timezone: "Asia/Bangkok"] + ).rows == [[~N[2022-12-12 05:00:00.123], "2022-12-12 12:00:00.123", "Asia/Bangkok"]] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime64(3)} AS d, toString(d), timeZone()", + %{"dt" => naive}, + settings: [session_timezone: "Europe/Berlin"] + ).rows == [[~N[2022-12-12 11:00:00.123], "2022-12-12 12:00:00.123", "Europe/Berlin"]] + end + + test "naive DateTime params with explicit timezone ignore session_timezone", %{pool: pool} do + naive = ~N[2022-12-12 12:00:00] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime('Asia/Bangkok')} AS d, toString(d)", + %{"dt" => naive}, + settings: [session_timezone: "UTC"] + ).rows == [ + [ + DateTime.new!(~D[2022-12-12], ~T[12:00:00], "Asia/Bangkok"), + "2022-12-12 12:00:00" + ] + ] + end + + test "naive DateTime64 params with explicit timezone ignore session_timezone", %{pool: pool} do + naive = ~N[2022-12-12 12:00:00.123] + + assert Ch.query!( + pool, + "SELECT {dt:DateTime64(3, 'Asia/Bangkok')} AS d, toString(d)", + %{"dt" => naive}, + settings: [session_timezone: "UTC"] + ).rows == [ + [ + DateTime.new!(~D[2022-12-12], ~T[12:00:00.123], "Asia/Bangkok"), + "2022-12-12 12:00:00.123" + ] + ] + end +end diff --git a/test/ch/query_string_test.exs b/test/ch/query_string_test.exs index 398ab3e5..515769d4 100644 --- a/test/ch/query_string_test.exs +++ b/test/ch/query_string_test.exs @@ -1,34 +1,260 @@ defmodule Ch.QueryStringTest do - use ExUnit.Case, - async: true, - parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + use ExUnit.Case, async: true + use ExUnitProperties - setup ctx do - {:ok, query_options: ctx[:query_options] || []} + describe "path/2" do + property "starts with / and only adds ? when there is a query string" do + check all params <- params(), + options <- query_options() do + path = Ch.HTTP.path(params, options) + query = URI.parse(path).query + + if params == %{} and Enum.empty?(options) do + assert path == "/" + assert query == nil + else + assert path =~ ~r"^/\\?" + assert query != nil + refute String.ends_with?(path, "?") + refute String.ends_with?(path, "&") + refute path =~ "?&" + end + end + end + + property "encodes params with param_ prefix and leaves options unprefixed" do + check all params <- params(), + options <- query_options() do + decoded = Ch.HTTP.path(params, options) |> query_string() |> URI.decode_query() + + for {key, value} <- params do + assert decoded["param_#{key}"] == expected_param(value) + end + + for {key, value} <- options do + assert decoded[to_string(key)] == to_string(value) + end + end + end + + property "accepts maps and keyword lists for options" do + check all options <- query_options() do + map_options = Map.new(options) + + assert Ch.HTTP.path(%{}, options) |> query_string() |> URI.decode_query() == + Ch.HTTP.path(%{}, map_options) |> query_string() |> URI.decode_query() + end + end + end + + describe "param encoding" do + property "escapes top-level string params as ClickHouse escaped text" do + check all string <- safe_string() do + decoded = decoded_param(string) + + assert decoded == expected_param(string) + + assert decoded == + string + |> String.replace("\\", "\\\\") + |> String.replace("\t", "\\\t") + |> String.replace("\n", "\\\n") + end + end + + property "encodes scalar params" do + check all value <- scalar_param() do + assert decoded_param(value) == expected_param(value) + end + end + + property "encodes array params" do + check all values <- list_of(array_scalar(), max_length: 8) do + assert decoded_param(values) == expected_param(values) + end + end + + property "encodes tuple params like arrays with parentheses" do + check all values <- list_of(array_scalar(), max_length: 8) do + tuple = List.to_tuple(values) + assert decoded_param(tuple) == expected_param(tuple) + end + end + + property "encodes map params as ClickHouse map literals" do + check all entries <- uniq_list_of({safe_string(), array_scalar()}, max_length: 8) do + map = Map.new(entries) + assert decoded_param(map) == expected_param(map) + end + end end - setup do - {:ok, conn: start_supervised!(Ch)} + describe "ClickHouse round-trip" do + setup do + {:ok, pool: start_supervised!(Ch)} + end + + # For more info see + # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters + # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting + property "string parameters round-trip through ClickHouse", %{pool: pool} do + check all string <- safe_string() do + assert Ch.query!(pool, "select {s:String}", %{"s" => string}).rows == [[string]] + end + end + + test "string parameters are escaped", %{pool: pool} do + for s <- ["\t", "\n", "\\", "'", "\b", "\f", "\r", "\0"] do + assert Ch.query!(pool, "select {s:String}", %{"s" => s}).rows == [[s]] + end + + assert Ch.query!(pool, "select splitByChar('\t', 'abc\t123')").rows == + [[["abc", "123"]]] + + assert Ch.query!(pool, "select splitByChar('\t', {arg1:String})", %{"arg1" => "abc\t123"}).rows == + [[["abc", "123"]]] + end end - # For more info see - # https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - # "escaped" format is the same as https://clickhouse.com/docs/en/interfaces/formats#tabseparated-data-formatting - test "binaries are escaped properly", %{conn: conn, query_options: query_options} do - for s <- ["\t", "\n", "\\", "'", "\b", "\f", "\r", "\0"] do - assert Ch.query!(conn, "select {s:String}", %{"s" => s}, query_options).rows == [[s]] + defp query_string(path) do + case URI.parse(path).query do + nil -> "" + query -> query end + end + + defp decoded_param(value) do + Ch.HTTP.path(%{"value" => value}) + |> query_string() + |> URI.decode_query() + |> Map.fetch!("param_value") + end + + defp params do + map_of(safe_key(), param(), max_length: 8) + end + + defp query_options do + uniq_list_of({safe_key(), one_of([integer(), boolean(), safe_string()])}, max_length: 8) + end + + defp param do + one_of([ + scalar_param(), + list_of(array_scalar(), max_length: 5), + map_of(safe_string(), array_scalar(), max_length: 5) + ]) + end + + defp scalar_param do + one_of([ + integer(), + float(), + boolean(), + constant(nil), + safe_string(), + decimal_gen(), + date_gen(), + naive_datetime_gen(), + time_gen() + ]) + end + + defp array_scalar do + one_of([ + integer(), + float(), + boolean(), + constant(nil), + safe_string(), + decimal_gen(), + date_gen(), + naive_datetime_gen(), + time_gen() + ]) + end + + defp safe_key do + string(:alphanumeric, min_length: 1, max_length: 16) + end + + defp safe_string do + string(:printable, max_length: 32) + end + + defp decimal_gen do + gen all sign <- member_of([1, -1]), + coef <- integer(0..999_999), + exp <- integer(-8..8) do + Decimal.new(sign, coef, exp) + end + end + + defp date_gen do + gen all days <- integer(0..36_500) do + Date.add(~D[1970-01-01], days) + end + end + + defp naive_datetime_gen do + gen all date <- date_gen(), + hour <- integer(0..23), + minute <- integer(0..59), + second <- integer(0..59), + microsecond <- integer(0..999_999) do + NaiveDateTime.new!(date, Time.new!(hour, minute, second, {microsecond, 6})) + end + end + + defp time_gen do + gen all hour <- integer(0..23), + minute <- integer(0..59), + second <- integer(0..59), + microsecond <- integer(0..999_999) do + Time.new!(hour, minute, second, {microsecond, 6}) + end + end + + defp expected_param(value) when is_integer(value), do: Integer.to_string(value) + defp expected_param(value) when is_float(value), do: Float.to_string(value) + defp expected_param(value) when is_boolean(value), do: Atom.to_string(value) + defp expected_param(nil), do: "\\N" + defp expected_param(%Decimal{} = value), do: Decimal.to_string(value, :scientific) + defp expected_param(%Date{} = value), do: Date.to_iso8601(value) + defp expected_param(%NaiveDateTime{} = value), do: NaiveDateTime.to_iso8601(value) + defp expected_param(%Time{} = value), do: Time.to_iso8601(value) + + defp expected_param(value) when is_binary(value) do + value + |> String.replace("\\", "\\\\") + |> String.replace("\t", "\\\t") + |> String.replace("\n", "\\\n") + end + + defp expected_param(value) when is_tuple(value) do + "(" <> (value |> Tuple.to_list() |> Enum.map_join(",", &expected_array_param/1)) <> ")" + end + + defp expected_param(value) when is_list(value) do + "[" <> Enum.map_join(value, ",", &expected_array_param/1) <> "]" + end + + defp expected_param(value) when is_map(value) do + "{" <> (value |> Map.to_list() |> Enum.map_join(",", &expected_map_param/1)) <> "}" + end + + defp expected_array_param(value) when is_binary(value) do + "'" <> (value |> String.replace("'", "''") |> String.replace("\\", "\\\\")) <> "'" + end + + defp expected_array_param(nil), do: "null" + + defp expected_array_param(%value{} = param) when value in [Date, NaiveDateTime], + do: "'" <> expected_param(param) <> "'" - # example from https://clickhouse.com/docs/en/interfaces/http#tabs-in-url-parameters - assert Ch.query!(conn, "select splitByChar('\t', 'abc\t123')", [], query_options).rows == - [[["abc", "123"]]] + defp expected_array_param(value), do: expected_param(value) - assert Ch.query!( - conn, - "select splitByChar('\t', {arg1:String})", - %{"arg1" => "abc\t123"}, - query_options - ).rows == - [[["abc", "123"]]] + defp expected_map_param({key, value}) do + expected_array_param(key) <> ":" <> expected_array_param(value) end end diff --git a/test/ch/query_test.exs b/test/ch/query_test.exs index e5e9a20a..879e3df8 100644 --- a/test/ch/query_test.exs +++ b/test/ch/query_test.exs @@ -1,59 +1,11 @@ defmodule Ch.QueryTest do - use ExUnit.Case, - async: true, - parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - - alias Ch.Query - - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - test "to_string" do - query = Query.build(["select ", 1 + ?0, ?+, 2 + ?0]) - assert to_string(query) == "select 1+2" - end - - describe "command" do - test "without command provided" do - assert Query.build("select 1+2").command == :select - assert Query.build("SELECT 1+2").command == :select - assert Query.build(" select 1+2").command == :select - assert Query.build("\t\n\t\nSELECT 1+2").command == :select - - assert Query.build(""" - - select 1+2 - """).command == :select - - assert Query.build(["select 1+2"]).command == :select - assert Query.build([?S, ?E, ?L | "ECT 1"]).command == :select - - assert Query.build("with insert as (select 1) select * from insert").command == :select - assert Query.build("insert into table(a, b) values(1, 2)").command == :insert - - assert Query.build("insert into table(a, b) select b, c from table2 where b = 'update'").command == - :insert - end - - test "with nil command provided" do - assert Query.build("select 1+2", command: nil).command == :select - end - - test "with command provided" do - assert Query.build("select 1+2", command: :custom).command == :custom - end - - @tag skip: true - test "TODO" do - assert Query.build("Select 1+2").command == :select - end - end + use ExUnit.Case, async: true # adapted from https://github.com/elixir-ecto/postgrex/blob/master/test/query_test.exs describe "query" do setup do - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} + pool = start_supervised!(Ch) + {:ok, pool: pool, conn: pool, query_options: []} end test "iodata", %{conn: conn, query_options: query_options} do @@ -319,36 +271,60 @@ defmodule Ch.QueryTest do test "decoded binaries copy behaviour", %{conn: conn, query_options: query_options} do text = "hello world" - assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text], query_options).rows + + assert [[bin]] = + Ch.query!(conn, "SELECT {text:String}", %{"text" => text}, query_options).rows + + assert bin == text assert :binary.referenced_byte_size(bin) == :binary.referenced_byte_size("hello world") # For OTP 20+ refc binaries up to 64 bytes might be copied during a GC text = String.duplicate("hello world", 6) - assert [[bin]] = Ch.query!(conn, "SELECT {$0:String}", [text], query_options).rows - assert :binary.referenced_byte_size(bin) == byte_size(text) + + assert [[bin]] = + Ch.query!(conn, "SELECT {text:String}", %{"text" => text}, query_options).rows + + assert bin == text end test "encode basic types", %{conn: conn, query_options: query_options} do # TODO # assert [[nil, nil]] = query("SELECT $1::text, $2::int", [nil, nil]) assert [[true, false]] = - Ch.query!(conn, "SELECT {$0:bool}, {$1:Bool}", [true, false], query_options).rows + Ch.query!( + conn, + "SELECT {a:Bool}, {b:Bool}", + %{"a" => true, "b" => false}, + query_options + ).rows - assert [["ẽ"]] = Ch.query!(conn, "SELECT {$0:char}", ["ẽ"], query_options).rows - assert [[42]] = Ch.query!(conn, "SELECT {$0:int}", [42], query_options).rows + assert [["ẽ"]] = Ch.query!(conn, "SELECT {s:String}", %{"s" => "ẽ"}, query_options).rows + assert [[42]] = Ch.query!(conn, "SELECT {i:Int32}", %{"i" => 42}, query_options).rows assert [[42.0, 43.0]] = - Ch.query!(conn, "SELECT {$0:float}, {$1:float}", [42, 43.0], query_options).rows + Ch.query!( + conn, + "SELECT {a:Float64}, {b:Float64}", + %{"a" => 42, "b" => 43.0}, + query_options + ).rows assert [[nil, nil]] = - Ch.query!(conn, "SELECT {$0:float}, {$1:float}", ["NaN", "nan"], query_options).rows + Ch.query!( + conn, + "SELECT {a:Float64}, {b:Float64}", + %{"a" => "NaN", "b" => "nan"}, + query_options + ).rows - assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["inf"], query_options).rows - assert [[nil]] = Ch.query!(conn, "SELECT {$0:float}", ["-inf"], query_options).rows - assert [["ẽric"]] = Ch.query!(conn, "SELECT {$0:varchar}", ["ẽric"], query_options).rows + assert [[nil]] = Ch.query!(conn, "SELECT {f:Float64}", %{"f" => "inf"}, query_options).rows + assert [[nil]] = Ch.query!(conn, "SELECT {f:Float64}", %{"f" => "-inf"}, query_options).rows + + assert [["ẽric"]] = + Ch.query!(conn, "SELECT {s:String}", %{"s" => "ẽric"}, query_options).rows assert [[<<1, 2, 3>>]] = - Ch.query!(conn, "SELECT {$0:bytea}", [<<1, 2, 3>>], query_options).rows + Ch.query!(conn, "SELECT {b:String}", %{"b" => <<1, 2, 3>>}, query_options).rows end test "encode numeric", %{conn: conn, query_options: query_options} do @@ -375,16 +351,20 @@ defmodule Ch.QueryTest do Enum.each(nums, fn {num, type} -> dec = Decimal.new(num, max_digits: :infinity, max_exponent: :infinity) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:#{type}}", [dec], query_options).rows + assert [[dec]] == Ch.query!(conn, "SELECT {d:#{type}}", %{"d" => dec}, query_options).rows end) end test "encode integers and floats as numeric", %{conn: conn, query_options: query_options} do dec = Decimal.new(1) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(1,0)}", [1], query_options).rows + + assert [[dec]] == + Ch.query!(conn, "SELECT {d:numeric(1,0)}", %{"d" => 1}, query_options).rows dec = Decimal.from_float(1.0) - assert [[dec]] == Ch.query!(conn, "SELECT {$0:numeric(2,1)}", [1.0], query_options).rows + + assert [[dec]] == + Ch.query!(conn, "SELECT {d:numeric(2,1)}", %{"d" => 1.0}, query_options).rows end @tag skip: true @@ -397,60 +377,59 @@ defmodule Ch.QueryTest do # TODO uuid = <<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15>> uuid_hex = "00010203-0405-0607-0809-0a0b0c0d0e0f" - assert [[^uuid]] = Ch.query!(conn, "SELECT {$0:UUID}", [uuid_hex], query_options).rows + + assert [[^uuid]] = + Ch.query!(conn, "SELECT {uuid:UUID}", %{"uuid" => uuid_hex}, query_options).rows end test "encode arrays", %{conn: conn, query_options: query_options} do - assert [[[]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[]], query_options).rows - assert [[[1]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1]], query_options).rows + assert [[[]]] = + Ch.query!(conn, "SELECT {a:Array(Int32)}", %{"a" => []}, query_options).rows + + assert [[[1]]] = + Ch.query!(conn, "SELECT {a:Array(Int32)}", %{"a" => [1]}, query_options).rows assert [[[1, 2]]] = - Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, 2]], query_options).rows + Ch.query!(conn, "SELECT {a:Array(Int32)}", %{"a" => [1, 2]}, query_options).rows - assert [[["1"]]] = Ch.query!(conn, "SELECT {$0:Array(String)}", [["1"]], query_options).rows - assert [[[true]]] = Ch.query!(conn, "SELECT {$0:Array(Bool)}", [[true]], query_options).rows + assert [[["1"]]] = + Ch.query!(conn, "SELECT {a:Array(String)}", %{"a" => ["1"]}, query_options).rows - assert [[[~D[2023-01-01]]]] = - Ch.query!(conn, "SELECT {$0:Array(Date)}", [[~D[2023-01-01]]], query_options).rows + assert [[[true]]] = + Ch.query!(conn, "SELECT {a:Array(Bool)}", %{"a" => [true]}, query_options).rows - assert [[[Ch.Test.to_clickhouse_naive(conn, ~N[2023-01-01 12:00:00])]]] == + assert [[[~D[2023-01-01]]]] = Ch.query!( conn, - "SELECT {$0:Array(DateTime)}", - [[~N[2023-01-01 12:00:00]]], + "SELECT {a:Array(Date)}", + %{"a" => [~D[2023-01-01]]}, query_options ).rows assert [[[~U[2023-01-01 12:00:00Z]]]] == Ch.query!( conn, - "SELECT {$0:Array(DateTime('UTC'))}", - [[~N[2023-01-01 12:00:00]]], + "SELECT {a:Array(DateTime('UTC'))}", + %{"a" => [~U[2023-01-01 12:00:00Z]]}, query_options ).rows - assert [[[~N[2023-01-01 12:00:00]]]] == + assert [[[[0], [1]]]] = Ch.query!( conn, - "SELECT {$0:Array(DateTime)}", - [[~U[2023-01-01 12:00:00Z]]], + "SELECT {a:Array(Array(Int32))}", + %{"a" => [[0], [1]]}, query_options ).rows - assert [[[~U[2023-01-01 12:00:00Z]]]] == + assert [[[[0]]]] = Ch.query!( conn, - "SELECT {$0:Array(DateTime('UTC'))}", - [[~U[2023-01-01 12:00:00Z]]], + "SELECT {a:Array(Array(Int32))}", + %{"a" => [[0]]}, query_options ).rows - assert [[[[0], [1]]]] = - Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0], [1]]], query_options).rows - - assert [[[[0]]]] = - Ch.query!(conn, "SELECT {$0:Array(Array(integer))}", [[[0]]], query_options).rows - # assert [[[1, nil, 3]]] = Ch.query!(conn, "SELECT {$0:Array(integer)}", [[1, nil, 3]], query_options).rows end @@ -460,28 +439,28 @@ defmodule Ch.QueryTest do # Ch.query!(conn, "SELECT {$0:inet4}::text", [{127, 0, 0, 1}], query_options).rows assert [[{127, 0, 0, 1}]] = - Ch.query!(conn, "SELECT {$0:text}::inet4", ["127.0.0.1"], query_options).rows + Ch.query!(conn, "SELECT {ip:String}::IPv4", %{"ip" => "127.0.0.1"}, query_options).rows assert [[{0, 0, 0, 0, 0, 0, 0, 1}]] = - Ch.query!(conn, "SELECT {$0:text}::inet6", ["::1"], query_options).rows + Ch.query!(conn, "SELECT {ip:String}::IPv6", %{"ip" => "::1"}, query_options).rows end test "result struct", %{conn: conn, query_options: query_options} do assert {:ok, res} = Ch.query(conn, "SELECT 123 AS a, 456 AS b", [], query_options) assert %Ch.Result{} = res - assert res.command == :select - assert res.columns == ["a", "b"] - assert res.num_rows == 1 + assert res.names == ["a", "b"] + assert res.rows == [[123, 456]] + assert is_list(res.headers) + assert is_binary(IO.iodata_to_binary(res.data)) end test "empty result struct", %{conn: conn, query_options: query_options} do - assert %Ch.Result{} = + assert %Ch.Result{names: ["number", "b"], rows: []} = res = Ch.query!(conn, "select number, 'a' as b from numbers(0)", [], query_options) - assert res.command == :select - assert res.columns == ["number", "b"] assert res.rows == [] - assert res.num_rows == 0 + assert is_list(res.headers) + assert is_binary(IO.iodata_to_binary(res.data)) end test "error struct", %{conn: conn, query_options: query_options} do @@ -489,7 +468,11 @@ defmodule Ch.QueryTest do end test "error code", %{conn: conn, query_options: query_options} do - assert {:error, %Ch.Error{code: 62}} = Ch.query(conn, "wat", [], query_options) + assert {:error, %Ch.Error{code: code, message: message}} = + Ch.query(conn, "wat", [], query_options) + + assert is_nil(code) or code == 62 + assert message =~ "Code: 62" end test "connection works after failure in execute", %{conn: conn, query_options: query_options} do @@ -512,19 +495,15 @@ defmodule Ch.QueryTest do assert_receive [[0]], :timer.seconds(1) end) end - - test "query struct interpolates to statement" do - assert "#{%Ch.Query{statement: "SELECT 1"}}" == "SELECT 1" - end end - test "query before and after idle ping", %{query_options: query_options} do - opts = [backoff_type: :stop, idle_interval: 1] + test "query before and after idle worker timeout" do + opts = [worker_idle_timeout: 1] {:ok, pid} = Ch.start_link(opts) - assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) + assert {:ok, _} = Ch.query(pid, "SELECT 42") :timer.sleep(20) - assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) + assert {:ok, _} = Ch.query(pid, "SELECT 42") :timer.sleep(20) - assert {:ok, _} = Ch.query(pid, "SELECT 42", [], query_options) + assert {:ok, _} = Ch.query(pid, "SELECT 42") end end diff --git a/test/ch/row_binary_test.exs b/test/ch/row_binary_test.exs index 3d5e453a..3eb3b362 100644 --- a/test/ch/row_binary_test.exs +++ b/test/ch/row_binary_test.exs @@ -164,26 +164,17 @@ defmodule Ch.RowBinaryTest do end end - test "utf8" do - # example from https://clickhouse.com/docs/en/sql-reference/functions/string-functions/#tovalidutf8 + test "strings preserve raw bytes" do value = "\x61\xF0\x80\x80\x80b" - bin = IO.iodata_to_binary(encode(:binary, value)) str = IO.iodata_to_binary(encode(:string, value)) - # encoding is the same since we don't want to modify the values implicitly - assert bin == str - - # but decoding is different based on what type is provided - assert decode_rows(str, [:string]) == [["a�b"]] - assert decode_rows(bin, [:string]) == [["a�b"]] - assert decode_rows(str, [:binary]) == [["\x61\xF0\x80\x80\x80b"]] - assert decode_rows(bin, [:binary]) == [["\x61\xF0\x80\x80\x80b"]] + assert decode_rows(str, [:string]) == [[value]] path = "/some/url" <> <<0xAE>> <> "-/" - assert decode_rows(<>, [:string]) == [["/some/url�-/"]] + assert decode_rows(<>, [:string]) == [[path]] path = <<0xAF>> <> "/some/url" <> <<0xAE, 0xFE>> <> "-/" <> <<0xFA>> - assert decode_rows(<>, [:string]) == [["�/some/url�-/�"]] + assert decode_rows(<>, [:string]) == [[path]] path = "/opportunity/category/جوائز-ومسابقات" assert decode_rows(<>, [:string]) == [[path]] @@ -777,19 +768,19 @@ defmodule Ch.RowBinaryTest do assert rows |> encode_rows(types) |> byte_by_byte(types) == rows end - test "long strings and binaries" do + test "long strings" do long_string = String.duplicate("a", 50_000) - long_binary = String.duplicate(<<0xA>>, 50_000) + long_byte_string = String.duplicate(<<0xA>>, 50_000) data = [ - [encode(:string, long_string), encode(:binary, long_binary)], - [encode(:string, long_string <> "b"), encode(:binary, long_binary <> <<0xB>>)] + [encode(:string, long_string), encode(:string, long_byte_string)], + [encode(:string, long_string <> "b"), encode(:string, long_byte_string <> <<0xB>>)] ] - assert byte_by_byte(data, [:string, :binary]) == [ - [long_string, long_binary], - [long_string <> "b", long_binary <> <<0xB>>] + assert byte_by_byte(data, [:string, :string]) == [ + [long_string, long_byte_string], + [long_string <> "b", long_byte_string <> <<0xB>>] ] end end diff --git a/test/ch/settings_test.exs b/test/ch/settings_test.exs index 69024b91..1aedfb07 100644 --- a/test/ch/settings_test.exs +++ b/test/ch/settings_test.exs @@ -1,26 +1,19 @@ defmodule Ch.SettingsTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + use ExUnit.Case, async: true - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - test "can pass default settings", %{query_options: query_options} do - assert {:ok, conn} = Ch.start_link(settings: [async_insert: 1]) - - assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "1"]]}} = - Ch.query(conn, "show settings like 'async_insert'", [], query_options) - end + test "can pass settings in options" do + pool = start_supervised!(Ch) - test "can overwrite default settings with options", %{query_options: query_options} do - assert {:ok, conn} = Ch.start_link(settings: [async_insert: 1]) + assert Ch.query!(pool, "show settings like 'async_insert'", _params = %{}, + settings: %{"async_insert" => 1} + ).rows == [ + ["async_insert", "Bool", "1"] + ] - assert {:ok, %{num_rows: 1, rows: [["async_insert", "Bool", "0"]]}} = - Ch.query( - conn, - "show settings like 'async_insert'", - [], - Keyword.merge(query_options, settings: [async_insert: 0]) - ) + assert Ch.query!(pool, "show settings like 'async_insert'", _params = %{}, + settings: %{"async_insert" => 0} + ).rows == [ + ["async_insert", "Bool", "0"] + ] end end diff --git a/test/ch/stream_test.exs b/test/ch/stream_test.exs deleted file mode 100644 index 06d95de3..00000000 --- a/test/ch/stream_test.exs +++ /dev/null @@ -1,84 +0,0 @@ -defmodule Ch.StreamTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - alias Ch.{Result, RowBinary} - - setup ctx do - {:ok, query_options: ctx[:query_options] || []} - end - - setup do - {:ok, conn: start_supervised!({Ch, database: Ch.Test.database()})} - end - - describe "enumerable Ch.stream/4" do - test "emits %Ch.Result{}", %{conn: conn, query_options: query_options} do - results = - DBConnection.run(conn, fn conn -> - conn - |> Ch.stream( - "select * from numbers({count:UInt64})", - %{"count" => 1_000_000}, - query_options - ) - |> Enum.into([]) - end) - - assert results |> Enum.map(fn %Result{rows: rows} -> rows end) |> List.flatten() == - Enum.to_list(0..999_999) - end - - test "raises on error", %{conn: conn, query_options: query_options} do - assert_raise Ch.Error, - ~r/Code: 62. DB::Exception: Syntax error: failed at position 8/, - fn -> - DBConnection.run(conn, fn conn -> - conn - |> Ch.stream("select ", %{"count" => 1_000_000}, query_options) - |> Enum.into([]) - end) - end - end - - test "large strings", %{conn: conn, query_options: query_options} do - results = - DBConnection.run(conn, fn conn -> - conn - |> Ch.stream( - "select repeat('abc', 500000) from numbers({count:UInt64})", - %{"count" => 10}, - query_options - ) - |> Enum.into([]) - end) - - expected_string = String.duplicate("abc", 500_000) - - assert results |> Enum.map(fn %Result{rows: rows} -> rows end) |> List.flatten() == - List.duplicate(expected_string, 10) - end - end - - describe "collectable Ch.stream/4" do - test "inserts chunks", %{conn: conn, query_options: query_options} do - Ch.query!(conn, "create table collect_stream(i UInt64) engine Memory") - on_exit(fn -> Ch.Test.query("DROP TABLE collect_stream") end) - - DBConnection.run(conn, fn conn -> - Stream.repeatedly(fn -> [:rand.uniform(100)] end) - |> Stream.chunk_every(100_000) - |> Stream.map(fn chunk -> RowBinary.encode_rows(chunk, _types = ["UInt64"]) end) - |> Stream.take(10) - |> Enum.into( - Ch.stream( - conn, - "insert into collect_stream(i) format RowBinary", - _params = [], - Keyword.merge(query_options, encode: false) - ) - ) - end) - - assert Ch.query!(conn, "select count(*) from collect_stream").rows == [[1_000_000]] - end - end -end diff --git a/test/ch/type_integration_test.exs b/test/ch/type_integration_test.exs new file mode 100644 index 00000000..2292ee8c --- /dev/null +++ b/test/ch/type_integration_test.exs @@ -0,0 +1,350 @@ +defmodule Ch.TypeIntegrationTest do + use ExUnit.Case, async: true + use ExUnitProperties + + alias Ch.RowBinary + + setup do + {:ok, pool: start_supervised!(Ch)} + end + + property "integer params round-trip across ClickHouse integer widths", %{pool: pool} do + check all {type, value} <- integer_param() do + assert Ch.query!(pool, "SELECT {value:#{type}}", %{"value" => value}).rows == [[value]] + end + end + + property "fixed string params are padded to their declared size", %{pool: pool} do + check all {size, value} <- fixed_string_param() do + padding = :binary.copy(<<0>>, size - byte_size(value)) + + assert Ch.query!(pool, "SELECT {value:FixedString(#{size})}", %{"value" => value}).rows == + [[value <> padding]] + end + end + + property "decimal params preserve Decimal(18, 4) scale", %{pool: pool} do + check all value <- decimal_param() do + assert Ch.query!(pool, "SELECT {value:Decimal(18, 4)}", %{"value" => value}).rows == + [[Decimal.round(value, 4)]] + end + end + + property "uuid params accept canonical text and decode to 16 bytes", %{pool: pool} do + check all {uuid_text, uuid_bin} <- uuid_param() do + assert Ch.query!(pool, "SELECT {value:UUID}, toString({value:UUID})", %{ + "value" => uuid_text + }).rows == [[uuid_bin, String.downcase(uuid_text)]] + end + end + + property "DateTime64 UTC params preserve microseconds", %{pool: pool} do + check all dt <- utc_datetime64() do + assert Ch.query!(pool, "SELECT {value:DateTime64(6, 'UTC')}", %{"value" => dt}).rows == + [[dt]] + end + end + + property "map and tuple params round-trip", %{pool: pool} do + check all map <- map_of(safe_string(), integer(0..255), max_length: 8), + tuple <- tuple_param() do + assert Ch.query!(pool, "SELECT {map:Map(String, UInt8)}, {tuple:Tuple(Int8, String)}", %{ + "map" => map, + "tuple" => tuple + }).rows == [[map, tuple]] + end + end + + test "fixed strings", %{pool: pool} do + assert Ch.query!( + pool, + "SELECT {empty:FixedString(2)}, {one:FixedString(2)}, {two:FixedString(2)}", + %{ + "empty" => "", + "one" => "a", + "two" => "aa" + } + ).rows == [[<<0, 0>>, "a" <> <<0>>, "aa"]] + + Help.query!("CREATE TABLE type_integration_fixed_string(a FixedString(3)) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_fixed_string") end) + + rowbinary = RowBinary.encode_rows([[""], ["a"], ["aa"], ["aaa"]], ["FixedString(3)"]) + Ch.query!(pool, ["INSERT INTO type_integration_fixed_string FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT * FROM type_integration_fixed_string").rows == [ + [<<0, 0, 0>>], + ["a" <> <<0, 0>>], + ["aa" <> <<0>>], + ["aaa"] + ] + end + + test "decimals", %{pool: pool} do + assert Ch.query!(pool, """ + SELECT + toDecimal32(2, 4), + toDecimal64(2, 4), + toDecimal128(2, 4), + toDecimal256(2, 4) + """).rows == [ + [ + Decimal.new("2.0000"), + Decimal.new("2.0000"), + Decimal.new("2.0000"), + Decimal.new("2.0000") + ] + ] + + Help.query!("CREATE TABLE type_integration_decimal(d Decimal32(4)) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_decimal") end) + + rowbinary = + RowBinary.encode_rows( + [[Decimal.new("2.66")], [Decimal.new("2.6666")], [Decimal.new("2.66666")]], + ["Decimal32(4)"] + ) + + Ch.query!(pool, ["INSERT INTO type_integration_decimal FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT * FROM type_integration_decimal").rows == [ + [Decimal.new("2.6600")], + [Decimal.new("2.6666")], + [Decimal.new("2.6667")] + ] + end + + test "booleans", %{pool: pool} do + Help.query!("CREATE TABLE type_integration_bool(a Int64, b Bool) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_bool") end) + + Ch.query!(pool, "INSERT INTO type_integration_bool VALUES (1, true), (2, 0), (5, 2)") + + rowbinary = RowBinary.encode_rows([[3, true], [4, false]], ["Int64", "Bool"]) + Ch.query!(pool, ["INSERT INTO type_integration_bool FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT *, a * b FROM type_integration_bool ORDER BY a").rows == [ + [1, true, 1], + [2, false, 0], + [3, true, 3], + [4, false, 0], + [5, true, 5] + ] + end + + property "Bool values inserted as RowBinary round-trip", %{pool: pool} do + Help.query!("CREATE TABLE type_integration_bool_property(id UInt8, b Bool) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_bool_property") end) + + check all rows <- bool_rows() do + Ch.query!(pool, "TRUNCATE TABLE type_integration_bool_property") + + rowbinary = RowBinary.encode_rows(rows, ["UInt8", "Bool"]) + + Ch.query!(pool, [ + "INSERT INTO type_integration_bool_property FORMAT RowBinary\n" | rowbinary + ]) + + assert Ch.query!(pool, "SELECT * FROM type_integration_bool_property ORDER BY id").rows == + Enum.sort_by(rows, &List.first/1) + end + end + + test "uuid", %{pool: pool} do + uuid = "417ddc5d-e556-4d27-95dd-a34d84e46a50" + uuid_bin = uuid |> String.replace("-", "") |> Base.decode16!(case: :lower) + + assert Ch.query!(pool, "SELECT {uuid:UUID}, toString({uuid:UUID})", %{"uuid" => uuid}).rows == + [[uuid_bin, uuid]] + + Help.query!("CREATE TABLE type_integration_uuid(x UUID, y String) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_uuid") end) + + Ch.query!(pool, "INSERT INTO type_integration_uuid SELECT generateUUIDv4(), 'Example 1'") + Ch.query!(pool, "INSERT INTO type_integration_uuid(y) VALUES ('Example 2')") + + rowbinary = RowBinary.encode_rows([[uuid_bin, "Example 3"]], ["UUID", "String"]) + Ch.query!(pool, ["INSERT INTO type_integration_uuid(x, y) FORMAT RowBinary\n" | rowbinary]) + + assert [ + [generated_uuid, "Example 1"], + [<<0::128>>, "Example 2"], + [^uuid_bin, "Example 3"] + ] = Ch.query!(pool, "SELECT * FROM type_integration_uuid ORDER BY y").rows + + assert byte_size(generated_uuid) == 16 + end + + test "enum8", %{pool: pool} do + Help.query!( + "CREATE TABLE type_integration_enum(i UInt8, x Enum('hello' = 1, 'world' = 2)) ENGINE Memory" + ) + + on_exit(fn -> Help.query!("DROP TABLE type_integration_enum") end) + + Ch.query!( + pool, + "INSERT INTO type_integration_enum VALUES (0, 'hello'), (1, 'world'), (2, 'hello')" + ) + + rowbinary = + RowBinary.encode_rows( + [[3, "hello"], [4, "world"], [5, 1], [6, 2]], + ["UInt8", "Enum8('hello' = 1, 'world' = 2)"] + ) + + Ch.query!(pool, ["INSERT INTO type_integration_enum(i, x) FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT *, CAST(x, 'Int8') FROM type_integration_enum ORDER BY i").rows == + [ + [0, "hello", 1], + [1, "world", 2], + [2, "hello", 1], + [3, "hello", 1], + [4, "world", 2], + [5, "hello", 1], + [6, "world", 2] + ] + end + + test "map and tuple", %{pool: pool} do + assert Ch.query!(pool, "SELECT {map:Map(String, UInt8)}, {tuple:Tuple(Int8, String)}", %{ + "map" => %{"pg" => 13, "hello" => 100}, + "tuple" => {-1, "abs"} + }).rows == [[%{"hello" => 100, "pg" => 13}, {-1, "abs"}]] + + Help.query!("CREATE TABLE type_integration_tuple(a Tuple(String, Int64)) ENGINE Memory") + on_exit(fn -> Help.query!("DROP TABLE type_integration_tuple") end) + + Ch.query!(pool, "INSERT INTO type_integration_tuple VALUES (('y', 10)), (('x', -10))") + rowbinary = RowBinary.encode_rows([[{"a", 20}], [{"b", 30}]], ["Tuple(String, Int64)"]) + Ch.query!(pool, ["INSERT INTO type_integration_tuple FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT a FROM type_integration_tuple ORDER BY a.1").rows == [ + [{"a", 20}], + [{"b", 30}], + [{"x", -10}], + [{"y", 10}] + ] + end + + test "datetime and datetime64 with timezone", %{pool: pool} do + Help.query!(""" + CREATE TABLE type_integration_datetime( + timestamp DateTime('Asia/Istanbul'), + precise DateTime64(3, 'Asia/Istanbul'), + event_id UInt8 + ) ENGINE Memory + """) + + on_exit(fn -> Help.query!("DROP TABLE type_integration_datetime") end) + + Ch.query!(pool, """ + INSERT INTO type_integration_datetime VALUES + (1546300800, 1546300800123, 1), + ('2019-01-01 00:00:00', '2019-01-01 00:00:00.123', 2) + """) + + assert Ch.query!( + pool, + "SELECT *, toString(timestamp), toString(precise) FROM type_integration_datetime ORDER BY event_id" + ).rows == + [ + [ + DateTime.new!(~D[2019-01-01], ~T[03:00:00], "Asia/Istanbul"), + DateTime.new!(~D[2019-01-01], ~T[03:00:00.123], "Asia/Istanbul"), + 1, + "2019-01-01 03:00:00", + "2019-01-01 03:00:00.123" + ], + [ + DateTime.new!(~D[2019-01-01], ~T[00:00:00], "Asia/Istanbul"), + DateTime.new!(~D[2019-01-01], ~T[00:00:00.123], "Asia/Istanbul"), + 2, + "2019-01-01 00:00:00", + "2019-01-01 00:00:00.123" + ] + ] + end + + defp integer_param do + one_of([ + typed_integer("Int8", -128..127), + typed_integer("Int16", -32_768..32_767), + typed_integer("Int32", -2_147_483_648..2_147_483_647), + typed_integer("Int64", -9_007_199_254_740_992..9_007_199_254_740_991), + typed_integer("UInt8", 0..255), + typed_integer("UInt16", 0..65_535), + typed_integer("UInt32", 0..4_294_967_295), + typed_integer("UInt64", 0..9_007_199_254_740_991) + ]) + end + + defp typed_integer(type, range) do + gen all value <- integer(range) do + {type, value} + end + end + + defp fixed_string_param do + gen all size <- integer(1..12), + value <- string(:alphanumeric, max_length: size) do + {size, value} + end + end + + defp decimal_param do + gen all sign <- member_of([1, -1]), + coef <- integer(0..999_999_999), + exp <- integer(-4..4) do + Decimal.new(sign, coef, exp) + end + end + + defp uuid_param do + gen all bytes <- binary(length: 16) do + <> = bytes + + uuid = + [a, b, c, d, e] + |> Enum.map_join("-", &Base.encode16(&1, case: :lower)) + + {uuid, bytes} + end + end + + defp utc_datetime64 do + gen all date <- date_gen(), + hour <- integer(0..23), + minute <- integer(0..59), + second <- integer(0..59), + microsecond <- integer(0..999_999) do + DateTime.new!(date, Time.new!(hour, minute, second, {microsecond, 6}), "Etc/UTC") + end + end + + defp tuple_param do + gen all n <- integer(-128..127), + string <- safe_string() do + {n, string} + end + end + + defp bool_rows do + gen all ids <- uniq_list_of(integer(0..255), max_length: 32), + values <- list_of(boolean(), length: length(ids)) do + Enum.zip_with(ids, values, fn id, value -> [id, value] end) + end + end + + defp date_gen do + gen all days <- integer(0..20_000) do + Date.add(~D[1970-01-01], days) + end + end + + defp safe_string do + string(:printable, max_length: 32) + end +end diff --git a/test/ch/variant_test.exs b/test/ch/variant_test.exs index fbd7144f..af0882ca 100644 --- a/test/ch/variant_test.exs +++ b/test/ch/variant_test.exs @@ -1,36 +1,30 @@ defmodule Ch.VariantTest do - use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] - import Ch.Test, only: [parameterize_query!: 2, parameterize_query!: 4] + use ExUnit.Case, async: true # https://clickhouse.com/docs/sql-reference/data-types/variant @moduletag :variant setup do - conn = start_supervised!({Ch, database: Ch.Test.database()}) - {:ok, conn: conn} + {:ok, pool: start_supervised!(Ch)} end - test "basic", ctx do - assert parameterize_query!(ctx, "select null::Variant(UInt64, String, Array(UInt64))").rows == + test "basic", %{pool: pool} do + assert Ch.query!(pool, "select null::Variant(UInt64, String, Array(UInt64))").rows == [[nil]] - assert parameterize_query!(ctx, "select [1]::Variant(UInt64, String, Array(UInt64))").rows == + assert Ch.query!(pool, "select [1]::Variant(UInt64, String, Array(UInt64))").rows == [[[1]]] - assert parameterize_query!(ctx, "select 0::Variant(UInt64, String, Array(UInt64))").rows == [ - [0] - ] + assert Ch.query!(pool, "select 0::Variant(UInt64, String, Array(UInt64))").rows == + [[0]] - assert parameterize_query!( - ctx, - "select 'Hello, World!'::Variant(UInt64, String, Array(UInt64))" - ).rows == + assert Ch.query!(pool, "select 'Hello, World!'::Variant(UInt64, String, Array(UInt64))").rows == [["Hello, World!"]] end # https://github.com/plausible/ch/issues/272 - test "ordering internal types", ctx do + test "ordering internal types", %{pool: pool} do test = %{ "'hello'" => "hello", "-10" => -10, @@ -40,27 +34,27 @@ defmodule Ch.VariantTest do } for {value, expected} <- test do - assert parameterize_query!( - ctx, + assert Ch.query!( + pool, "select #{value}::Variant(String, Int32, Bool, Map(String, Nullable(String)))" ).rows == [[expected]] end end - test "with a table", ctx do + test "with a table", %{pool: pool} do # https://clickhouse.com/docs/sql-reference/data-types/variant#creating-variant - parameterize_query!(ctx, """ - CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; + Help.query!(""" + CREATE TABLE variant_test_table (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) - on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) + on_exit(fn -> Help.query!("DROP TABLE variant_test_table") end) - parameterize_query!( - ctx, - "INSERT INTO variant_test VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" + Ch.query!( + pool, + "INSERT INTO variant_test_table VALUES (NULL), (42), ('Hello, World!'), ([1, 2, 3]);" ) - assert parameterize_query!(ctx, "SELECT v FROM variant_test").rows == [ + assert Ch.query!(pool, "SELECT v FROM variant_test_table").rows == [ [nil], [42], ["Hello, World!"], @@ -68,9 +62,9 @@ defmodule Ch.VariantTest do ] # https://clickhouse.com/docs/sql-reference/data-types/variant#reading-variant-nested-types-as-subcolumns - assert parameterize_query!( - ctx, - "SELECT v, v.String, v.UInt64, v.`Array(UInt64)` FROM variant_test;" + assert Ch.query!( + pool, + "SELECT v, v.String, v.UInt64, v.`Array(UInt64)` FROM variant_test_table;" ).rows == [ [nil, nil, nil, []], @@ -79,9 +73,9 @@ defmodule Ch.VariantTest do [[1, 2, 3], nil, nil, [1, 2, 3]] ] - assert parameterize_query!( - ctx, - "SELECT v, variantElement(v, 'String'), variantElement(v, 'UInt64'), variantElement(v, 'Array(UInt64)') FROM variant_test;" + assert Ch.query!( + pool, + "SELECT v, variantElement(v, 'String'), variantElement(v, 'UInt64'), variantElement(v, 'Array(UInt64)') FROM variant_test_table;" ).rows == [ [nil, nil, nil, []], [42, nil, 42, []], @@ -90,25 +84,19 @@ defmodule Ch.VariantTest do ] end - test "rowbinary", ctx do - parameterize_query!(ctx, """ - CREATE TABLE variant_test (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; + test "rowbinary", %{pool: pool} do + Help.query!(""" + CREATE TABLE variant_test_rowbinary (v Variant(UInt64, String, Array(UInt64))) ENGINE = Memory; """) - on_exit(fn -> Ch.Test.query("DROP TABLE variant_test") end) + on_exit(fn -> Help.query!("DROP TABLE variant_test_rowbinary") end) - parameterize_query!( - ctx, - "INSERT INTO variant_test FORMAT RowBinary", - [[nil], [42], ["Hello, World!"], [[1, 2, 3]]], - types: ["Variant(UInt64, String, Array(UInt64))"] - ) + rows = [[nil], [42], ["Hello, World!"], [[1, 2, 3]]] - assert parameterize_query!(ctx, "SELECT v FROM variant_test").rows == [ - [nil], - [42], - ["Hello, World!"], - [[1, 2, 3]] - ] + rowbinary = Ch.RowBinary.encode_rows(rows, ["Variant(UInt64, String, Array(UInt64))"]) + + Ch.query!(pool, ["INSERT INTO variant_test_rowbinary FORMAT RowBinary\n" | rowbinary]) + + assert Ch.query!(pool, "SELECT v FROM variant_test_rowbinary").rows == rows end end diff --git a/test/support/help.ex b/test/support/help.ex new file mode 100644 index 00000000..eff84a56 --- /dev/null +++ b/test/support/help.ex @@ -0,0 +1,35 @@ +defmodule Help do + @moduledoc false + + @pool Ch.TestPool + + def session_id(%{module: module, test: test}) do + rand = + Base.hex_encode32( + << + System.system_time(:nanosecond)::64, + :erlang.phash2(self(), 16_777_216)::24, + :erlang.unique_integer()::32 + >>, + case: :lower + ) + + "#{module}-#{test}-#{rand}" + end + + def start_link_pool(url) do + Ch.start_link(name: @pool, url: url, pool_size: 100) + end + + def query(statement, params \\ %{}, options \\ []) do + Ch.query(@pool, statement, params, options) + end + + def query!(statement, params \\ %{}, options \\ []) do + Ch.query!(@pool, statement, params, options) + end + + def to_maps(%{names: names, rows: rows}) do + Enum.map(rows, fn row -> names |> Enum.zip(row) |> Map.new() end) + end +end diff --git a/test/support/test.ex b/test/support/test.ex deleted file mode 100644 index 301e9ab1..00000000 --- a/test/support/test.ex +++ /dev/null @@ -1,123 +0,0 @@ -defmodule Ch.Test do - @moduledoc false - - def database, do: Application.fetch_env!(:ch, :database) - - # makes a query in a short lived process so that pool automatically exits once finished - def query(sql, params \\ [], opts \\ []) do - task = - Task.async(fn -> - {:ok, pid} = Ch.start_link(opts) - opts = Keyword.put_new_lazy(opts, :database, &database/0) - Ch.query!(pid, sql, params, opts) - end) - - Task.await(task) - end - - # helper for ExUnit.Case :parameterize - def parameterize_query_options(ctx, options \\ []) do - if default_options = ctx[:query_options] do - Keyword.merge(default_options, options) - else - options - end - end - - def parameterize_query(ctx, sql, params \\ [], options \\ []) do - Ch.query( - ctx.conn, - sql, - params, - parameterize_query_options(ctx, options) - ) - end - - def parameterize_query!(ctx, sql, params \\ [], options \\ []) do - Ch.query!( - ctx.conn, - sql, - params, - parameterize_query_options(ctx, options) - ) - end - - # TODO packet: :http? - def intercept_packets(socket, buffer \\ <<>>) do - receive do - {:tcp, ^socket, packet} -> - buffer = buffer <> packet - - if complete?(buffer) do - buffer - else - intercept_packets(socket, buffer) - end - end - end - - defp complete?(buffer) do - with {:ok, rest} <- eat_status(buffer), - {:ok, content_length, rest} <- eat_headers(rest) do - verify_body(content_length, rest) - else - _ -> false - end - end - - defp eat_status(buffer) do - case :erlang.decode_packet(:http_bin, buffer, []) do - {:ok, _, rest} -> {:ok, rest} - {:more, _} -> {:more, buffer} - end - end - - defp eat_headers(buffer, content_length \\ nil) do - case :erlang.decode_packet(:httph_bin, buffer, []) do - {:ok, {_, _, :"Content-Length", _, content_length}, rest} -> - eat_headers(rest, String.to_integer(content_length)) - - {:ok, {_, _, :"Transfer-Encoding", _, "chunked"}, rest} -> - eat_headers(rest, :chunked) - - {:ok, :http_eoh, rest} -> - {:ok, content_length, rest} - - {:ok, _, rest} -> - eat_headers(rest, content_length) - - {:more, _} -> - {:more, buffer} - end - end - - defp verify_body(:chunked, chunks) do - String.ends_with?(chunks, "\r\n0\r\n\r\n") - end - - defp verify_body(content_length, body) do - byte_size(body) == content_length - end - - # shifts naive datetimes for non-utc timezones into utc to match ClickHouse behaviour - # see https://clickhouse.com/docs/en/sql-reference/data-types/datetime#usage-remarks - def to_clickhouse_naive(conn, %NaiveDateTime{} = naive_datetime) do - case Ch.query!(conn, "select timezone()").rows do - [["UTC"]] -> - naive_datetime - - [[timezone]] -> - naive_datetime - |> DateTime.from_naive!(timezone) - |> DateTime.shift_zone!("Etc/UTC") - |> DateTime.to_naive() - end - end - - def clickhouse_tz(conn) do - case Ch.query!(conn, "select timezone()").rows do - [["UTC"]] -> "Etc/UTC" - [[timezone]] -> timezone - end - end -end diff --git a/test/test_helper.exs b/test/test_helper.exs index 97caedd8..f4c80f85 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,46 +1,39 @@ -clickhouse_available? = - case :httpc.request(:get, {~c"http://localhost:8123/ping", []}, [], []) do - {:ok, {{_version, _status = 200, _reason}, _headers, ~c"Ok.\n"}} -> - true +url = "http://localhost:8123" - {:error, {:failed_connect, [{:to_address, _to_address}, {:inet, [:inet], :econnrefused}]}} -> - false - end - -unless clickhouse_available? do - Mix.shell().error(""" - ClickHouse is not detected at localhost:8123! Please start the local container with the following command: +{:ok, _pid} = Help.start_link_pool(url) - docker compose up -d clickhouse - """) +version = + case Help.query("select version()") do + {:ok, %{rows: [[version]]}} -> + version - System.halt(1) -end + {:error, reason} -> + Mix.shell().error(""" + ClickHouse is not detected at #{url}: #{Exception.message(reason)} -Calendar.put_time_zone_database(Tz.TimeZoneDatabase) -default_test_db = System.get_env("CH_DATABASE", "ch_elixir_test") -Application.put_env(:ch, :database, default_test_db) + Please start the container with the following command: -Ch.Test.query( - "DROP DATABASE IF EXISTS {db:Identifier}", - %{"db" => default_test_db}, - database: "default" -) + docker compose up -d clickhouse + """) -Ch.Test.query( - "CREATE DATABASE {db:Identifier}", - %{"db" => default_test_db}, - database: "default" -) - -%{rows: [[ch_version]]} = Ch.Test.query("SELECT version()") + System.halt(1) + end -extra_exclude = - if ch_version >= "25" do +exclude = + if version >= "25" do [] else # Time, Variant, JSON, and Dynamic types are not supported in older ClickHouse versions we have in the CI [:time, :variant, :json, :dynamic] end -ExUnit.start(exclude: [:slow | extra_exclude]) +assert_receive_timeout = + if System.get_env("CI") do + to_timeout(second: 5) + else + to_timeout(second: 1) + end + +Calendar.put_time_zone_database(Tz.TimeZoneDatabase) + +ExUnit.start(exclude: exclude, assert_receive_timeout: assert_receive_timeout)