diff --git a/Makefile b/Makefile index 5439ae970..e185bff77 100644 --- a/Makefile +++ b/Makefile @@ -134,10 +134,11 @@ GO_TEST_FLAGS ?= -coverprofile=.coverprofile .PHONY: test test: check-license-headers check-codegen gotest check-fmtprints check-todos -GO_TEST_PATH ?= $(shell $(GO) list ./... | grep -v v2/integration) +GO_TEST_PATH ?= $(shell $(GO) list ./... | grep -v v2/integration | tr '\n' ' ') .PHONY: gotest gotest: $(GO) test -timeout=5m -v ${GO_TEST_FLAGS} $(GO_TEST_PATH) + @echo "All tests passed successfully." .PHONY: data-race-test data-race-test: diff --git a/docs/developer/environment/README.md b/docs/developer/environment/README.md index 7bbf7bb6a..a5bbc38d2 100644 --- a/docs/developer/environment/README.md +++ b/docs/developer/environment/README.md @@ -31,3 +31,19 @@ for verification purposes. You can stop the developer environment by running `make developer-stop`. To delete the developer environment, run `make developer-delete` which will destroy all data. + +## Included backends + +The Compose file brings up Prometheus, InfluxDB 2.x, InfluxDB 3.x Core (on +`:8181`, seeded by the telegraf container) and ClickHouse alongside Grafana. +Trickster's dev config registers a matching backend for each, so Grafana can +query the upstream directly or via Trickster for a side-by-side comparison. + +For InfluxDB 3.x, Trickster also exposes an Apache Arrow Flight SQL (gRPC) +proxy on `:8485` (`flight_port` in the backend config). The +`Trickster InfluxDB 3` Grafana dashboard exercises both the HTTP InfluxQL +endpoint and Flight SQL via the InfluxDB datasource in SQL mode, with direct +and Trickster-cached targets on the same panel. An equivalent +`ClickHouse (Grafana Plugin)` dashboard is included for ClickHouse. The +`verify-influxdb3.sh` script runs a quick end-to-end check of the v3 HTTP +and Flight SQL paths through Trickster. diff --git a/docs/developer/environment/docker-compose-data/dashboards/trickster-clickhouse-grafana.json b/docs/developer/environment/docker-compose-data/dashboards/trickster-clickhouse-grafana.json new file mode 100644 index 000000000..33a0ef6ff --- /dev/null +++ b/docs/developer/environment/docker-compose-data/dashboards/trickster-clickhouse-grafana.json @@ -0,0 +1,183 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 80 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 22, "w": 12, "x": 0, "y": 0 }, + "id": 1, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${datasource}" + }, + "editorType": "sql", + "format": 1, + "meta": { "builderOptions": { "database": "default", "table": "trips" } }, + "queryType": "sql", + "rawSql": "SELECT toStartOfFiveMinute(pickup_datetime) AS t, count() AS Count FROM default.trips WHERE $__timeFilter(pickup_datetime) GROUP BY t ORDER BY t", + "refId": "A" + } + ], + "title": "# Trips Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green" }, + { "color": "red", "value": 80 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 22, "w": 12, "x": 12, "y": 0 }, + "id": 2, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { + "type": "grafana-clickhouse-datasource", + "uid": "${datasource}" + }, + "editorType": "sql", + "format": 1, + "meta": { "builderOptions": { "database": "default", "table": "trips" } }, + "queryType": "sql", + "rawSql": "WITH sumIf(trip_id, payment_type = 'CSH') AS cash_trips, sumIf(trip_id, payment_type != 'CSH') AS non_cash_trips SELECT toStartOfFiveMinute(pickup_datetime) AS t, non_cash_trips / (cash_trips + non_cash_trips) * 100 AS \"Card Use Rate\" FROM default.trips WHERE $__timeFilter(pickup_datetime) GROUP BY t ORDER BY t", + "refId": "A" + } + ], + "title": "% Trips Paid w/ Credit Card", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5s", + "schemaVersion": 41, + "tags": [], + "templating": { + "list": [ + { + "allowCustomValue": false, + "current": { + "text": "clickhouse-grafana-direct", + "value": "clickhouse-grafana-direct" + }, + "description": "", + "name": "datasource", + "options": [], + "query": "grafana-clickhouse-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "ClickHouse (Grafana Plugin)", + "uid": "clickhouse-grafana-plugin", + "version": 1 +} diff --git a/docs/developer/environment/docker-compose-data/dashboards/trickster-influxdb3.json b/docs/developer/environment/docker-compose-data/dashboards/trickster-influxdb3.json new file mode 100644 index 000000000..0094fc120 --- /dev/null +++ b/docs/developer/environment/docker-compose-data/dashboards/trickster-influxdb3.json @@ -0,0 +1,266 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Exercises both HTTP InfluxQL (cached via /query) and Flight SQL (cached via gRPC passthrough). HTTP SQL (/api/v3/query_sql) is tested via curl — Grafana does not speak HTTP for SQL mode.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "type": "row", + "title": "$measurement", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "panels": [] + }, + { + "title": "HTTP InfluxQL — ${measurement} row count (Direct vs Trickster)", + "type": "timeseries", + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "gridPos": { "h": 10, "w": 12, "x": 0, "y": 1 }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, + "pointSize": 5, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byFrameRefID", "options": "Direct" }, + "properties": [ + { "id": "displayName", "value": "Direct (InfluxDB 3)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + }, + { + "matcher": { "id": "byFrameRefID", "options": "Trickster" }, + "properties": [ + { "id": "displayName", "value": "Trickster (cached)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "custom.lineWidth", "value": 3 } + ] + } + ] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_direct_ql" }, + "query": "SELECT count(*) FROM \"${measurement}\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": true, "resultFormat": "time_series", "refId": "Direct" + }, + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_ql" }, + "query": "SELECT count(*) FROM \"${measurement}\" WHERE $timeFilter GROUP BY time($__interval) fill(0)", + "rawQuery": true, "resultFormat": "time_series", "refId": "Trickster" + } + ] + }, + { + "title": "Flight SQL — ${measurement} row count (Direct vs Trickster)", + "type": "timeseries", + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "gridPos": { "h": 10, "w": 12, "x": 12, "y": 1 }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, + "pointSize": 5, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { "id": "byFrameRefID", "options": "Direct" }, + "properties": [ + { "id": "displayName", "value": "Direct (InfluxDB 3)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + }, + { + "matcher": { "id": "byFrameRefID", "options": "Trickster" }, + "properties": [ + { "id": "displayName", "value": "Trickster (cached)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "custom.lineWidth", "value": 3 } + ] + } + ] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_direct_flight" }, + "rawSql": "SELECT date_bin(INTERVAL '${__interval_ms} milliseconds', time) AS time, count(*) AS count FROM ${measurement} WHERE time >= $__timeFrom AND time < $__timeTo GROUP BY 1 ORDER BY 1", + "resultFormat": "time_series", "refId": "Direct" + }, + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "rawSql": "SELECT date_bin(INTERVAL '${__interval_ms} milliseconds', time) AS time, count(*) AS count FROM ${measurement} WHERE time >= $__timeFrom AND time < $__timeTo GROUP BY 1 ORDER BY 1", + "resultFormat": "time_series", "refId": "Trickster" + } + ] + }, + { + "title": "${measurement}.${field} over time (Direct vs Trickster)", + "type": "timeseries", + "datasource": { "type": "datasource", "uid": "-- Mixed --" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 11 }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "lineWidth": 2, "fillOpacity": 10, + "pointSize": 5, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" } + } + }, + "overrides": [ + { + "matcher": { "id": "byFrameRefID", "options": "Direct" }, + "properties": [ + { "id": "displayName", "value": "Direct (InfluxDB 3)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } + ] + }, + { + "matcher": { "id": "byFrameRefID", "options": "Trickster" }, + "properties": [ + { "id": "displayName", "value": "Trickster (cached)" }, + { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }, + { "id": "custom.lineStyle", "value": { "fill": "dash", "dash": [10, 10] } }, + { "id": "custom.lineWidth", "value": 3 } + ] + } + ] + }, + "options": { + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "mean"] } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_direct_flight" }, + "rawSql": "SELECT date_bin(INTERVAL '${__interval_ms} milliseconds', time) AS time, avg(\"${field}\") AS \"${field}\" FROM ${measurement} WHERE time >= $__timeFrom AND time < $__timeTo GROUP BY 1 ORDER BY 1", + "resultFormat": "time_series", "refId": "Direct" + }, + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "rawSql": "SELECT date_bin(INTERVAL '${__interval_ms} milliseconds', time) AS time, avg(\"${field}\") AS \"${field}\" FROM ${measurement} WHERE time >= $__timeFrom AND time < $__timeTo GROUP BY 1 ORDER BY 1", + "resultFormat": "time_series", "refId": "Trickster" + } + ] + }, + { + "title": "Flight SQL — Latest ${measurement} rows (via Trickster)", + "type": "table", + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 21 }, + "description": "Latest 20 rows of the selected measurement. Exercises a non-aggregated Flight SQL query through Trickster's cache.", + "fieldConfig": { + "defaults": { "custom": { "align": "left" } }, + "overrides": [] + }, + "options": { "showHeader": true, "cellHeight": "sm" }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "rawSql": "SELECT * FROM (SELECT * FROM ${measurement} WHERE time >= $__timeFrom AND time < $__timeTo ORDER BY time DESC LIMIT 20) ORDER BY time ASC", + "resultFormat": "table", + "refId": "Latest" + } + ] + }, + { + "type": "row", + "title": "metadata", + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, + "panels": [] + }, + { + "title": "Flight SQL — Tables in trickster DB (information_schema)", + "type": "table", + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 32 }, + "description": "Exercises InfluxDB 3's information_schema via Trickster's Flight SQL proxy. Proves the Flight SQL query path end-to-end with a non-time-series result set.", + "fieldConfig": { + "defaults": { "custom": { "align": "left" } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "time" }, + "properties": [{ "id": "custom.hidden", "value": true }] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm" + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "rawSql": "SELECT now() AS time, table_catalog, table_schema, table_name, table_type FROM information_schema.tables WHERE table_schema IN ('iox', 'system') ORDER BY table_schema, table_name", + "resultFormat": "table", + "refId": "Tables" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["influxdb", "influxdb3", "trickster"], + "templating": { + "list": [ + { + "name": "measurement", + "label": "Measurement (via Flight SQL)", + "type": "query", + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "query": "SELECT DISTINCT table_name FROM information_schema.tables WHERE table_schema = 'iox' ORDER BY table_name", + "refresh": 1, + "multi": false, + "includeAll": false, + "current": { "value": "cpu", "text": "cpu" } + }, + { + "name": "field", + "label": "Field (numeric columns of selected measurement)", + "type": "query", + "datasource": { "type": "influxdb", "uid": "ds_infl3_trk_flight" }, + "query": "SELECT column_name FROM information_schema.columns WHERE table_schema = 'iox' AND table_name = '${measurement}' AND data_type IN ('Float64', 'Int64', 'UInt64') ORDER BY column_name", + "refresh": 1, + "multi": false, + "includeAll": false + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Trickster - InfluxDB 3.x", + "uid": "trickster-influxdb3", + "version": 4 +} diff --git a/docs/developer/environment/docker-compose-data/grafana-config/provisioning/datasources/datasource.yaml b/docs/developer/environment/docker-compose-data/grafana-config/provisioning/datasources/datasource.yaml index e567c6f5c..f31f3ec6c 100644 --- a/docs/developer/environment/docker-compose-data/grafana-config/provisioning/datasources/datasource.yaml +++ b/docs/developer/environment/docker-compose-data/grafana-config/provisioning/datasources/datasource.yaml @@ -197,6 +197,74 @@ datasources: secureJsonData: token: trickster-dev-token +# InfluxDB 3.x (SQL via HTTP — not used by Grafana, here for parity) +- name: "1: InfluxDB3 | Direct | SQL" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_direct_sql + url: http://influxdb3:8181 + editable: true + jsonData: + version: SQL + dbName: trickster + +- name: "2: InfluxDB3 | Trickster | SQL" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_trk_sql + url: http://host.docker.internal:8480/influx3 + editable: true + jsonData: + version: SQL + dbName: trickster + +- name: "3: InfluxDB3 | Direct | InfluxQL" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_direct_ql + url: http://influxdb3:8181 + editable: true + jsonData: + dbName: trickster + +- name: "4: InfluxDB3 | Trickster | InfluxQL" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_trk_ql + url: http://host.docker.internal:8480/influx3 + editable: true + jsonData: + dbName: trickster + +# InfluxDB 3.x via Flight SQL (gRPC) — Trickster proxies on port 8485 +- name: "5: InfluxDB3 | Trickster | SQL (Flight)" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_trk_flight + url: http://host.docker.internal:8485 + editable: true + jsonData: + version: SQL + dbName: trickster + insecureGrpc: true + +- name: "6: InfluxDB3 | Direct | SQL (Flight)" + type: influxdb + access: proxy + orgId: 1 + uid: ds_infl3_direct_flight + url: http://influxdb3:8181 + editable: true + jsonData: + version: SQL + dbName: trickster + insecureGrpc: true + # ClickHouse - name: clickhouse-direct type: vertamedia-clickhouse-datasource @@ -230,4 +298,30 @@ datasources: defaultDatabase: default username: default password: "" - usePOST: true \ No newline at end of file + usePOST: true + +# ClickHouse (Official Grafana Plugin) +- name: clickhouse-grafana-direct + type: grafana-clickhouse-datasource + access: proxy + editable: true + isDefault: false + jsonData: + host: clickhouse + port: 8123 + protocol: http + defaultDatabase: default + username: default + +- name: clickhouse-grafana-trickster + type: grafana-clickhouse-datasource + access: proxy + editable: true + isDefault: false + jsonData: + host: host.docker.internal + port: 8480 + path: /click1/ + protocol: http + defaultDatabase: default + username: default \ No newline at end of file diff --git a/docs/developer/environment/docker-compose-data/telegraf-config/telegraf.conf b/docs/developer/environment/docker-compose-data/telegraf-config/telegraf.conf index 42ad0e27f..a67988675 100755 --- a/docs/developer/environment/docker-compose-data/telegraf-config/telegraf.conf +++ b/docs/developer/environment/docker-compose-data/telegraf-config/telegraf.conf @@ -18,8 +18,13 @@ [[inputs.net]] -[[outputs.influxdb_v2]] +[[outputs.influxdb_v2]] urls = ["http://influxdb2:8086"] token = "trickster-dev-token" organization = "trickster-dev" bucket = "trickster" + +[[outputs.influxdb]] + urls = ["http://influxdb3:8181"] + database = "trickster" + skip_database_creation = true diff --git a/docs/developer/environment/docker-compose.yml b/docs/developer/environment/docker-compose.yml index c5721c38f..aaef0b83a 100644 --- a/docs/developer/environment/docker-compose.yml +++ b/docs/developer/environment/docker-compose.yml @@ -22,7 +22,7 @@ services: image: grafana/grafana:11.6.0 user: nobody environment: - - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource + - GF_INSTALL_PLUGINS=vertamedia-clickhouse-datasource,grafana-clickhouse-datasource volumes: - ./docker-compose-data/grafana-config:/etc/grafana - ./docker-compose-data/dashboards:/var/lib/grafana/dashboards @@ -91,6 +91,24 @@ services: # trying to setup an influxdb instance with the influxdb_cli service. - influxdb2 + # influxdb 3.x + influxdb3: + image: quay.io/influxdb/influxdb3-core:latest + ports: + - 8181:8181 + volumes: + - influxdb3-data:/var/lib/influxdb3 + build: + context: . + network: host + restart: always + command: + - serve + - --node-id=dev01 + - --object-store=file + - --data-dir=/var/lib/influxdb3 + - --without-auth + # clickhouse clickhouse: image: clickhouse/clickhouse-server:25.3 @@ -155,7 +173,9 @@ services: - ./docker-compose-data/telegraf-config/telegraf.conf:/etc/telegraf/telegraf.conf depends_on: - influxdb2 + - influxdb3 volumes: influxdbv2: + influxdb3-data: prometheus-data: diff --git a/docs/developer/environment/trickster-config/trickster.yaml b/docs/developer/environment/trickster-config/trickster.yaml index 0e2df8cad..8eeac534d 100644 --- a/docs/developer/environment/trickster-config/trickster.yaml +++ b/docs/developer/environment/trickster-config/trickster.yaml @@ -117,7 +117,7 @@ backends: cache_name: redis flux1: provider: influxdb - origin_url: 'http://127.0.0.1:8186/' + origin_url: 'http://127.0.0.1:8086/' cache_name: mem1 backfill_tolerance: 30s timeseries_retention_factor: 5184000 @@ -127,6 +127,14 @@ backends: cache_name: mem1 backfill_tolerance: 30s timeseries_retention_factor: 5184000 + influx3: + provider: influxdb + origin_url: 'http://127.0.0.1:8181/' + cache_name: mem1 + backfill_tolerance: 30s + timeseries_retention_factor: 5184000 + flight_port: 8485 + flight_upstream_address: '127.0.0.1:8181' sim1: provider: prometheus origin_url: 'http://127.0.0.1:8482/prometheus' diff --git a/docs/developer/environment/verify-influxdb3.sh b/docs/developer/environment/verify-influxdb3.sh new file mode 100755 index 000000000..dc7c2e73d --- /dev/null +++ b/docs/developer/environment/verify-influxdb3.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Exercises all three InfluxDB 3.x query paths through Trickster: +# 1. HTTP InfluxQL via /api/v3/query_influxql +# 2. HTTP SQL via /api/v3/query_sql +# 3. Flight SQL via gRPC (requires grpcurl) +# +# Usage: ./verify-influxdb3.sh [base_url] +# Default base_url is http://localhost:8480/influx3 + +set -euo pipefail + +BASE="${1:-http://localhost:8480/influx3}" +DIRECT="http://localhost:8181" +TOKEN="trickster-dev-token" +DB="trickster" + +echo "== 1. HTTP InfluxQL via Trickster ==" +curl -fsS -H "Authorization: Bearer ${TOKEN}" \ + --data-urlencode "q=SELECT mean(\"usage_idle\") FROM \"cpu\" WHERE \"cpu\" = 'cpu-total' AND time > now() - 5m GROUP BY time(10s)" \ + --data-urlencode "db=${DB}" \ + "${BASE}/query" | head -c 500 +echo "" + +echo "== 2. HTTP SQL via Trickster (/api/v3/query_sql) ==" +NOW=$(date +%s) +FIVE_MIN_AGO=$((NOW - 300)) +curl -fsS -H "Authorization: Bearer ${TOKEN}" \ + --data-urlencode "q=SELECT date_bin(INTERVAL '10 seconds', time) AS time, avg(usage_idle) FROM cpu WHERE cpu = 'cpu-total' AND time >= ${FIVE_MIN_AGO} AND time < ${NOW} GROUP BY 1 ORDER BY 1" \ + --data-urlencode "db=${DB}" \ + --data-urlencode "format=json" \ + "${BASE}/api/v3/query_sql" | head -c 500 +echo "" + +echo "== 3. Flight SQL via Trickster (port 8485) ==" +if command -v grpcurl >/dev/null 2>&1; then + grpcurl -plaintext -H "authorization: Bearer ${TOKEN}" \ + -d "{\"cmd\": {\"query\": \"SELECT avg(usage_idle) FROM cpu WHERE cpu = 'cpu-total' LIMIT 5\"}}" \ + localhost:8485 arrow.flight.protocol.FlightService/GetFlightInfo | head -c 500 || true +else + echo "grpcurl not installed — skipping Flight SQL test" +fi +echo "" + +echo "== Cache-hit check (re-run HTTP SQL) ==" +curl -fsS -H "Authorization: Bearer ${TOKEN}" \ + --data-urlencode "q=SELECT date_bin(INTERVAL '10 seconds', time) AS time, avg(usage_idle) FROM cpu WHERE cpu = 'cpu-total' AND time >= ${FIVE_MIN_AGO} AND time < ${NOW} GROUP BY 1 ORDER BY 1" \ + --data-urlencode "db=${DB}" \ + --data-urlencode "format=json" \ + -D- \ + "${BASE}/api/v3/query_sql" | grep -i "x-trickster\|status" | head -5 diff --git a/docs/influxdb.md b/docs/influxdb.md index d621afd83..0c75b03d5 100644 --- a/docs/influxdb.md +++ b/docs/influxdb.md @@ -8,10 +8,86 @@ Trickster is tested with the built-in [InfluxDB DataSource Plugin for Grafana](h Trickster uses InfluxDB-provided packages to parse and normalize queries for caching and acceleration. If you find query or response structures that are not yet supported, or providing inconsistent or unexpected results, we'd love for you to report those so we can further improve our InfluxDB support. -Trickster supports integrations with InfluxDB 1.x and 2.0. +Trickster supports integrations with InfluxDB 1.x, 2.x, and 3.x. -### A note on Flux Language Support: +## InfluxDB 3.x Support -Trickster supports the Flux Query Language for general/basic usage. Trickster does not support advanced union-style queries (e.g., with multiple `from` clauses). In this rare use case, these responses will currently provide invalid data, however, a subsequent beta will proxy unsupported requests. +Trickster supports InfluxDB 3.x via both the native v3 API endpoints and the v1/v2 compatibility endpoints. -Trickster currently does not properly handle schema changes within a response CSV body (e.g., multiple CSVs in the same document with their own #annotation and header rows). We will fully support this use case in a future beta. +### Supported v3 Endpoints + +- `GET/POST /api/v3/query_sql` — SQL queries with delta-proxy caching +- `GET/POST /api/v3/query_influxql` — InfluxQL queries via the native v3 API +- `POST /api/v3/write_lp` — line protocol writes (proxied, not cached) + +### SQL Query Caching + +SQL queries using `date_bin()` for time binning are parsed for time range extraction and step detection, enabling delta-proxy caching. Trickster extracts time ranges from `WHERE` clauses and intervals from `date_bin(INTERVAL '...', time)` in `SELECT`. + +Example query that Trickster will accelerate: + +```sql +SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(temperature) +FROM weather +WHERE time >= '2024-01-01 00:00:00' AND time < '2024-01-02 00:00:00' +GROUP BY 1 +``` + +Non-SELECT queries and queries without parseable time ranges are proxied without caching. + +### Response Formats + +Trickster supports the following v3 response formats, controlled by the `format` query parameter: + +- `json` (default) — JSON array of objects +- `jsonl` — JSON Lines (one JSON object per line) +- `csv` — standard CSV with header row + +The `parquet` and `pretty` formats are not supported for caching and will be proxied through. + +### v1/v2 Compatibility + +InfluxDB 3.x ships with v1 and v2 compatibility endpoints. Trickster's existing InfluxQL support works against these endpoints with no additional configuration — just point Trickster at the v3 instance and query via `/query`. + +### Flight SQL (gRPC) + +InfluxDB 3.x exposes SQL via Apache Arrow Flight SQL on gRPC in addition to HTTP. Grafana's InfluxDB datasource in SQL mode, the Python/Rust/Java SDKs, and ADBC all default to Flight SQL, so HTTP-only caching misses a significant fraction of real-world query traffic. + +Trickster can expose a Flight SQL server that proxies to an upstream Flight SQL endpoint and caches the Arrow IPC byte stream. Enable it per-backend: + +```yaml +backends: + influx3: + provider: influxdb + origin_url: 'http://influxdb3:8181/' + flight_port: 8485 # listen on gRPC :8485 + flight_upstream_address: 'influxdb3:8181' # optional, defaults to origin_url host +``` + +The `authorization` and `database` headers are forwarded from the client through to the upstream. Queries are cached keyed by the full SQL statement; re-running the same query returns the cached Arrow response. + +Flight SQL listeners participate in graceful shutdown: on SIGTERM the daemon calls `GracefulStop` on each registered gRPC server with a 5-second drain before forcing closure. Config reload (SIGHUP) replaces the listener in-place — the old server is drained and the new one binds the same port. + +#### Metadata RPCs + +ADBC clients (Grafana's SQL datasource, the Python `adbc_driver_flightsql`, etc.) probe the following RPCs on connect to populate schema browsers and query editors. Trickster proxies these to the upstream and caches the Arrow IPC response: + +- `GetFlightInfoTables` / `DoGetTables` +- `GetFlightInfoCatalogs` / `DoGetCatalogs` +- `GetFlightInfoSchemas` / `DoGetDBSchemas` +- `GetFlightInfoTableTypes` / `DoGetTableTypes` +- `GetFlightInfoSqlInfo` / `DoGetSqlInfo` + +#### Prepared Statements + +Trickster proxies the full prepared statement lifecycle: `CreatePreparedStatement`, `DoPutPreparedStatementQuery` (parameter binding), `DoGetPreparedStatement`, `ClosePreparedStatement`. Cache keys incorporate the bound parameter hash so two clients running the same statement with different values don't alias. + +Note: InfluxDB 3 Core 3.10 reports a parameter schema at prepare time but does not resolve bound values during query planning (upstream limitation). Parameterless prepared statements work end-to-end; parameterized ones require a newer Core or Enterprise build. + +## Flux Language Support + +Trickster supports the Flux Query Language for general/basic usage with InfluxDB 1.x and 2.x. Flux is not supported in InfluxDB 3.x. + +The delta-proxy cache accepts `now()` as a `range()` bound and handles queries with `aggregateWindow(every: ...)` -- the common Grafana shape. Multi-table Flux CSV responses (one table per series in the result set) are also read correctly. + +Trickster does not support advanced union-style queries (e.g., with multiple `from` clauses). In this rare use case, these responses will currently provide invalid data, however, a subsequent beta will proxy unsupported requests. diff --git a/docs/new-changed-2.0.md b/docs/new-changed-2.0.md index 6eaf59c93..2882db559 100644 --- a/docs/new-changed-2.0.md +++ b/docs/new-changed-2.0.md @@ -25,7 +25,7 @@ Trickster 2.0 is a near-complete rewrite of the project with performance, durabi - **Request Rewriters**: You can now chain a collection of [request rewriters](./request_rewriters.md) for more robust request transformation possibilities - **Cache Purging**: We now support purging specific cache items by Key (on the public ports) or Path (on the mgmt port). Read more in the [cache documentation](./caches.md) - **Simulated Latency**: You can use Trickster for [Simulated Latency](./simulated-latency.md) in lab environments -- We've added support for InfluxDB 2.0 and Flux Query Language and are targeting support for InfluxDB 3.0 in Trickster v2.1.x. +- We've added support for InfluxDB 2.0 and Flux Query Language, plus InfluxDB 3.x via the native v3 HTTP API (SQL and InfluxQL) and an optional Apache Arrow Flight SQL (gRPC) proxy. See the [InfluxDB support doc](./influxdb.md) for details. ### Configuration & Security - **YAML Configuration**: Trickster 2.0 uses YAML for configuration instead of TOML diff --git a/docs/roadmap.md b/docs/roadmap.md index f01fadcb6..42eb614ef 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -33,7 +33,7 @@ The roadmap for Trickster in 2025 focuses on delivering Trickster versions 2.0 a - [ ] Kube Gateway API support - [ ] More easily-importable Trickster packages by other projects - [ ] Support for MySQL as Time Series - - [ ] Support for InfluxDB 3.0 + - [x] Support for InfluxDB 3.x (HTTP v3 API + Flight SQL) - [ ] Support for Autodiscovery (e.g., Kubernetes Pod Annotations) - [ ] Object Pooling where possible to improve memory management diff --git a/docs/supported-backend-providers.md b/docs/supported-backend-providers.md index 57c5db37a..88ba86141 100644 --- a/docs/supported-backend-providers.md +++ b/docs/supported-backend-providers.md @@ -16,7 +16,7 @@ Trickster fully supports the [Prometheus HTTP API (v1)](https://prometheus.io/do ### InfluxDB -Trickster supports for InfluxDB. Specify `'influxdb'` as the Provider when configuring Trickster. +Trickster supports InfluxDB 1.x, 2.x, and 3.x. Specify `'influxdb'` as the Provider when configuring Trickster. See the [InfluxDB Support Document](./influxdb.md) for more information. diff --git a/examples/conf/example.full.yaml b/examples/conf/example.full.yaml index 87d2a29c0..7f629caef 100644 --- a/examples/conf/example.full.yaml +++ b/examples/conf/example.full.yaml @@ -529,6 +529,25 @@ backends: # healthcheck: # interval: 1s +# # InfluxDB 3.x backend example. See /docs/influxdb.md for full details +# # (v3 HTTP API: /api/v3/query_sql, /api/v3/query_influxql, /api/v3/write_lp; +# # v1/v2 compatibility endpoints also work against the same backend). +# +# influx3: +# is_default: false +# provider: influxdb +# origin_url: 'http://influxdb3:8181/' +# +# # flight_port enables an Apache Arrow Flight SQL (gRPC) listener on the given port. +# # When > 0, Trickster proxies Flight SQL queries, metadata RPCs (GetTables/Schemas/etc.) +# # and the full prepared-statement lifecycle, caching the Arrow IPC byte stream. +# # Leave unset or 0 to disable Flight SQL for this backend. InfluxDB 3.x only. +# flight_port: 8485 +# +# # flight_upstream_address overrides the upstream Flight SQL address. +# # Defaults to the host:port from origin_url when empty. +# flight_upstream_address: 'influxdb3:8181' + # # Application Load Balancer Backend configuration options, see /docs/alb.md for more information # # alb01: diff --git a/go.mod b/go.mod index a4b3bd218..c043c9ebc 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.2 require ( github.com/alicebob/miniredis/v2 v2.37.0 github.com/andybalholm/brotli v1.2.1 + github.com/apache/arrow-go/v18 v18.5.2 github.com/dgraph-io/badger/v4 v4.9.1 github.com/dgraph-io/ristretto/v2 v2.4.0 github.com/influxdata/influxdb v1.12.4 @@ -27,6 +28,7 @@ require ( golang.org/x/crypto v0.50.0 golang.org/x/net v0.53.0 golang.org/x/sync v0.20.0 + google.golang.org/grpc v1.80.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) @@ -84,7 +86,7 @@ require ( github.com/curioswitch/go-reassign v0.3.0 // indirect github.com/daixiang0/gci v0.13.7 // indirect github.com/dave/dst v0.27.3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dlclark/regexp2 v1.11.5 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -108,6 +110,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/godoc-lint/godoc-lint v0.11.2 // indirect github.com/gofrs/flock v0.13.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect @@ -146,6 +149,7 @@ require ( github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect github.com/kisielk/errcheck v1.10.0 // indirect github.com/kkHAIKE/contextcheck v1.1.6 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kulti/thelper v0.7.1 // indirect github.com/kunwardeep/paralleltest v1.0.15 // indirect github.com/lasiar/canonicalheader v1.1.2 // indirect @@ -181,7 +185,8 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pierrec/lz4/v4 v4.1.25 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/quasilyte/go-ruleguard v0.4.5 // indirect @@ -228,6 +233,7 @@ require ( github.com/yeya24/promlinter v0.3.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect gitlab.com/bosi/decorder v0.4.2 // indirect go-simpler.org/musttag v0.14.0 // indirect go-simpler.org/sloglint v0.11.1 // indirect @@ -242,6 +248,7 @@ require ( go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect golang.org/x/mod v0.35.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect @@ -250,9 +257,9 @@ require ( golang.org/x/text v0.36.0 // indirect golang.org/x/tools v0.44.0 // indirect golang.org/x/vuln v1.2.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect - google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 26d318c87..10895f1ab 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,10 @@ github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEW github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.2 h1:3uoHjoaEie5eVsxx/Bt64hKwZx4STb+beAkqKOlq/lY= +github.com/apache/arrow-go/v18 v18.5.2/go.mod h1:yNoizNTT4peTciJ7V01d2EgOkE1d0fQ1vZcFOsVtFsw= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= github.com/ashanbrown/makezero/v2 v2.1.0 h1:snuKYMbqosNokUKm+R6/+vOPs8yVAi46La7Ck6QYSaE= @@ -168,8 +172,9 @@ github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEy github.com/dave/jennifer v1.7.1 h1:B4jJJDHelWcDhlRQxWeo0Npa/pYKBLrirAQoTN45txo= github.com/dave/jennifer v1.7.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8= github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY= github.com/dgraph-io/badger/v4 v4.9.1 h1:DocZXZkg5JJHJPtUErA0ibyHxOVUDVoXLSCV6t8NC8w= @@ -250,6 +255,8 @@ github.com/go-xmlfmt/xmlfmt v1.1.3 h1:t8Ey3Uy7jDSEisW2K3somuMKIpzktkWptA0iFCnRUW github.com/go-xmlfmt/xmlfmt v1.1.3/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godoc-lint/godoc-lint v0.11.2 h1:Bp0FkJWoSdNsBikdNgIcgtaoo+xz6I/Y9s5WSBQUeeM= github.com/godoc-lint/godoc-lint v0.11.2/go.mod h1:iVpGdL1JCikNH2gGeAn3Hh+AgN5Gx/I/cxV+91L41jo= github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= @@ -284,6 +291,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/asciicheck v0.5.0 h1:jczN/BorERZwK8oiFBOGvlGPknhvq0bjnysTj4nUfo0= github.com/golangci/asciicheck v0.5.0/go.mod h1:5RMNAInbNFw2krqN6ibBxN/zfRFa9S6tA1nPdM0l8qQ= github.com/golangci/dupl v0.0.0-20250308024227-f665c8d69b32 h1:WUvBfQL6EW/40l6OmeSBYQJNSif4O11+bmWEz+C7FYw= @@ -409,10 +418,12 @@ github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -472,6 +483,10 @@ github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -491,6 +506,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakabonne/nestif v0.3.1 h1:wm28nZjhQY5HyYPx+weN3Q65k6ilSBxDb8v5S81B81U= github.com/nakabonne/nestif v0.3.1/go.mod h1:9EtoZochLn5iUprVDmDjqGKPofoUEBL8U4Ngq6aY7OE= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhKRf3Swg= github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= @@ -514,11 +531,14 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= +github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -559,6 +579,8 @@ github.com/raeperd/recvcheck v0.2.0 h1:GnU+NsbiCqdC2XX5+vMZzP+jAJC5fht7rcVTAhX74 github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtzBnWNocnYU= github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -672,6 +694,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo= @@ -750,8 +774,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= -golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 h1:qWFG1Dj7TBjOjOvhEOkmyGPVoquqUKnIU0lEVLp8xyk= @@ -987,6 +1011,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1102,6 +1128,14 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= mvdan.cc/gofumpt v0.9.2 h1:zsEMWL8SVKGHNztrx6uZrXdp7AX8r421Vvp23sz7ik4= mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= diff --git a/integration/go.mod b/integration/go.mod index b8d78630b..1358ac54f 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -6,6 +6,7 @@ replace github.com/trickstercache/trickster/v2 => ../ require ( github.com/ClickHouse/clickhouse-go/v2 v2.45.0 + github.com/apache/arrow-go/v18 v18.5.2 github.com/influxdata/influxdb-client-go/v2 v2.14.0 github.com/klauspost/compress v1.18.6 github.com/prometheus/client_golang v1.23.2 @@ -13,6 +14,7 @@ require ( github.com/stretchr/testify v1.11.1 github.com/trickstercache/trickster/v2 v2.0.0-00010101000000-000000000000 golang.org/x/crypto v0.50.0 + google.golang.org/grpc v1.80.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -24,7 +26,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgraph-io/badger/v4 v4.9.1 // indirect github.com/dgraph-io/ristretto/v2 v2.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -32,6 +34,7 @@ require ( github.com/go-faster/errors v0.7.1 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/uuid v1.6.0 // indirect @@ -42,6 +45,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -50,7 +54,7 @@ require ( github.com/paulmach/orb v0.12.0 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.25 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect @@ -58,6 +62,7 @@ require ( github.com/segmentio/asm v1.2.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.etcd.io/bbolt v1.4.3 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.68.0 // indirect @@ -72,14 +77,18 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect + golang.org/x/mod v0.35.0 // indirect golang.org/x/net v0.53.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.43.0 // indirect + golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa // indirect golang.org/x/text v0.36.0 // indirect + golang.org/x/tools v0.44.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260427160629-7cedc36a6bc4 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect - google.golang.org/grpc v1.80.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/integration/go.sum b/integration/go.sum index a3ae849f7..182abd44f 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -47,6 +47,10 @@ github.com/alicebob/miniredis/v2 v2.37.0 h1:RheObYW32G1aiJIj81XVt78ZHJpHonHLHW7O github.com/alicebob/miniredis/v2 v2.37.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.5.2 h1:3uoHjoaEie5eVsxx/Bt64hKwZx4STb+beAkqKOlq/lY= +github.com/apache/arrow-go/v18 v18.5.2/go.mod h1:yNoizNTT4peTciJ7V01d2EgOkE1d0fQ1vZcFOsVtFsw= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -73,8 +77,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v4 v4.9.1 h1:DocZXZkg5JJHJPtUErA0ibyHxOVUDVoXLSCV6t8NC8w= github.com/dgraph-io/badger/v4 v4.9.1/go.mod h1:5/MEx97uzdPUHR4KtkNt8asfI2T4JiEiQlV7kWUo8c0= github.com/dgraph-io/ristretto/v2 v2.4.0 h1:I/w09yLjhdcVD2QV192UJcq8dPBaAJb9pOuMyNy0XlU= @@ -108,6 +113,8 @@ github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ4 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= @@ -141,6 +148,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs= @@ -203,11 +212,13 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= +github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -220,7 +231,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -234,6 +251,8 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= @@ -246,8 +265,9 @@ github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcR github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -276,6 +296,8 @@ github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEy github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/redis/go-redis/v9 v9.19.0 h1:XPVaaPSnG6RhYf7p+rmSa9zZfeVAnWsH5h3lxthOm/k= github.com/redis/go-redis/v9 v9.19.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= @@ -289,6 +311,8 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -313,6 +337,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo= @@ -374,6 +400,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU= +golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -394,6 +422,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -488,6 +518,8 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa h1:efT73AJZfAAUV7SOip6pWGkwJDzIGiKBZGVzHYa+ve4= +golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -543,10 +575,14 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -659,6 +695,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.1 h1:eFJ2ShBLIEnUWlLy12raN0Z1plqmFX9Qe3rjQTKt6sU= +modernc.org/sqlite v1.46.1/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/integration/harness_test.go b/integration/harness_test.go index cf513b48e..c8ba2dac1 100644 --- a/integration/harness_test.go +++ b/integration/harness_test.go @@ -165,6 +165,14 @@ type cacheProviderCase struct { } func writeTestConfig(t *testing.T, frontPort, metricsPort, mgmtPort int) string { + t.Helper() + return writeTestConfigWithFlight(t, frontPort, metricsPort, mgmtPort, 0) +} + +// writeTestConfigWithFlight writes a per-test trickster config and overrides +// the influx3 backend's flight_port. Pass 0 to disable the Flight SQL listener +// for this test; pass a unique port to avoid collisions across parallel tests. +func writeTestConfigWithFlight(t *testing.T, frontPort, metricsPort, mgmtPort, flightPort int) string { t.Helper() b, err := os.ReadFile("../docs/developer/environment/trickster-config/trickster.yaml") require.NoError(t, err) @@ -176,6 +184,9 @@ func writeTestConfig(t *testing.T, frontPort, metricsPort, mgmtPort int) string c.MgmtConfig = mgmt.New() } c.MgmtConfig.ListenPort = mgmtPort + if bo, ok := c.Backends["influx3"]; ok && bo != nil { + bo.FlightPort = flightPort + } out, err := yaml.Marshal(&c) require.NoError(t, err) path := filepath.Join(t.TempDir(), "trickster.yaml") diff --git a/integration/influxdb3_flight_test.go b/integration/influxdb3_flight_test.go new file mode 100644 index 000000000..e027d4959 --- /dev/null +++ b/integration/influxdb3_flight_test.go @@ -0,0 +1,169 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +// TestInfluxDB3FlightSQL exercises the Flight SQL gRPC proxy end-to-end using +// the same ADBC-shaped wire format that Grafana's SQL datasource speaks. +// Covers the query path (cache miss + hit) and the metadata RPCs that ADBC +// clients probe on connect. +func TestInfluxDB3FlightSQL(t *testing.T) { + // Unique flight port per test to avoid collisions across parallel runs. + flightPort := 18585 + cfg := writeTestConfigWithFlight(t, 8593, 8594, 8595, flightPort) + influxAddr := "127.0.0.1:8593" + h := tricksterHarness{ConfigPath: cfg, BaseAddr: influxAddr, MetricsAddr: "127.0.0.1:8594"} + h.start(t) + waitForInfluxDB3Data(t, "127.0.0.1:8181") + + tricksterFlightAddr := fmt.Sprintf("127.0.0.1:%d", flightPort) + + // Wait for the Flight listener to accept connections — it starts in a + // goroutine during backend construction so there can be a brief lag. + var client *flightsql.Client + require.Eventually(t, func() bool { + c, err := flightsql.NewClientCtx(context.Background(), tricksterFlightAddr, nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return false + } + client = c + return true + }, 10*time.Second, 250*time.Millisecond, "flight sql listener never became ready") + t.Cleanup(func() { client.Close() }) + + // All calls need the `database` header to tell v3 which DB to query. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + ctx = metadata.AppendToOutgoingContext(ctx, "database", "trickster") + + t.Run("execute", func(t *testing.T) { + q := "SELECT avg(usage_idle) AS usage_idle FROM cpu WHERE cpu = 'cpu-total' LIMIT 10" + info, err := client.Execute(ctx, q) + require.NoError(t, err) + require.NotEmpty(t, info.Endpoint) + + reader, err := client.DoGet(ctx, info.Endpoint[0].Ticket) + require.NoError(t, err) + defer reader.Release() + var rows int64 + for reader.Next() { + rows += reader.RecordBatch().NumRows() + } + require.NoError(t, reader.Err()) + require.Greater(t, rows, int64(0), "expected rows from upstream") + }) + + t.Run("execute_cache_hit", func(t *testing.T) { + // Same exact query text — second Execute should hit the in-memory cache. + q := "SELECT host, avg(usage_idle) AS usage_idle FROM cpu WHERE cpu = 'cpu-total' GROUP BY host LIMIT 5" + for range 2 { + info, err := client.Execute(ctx, q) + require.NoError(t, err) + reader, err := client.DoGet(ctx, info.Endpoint[0].Ticket) + require.NoError(t, err) + for reader.Next() { + } + reader.Release() + } + // If we got here without error, the proxy handled a repeat. Deeper + // cache-hit assertions would require exposing cache counters; the + // passing test shows correctness of the passthrough + caching path. + }) + + t.Run("get_tables", func(t *testing.T) { + info, err := client.GetTables(ctx, &flightsql.GetTablesOpts{}) + require.NoError(t, err, "GetTables should succeed (not Unimplemented)") + require.NotEmpty(t, info.Endpoint) + + reader, err := client.DoGet(ctx, info.Endpoint[0].Ticket) + require.NoError(t, err) + defer reader.Release() + var rows int64 + for reader.Next() { + rows += reader.RecordBatch().NumRows() + } + require.NoError(t, reader.Err()) + require.Greater(t, rows, int64(0), "expected at least one table in v3 instance") + }) + + t.Run("get_catalogs", func(t *testing.T) { + info, err := client.GetCatalogs(ctx) + require.NoError(t, err, "GetCatalogs should succeed (not Unimplemented)") + require.NotEmpty(t, info.Endpoint) + }) + + t.Run("get_table_types", func(t *testing.T) { + info, err := client.GetTableTypes(ctx) + require.NoError(t, err, "GetTableTypes should succeed (not Unimplemented)") + require.NotEmpty(t, info.Endpoint) + }) + + t.Run("get_db_schemas", func(t *testing.T) { + info, err := client.GetDBSchemas(ctx, &flightsql.GetDBSchemasOpts{}) + require.NoError(t, err, "GetDBSchemas should succeed (not Unimplemented)") + require.NotEmpty(t, info.Endpoint) + }) + + t.Run("get_sql_info", func(t *testing.T) { + info, err := client.GetSqlInfo(ctx, []flightsql.SqlInfo{ + flightsql.SqlInfoFlightSqlServerName, + }) + require.NoError(t, err, "GetSqlInfo should succeed (not Unimplemented)") + require.NotEmpty(t, info.Endpoint) + }) + + t.Run("prepared_statement", func(t *testing.T) { + ps, err := client.Prepare(ctx, "SELECT avg(usage_idle) FROM cpu WHERE cpu = 'cpu-total' LIMIT 5") + require.NoError(t, err, "Prepare should succeed (not Unimplemented)") + defer ps.Close(ctx) + + info, err := ps.Execute(ctx) + require.NoError(t, err) + require.NotEmpty(t, info.Endpoint) + + reader, err := client.DoGet(ctx, info.Endpoint[0].Ticket) + require.NoError(t, err) + defer reader.Release() + var rows int64 + for reader.Next() { + rows += reader.RecordBatch().NumRows() + } + require.NoError(t, reader.Err()) + require.Greater(t, rows, int64(0), "expected rows from prepared statement") + }) + + // prepared_statement_with_params is covered by unit tests + // (TestPreparedStatement_Parameterized in pkg/backends/influxdb/flight/) + // using a fake upstream. An integration test against a real InfluxDB 3 + // Core instance is not included: Core 3.10 recognizes Flight SQL placeholders + // at Prepare time (returns a parameter schema) but does not resolve the + // bound values during query planning ("No value found for placeholder"), + // so the failure mode is in upstream plan resolution, not our proxy. +} diff --git a/integration/influxdb3_http_test.go b/integration/influxdb3_http_test.go new file mode 100644 index 000000000..5528596ca --- /dev/null +++ b/integration/influxdb3_http_test.go @@ -0,0 +1,126 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "fmt" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestInfluxDB3HTTP exercises the v3 HTTP endpoints (`/api/v3/query_sql` and +// `/api/v3/query_influxql`) through Trickster's influx3 backend. Verifies that +// time-range queries are cached and non-cacheable queries pass through. +func TestInfluxDB3HTTP(t *testing.T) { + cfg := writeTestConfigWithFlight(t, 8590, 8591, 8592, 0) // flight disabled for http tests + influxAddr := "127.0.0.1:8590" + h := tricksterHarness{ConfigPath: cfg, BaseAddr: influxAddr, MetricsAddr: "127.0.0.1:8591"} + h.start(t) + waitForInfluxDB3Data(t, "127.0.0.1:8181") + + baseURL := "http://" + influxAddr + "/influx3" + now := time.Now().Unix() + fiveMinAgo := now - 300 + + doGet := func(t *testing.T, path string, params url.Values) (*http.Response, []byte) { + t.Helper() + u := baseURL + path + "?" + params.Encode() + resp, err := http.Get(u) + require.NoError(t, err) + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return resp, b + } + + t.Run("sql_cacheable", func(t *testing.T) { + q := fmt.Sprintf( + "SELECT date_bin(INTERVAL '10 seconds', time) AS time, avg(usage_idle) AS usage_idle "+ + "FROM cpu WHERE cpu = 'cpu-total' AND time >= %d AND time < %d GROUP BY 1 ORDER BY 1", + fiveMinAgo, now) + params := url.Values{"q": {q}, "db": {"trickster"}, "format": {"json"}} + + // first call — populates cache + resp1, body1 := doGet(t, "/api/v3/query_sql", params) + require.Equal(t, http.StatusOK, resp1.StatusCode, "body: %s", string(body1)) + require.NotEmpty(t, body1) + hdr1 := parseTricksterResult(resp1.Header.Get("X-Trickster-Result")) + require.NotEmpty(t, hdr1["engine"], "expected engine on first call") + + // second call — should be a cache hit + resp2, body2 := doGet(t, "/api/v3/query_sql", params) + require.Equal(t, http.StatusOK, resp2.StatusCode, "body: %s", string(body2)) + hdr2 := parseTricksterResult(resp2.Header.Get("X-Trickster-Result")) + require.Contains(t, hdr2, "status", "expected status in X-Trickster-Result on second call") + }) + + t.Run("sql_jsonl_format", func(t *testing.T) { + q := fmt.Sprintf( + "SELECT date_bin(INTERVAL '10 seconds', time) AS time, avg(usage_idle) AS usage_idle "+ + "FROM cpu WHERE cpu = 'cpu-total' AND time >= %d AND time < %d GROUP BY 1 ORDER BY 1", + fiveMinAgo, now) + params := url.Values{"q": {q}, "db": {"trickster"}, "format": {"jsonl"}} + resp, body := doGet(t, "/api/v3/query_sql", params) + require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(body)) + require.NotEmpty(t, body) + // JSONL: each line a JSON object (or empty body if no rows) + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + if line == "" { + continue + } + require.True(t, strings.HasPrefix(line, "{") && strings.HasSuffix(line, "}"), + "expected JSONL line, got: %s", line) + } + }) + + t.Run("sql_csv_format", func(t *testing.T) { + q := fmt.Sprintf( + "SELECT date_bin(INTERVAL '10 seconds', time) AS time, avg(usage_idle) AS usage_idle "+ + "FROM cpu WHERE cpu = 'cpu-total' AND time >= %d AND time < %d GROUP BY 1 ORDER BY 1", + fiveMinAgo, now) + params := url.Values{"q": {q}, "db": {"trickster"}, "format": {"csv"}} + resp, body := doGet(t, "/api/v3/query_sql", params) + require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(body)) + require.NotEmpty(t, body) + // CSV: first line is header, should contain the column names + firstLine := strings.SplitN(string(body), "\n", 2)[0] + require.Contains(t, firstLine, "time", "expected time column in CSV header") + }) + + t.Run("sql_non_cacheable", func(t *testing.T) { + // Query without date_bin or time-range WHERE falls through to proxy. + params := url.Values{"q": {"SELECT 1"}, "db": {"trickster"}, "format": {"json"}} + resp, body := doGet(t, "/api/v3/query_sql", params) + require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(body)) + }) + + t.Run("influxql_v3_native", func(t *testing.T) { + q := `SELECT mean("usage_idle") FROM "cpu" WHERE "cpu" = 'cpu-total' AND time > now() - 5m GROUP BY time(10s)` + params := url.Values{"q": {q}, "db": {"trickster"}, "format": {"json"}} + resp, body := doGet(t, "/api/v3/query_influxql", params) + require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", string(body)) + require.NotEmpty(t, body) + hdr := parseTricksterResult(resp.Header.Get("X-Trickster-Result")) + require.NotEmpty(t, hdr["engine"], "expected engine on v3 InfluxQL response") + }) +} diff --git a/integration/influxdb_sdk_test.go b/integration/influxdb_sdk_test.go index 43789a40d..d2d34a977 100644 --- a/integration/influxdb_sdk_test.go +++ b/integration/influxdb_sdk_test.go @@ -18,6 +18,9 @@ package integration import ( "context" + "io" + "net/http" + "strings" "testing" "time" @@ -72,4 +75,38 @@ func TestInfluxDBSDK(t *testing.T) { } require.NoError(t, result2.Err()) }) + + // cache_hit_header uses raw HTTP to inspect X-Trickster-Result since the v2 + // SDK hides response headers. Asserts the second identical query lands in + // the delta-proxy cache — regression coverage for the `now()` range-bound + // handling in flux.parseRange (which previously flowed to HTTPProxy). + t.Run("cache_hit_header", func(t *testing.T) { + fluxURL := "http://" + influxAddr + "/flux2/api/v2/query?org=trickster-dev" + body := `{"query": "from(bucket: \"trickster\") |> range(start: -1h, stop: now()) |> aggregateWindow(every: 1m, fn: mean) |> limit(n: 5)", "type": "flux"}` + do := func() *http.Response { + req, err := http.NewRequest("POST", fluxURL, strings.NewReader(body)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Token trickster-dev-token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + _, _ = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + return resp + } + r1 := do() + require.Equal(t, http.StatusOK, r1.StatusCode) + hdr1 := parseTricksterResult(r1.Header.Get("X-Trickster-Result")) + require.Equal(t, "DeltaProxyCache", hdr1["engine"], + "expected DeltaProxyCache engine (got %q) — now() range bound may not be parsed", + r1.Header.Get("X-Trickster-Result")) + + r2 := do() + require.Equal(t, http.StatusOK, r2.StatusCode) + hdr2 := parseTricksterResult(r2.Header.Get("X-Trickster-Result")) + // "hit" = full cache hit, "phit" = partial (next step boundary advanced). + require.Contains(t, []string{"hit", "phit"}, hdr2["status"], + "expected cache hit on second call, got %q (header: %q)", + hdr2["status"], r2.Header.Get("X-Trickster-Result")) + }) } diff --git a/integration/influxdb_test.go b/integration/influxdb_test.go index 4268efc71..0a7dba6a0 100644 --- a/integration/influxdb_test.go +++ b/integration/influxdb_test.go @@ -19,6 +19,7 @@ package integration import ( "io" "net/http" + "net/url" "strings" "testing" @@ -75,4 +76,38 @@ func TestInfluxDB(t *testing.T) { require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "expected 401 for wrong token, got %d: %s", resp.StatusCode, string(body)) }) + + // v1 InfluxQL goes through /flux2/query against InfluxDB 2's v1-compat + // endpoint. Verifies Trickster's v1 InfluxQL handler + cache path. + t.Run("influxql_select", func(t *testing.T) { + q := `SELECT mean("usage_idle") FROM "cpu" WHERE "cpu" = 'cpu-total' AND time > now() - 5m GROUP BY time(10s)` + u := "http://" + influxAddr + "/flux2/query?db=trickster&q=" + url.QueryEscape(q) + req, err := http.NewRequest("GET", u, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Token trickster-dev-token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode, "unexpected status: %s", string(b)) + hdr := parseTricksterResult(resp.Header.Get("X-Trickster-Result")) + require.NotEmpty(t, hdr["engine"], "expected engine header on v1 InfluxQL response") + }) + + // Non-cacheable InfluxQL (SHOW MEASUREMENTS) should passthrough to upstream. + t.Run("influxql_show_measurements", func(t *testing.T) { + u := "http://" + influxAddr + "/flux2/query?db=trickster&q=" + url.QueryEscape("SHOW MEASUREMENTS") + req, err := http.NewRequest("GET", u, nil) + require.NoError(t, err) + req.Header.Set("Authorization", "Token trickster-dev-token") + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode, + "expected SHOW MEASUREMENTS to proxy through: %s", string(b)) + require.Contains(t, string(b), "cpu", "expected cpu measurement in response") + }) } diff --git a/integration/main_test.go b/integration/main_test.go index d87b931c1..6220fd1f5 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -190,6 +190,33 @@ func waitForInfluxDBData(t *testing.T, influxAddr string) { }, 30*time.Second, 2*time.Second, "InfluxDB data never became available") } +// waitForInfluxDB3Data polls the v3 SQL endpoint until rows land in the cpu +// table (seeded by telegraf via v1-compat writes). +func waitForInfluxDB3Data(t *testing.T, influxAddr string) { + t.Helper() + require.EventuallyWithT(t, func(collect *assert.CollectT) { + req, err := http.NewRequest("POST", + "http://"+influxAddr+"/api/v3/query_sql", + strings.NewReader(`{"q": "SELECT cpu FROM cpu LIMIT 1", "db": "trickster"}`)) + if !assert.NoError(collect, err) { + return + } + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if !assert.NoError(collect, err) { + return + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if !assert.NoError(collect, err) { + return + } + // v3 returns [] when no rows, [{...}] when rows exist + assert.Greater(collect, len(strings.TrimSpace(string(b))), 2, + "waiting for Telegraf to write data to InfluxDB 3") + }, 60*time.Second, 2*time.Second, "InfluxDB 3 data never became available") +} + type promResponse struct { Status string `json:"status"` Data json.RawMessage `json:"data"` diff --git a/pkg/backends/influxdb/flight/client.go b/pkg/backends/influxdb/flight/client.go new file mode 100644 index 000000000..2101952c6 --- /dev/null +++ b/pkg/backends/influxdb/flight/client.go @@ -0,0 +1,278 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package flight + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "sync" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/flight" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +// UpstreamConfig configures the upstream Flight SQL client. +type UpstreamConfig struct { + // Address is the upstream Flight SQL endpoint (host:port). + Address string + // Database is the v3 database name, sent via the `database` metadata header. + Database string + // BearerToken is the optional auth token. + BearerToken string +} + +// FlightSQLClient is the default UpstreamClient implementation that talks to a +// Flight SQL server over gRPC and returns IPC-encoded bytes. +type FlightSQLClient struct { + cfg UpstreamConfig + client *flightsql.Client + conn *grpc.ClientConn + alloc memory.Allocator + + // prepared statements are tracked by their handle bytes so we can look up + // the client-side object when Execute / Close is later called. + preparedMu sync.Mutex + prepared map[string]*flightsql.PreparedStatement +} + +// NewFlightSQLClient dials the upstream Flight SQL endpoint. +func NewFlightSQLClient(cfg UpstreamConfig) (*FlightSQLClient, error) { + conn, err := grpc.NewClient(cfg.Address, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return nil, fmt.Errorf("grpc dial: %w", err) + } + c, err := flightsql.NewClientCtx(context.Background(), + cfg.Address, nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("flightsql client: %w", err) + } + return &FlightSQLClient{ + cfg: cfg, + client: c, + conn: conn, + alloc: memory.DefaultAllocator, + prepared: make(map[string]*flightsql.PreparedStatement), + }, nil +} + +// PrepareStatement creates a prepared statement upstream and returns its handle. +// The handle is opaque bytes minted by the upstream; clients use it as the +// round-trip identifier for Execute / Close. +func (c *FlightSQLClient) PrepareStatement(ctx context.Context, + query string, +) ([]byte, error) { + ctx = c.withAuth(ctx) + ps, err := c.client.Prepare(ctx, query) + if err != nil { + return nil, fmt.Errorf("flight prepare: %w", err) + } + handle := ps.Handle() + c.preparedMu.Lock() + c.prepared[string(handle)] = ps + c.preparedMu.Unlock() + return handle, nil +} + +// ExecutePrepared runs a previously-prepared statement upstream and returns +// the IPC-encoded response bytes. Parameter binding is not yet supported — +// clients that rely on bound params will see the same data each call. +func (c *FlightSQLClient) ExecutePrepared(ctx context.Context, + handle []byte, +) ([]byte, error) { + ctx = c.withAuth(ctx) + c.preparedMu.Lock() + ps, ok := c.prepared[string(handle)] + c.preparedMu.Unlock() + if !ok { + return nil, errors.New("unknown prepared statement handle") + } + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return ps.Execute(ctx) + }) +} + +// SetPreparedStatementParams binds parameter values on the upstream prepared +// statement. The next ExecutePrepared call against this handle will use them. +func (c *FlightSQLClient) SetPreparedStatementParams(_ context.Context, + handle []byte, params arrow.RecordBatch, +) error { + c.preparedMu.Lock() + ps, ok := c.prepared[string(handle)] + c.preparedMu.Unlock() + if !ok { + return errors.New("unknown prepared statement handle") + } + ps.SetParameters(params) + return nil +} + +// ClosePrepared releases the upstream prepared statement. +func (c *FlightSQLClient) ClosePrepared(ctx context.Context, handle []byte) error { + ctx = c.withAuth(ctx) + c.preparedMu.Lock() + ps, ok := c.prepared[string(handle)] + if ok { + delete(c.prepared, string(handle)) + } + c.preparedMu.Unlock() + if !ok { + return nil + } + return ps.Close(ctx) +} + +// Execute runs a SQL query against the upstream and returns the IPC-encoded +// bytes (schema + record batches) of the entire response. Results are buffered +// to enable caching. +func (c *FlightSQLClient) Execute(ctx context.Context, query string) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.Execute(ctx, query) + }) +} + +// GetCatalogs returns IPC bytes for the upstream's catalog list. +func (c *FlightSQLClient) GetCatalogs(ctx context.Context) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.GetCatalogs(ctx) + }) +} + +// GetDBSchemas returns IPC bytes for the upstream's DB schema list. +func (c *FlightSQLClient) GetDBSchemas(ctx context.Context, + opts *flightsql.GetDBSchemasOpts, +) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.GetDBSchemas(ctx, opts) + }) +} + +// GetTables returns IPC bytes for the upstream's table list. +func (c *FlightSQLClient) GetTables(ctx context.Context, + opts *flightsql.GetTablesOpts, +) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.GetTables(ctx, opts) + }) +} + +// GetTableTypes returns IPC bytes for the upstream's supported table types. +func (c *FlightSQLClient) GetTableTypes(ctx context.Context) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.GetTableTypes(ctx) + }) +} + +// GetSqlInfo returns IPC bytes for the upstream's SQL info records. +func (c *FlightSQLClient) GetSqlInfo(ctx context.Context, + info []flightsql.SqlInfo, +) ([]byte, error) { + ctx = c.withAuth(ctx) + return c.fetchAsIPC(ctx, func(ctx context.Context) (*flight.FlightInfo, error) { + return c.client.GetSqlInfo(ctx, info) + }) +} + +// fetchAsIPC calls a FlightInfo-returning function, resolves the first endpoint +// ticket via DoGet, and buffers the resulting record batches into IPC bytes. +func (c *FlightSQLClient) fetchAsIPC(ctx context.Context, + getInfo func(context.Context) (*flight.FlightInfo, error), +) ([]byte, error) { + info, err := getInfo(ctx) + if err != nil { + return nil, fmt.Errorf("flight request: %w", err) + } + if len(info.Endpoint) == 0 { + return nil, errors.New("flight info has no endpoints") + } + reader, err := c.client.DoGet(ctx, info.Endpoint[0].Ticket) + if err != nil { + return nil, fmt.Errorf("flight doGet: %w", err) + } + defer reader.Release() + + var buf bytes.Buffer + w := ipc.NewWriter(&buf, ipc.WithSchema(reader.Schema())) + for reader.Next() { + rec := reader.RecordBatch() + if err := w.Write(rec); err != nil { + return nil, fmt.Errorf("ipc write: %w", err) + } + } + if err := reader.Err(); err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("flight read: %w", err) + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("ipc close: %w", err) + } + return buf.Bytes(), nil +} + +// withAuth adds the bearer token and database headers to the outgoing context. +// Inbound metadata (from a client calling our server) is forwarded through — +// this lets the end client's `database` and `authorization` headers flow to +// the upstream without reconfiguration. +func (c *FlightSQLClient) withAuth(ctx context.Context) context.Context { + out := metadata.MD{} + if in, ok := metadata.FromIncomingContext(ctx); ok { + for _, h := range []string{"authorization", "database", "bucket-name"} { + if v := in.Get(h); len(v) > 0 { + out.Set(h, v...) + } + } + } + if c.cfg.BearerToken != "" && len(out.Get("authorization")) == 0 { + out.Set("authorization", "Bearer "+c.cfg.BearerToken) + } + if c.cfg.Database != "" && len(out.Get("database")) == 0 { + out.Set("database", c.cfg.Database) + } + if len(out) == 0 { + return ctx + } + return metadata.NewOutgoingContext(ctx, out) +} + +// Close releases the gRPC connection. +func (c *FlightSQLClient) Close() error { + if c.client != nil { + _ = c.client.Close() + } + if c.conn != nil { + return c.conn.Close() + } + return nil +} + +// Compile-time check +var _ UpstreamClient = (*FlightSQLClient)(nil) diff --git a/pkg/backends/influxdb/flight/listener.go b/pkg/backends/influxdb/flight/listener.go new file mode 100644 index 000000000..d661f3a0d --- /dev/null +++ b/pkg/backends/influxdb/flight/listener.go @@ -0,0 +1,184 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package flight + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "time" + + "github.com/apache/arrow-go/v18/arrow/flight" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "google.golang.org/grpc" +) + +// ListenerConfig configures the Flight SQL listener. +type ListenerConfig struct { + Address string // e.g., "0.0.0.0" + Port int + Name string // optional identifier for logs/metrics +} + +// Listener wraps a gRPC server that exposes a Flight SQL service. +type Listener struct { + server *grpc.Server + lis net.Listener + name string +} + +var ( + registryMu sync.Mutex + registry = map[string]*Listener{} +) + +// replaceExistingTimeout bounds how long a same-named listener gets to drain +// when Start is called for a backend that's being reloaded. +const replaceExistingTimeout = 2 * time.Second + +// Start begins accepting Flight SQL connections on the configured address +// and registers the listener for coordinated shutdown via ShutdownAll. +// If a listener with the same Name is already registered (e.g., during config +// reload), it is gracefully stopped first so the port can be rebound. +func Start(cfg ListenerConfig, srv *Server) (*Listener, error) { + if srv == nil { + return nil, errors.New("server is nil") + } + if cfg.Name != "" { + registryMu.Lock() + existing, ok := registry[cfg.Name] + if ok { + delete(registry, cfg.Name) + } + registryMu.Unlock() + if ok { + _ = existing.Stop(replaceExistingTimeout) + } + } + addr := fmt.Sprintf("%s:%d", cfg.Address, cfg.Port) + lis, err := listenWithRetry(addr, 500*time.Millisecond) + if err != nil { + return nil, fmt.Errorf("flight listener: %w", err) + } + grpcSrv := grpc.NewServer() + flight.RegisterFlightServiceServer(grpcSrv, flightsql.NewFlightServer(srv)) + go func() { + _ = grpcSrv.Serve(lis) + }() + l := &Listener{server: grpcSrv, lis: lis, name: cfg.Name} + registryMu.Lock() + key := cfg.Name + if key == "" { + key = addr // fall back to address for unnamed listeners + } + registry[key] = l + registryMu.Unlock() + return l, nil +} + +// Stop gracefully shuts down the Flight SQL server, draining active streams +// until drainTimeout elapses, then forcing connection closure. Pass 0 for +// immediate force-stop. Blocks until the underlying socket is fully closed +// so callers can rebind the port safely. +func (l *Listener) Stop(drainTimeout time.Duration) error { + if l.server == nil { + return nil + } + if drainTimeout <= 0 { + l.server.Stop() + } else { + done := make(chan struct{}) + go func() { + l.server.GracefulStop() + close(done) + }() + select { + case <-done: + case <-time.After(drainTimeout): + l.server.Stop() + } + } + // Belt-and-suspenders: ensure the net.Listener is closed. gRPC closes it + // during Stop/GracefulStop, but close is idempotent and this makes the + // port-release deterministic for immediate rebind. + if l.lis != nil { + _ = l.lis.Close() + } + return nil +} + +// Addr returns the listener's bound address, useful for tests. +func (l *Listener) Addr() net.Addr { + if l.lis == nil { + return nil + } + return l.lis.Addr() +} + +// Name returns the listener's name. +func (l *Listener) Name() string { return l.name } + +// listenWithRetry retries net.Listen for up to maxWait to tolerate brief +// kernel-level socket release lag when rebinding a port during config reload. +func listenWithRetry(addr string, maxWait time.Duration) (net.Listener, error) { + deadline := time.Now().Add(maxWait) + backoff := 10 * time.Millisecond + var lastErr error + for { + lis, err := net.Listen("tcp", addr) + if err == nil { + return lis, nil + } + lastErr = err + if time.Now().After(deadline) { + return nil, lastErr + } + time.Sleep(backoff) + if backoff < 100*time.Millisecond { + backoff *= 2 + } + } +} + +// ShutdownAll gracefully stops all Flight SQL listeners registered via Start +// and clears the registry. Returns the first non-nil error encountered. +// Pass ctx.Deadline() or derive a timeout to bound the drain. +func ShutdownAll(ctx context.Context) error { + registryMu.Lock() + ls := make([]*Listener, 0, len(registry)) + for _, l := range registry { + ls = append(ls, l) + } + registry = map[string]*Listener{} + registryMu.Unlock() + + drain := 5 * time.Second + if dl, ok := ctx.Deadline(); ok { + if remaining := time.Until(dl); remaining > 0 { + drain = remaining + } + } + var firstErr error + for _, l := range ls { + if err := l.Stop(drain); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} diff --git a/pkg/backends/influxdb/flight/listener_test.go b/pkg/backends/influxdb/flight/listener_test.go new file mode 100644 index 000000000..0115dd724 --- /dev/null +++ b/pkg/backends/influxdb/flight/listener_test.go @@ -0,0 +1,218 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package flight + +import ( + "context" + "net" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// TestEndToEnd starts a Flight SQL server, connects a client, executes a +// query, and verifies the results stream back correctly through the full +// gRPC + Arrow IPC pipeline. +func TestEndToEnd(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + srv := NewServer(up, newMemCache()) + + lis, err := Start(ListenerConfig{Address: "127.0.0.1", Port: 0}, srv) + if err != nil { + t.Fatal(err) + } + defer lis.Stop(0) + + addr := lis.Addr().String() + client, err := flightsql.NewClientCtx(context.Background(), addr, nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("client dial: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + info, err := client.Execute(ctx, "SELECT * FROM cpu") + if err != nil { + t.Fatalf("execute: %v", err) + } + if len(info.Endpoint) == 0 { + t.Fatal("no endpoints returned") + } + + reader, err := client.DoGet(ctx, info.Endpoint[0].Ticket) + if err != nil { + t.Fatalf("doGet: %v", err) + } + defer reader.Release() + + var rows int64 + for reader.Next() { + rec := reader.Record() + rows += rec.NumRows() + } + if err := reader.Err(); err != nil { + t.Fatalf("read: %v", err) + } + if rows != 2 { + t.Errorf("expected 2 rows, got %d", rows) + } + if up.callCount != 1 { + t.Errorf("expected 1 upstream call, got %d", up.callCount) + } +} + +// TestEndToEnd_Metadata verifies metadata RPCs (GetTables, GetCatalogs, etc.) +// flow through the full gRPC pipeline and reach the upstream. +func TestEndToEnd_Metadata(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + srv := NewServer(up, newMemCache()) + + lis, err := Start(ListenerConfig{Address: "127.0.0.1", Port: 0}, srv) + if err != nil { + t.Fatal(err) + } + defer lis.Stop(0) + + client, err := flightsql.NewClientCtx(context.Background(), lis.Addr().String(), nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("client dial: %v", err) + } + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + t.Run("GetCatalogs", func(t *testing.T) { + info, err := client.GetCatalogs(ctx) + if err != nil { + t.Fatalf("GetCatalogs: %v", err) + } + if info == nil || len(info.Endpoint) == 0 { + t.Fatal("missing endpoint") + } + }) + + t.Run("GetTables", func(t *testing.T) { + info, err := client.GetTables(ctx, &flightsql.GetTablesOpts{}) + if err != nil { + t.Fatalf("GetTables: %v", err) + } + if info == nil || len(info.Endpoint) == 0 { + t.Fatal("missing endpoint") + } + }) + + t.Run("GetTableTypes", func(t *testing.T) { + info, err := client.GetTableTypes(ctx) + if err != nil { + t.Fatalf("GetTableTypes: %v", err) + } + if info == nil || len(info.Endpoint) == 0 { + t.Fatal("missing endpoint") + } + }) + + t.Run("GetDBSchemas", func(t *testing.T) { + info, err := client.GetDBSchemas(ctx, &flightsql.GetDBSchemasOpts{}) + if err != nil { + t.Fatalf("GetDBSchemas: %v", err) + } + if info == nil || len(info.Endpoint) == 0 { + t.Fatal("missing endpoint") + } + }) + + t.Run("GetSqlInfo", func(t *testing.T) { + info, err := client.GetSqlInfo(ctx, []flightsql.SqlInfo{ + flightsql.SqlInfoFlightSqlServerName, + }) + if err != nil { + t.Fatalf("GetSqlInfo: %v", err) + } + if info == nil || len(info.Endpoint) == 0 { + t.Fatal("missing endpoint") + } + }) +} + +// TestStart_ReplacesExistingByName simulates a config reload: calling Start +// twice with the same Name + Port should stop the first listener so the +// second can bind without EADDRINUSE. +func TestStart_ReplacesExistingByName(t *testing.T) { + srv := NewServer(&fakeUpstream{ipcBytes: buildTestIPC(t)}, newMemCache()) + // pick an explicit port so both calls target the same bind address + first, err := Start(ListenerConfig{Address: "127.0.0.1", Port: 0, Name: "backend-a"}, srv) + if err != nil { + t.Fatal(err) + } + port := first.Addr().(*net.TCPAddr).Port + + second, err := Start(ListenerConfig{Address: "127.0.0.1", Port: port, Name: "backend-a"}, srv) + if err != nil { + t.Fatalf("second Start on same name+port should replace, got: %v", err) + } + defer second.Stop(0) + + // first listener should be stopped; RPC against it should fail + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := ShutdownAll(ctx); err != nil { + t.Errorf("ShutdownAll: %v", err) + } +} + +// TestShutdownAll verifies that registered listeners are stopped and +// accepting connections afterward fails. +func TestShutdownAll(t *testing.T) { + srv := NewServer(&fakeUpstream{ipcBytes: buildTestIPC(t)}, newMemCache()) + lis, err := Start(ListenerConfig{Address: "127.0.0.1", Port: 0, Name: "test"}, srv) + if err != nil { + t.Fatal(err) + } + addr := lis.Addr().String() + + if err := ShutdownAll(context.Background()); err != nil { + t.Fatalf("ShutdownAll: %v", err) + } + + // After shutdown, a new dial to the same address should fail quickly. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err = flightsql.NewClientCtx(ctx, addr, nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + // dial may succeed (lazy), so also try a real RPC to confirm the server is gone. + if err == nil { + c, _ := flightsql.NewClientCtx(ctx, addr, nil, nil, + grpc.WithTransportCredentials(insecure.NewCredentials())) + if c != nil { + defer c.Close() + _, rpcErr := c.Execute(ctx, "SELECT 1") + if rpcErr == nil { + t.Error("expected RPC to fail after ShutdownAll") + } + } + } +} diff --git a/pkg/backends/influxdb/flight/metadata_test.go b/pkg/backends/influxdb/flight/metadata_test.go new file mode 100644 index 000000000..3a407e544 --- /dev/null +++ b/pkg/backends/influxdb/flight/metadata_test.go @@ -0,0 +1,350 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package flight + +import ( + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/flight" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "github.com/apache/arrow-go/v18/arrow/memory" +) + +// TestDoGetCatalogs verifies the catalogs proxy caches IPC bytes. +func TestDoGetCatalogs(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + for range 2 { + schema, ch, err := srv.DoGetCatalogs(context.Background()) + if err != nil { + t.Fatalf("DoGetCatalogs: %v", err) + } + if schema == nil { + t.Fatal("expected schema") + } + for chunk := range ch { + chunk.Data.Release() + } + } + if up.catalogCalls != 1 { + t.Errorf("expected 1 upstream catalog call (2nd cached), got %d", up.catalogCalls) + } +} + +// TestDoGetTables verifies tables proxy caches and uses a key that reflects filters. +func TestDoGetTables(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + cmd1 := fakeGetTables{} + cmd2 := fakeGetTables{tableNameFilter: "cpu"} + + // same filter twice → 1 upstream call + for range 2 { + _, ch, err := srv.DoGetTables(context.Background(), cmd1) + if err != nil { + t.Fatalf("DoGetTables: %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + // different filter → another upstream call (different cache key) + _, ch, err := srv.DoGetTables(context.Background(), cmd2) + if err != nil { + t.Fatalf("DoGetTables (filtered): %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + + if up.tablesCalls != 2 { + t.Errorf("expected 2 tables calls (2nd cached, 3rd different key), got %d", up.tablesCalls) + } +} + +// TestDoGetDBSchemas verifies DB schemas proxy. +func TestDoGetDBSchemas(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + _, ch, err := srv.DoGetDBSchemas(context.Background(), fakeGetDBSchemas{}) + if err != nil { + t.Fatalf("DoGetDBSchemas: %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + if up.dbSchemaCalls != 1 { + t.Errorf("expected 1 dbSchemas call, got %d", up.dbSchemaCalls) + } +} + +// TestDoGetTableTypes verifies table types proxy + caching. +func TestDoGetTableTypes(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + for range 3 { + _, ch, err := srv.DoGetTableTypes(context.Background()) + if err != nil { + t.Fatalf("DoGetTableTypes: %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + if up.tableTypesCalls != 1 { + t.Errorf("expected 1 tableTypes call (cached), got %d", up.tableTypesCalls) + } +} + +// TestDoGetSqlInfo verifies SQL info proxy with info slice as cache key component. +func TestDoGetSqlInfo(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + _, ch, err := srv.DoGetSqlInfo(context.Background(), fakeGetSqlInfo{info: []uint32{1, 2, 3}}) + if err != nil { + t.Fatalf("DoGetSqlInfo: %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + if up.sqlInfoCalls != 1 { + t.Errorf("expected 1 sqlInfo call, got %d", up.sqlInfoCalls) + } +} + +// TestPreparedStatement_Parameterized verifies that two Execute calls with +// different bound parameter values each hit upstream (distinct cache keys), +// while repeating the same params reuses the cache. +func TestPreparedStatement_Parameterized(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + // Create a prepared statement + res, err := srv.CreatePreparedStatement(context.Background(), + fakeCreatePrepReq{query: "SELECT * FROM cpu WHERE cpu = ?"}) + if err != nil { + t.Fatal(err) + } + + // Build two distinct parameter records + recA := buildParamRecord(t, "cpu-total") + defer recA.Release() + recB := buildParamRecord(t, "cpu0") + defer recB.Release() + + cmd := fakePrepQuery{handle: res.Handle} + + // Bind params A, execute twice (2nd should cache-hit) + if _, err := srv.DoPutPreparedStatementQuery(context.Background(), cmd, + &fakeMessageReader{rec: recA}, nil); err != nil { + t.Fatalf("DoPut A: %v", err) + } + for range 2 { + _, ch, err := srv.DoGetPreparedStatement(context.Background(), cmd) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + if up.executePreparedCalls != 1 { + t.Errorf("phase A: expected 1 upstream execute, got %d", up.executePreparedCalls) + } + + // Rebind with params B, execute — should NOT cache-hit since key differs + if _, err := srv.DoPutPreparedStatementQuery(context.Background(), cmd, + &fakeMessageReader{rec: recB}, nil); err != nil { + t.Fatalf("DoPut B: %v", err) + } + _, ch, err := srv.DoGetPreparedStatement(context.Background(), cmd) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + if up.executePreparedCalls != 2 { + t.Errorf("phase B: expected 2 upstream executes (different params), got %d", + up.executePreparedCalls) + } + if up.setParamsCalls != 2 { + t.Errorf("expected 2 SetParams calls, got %d", up.setParamsCalls) + } +} + +// buildParamRecord creates a single-column, single-row string record for use +// as a parameter binding. +func buildParamRecord(t *testing.T, val string) arrow.RecordBatch { + t.Helper() + schema := arrow.NewSchema([]arrow.Field{ + {Name: "p1", Type: arrow.BinaryTypes.String}, + }, nil) + b := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer b.Release() + b.Field(0).(*array.StringBuilder).Append(val) + return b.NewRecord() +} + +// fakeMessageReader adapts a single RecordBatch as a flight.MessageReader. +type fakeMessageReader struct { + rec arrow.RecordBatch + done bool +} + +func (f *fakeMessageReader) Next() bool { + if f.done { + return false + } + f.done = true + return true +} +func (f *fakeMessageReader) RecordBatch() arrow.RecordBatch { return f.rec } +func (f *fakeMessageReader) Record() arrow.RecordBatch { return f.rec } +func (f *fakeMessageReader) Schema() *arrow.Schema { return f.rec.Schema() } +func (f *fakeMessageReader) Err() error { return nil } +func (f *fakeMessageReader) Release() {} +func (f *fakeMessageReader) Retain() {} +func (f *fakeMessageReader) Read() (arrow.RecordBatch, error) { return nil, nil } +func (f *fakeMessageReader) Chunk() flight.StreamChunk { return flight.StreamChunk{Data: f.rec} } +func (f *fakeMessageReader) LatestFlightDescriptor() *flight.FlightDescriptor { + return nil +} +func (f *fakeMessageReader) LatestAppMetadata() []byte { return nil } + +// TestPreparedStatement covers the full prepare → execute → close cycle +// proxied to the upstream, including cache reuse on repeated Execute. +func TestPreparedStatement(t *testing.T) { + up := &fakeUpstream{ipcBytes: buildTestIPC(t)} + srv := NewServer(up, newMemCache()) + + // Create + req := fakeCreatePrepReq{query: "SELECT * FROM cpu WHERE cpu = ?"} + res, err := srv.CreatePreparedStatement(context.Background(), req) + if err != nil { + t.Fatalf("CreatePreparedStatement: %v", err) + } + if len(res.Handle) == 0 { + t.Fatal("expected non-empty handle") + } + if up.prepareCalls != 1 { + t.Errorf("expected 1 prepare call, got %d", up.prepareCalls) + } + + // Execute twice — second should hit cache + cmd := fakePrepQuery{handle: res.Handle} + for range 2 { + _, ch, err := srv.DoGetPreparedStatement(context.Background(), cmd) + if err != nil { + t.Fatalf("DoGetPreparedStatement: %v", err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + if up.executePreparedCalls != 1 { + t.Errorf("expected 1 upstream execute (2nd cached), got %d", up.executePreparedCalls) + } + + // Close + closeReq := fakeClosePrepReq{handle: res.Handle} + if err := srv.ClosePreparedStatement(context.Background(), closeReq); err != nil { + t.Errorf("ClosePreparedStatement: %v", err) + } + if up.closePreparedCalls != 1 { + t.Errorf("expected 1 close call, got %d", up.closePreparedCalls) + } +} + +func TestGetFlightInfoPreparedStatement(t *testing.T) { + srv := NewServer(&fakeUpstream{}, newMemCache()) + desc := &flight.FlightDescriptor{Cmd: []byte("some-cmd-bytes")} + info, err := srv.GetFlightInfoPreparedStatement(context.Background(), + fakePrepQuery{handle: []byte("h")}, desc) + if err != nil { + t.Fatal(err) + } + if len(info.Endpoint) != 1 { + t.Fatalf("expected 1 endpoint, got %d", len(info.Endpoint)) + } +} + +// fakeCreatePrepReq implements flightsql.ActionCreatePreparedStatementRequest. +type fakeCreatePrepReq struct { + query string +} + +func (f fakeCreatePrepReq) GetQuery() string { return f.query } +func (f fakeCreatePrepReq) GetTransactionId() []byte { return nil } + +// fakePrepQuery implements flightsql.PreparedStatementQuery. +type fakePrepQuery struct { + handle []byte +} + +func (f fakePrepQuery) GetPreparedStatementHandle() []byte { return f.handle } + +// fakeClosePrepReq implements flightsql.ActionClosePreparedStatementRequest. +type fakeClosePrepReq struct { + handle []byte +} + +func (f fakeClosePrepReq) GetPreparedStatementHandle() []byte { return f.handle } + +// fakeGetTables implements flightsql.GetTables for tests. +type fakeGetTables struct { + catalog *string + dbSchemaFilter *string + tableNameFilter string + tableTypes []string + includeSchema bool +} + +func (f fakeGetTables) GetCatalog() *string { return f.catalog } +func (f fakeGetTables) GetDBSchemaFilterPattern() *string { return f.dbSchemaFilter } +func (f fakeGetTables) GetTableNameFilterPattern() *string { return &f.tableNameFilter } +func (f fakeGetTables) GetTableTypes() []string { return f.tableTypes } +func (f fakeGetTables) GetIncludeSchema() bool { return f.includeSchema } + +// fakeGetDBSchemas implements flightsql.GetDBSchemas. +type fakeGetDBSchemas struct { + catalog *string + dbSchemaFilter *string +} + +func (f fakeGetDBSchemas) GetCatalog() *string { return f.catalog } +func (f fakeGetDBSchemas) GetDBSchemaFilterPattern() *string { return f.dbSchemaFilter } + +// fakeGetSqlInfo implements flightsql.GetSqlInfo. +type fakeGetSqlInfo struct { + info []uint32 +} + +func (f fakeGetSqlInfo) GetInfo() []uint32 { return f.info } + +// unused interface guards +var _ flightsql.GetTables = fakeGetTables{} +var _ flightsql.GetDBSchemas = fakeGetDBSchemas{} +var _ flightsql.GetSqlInfo = fakeGetSqlInfo{} diff --git a/pkg/backends/influxdb/flight/server.go b/pkg/backends/influxdb/flight/server.go new file mode 100644 index 000000000..148c24eb9 --- /dev/null +++ b/pkg/backends/influxdb/flight/server.go @@ -0,0 +1,502 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package flight provides an Apache Arrow Flight SQL server that proxies +// queries to an upstream InfluxDB 3.x Flight SQL endpoint, caching IPC byte +// streams keyed by the tokenized SQL statement. +package flight + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/flight" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql/schema_ref" + "github.com/apache/arrow-go/v18/arrow/ipc" + "github.com/apache/arrow-go/v18/arrow/memory" + "google.golang.org/grpc/metadata" +) + +// tenantKey derives a stable per-tenant cache namespace from the incoming +// gRPC metadata. client.withAuth forwards `authorization`, `database`, and +// `bucket-name` to the upstream for per-request scoping, so cache keys must +// mirror that scope to avoid returning one tenant's data to another. +// Authorization is hashed so bearer tokens don't leak into cache keys. +func tenantKey(ctx context.Context) string { + md, _ := metadata.FromIncomingContext(ctx) + db := mdFirst(md, "database") + bucket := mdFirst(md, "bucket-name") + auth := mdFirst(md, "authorization") + authPart := "" + if auth != "" { + sum := sha256.Sum256([]byte(auth)) + authPart = hex.EncodeToString(sum[:8]) + } + return db + "|" + bucket + "|" + authPart +} + +func mdFirst(md metadata.MD, key string) string { + if md == nil { + return "" + } + v := md.Get(key) + if len(v) == 0 { + return "" + } + return v[0] +} + +// Server is a Flight SQL server that acts as a caching proxy to an upstream +// Flight SQL service (e.g., InfluxDB 3.x). +type Server struct { + flightsql.BaseServer + + upstream UpstreamClient + cache Cache + alloc memory.Allocator + + // paramHashes tracks the most recent bound parameter hash per prepared + // statement handle. Used as part of the DoGetPreparedStatement cache key + // so two clients executing the same prepared statement with different + // parameter values don't alias each other's cache entries. + paramMu sync.Mutex + paramHashes map[string]string +} + +// UpstreamClient is the minimum surface the server needs from a Flight SQL +// client implementation. This lets us swap in a fake for tests. +// Each method returns the IPC-encoded bytes (schema + record batches) of the +// upstream response so callers can cache the whole stream verbatim. +type UpstreamClient interface { + Execute(ctx context.Context, query string) ([]byte, error) + GetCatalogs(ctx context.Context) ([]byte, error) + GetDBSchemas(ctx context.Context, opts *flightsql.GetDBSchemasOpts) ([]byte, error) + GetTables(ctx context.Context, opts *flightsql.GetTablesOpts) ([]byte, error) + GetTableTypes(ctx context.Context) ([]byte, error) + GetSqlInfo(ctx context.Context, info []flightsql.SqlInfo) ([]byte, error) + PrepareStatement(ctx context.Context, query string) ([]byte, error) + SetPreparedStatementParams(ctx context.Context, handle []byte, params arrow.RecordBatch) error + ExecutePrepared(ctx context.Context, handle []byte) ([]byte, error) + ClosePrepared(ctx context.Context, handle []byte) error + Close() error +} + +// Cache stores serialized Arrow IPC byte streams keyed by query. +type Cache interface { + Get(key string) ([]byte, bool) + Set(key string, data []byte, ttl time.Duration) +} + +// NewServer constructs a Flight SQL server with the given upstream and cache. +func NewServer(upstream UpstreamClient, cache Cache) *Server { + return &Server{ + upstream: upstream, + cache: cache, + alloc: memory.DefaultAllocator, + paramHashes: make(map[string]string), + } +} + +// GetFlightInfoStatement handles a SQL query request. It returns a FlightInfo +// with a single endpoint whose ticket carries the query text. The actual +// execution happens in DoGetStatement when the client fetches the ticket. +func (s *Server) GetFlightInfoStatement(_ context.Context, + cmd flightsql.StatementQuery, desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + ticket, err := flightsql.CreateStatementQueryTicket([]byte(cmd.GetQuery())) + if err != nil { + return nil, err + } + return &flight.FlightInfo{ + FlightDescriptor: desc, + Endpoint: []*flight.FlightEndpoint{ + {Ticket: &flight.Ticket{Ticket: ticket}}, + }, + TotalRecords: -1, + TotalBytes: -1, + }, nil +} + +// DoGetStatement executes the query (cache-first, upstream on miss) and streams +// the Arrow IPC record batches back to the client. +func (s *Server) DoGetStatement(ctx context.Context, + ticket flightsql.StatementQueryTicket, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + query := string(ticket.GetStatementHandle()) + key := tenantKey(ctx) + ":stmt:" + query + + ipcBytes, cached := s.cacheGet(key) + if !cached { + b, err := s.upstream.Execute(ctx, query) + if err != nil { + return nil, nil, fmt.Errorf("upstream execute: %w", err) + } + ipcBytes = b + s.cacheSet(key, ipcBytes) + } + + return streamIPCBytes(ipcBytes) +} + +// flightInfoForCommand constructs a FlightInfo for metadata RPCs. The ticket +// is the command proto bytes from the descriptor; the server framework decodes +// and routes to the appropriate DoGetX method. +func (s *Server) flightInfoForCommand(desc *flight.FlightDescriptor, + schema *arrow.Schema, +) *flight.FlightInfo { + return &flight.FlightInfo{ + Endpoint: []*flight.FlightEndpoint{{Ticket: &flight.Ticket{Ticket: desc.Cmd}}}, + FlightDescriptor: desc, + Schema: flight.SerializeSchema(schema, s.alloc), + TotalRecords: -1, + TotalBytes: -1, + } +} + +// fetchMetadata centralizes the cache-then-upstream pattern for metadata RPCs. +// key should be a stable, collision-resistant identifier for the request. +func (s *Server) fetchMetadata(ctx context.Context, key string, + fetch func(context.Context) ([]byte, error), +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + ipcBytes, cached := s.cacheGet(key) + if !cached { + b, err := fetch(ctx) + if err != nil { + return nil, nil, fmt.Errorf("upstream %s: %w", key, err) + } + ipcBytes = b + s.cacheSet(key, ipcBytes) + } + return streamIPCBytes(ipcBytes) +} + +// GetFlightInfoCatalogs returns a FlightInfo describing the catalog list. +func (s *Server) GetFlightInfoCatalogs(_ context.Context, + desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + return s.flightInfoForCommand(desc, schema_ref.Catalogs), nil +} + +// DoGetCatalogs streams the upstream's catalog list (cache-first). +func (s *Server) DoGetCatalogs(ctx context.Context, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + return s.fetchMetadata(ctx, tenantKey(ctx)+":meta:catalogs", + s.upstream.GetCatalogs) +} + +// GetFlightInfoSchemas returns a FlightInfo describing DB schemas. +func (s *Server) GetFlightInfoSchemas(_ context.Context, + _ flightsql.GetDBSchemas, desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + return s.flightInfoForCommand(desc, schema_ref.DBSchemas), nil +} + +// DoGetDBSchemas streams the upstream's DB schema list (cache-first). +func (s *Server) DoGetDBSchemas(ctx context.Context, + cmd flightsql.GetDBSchemas, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + opts := &flightsql.GetDBSchemasOpts{ + Catalog: cmd.GetCatalog(), + DbSchemaFilterPattern: cmd.GetDBSchemaFilterPattern(), + } + key := tenantKey(ctx) + ":meta:dbschemas:" + strings.Join([]string{ + deref(cmd.GetCatalog()), + deref(cmd.GetDBSchemaFilterPattern()), + }, "|") + return s.fetchMetadata(ctx, key, func(ctx context.Context) ([]byte, error) { + return s.upstream.GetDBSchemas(ctx, opts) + }) +} + +// GetFlightInfoTables returns a FlightInfo describing the table list. +func (s *Server) GetFlightInfoTables(_ context.Context, + cmd flightsql.GetTables, desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + schema := schema_ref.Tables + if cmd.GetIncludeSchema() { + schema = schema_ref.TablesWithIncludedSchema + } + return s.flightInfoForCommand(desc, schema), nil +} + +// DoGetTables streams the upstream's table list (cache-first). +func (s *Server) DoGetTables(ctx context.Context, + cmd flightsql.GetTables, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + tableTypes := cmd.GetTableTypes() + opts := &flightsql.GetTablesOpts{ + Catalog: cmd.GetCatalog(), + DbSchemaFilterPattern: cmd.GetDBSchemaFilterPattern(), + TableNameFilterPattern: cmd.GetTableNameFilterPattern(), + TableTypes: tableTypes, + IncludeSchema: cmd.GetIncludeSchema(), + } + key := tenantKey(ctx) + ":meta:tables:" + strings.Join([]string{ + deref(cmd.GetCatalog()), + deref(cmd.GetDBSchemaFilterPattern()), + deref(cmd.GetTableNameFilterPattern()), + strings.Join(tableTypes, ","), + strconv.FormatBool(cmd.GetIncludeSchema()), + }, "|") + return s.fetchMetadata(ctx, key, func(ctx context.Context) ([]byte, error) { + return s.upstream.GetTables(ctx, opts) + }) +} + +// GetFlightInfoTableTypes returns a FlightInfo describing table types. +func (s *Server) GetFlightInfoTableTypes(_ context.Context, + desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + return s.flightInfoForCommand(desc, schema_ref.TableTypes), nil +} + +// DoGetTableTypes streams the upstream's table types (cache-first). +func (s *Server) DoGetTableTypes(ctx context.Context, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + return s.fetchMetadata(ctx, tenantKey(ctx)+":meta:tabletypes", + s.upstream.GetTableTypes) +} + +// GetFlightInfoSqlInfo returns a FlightInfo describing SQL info. BaseServer's +// default implementation fails with NotFound unless info is locally registered +// via RegisterSqlInfo; we override to always route through to DoGetSqlInfo so +// the response reflects upstream capabilities. +func (s *Server) GetFlightInfoSqlInfo(_ context.Context, + _ flightsql.GetSqlInfo, desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + return s.flightInfoForCommand(desc, schema_ref.SqlInfo), nil +} + +// DoGetSqlInfo streams upstream SQL info records (cache-first). The default +// BaseServer GetFlightInfoSqlInfo/DoGetSqlInfo use locally-registered info; we +// intercept DoGetSqlInfo and proxy to upstream instead so values reflect the +// actual upstream capabilities (version, dialect, etc.). +func (s *Server) DoGetSqlInfo(ctx context.Context, + cmd flightsql.GetSqlInfo, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + rawInfo := cmd.GetInfo() + info := make([]flightsql.SqlInfo, len(rawInfo)) + for i, v := range rawInfo { + info[i] = flightsql.SqlInfo(v) + } + return s.fetchMetadata(ctx, fmt.Sprintf("%s:meta:sqlinfo:%v", tenantKey(ctx), rawInfo), + func(ctx context.Context) ([]byte, error) { + return s.upstream.GetSqlInfo(ctx, info) + }) +} + +// deref returns the dereferenced string or empty when nil. +func deref(p *string) string { + if p == nil { + return "" + } + return *p +} + +// CreatePreparedStatement proxies prepared-statement creation to the upstream +// and passes the handle back to the client. No caching at the prepare stage — +// caching happens on the DoGetPreparedStatement path keyed by the handle. +func (s *Server) CreatePreparedStatement(ctx context.Context, + req flightsql.ActionCreatePreparedStatementRequest, +) (flightsql.ActionCreatePreparedStatementResult, error) { + handle, err := s.upstream.PrepareStatement(ctx, req.GetQuery()) + if err != nil { + return flightsql.ActionCreatePreparedStatementResult{}, + fmt.Errorf("upstream prepare: %w", err) + } + return flightsql.ActionCreatePreparedStatementResult{Handle: handle}, nil +} + +// GetFlightInfoPreparedStatement returns a FlightInfo whose ticket is the +// command proto bytes from the descriptor, same pattern as metadata RPCs. +// The framework decodes and routes to DoGetPreparedStatement. +func (s *Server) GetFlightInfoPreparedStatement(_ context.Context, + _ flightsql.PreparedStatementQuery, desc *flight.FlightDescriptor, +) (*flight.FlightInfo, error) { + // No static schema available pre-execution; pass nil to let Arrow infer + // from the response stream. The real schema surfaces in DoGet. + return &flight.FlightInfo{ + Endpoint: []*flight.FlightEndpoint{ + {Ticket: &flight.Ticket{Ticket: desc.Cmd}}, + }, + FlightDescriptor: desc, + TotalRecords: -1, + TotalBytes: -1, + }, nil +} + +// DoPutPreparedStatementQuery receives parameter bindings from the client and +// forwards them to the upstream prepared statement. The parameter hash is +// recorded against the handle so DoGet cache keys reflect the bound values. +func (s *Server) DoPutPreparedStatementQuery(ctx context.Context, + cmd flightsql.PreparedStatementQuery, + reader flight.MessageReader, _ flight.MetadataWriter, +) ([]byte, error) { + handle := cmd.GetPreparedStatementHandle() + if !reader.Next() { + // no record batches sent — treat as clearing params + s.setParamHash(handle, "") + return handle, nil + } + rec := reader.RecordBatch() + rec.Retain() + defer rec.Release() + if err := s.upstream.SetPreparedStatementParams(ctx, handle, rec); err != nil { + return nil, fmt.Errorf("upstream set params: %w", err) + } + hash, err := hashRecordBatch(rec) + if err != nil { + return nil, fmt.Errorf("hash params: %w", err) + } + s.setParamHash(handle, hash) + return handle, nil +} + +// DoGetPreparedStatement executes the upstream prepared statement and streams +// its Arrow IPC output. Cache key includes the bound parameter hash so two +// clients running the same statement with different params don't collide. +func (s *Server) DoGetPreparedStatement(ctx context.Context, + cmd flightsql.PreparedStatementQuery, +) (*arrow.Schema, <-chan flight.StreamChunk, error) { + handle := cmd.GetPreparedStatementHandle() + key := tenantKey(ctx) + ":prep:" + string(handle) + ":" + s.paramHash(handle) + return s.fetchMetadata(ctx, key, func(ctx context.Context) ([]byte, error) { + return s.upstream.ExecutePrepared(ctx, handle) + }) +} + +// ClosePreparedStatement releases the upstream handle. +func (s *Server) ClosePreparedStatement(ctx context.Context, + req flightsql.ActionClosePreparedStatementRequest, +) error { + handle := req.GetPreparedStatementHandle() + s.paramMu.Lock() + delete(s.paramHashes, string(handle)) + s.paramMu.Unlock() + return s.upstream.ClosePrepared(ctx, handle) +} + +func (s *Server) setParamHash(handle []byte, hash string) { + s.paramMu.Lock() + defer s.paramMu.Unlock() + if hash == "" { + delete(s.paramHashes, string(handle)) + return + } + s.paramHashes[string(handle)] = hash +} + +func (s *Server) paramHash(handle []byte) string { + s.paramMu.Lock() + defer s.paramMu.Unlock() + return s.paramHashes[string(handle)] +} + +// hashRecordBatch returns a stable hex-encoded hash of an Arrow RecordBatch +// by writing its IPC-encoded bytes through sha256. +func hashRecordBatch(rec arrow.RecordBatch) (string, error) { + var buf bytes.Buffer + w := ipc.NewWriter(&buf, ipc.WithSchema(rec.Schema())) + if err := w.Write(rec); err != nil { + return "", err + } + if err := w.Close(); err != nil { + return "", err + } + sum := sha256.Sum256(buf.Bytes()) + return hex.EncodeToString(sum[:]), nil +} + +// streamIPCBytes parses serialized Arrow IPC bytes and returns the schema +// plus a channel of stream chunks the Flight server framework will write out. +func streamIPCBytes(b []byte) (*arrow.Schema, <-chan flight.StreamChunk, error) { + r, err := ipc.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, nil, fmt.Errorf("ipc reader: %w", err) + } + schema := r.Schema() + ch := make(chan flight.StreamChunk) + go func() { + defer close(ch) + defer r.Release() + for r.Next() { + rec := r.RecordBatch() + rec.Retain() + ch <- flight.StreamChunk{Data: rec} + } + }() + return schema, ch, nil +} + +func (s *Server) cacheGet(query string) ([]byte, bool) { + if s.cache == nil { + return nil, false + } + return s.cache.Get(query) +} + +func (s *Server) cacheSet(query string, data []byte) { + if s.cache == nil { + return + } + s.cache.Set(query, data, 60*time.Second) +} + +// EncodeRecords serializes a slice of Arrow records into IPC bytes with their +// shared schema. Useful when constructing cached results from non-Arrow sources. +func EncodeRecords(schema *arrow.Schema, records []arrow.RecordBatch) ([]byte, error) { + var buf bytes.Buffer + w := ipc.NewWriter(&buf, ipc.WithSchema(schema)) + for _, rec := range records { + if err := w.Write(rec); err != nil { + return nil, err + } + } + if err := w.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// DecodeRecords parses IPC bytes into the schema and a slice of records. +// Primarily used in tests. +func DecodeRecords(b []byte) (*arrow.Schema, []arrow.RecordBatch, error) { + r, err := ipc.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, nil, err + } + defer r.Release() + var recs []arrow.RecordBatch + for r.Next() { + rec := r.RecordBatch() + rec.Retain() + recs = append(recs, rec) + } + return r.Schema(), recs, r.Err() +} + +// Compile-time interface checks +var _ array.Builder = (*array.StringBuilder)(nil) diff --git a/pkg/backends/influxdb/flight/server_test.go b/pkg/backends/influxdb/flight/server_test.go new file mode 100644 index 000000000..01ae146ae --- /dev/null +++ b/pkg/backends/influxdb/flight/server_test.go @@ -0,0 +1,429 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package flight + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/flight/flightsql" + "github.com/apache/arrow-go/v18/arrow/memory" + "google.golang.org/grpc/metadata" +) + +func ctxWithTenant(auth, db string) context.Context { + md := metadata.MD{} + if auth != "" { + md.Set("authorization", auth) + } + if db != "" { + md.Set("database", db) + } + return metadata.NewIncomingContext(context.Background(), md) +} + +// fakeUpstream is a simple UpstreamClient for tests that returns pre-canned IPC bytes. +// All RPC methods return the same ipcBytes payload — tests only care about call +// counts and cache behavior, not differentiated metadata content. +type fakeUpstream struct { + mu sync.Mutex + callCount int + ipcBytes []byte + lastQuery string + returnErr error + + // per-method counters so tests can verify specific RPCs were hit + executeCalls int + catalogCalls int + dbSchemaCalls int + tablesCalls int + tableTypesCalls int + sqlInfoCalls int + prepareCalls int + setParamsCalls int + executePreparedCalls int + closePreparedCalls int +} + +func (f *fakeUpstream) Execute(_ context.Context, query string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.executeCalls++ + f.lastQuery = query + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) GetCatalogs(_ context.Context) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.catalogCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) GetDBSchemas(_ context.Context, _ *flightsql.GetDBSchemasOpts) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.dbSchemaCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) GetTables(_ context.Context, _ *flightsql.GetTablesOpts) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.tablesCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) GetTableTypes(_ context.Context) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.tableTypesCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) GetSqlInfo(_ context.Context, _ []flightsql.SqlInfo) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.sqlInfoCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) PrepareStatement(_ context.Context, _ string) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.prepareCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return []byte("fake-handle"), nil +} + +func (f *fakeUpstream) SetPreparedStatementParams(_ context.Context, _ []byte, _ arrow.RecordBatch) error { + f.mu.Lock() + defer f.mu.Unlock() + f.setParamsCalls++ + return nil +} + +func (f *fakeUpstream) ExecutePrepared(_ context.Context, _ []byte) ([]byte, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.callCount++ + f.executePreparedCalls++ + if f.returnErr != nil { + return nil, f.returnErr + } + return f.ipcBytes, nil +} + +func (f *fakeUpstream) ClosePrepared(_ context.Context, _ []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.closePreparedCalls++ + return nil +} + +func (f *fakeUpstream) Close() error { return nil } + +// memCache is a simple in-memory implementation of Cache for tests. +type memCache struct { + mu sync.Mutex + data map[string][]byte +} + +func newMemCache() *memCache { + return &memCache{data: make(map[string][]byte)} +} + +func (c *memCache) Get(key string) ([]byte, bool) { + c.mu.Lock() + defer c.mu.Unlock() + b, ok := c.data[key] + return b, ok +} + +func (c *memCache) Set(key string, data []byte, _ time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.data[key] = data +} + +// buildTestIPC creates a small Arrow record and encodes it to IPC bytes. +func buildTestIPC(t *testing.T) []byte { + t.Helper() + mem := memory.DefaultAllocator + schema := arrow.NewSchema([]arrow.Field{ + {Name: "time", Type: arrow.FixedWidthTypes.Timestamp_ns}, + {Name: "value", Type: arrow.PrimitiveTypes.Float64}, + }, nil) + b := array.NewRecordBuilder(mem, schema) + defer b.Release() + b.Field(0).(*array.TimestampBuilder).AppendValues( + []arrow.Timestamp{1000000000, 2000000000}, nil) + b.Field(1).(*array.Float64Builder).AppendValues( + []float64{1.5, 2.7}, nil) + rec := b.NewRecord() + defer rec.Release() + data, err := EncodeRecords(schema, []arrow.RecordBatch{rec}) + if err != nil { + t.Fatalf("EncodeRecords: %v", err) + } + return data +} + +func TestDoGetStatement_UpstreamHit(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + cache := newMemCache() + srv := NewServer(up, cache) + + sqt := statementTicket("SELECT * FROM cpu") + + schema, ch, err := srv.DoGetStatement(context.Background(), sqt) + if err != nil { + t.Fatalf("DoGetStatement: %v", err) + } + if schema == nil { + t.Fatal("expected non-nil schema") + } + if len(schema.Fields()) != 2 { + t.Errorf("expected 2 fields, got %d", len(schema.Fields())) + } + + rowCount := 0 + for chunk := range ch { + rowCount += int(chunk.Data.NumRows()) + chunk.Data.Release() + } + if rowCount != 2 { + t.Errorf("expected 2 rows, got %d", rowCount) + } + if up.callCount != 1 { + t.Errorf("expected 1 upstream call, got %d", up.callCount) + } +} + +func TestDoGetStatement_CacheHit(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + cache := newMemCache() + srv := NewServer(up, cache) + + query := "SELECT * FROM cpu" + sqt := statementTicket(query) + + // First call populates cache. + _, ch, err := srv.DoGetStatement(context.Background(), sqt) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + + // Second call should hit cache, not upstream. + _, ch2, err := srv.DoGetStatement(context.Background(), sqt) + if err != nil { + t.Fatal(err) + } + for chunk := range ch2 { + chunk.Data.Release() + } + + if up.callCount != 1 { + t.Errorf("expected 1 upstream call (2nd should hit cache), got %d", up.callCount) + } +} + +func TestDoGetStatement_TenantIsolation(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + srv := NewServer(up, newMemCache()) + query := "SELECT * FROM cpu" + sqt := statementTicket(query) + + ctxA := ctxWithTenant("Bearer tokenA", "tenant_a") + ctxB := ctxWithTenant("Bearer tokenB", "tenant_b") + + _, ch, err := srv.DoGetStatement(ctxA, sqt) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + _, ch2, err := srv.DoGetStatement(ctxB, sqt) + if err != nil { + t.Fatal(err) + } + for chunk := range ch2 { + chunk.Data.Release() + } + + if up.executeCalls != 2 { + t.Errorf("expected 2 upstream Execute calls (one per tenant), got %d", + up.executeCalls) + } +} + +func TestDoGetStatement_SameTenantHits(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + srv := NewServer(up, newMemCache()) + query := "SELECT * FROM cpu" + sqt := statementTicket(query) + + ctx := ctxWithTenant("Bearer tokenA", "tenant_a") + + for range 2 { + _, ch, err := srv.DoGetStatement(ctx, sqt) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + + if up.executeCalls != 1 { + t.Errorf("expected 1 upstream call (2nd hits cache), got %d", + up.executeCalls) + } +} + +func TestDoGetCatalogs_TenantIsolation(t *testing.T) { + ipcBytes := buildTestIPC(t) + up := &fakeUpstream{ipcBytes: ipcBytes} + srv := NewServer(up, newMemCache()) + + ctxA := ctxWithTenant("Bearer tokenA", "tenant_a") + ctxB := ctxWithTenant("Bearer tokenB", "tenant_b") + + for _, ctx := range []context.Context{ctxA, ctxB} { + _, ch, err := srv.DoGetCatalogs(ctx) + if err != nil { + t.Fatal(err) + } + for chunk := range ch { + chunk.Data.Release() + } + } + + if up.catalogCalls != 2 { + t.Errorf("expected 2 upstream GetCatalogs calls, got %d", up.catalogCalls) + } +} + +func TestDoGetStatement_UpstreamError(t *testing.T) { + up := &fakeUpstream{returnErr: fmt.Errorf("boom")} + cache := newMemCache() + srv := NewServer(up, cache) + + sqt := statementTicket("SELECT 1") + + _, _, err := srv.DoGetStatement(context.Background(), sqt) + if err == nil { + t.Fatal("expected error") + } +} + +func TestGetFlightInfoStatement(t *testing.T) { + srv := NewServer(&fakeUpstream{}, newMemCache()) + cmd := fakeStatementQuery{query: "SELECT 1"} + info, err := srv.GetFlightInfoStatement(context.Background(), cmd, nil) + if err != nil { + t.Fatal(err) + } + if info == nil || len(info.Endpoint) != 1 { + t.Fatal("expected 1 endpoint") + } + if info.Endpoint[0].Ticket == nil { + t.Fatal("expected ticket") + } +} + +// fakeStatementQuery implements flightsql.StatementQuery for tests. +type fakeStatementQuery struct { + query string +} + +func (f fakeStatementQuery) GetQuery() string { return f.query } +func (f fakeStatementQuery) GetTransactionId() []byte { return nil } + +// fakeStatementTicket implements flightsql.StatementQueryTicket for tests. +type fakeStatementTicket struct { + handle []byte +} + +func (f fakeStatementTicket) GetStatementHandle() []byte { return f.handle } + +func statementTicket(query string) flightsql.StatementQueryTicket { + return fakeStatementTicket{handle: []byte(query)} +} + +func TestRoundTripIPC(t *testing.T) { + ipcBytes := buildTestIPC(t) + schema, recs, err := DecodeRecords(ipcBytes) + if err != nil { + t.Fatal(err) + } + defer func() { + for _, r := range recs { + r.Release() + } + }() + if len(schema.Fields()) != 2 { + t.Errorf("schema field count: got %d, want 2", len(schema.Fields())) + } + var total int64 + for _, r := range recs { + total += r.NumRows() + } + if total != 2 { + t.Errorf("total rows: got %d, want 2", total) + } +} diff --git a/pkg/backends/influxdb/flux/flux.go b/pkg/backends/influxdb/flux/flux.go index 2f48046c3..92d7d8430 100644 --- a/pkg/backends/influxdb/flux/flux.go +++ b/pkg/backends/influxdb/flux/flux.go @@ -326,6 +326,9 @@ func parseRange(input string) (timeseries.Extent, error) { } func tryParseTimeField(s string) (time.Time, error) { + if s == "now()" { + return time.Now(), nil + } var t time.Time var erd, eat, eut error if t, erd = tryParseRelativeDuration(s); erd == nil { @@ -364,10 +367,32 @@ func tryParseUnixTimestamp(s string) (time.Time, error) { return time.Unix(int64(unix), 0).UTC(), nil } +// tokenizeRangeLine replaces the body of `|> range(...)` with the placeholder, +// correctly matching the closing paren at the same nesting depth (so nested +// function calls like `now()` inside range() don't confuse the match). func tokenizeRangeLine(input string, funcStart int) string { - i := strings.Index(input[funcStart:], ")") - if i < 0 { + open := funcStart + len(FuncRange) - 1 // index of the `(` in `|> range(` + if open >= len(input) || input[open] != '(' { + return input + } + depth := 1 + close := -1 + for j := open + 1; j < len(input); j++ { + switch input[j] { + case '(': + depth++ + case ')': + depth-- + if depth == 0 { + close = j + } + } + if close >= 0 { + break + } + } + if close < 0 { return input } - return input[:funcStart+len(FuncRange)] + TokenPlaceholderTimeRange + input[funcStart+i:] + return input[:open+1] + TokenPlaceholderTimeRange + input[close:] } diff --git a/pkg/backends/influxdb/flux/flux_test.go b/pkg/backends/influxdb/flux/flux_test.go index 79ce652f5..42761acab 100644 --- a/pkg/backends/influxdb/flux/flux_test.go +++ b/pkg/backends/influxdb/flux/flux_test.go @@ -80,6 +80,33 @@ func TestParseQuery(t *testing.T) { } } +// TestParseQuery_Now verifies that `now()` as a range bound is accepted and +// resolved to the current time. This is Grafana's default `stop` value for +// Flux queries — without this, aggregateWindow dashboards fall through to +// HTTPProxy instead of being delta-proxy cached. +func TestParseQuery_Now(t *testing.T) { + before := time.Now() + q := `from(bucket: "trickster") |> range(start: -1h, stop: now()) |> aggregateWindow(every: 1m, fn: mean) |> limit(n: 5)` + _, e, d, err := ParseQuery(q) + if err != nil { + t.Fatalf("ParseQuery(now()): %v", err) + } + if d != time.Minute { + t.Errorf("step: got %v, want 1m", d) + } + if e.Start.IsZero() || e.End.IsZero() { + t.Fatalf("extent should be populated, got %+v", e) + } + if e.End.Before(before) { + t.Errorf("now() should resolve to ~now, got %v (before=%v)", e.End, before) + } + // start is relative -1h, end is now: window should be ~1h. + window := e.End.Sub(e.Start) + if window < 55*time.Minute || window > 65*time.Minute { + t.Errorf("expected ~1h window, got %v", window) + } +} + func TestParseTimeRangeQuery(t *testing.T) { b, _ := json.Marshal(JSONRequestBody{ Query: testFluxQuery1, diff --git a/pkg/backends/influxdb/flux/unmarshal.go b/pkg/backends/influxdb/flux/unmarshal.go index 435ddc081..95b3253a2 100644 --- a/pkg/backends/influxdb/flux/unmarshal.go +++ b/pkg/backends/influxdb/flux/unmarshal.go @@ -51,7 +51,14 @@ func UnmarshalTimeseriesReader(reader io.Reader, if trq == nil { return nil, timeseries.ErrNoTimerangeQuery } - rows, err := csv.NewReader(reader).ReadAll() + // Flux CSV responses may contain multiple result tables separated by blank + // lines, each with its own schema/column count. Allow variable fields per + // record so multi-table responses read without error; the downstream + // parser handles the single-table case and ignores structural rows it + // doesn't recognize. + cr := csv.NewReader(reader) + cr.FieldsPerRecord = -1 + rows, err := cr.ReadAll() if err != nil { return nil, err } diff --git a/pkg/backends/influxdb/handler_query.go b/pkg/backends/influxdb/handler_query.go index ef6132bd8..5ccc7dc76 100644 --- a/pkg/backends/influxdb/handler_query.go +++ b/pkg/backends/influxdb/handler_query.go @@ -18,11 +18,13 @@ package influxdb import ( "net/http" + "slices" "strings" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/flux" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/influxql" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + isql "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/sql" "github.com/trickstercache/trickster/v2/pkg/errors" "github.com/trickstercache/trickster/v2/pkg/proxy/engines" "github.com/trickstercache/trickster/v2/pkg/proxy/params" @@ -35,6 +37,16 @@ import ( func (c *Client) QueryHandler(w http.ResponseWriter, r *http.Request) { f := iofmt.Detect(r) switch { + case f.IsV3SQL(): + if !isV3SelectQuery(r) { + c.ProxyHandler(w, r) + return + } + case f.IsV3InfluxQL(): + if !isV3SelectQuery(r) { + c.ProxyHandler(w, r) + return + } case f.IsInfluxQL(): qp, _, _ := params.GetRequestValues(r) // skip non-selects @@ -54,12 +66,25 @@ func (c *Client) QueryHandler(w http.ResponseWriter, r *http.Request) { engines.DeltaProxyCacheRequest(w, r, c.Modeler()) } +// isV3SelectQuery checks if a v3 request contains a SELECT query. +func isV3SelectQuery(r *http.Request) bool { + q, err := isql.ExtractQuery(r) + if err != nil || q == "" { + return false + } + return slices.Contains(strings.Fields(strings.ToLower(q)), "select") +} + // ParseTimeRangeQuery parses the key parts of a TimeRangeQuery from the inbound HTTP Request func (c *Client) ParseTimeRangeQuery(r *http.Request) (*timeseries.TimeRangeQuery, *timeseries.RequestOptions, bool, error, ) { f := iofmt.Detect(r) switch { + case f.IsV3SQL(): + return isql.ParseTimeRangeQuery(r, f) + case f.IsV3InfluxQL(): + return parseV3InfluxQL(r, f) case f.IsInfluxQL(): return influxql.ParseTimeRangeQuery(r, f) case f.IsFlux(): @@ -67,3 +92,27 @@ func (c *Client) ParseTimeRangeQuery(r *http.Request) (*timeseries.TimeRangeQuer } return nil, nil, false, errors.ErrBadRequest } + +// parseV3InfluxQL reuses the v1 InfluxQL parser but sets v3 output format +func parseV3InfluxQL(r *http.Request, f iofmt.Format, +) (*timeseries.TimeRangeQuery, *timeseries.RequestOptions, bool, error) { + // v3 InfluxQL uses the same query language as v1, but the query arrives + // in the "q" param (GET or POST body) and the response format is determined + // by the "format" param. We reuse the v1 parser by temporarily swapping + // the format to InfluxQL, then override the OutputFormat for v3 response. + v1f := iofmt.InfluxqlGet + if r.Method == http.MethodPost { + v1f = iofmt.InfluxqlPost + } + trq, rlo, canOPC, err := influxql.ParseTimeRangeQuery(r, v1f) + if err != nil { + return trq, rlo, canOPC, err + } + if rlo != nil { + rlo.OutputFormat = iofmt.V3OutputFormat(r) + } + if trq != nil && trq.ParsedQuery != nil { + trq.ParsedQuery = &isql.V3InfluxQLQuery{Inner: trq.ParsedQuery} + } + return trq, rlo, canOPC, nil +} diff --git a/pkg/backends/influxdb/handler_query_test.go b/pkg/backends/influxdb/handler_query_test.go index 69ed079fd..246097977 100644 --- a/pkg/backends/influxdb/handler_query_test.go +++ b/pkg/backends/influxdb/handler_query_test.go @@ -17,6 +17,7 @@ package influxdb import ( + "bytes" "io" "net/http" "net/url" @@ -112,6 +113,78 @@ func TestQueryHandlerNotSelect(t *testing.T) { } } +func TestIsV3SelectQuery(t *testing.T) { + sql := "SELECT * FROM cpu WHERE time >= 1704067200 AND time < 1704070800" + cases := []struct { + name string + method string + rawQuery string + contentType string + body string + want bool + }{ + { + name: "GET with q param", + method: http.MethodGet, + rawQuery: url.Values{"q": {sql}}.Encode(), + want: true, + }, + { + name: "POST JSON body with q field", + method: http.MethodPost, + contentType: "application/json", + body: `{"q":"` + sql + `"}`, + want: true, + }, + { + name: "POST form-urlencoded body", + method: http.MethodPost, + contentType: "application/x-www-form-urlencoded", + body: url.Values{"q": {sql}}.Encode(), + want: true, + }, + { + name: "POST raw SQL body", + method: http.MethodPost, + body: sql, + want: true, + }, + { + name: "POST non-select raw body", + method: http.MethodPost, + body: "CREATE TABLE foo (id INT)", + want: false, + }, + { + name: "POST JSON non-select", + method: http.MethodPost, + contentType: "application/json", + body: `{"q":"DROP TABLE foo"}`, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := &http.Request{ + Method: tc.method, + URL: &url.URL{Path: "/api/v3/query_sql", RawQuery: tc.rawQuery}, + Header: http.Header{}, + } + if tc.body != "" { + r.Body = io.NopCloser(bytes.NewReader([]byte(tc.body))) + r.ContentLength = int64(len(tc.body)) + } + if tc.contentType != "" { + r.Header.Set("Content-Type", tc.contentType) + } + got := isV3SelectQuery(r) + if got != tc.want { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} + func TestParseTimeRangeQueryMissingQuery(t *testing.T) { expected := errors.ErrBadRequest req := &http.Request{URL: &url.URL{ diff --git a/pkg/backends/influxdb/influxdb.go b/pkg/backends/influxdb/influxdb.go index d0ba63811..7900bf652 100644 --- a/pkg/backends/influxdb/influxdb.go +++ b/pkg/backends/influxdb/influxdb.go @@ -19,11 +19,16 @@ package influxdb import ( "net/http" + "net/url" + "time" "github.com/trickstercache/trickster/v2/pkg/backends" + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/flight" bo "github.com/trickstercache/trickster/v2/pkg/backends/options" "github.com/trickstercache/trickster/v2/pkg/backends/providers/registry/types" "github.com/trickstercache/trickster/v2/pkg/cache" + "github.com/trickstercache/trickster/v2/pkg/observability/logging" + "github.com/trickstercache/trickster/v2/pkg/observability/logging/logger" ) var _ backends.TimeseriesBackend = (*Client)(nil) @@ -37,14 +42,79 @@ var _ types.NewBackendClientFunc = NewClient // NewClient returns a new Client Instance func NewClient(name string, o *bo.Options, router http.Handler, - cache cache.Cache, _ backends.Backends, _ types.Lookup, + c cache.Cache, _ backends.Backends, _ types.Lookup, ) (backends.Backend, error) { if o != nil { o.FastForwardDisable = true } - c := &Client{} - b, err := backends.NewTimeseriesBackend(name, o, c.RegisterHandlers, - router, cache, NewModeler()) - c.TimeseriesBackend = b - return c, err + client := &Client{} + b, err := backends.NewTimeseriesBackend(name, o, client.RegisterHandlers, + router, c, NewModeler()) + client.TimeseriesBackend = b + if err == nil && o != nil && o.FlightPort > 0 { + if ferr := startFlightListener(name, o, c); ferr != nil { + logger.Error("flight sql listener startup failed", + logging.Pairs{"backend": name, "port": o.FlightPort, "detail": ferr}) + } + } + return client, err +} + +// startFlightListener launches a Flight SQL server on o.FlightPort that +// proxies queries to the backend's upstream Flight SQL endpoint. The listener +// is registered in the flight package's registry; the daemon drains it on +// SIGTERM via flight.ShutdownAll, and calls with a reused Name replace the +// existing listener (supporting config reload). +func startFlightListener(name string, o *bo.Options, c cache.Cache) error { + upstream := o.FlightUpstreamAddress + if upstream == "" { + u, err := url.Parse(o.OriginURL) + if err != nil { + return err + } + upstream = u.Host + } + fc, err := flight.NewFlightSQLClient(flight.UpstreamConfig{ + Address: upstream, + }) + if err != nil { + return err + } + srv := flight.NewServer(fc, newFlightCache(c)) + lis, err := flight.Start(flight.ListenerConfig{ + Address: "0.0.0.0", + Port: o.FlightPort, + Name: name, + }, srv) + if err != nil { + return err + } + logger.Info("flight sql listener started", + logging.Pairs{"backend": name, "address": lis.Addr().String()}) + return nil +} + +// flightCacheAdapter adapts a Trickster cache.Cache to the flight.Cache +// interface (Get/Set vs Retrieve/Store). Without this, Flight SQL requests +// pass through to upstream unconditionally and the caching path advertised +// in docs/influxdb.md never engages. +type flightCacheAdapter struct{ c cache.Cache } + +func newFlightCache(c cache.Cache) flight.Cache { + if c == nil { + return nil + } + return &flightCacheAdapter{c: c} +} + +func (a *flightCacheAdapter) Get(key string) ([]byte, bool) { + b, _, err := a.c.Retrieve(key) + if err != nil || len(b) == 0 { + return nil, false + } + return b, true +} + +func (a *flightCacheAdapter) Set(key string, data []byte, ttl time.Duration) { + _ = a.c.Store(key, data, ttl) } diff --git a/pkg/backends/influxdb/influxdb_test.go b/pkg/backends/influxdb/influxdb_test.go index 95e80345f..4081e924d 100644 --- a/pkg/backends/influxdb/influxdb_test.go +++ b/pkg/backends/influxdb/influxdb_test.go @@ -18,6 +18,7 @@ package influxdb import ( "testing" + "time" "github.com/trickstercache/trickster/v2/pkg/backends" bo "github.com/trickstercache/trickster/v2/pkg/backends/options" @@ -78,3 +79,33 @@ func TestNewClient(t *testing.T) { t.Errorf("expected %s got %s", "TEST_CLIENT", c.Configuration().Provider) } } + +func TestNewFlightCacheAdapter(t *testing.T) { + if newFlightCache(nil) != nil { + t.Fatal("newFlightCache(nil) should be nil") + } + + conf, err := config.Load([]string{"-provider", providers.InfluxDB, "-origin-url", "http://1"}) + if err != nil { + t.Fatalf("config: %v", err) + } + caches := cr.LoadCachesFromConfig(conf) + defer cr.CloseCaches(caches) + c := caches["default"] + if c == nil { + t.Fatal("missing default cache") + } + + fc := newFlightCache(c) + if fc == nil { + t.Fatal("expected non-nil adapter") + } + if _, ok := fc.Get("missing-key"); ok { + t.Error("expected miss on empty cache") + } + fc.Set("k1", []byte("hello"), time.Minute) + b, ok := fc.Get("k1") + if !ok || string(b) != "hello" { + t.Errorf("round-trip failed: ok=%v b=%q", ok, b) + } +} diff --git a/pkg/backends/influxdb/iofmt/iofmt.go b/pkg/backends/influxdb/iofmt/iofmt.go index 9123e82e8..22e9569e8 100644 --- a/pkg/backends/influxdb/iofmt/iofmt.go +++ b/pkg/backends/influxdb/iofmt/iofmt.go @@ -21,6 +21,7 @@ package iofmt import ( "errors" "net/http" + "strings" "github.com/trickstercache/trickster/v2/pkg/proxy/headers" ) @@ -35,6 +36,8 @@ const ( isFlux // 4 isFluxInputJSON // 8 isFluxOutputJSON // 16 + isV3SQL // 32 + isV3InfluxQL // 64 InfluxqlGet = isInfluxql InfluxqlPost = isInfluxql + isInfluxqlPost @@ -44,6 +47,16 @@ const ( FluxRawJSON = isFlux + isFluxOutputJSON FluxRawCsv = isFlux + + V3SQL = isV3SQL + V3InfluxQL = isV3InfluxQL +) + +// V3 output format constants stored in RequestOptions.OutputFormat +const ( + V3OutputJSON byte = 32 + V3OutputJSONL byte = 33 + V3OutputCSV byte = 34 ) var ErrSupportedQueryLanguage = errors.New("unsupported query language") @@ -64,11 +77,44 @@ func (f Format) IsFluxOutputJSON() bool { return f&isFluxOutputJSON == isFluxOutputJSON } +func (f Format) IsV3SQL() bool { + return f&isV3SQL == isV3SQL +} + +func (f Format) IsV3InfluxQL() bool { + return f&isV3InfluxQL == isV3InfluxQL +} + +func (f Format) IsV3() bool { + return f.IsV3SQL() || f.IsV3InfluxQL() +} + func (f Format) IsPost() bool { return f&isFlux == isFlux || f&isInfluxqlPost == isInfluxqlPost } +// V3OutputFormat returns the v3 output format byte from the request's format param. +func V3OutputFormat(r *http.Request) byte { + switch strings.ToLower(r.URL.Query().Get("format")) { + case "jsonl": + return V3OutputJSONL + case "csv": + return V3OutputCSV + default: + return V3OutputJSON + } +} + func Detect(r *http.Request) Format { + if r.URL != nil { + p := r.URL.Path + switch { + case strings.HasSuffix(p, "/api/v3/query_sql"): + return V3SQL + case strings.HasSuffix(p, "/api/v3/query_influxql"): + return V3InfluxQL + } + } if r.Method == http.MethodGet { return InfluxqlGet } diff --git a/pkg/backends/influxdb/iofmt/iofmt_test.go b/pkg/backends/influxdb/iofmt/iofmt_test.go index 24dfb9a45..b7c24a2be 100644 --- a/pkg/backends/influxdb/iofmt/iofmt_test.go +++ b/pkg/backends/influxdb/iofmt/iofmt_test.go @@ -18,6 +18,7 @@ package iofmt import ( "net/http" + "net/url" "testing" "github.com/trickstercache/trickster/v2/pkg/proxy/headers" @@ -154,3 +155,67 @@ func TestDetect(t *testing.T) { }) } } + +func TestDetectV3(t *testing.T) { + tests := []struct { + name string + path string + method string + expected Format + }{ + {"v3 SQL GET", "/api/v3/query_sql", http.MethodGet, V3SQL}, + {"v3 SQL POST", "/api/v3/query_sql", http.MethodPost, V3SQL}, + {"v3 InfluxQL GET", "/api/v3/query_influxql", http.MethodGet, V3InfluxQL}, + {"v3 InfluxQL POST", "/api/v3/query_influxql", http.MethodPost, V3InfluxQL}, + {"v1 query GET", "/query", http.MethodGet, InfluxqlGet}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &http.Request{ + Method: tt.method, + URL: &url.URL{Path: tt.path}, + } + out := Detect(r) + if out != tt.expected { + t.Errorf("expected %d got %d", tt.expected, out) + } + }) + } +} + +func TestIsV3(t *testing.T) { + if !V3SQL.IsV3() { + t.Error("V3SQL.IsV3() should be true") + } + if !V3InfluxQL.IsV3() { + t.Error("V3InfluxQL.IsV3() should be true") + } + if InfluxqlGet.IsV3() { + t.Error("InfluxqlGet.IsV3() should be false") + } + if FluxRawCsv.IsV3() { + t.Error("FluxRawCsv.IsV3() should be false") + } +} + +func TestV3OutputFormat(t *testing.T) { + tests := []struct { + format string + expected byte + }{ + {"json", V3OutputJSON}, + {"jsonl", V3OutputJSONL}, + {"csv", V3OutputCSV}, + {"", V3OutputJSON}, + {"parquet", V3OutputJSON}, + } + for _, tt := range tests { + t.Run(tt.format, func(t *testing.T) { + r := &http.Request{URL: &url.URL{RawQuery: "format=" + tt.format}} + out := V3OutputFormat(r) + if out != tt.expected { + t.Errorf("expected %d got %d", tt.expected, out) + } + }) + } +} diff --git a/pkg/backends/influxdb/routes.go b/pkg/backends/influxdb/routes.go index 65e81d2fe..e4281c4d9 100644 --- a/pkg/backends/influxdb/routes.go +++ b/pkg/backends/influxdb/routes.go @@ -61,6 +61,24 @@ func (c *Client) DefaultPathConfigs(_ *bo.Options) po.List { MatchTypeName: matching.PathMatchNameExact, MatchType: matching.PathMatchTypeExact, }, + { + Path: "/" + apiv3QuerySQL, + HandlerName: mnQuery, + Methods: methods.GetAndPost(), + CacheKeyParams: []string{influxql.ParamDB, influxql.ParamQuery, "format"}, + CacheKeyHeaders: []string{}, + MatchTypeName: matching.PathMatchNameExact, + MatchType: matching.PathMatchTypeExact, + }, + { + Path: "/" + apiv3QueryInfluxQL, + HandlerName: mnQuery, + Methods: methods.GetAndPost(), + CacheKeyParams: []string{influxql.ParamDB, influxql.ParamQuery, "format"}, + CacheKeyHeaders: []string{}, + MatchTypeName: matching.PathMatchNameExact, + MatchType: matching.PathMatchTypeExact, + }, { Path: "/", HandlerName: providers.Proxy, diff --git a/pkg/backends/influxdb/routes_test.go b/pkg/backends/influxdb/routes_test.go index 1683a114f..4a8d7db43 100644 --- a/pkg/backends/influxdb/routes_test.go +++ b/pkg/backends/influxdb/routes_test.go @@ -65,7 +65,7 @@ func TestDefaultPathConfigs(t *testing.T) { t.Errorf("expected to find path named: %s", "/") } - const expectedLen = 3 + const expectedLen = 5 if len(rsc.BackendOptions.Paths) != expectedLen { t.Errorf("expected ordered length to be: %d, got: %d", expectedLen, len(rsc.BackendOptions.Paths)) } diff --git a/pkg/backends/influxdb/serialization.go b/pkg/backends/influxdb/serialization.go index f7735ad7c..8e2fe025b 100644 --- a/pkg/backends/influxdb/serialization.go +++ b/pkg/backends/influxdb/serialization.go @@ -23,6 +23,7 @@ import ( "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/flux" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/influxql" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + isql "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/sql" "github.com/trickstercache/trickster/v2/pkg/errors" "github.com/trickstercache/trickster/v2/pkg/timeseries" "github.com/trickstercache/trickster/v2/pkg/timeseries/dataset" @@ -46,6 +47,9 @@ func UnmarshalTimeseries(data []byte, if len(data) == 0 || trq == nil { return nil, errors.ErrBadRequest } + if isV3Query(trq) { + return isql.UnmarshalTimeseries(data, trq) + } if strings.Contains(strings.ToLower(trq.Statement), flux.FuncRange) { return flux.UnmarshalTimeseries(data, trq) } @@ -58,6 +62,9 @@ func UnmarshalTimeseriesReader(reader io.Reader, if reader == nil || trq == nil { return nil, errors.ErrBadRequest } + if isV3Query(trq) { + return isql.UnmarshalTimeseriesReader(reader, trq) + } if strings.Contains(strings.ToLower(trq.Statement), flux.FuncRange) { return flux.UnmarshalTimeseriesReader(reader, trq) } @@ -70,6 +77,9 @@ func MarshalTimeseries(ts timeseries.Timeseries, if ts == nil || rlo == nil { return nil, errors.ErrBadRequest } + if rlo.OutputFormat >= iofmt.V3OutputJSON { + return isql.MarshalTimeseries(ts, rlo, status) + } if iofmt.Format(rlo.OutputFormat).IsInfluxQL() { return influxql.MarshalTimeseries(ts, rlo, status) } @@ -82,8 +92,24 @@ func MarshalTimeseriesWriter(ts timeseries.Timeseries, if ts == nil || rlo == nil || w == nil { return errors.ErrBadRequest } + if rlo.OutputFormat >= iofmt.V3OutputJSON { + return isql.MarshalTimeseriesWriter(ts, rlo, status, w) + } if rlo.OutputFormat < 4 { return influxql.MarshalTimeseriesWriter(ts, rlo, status, w) } return flux.MarshalTimeseriesWriter(ts, rlo, status, w) } + +// isV3Query returns true if the parsed query is a v3 query type (either the +// native SQL query or a v3 InfluxQL wrapper around the v1 parse tree). +func isV3Query(trq *timeseries.TimeRangeQuery) bool { + if trq.ParsedQuery == nil { + return false + } + switch trq.ParsedQuery.(type) { + case *isql.Query, *isql.V3InfluxQLQuery: + return true + } + return false +} diff --git a/pkg/backends/influxdb/sql/lexing.go b/pkg/backends/influxdb/sql/lexing.go new file mode 100644 index 000000000..2af92347d --- /dev/null +++ b/pkg/backends/influxdb/sql/lexing.go @@ -0,0 +1,49 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "github.com/trickstercache/trickster/v2/pkg/parsing/lex" + lsql "github.com/trickstercache/trickster/v2/pkg/parsing/lex/sql" + "github.com/trickstercache/trickster/v2/pkg/parsing/token" +) + +// DataFusion SQL token types +const ( + // Unordered keywords / functions + tokenDateBin token.Typ = iota + (lsql.TokenSQLFunction + 40) + tokenDateTrunc + tokenInterval +) + +// token values +const ( + tokenValInterval = "interval" +) + +var dfKey = map[string]token.Typ{ + "date_bin": tokenDateBin, + "date_trunc": tokenDateTrunc, + tokenValInterval: tokenInterval, +} + +// LexerOptions returns DataFusion-crafted Lexer Options +func LexerOptions() *lex.Options { + return &lex.Options{ + CustomKeywords: dfKey, + } +} diff --git a/pkg/backends/influxdb/sql/marshal.go b/pkg/backends/influxdb/sql/marshal.go new file mode 100644 index 000000000..ece017957 --- /dev/null +++ b/pkg/backends/influxdb/sql/marshal.go @@ -0,0 +1,161 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "strconv" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + "github.com/trickstercache/trickster/v2/pkg/timeseries" + "github.com/trickstercache/trickster/v2/pkg/timeseries/dataset" +) + +// MarshalTimeseries converts a Timeseries into a v3 response body +func MarshalTimeseries(ts timeseries.Timeseries, + rlo *timeseries.RequestOptions, _ int, +) ([]byte, error) { + buf := new(bytes.Buffer) + if err := MarshalTimeseriesWriter(ts, rlo, 0, buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// MarshalTimeseriesWriter converts a Timeseries into a v3 response body via io.Writer +func MarshalTimeseriesWriter(ts timeseries.Timeseries, + rlo *timeseries.RequestOptions, _ int, w io.Writer, +) error { + if ts == nil { + return timeseries.ErrUnknownFormat + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + return timeseries.ErrUnknownFormat + } + var of byte + if rlo != nil { + of = rlo.OutputFormat + } + switch of { + case iofmt.V3OutputJSONL: + return marshalJSONL(w, ds) + case iofmt.V3OutputCSV: + return marshalCSV(w, ds) + default: + return marshalJSON(w, ds) + } +} + +func marshalJSON(w io.Writer, ds *dataset.DataSet) error { + rows := dataSetToRows(ds) + enc := json.NewEncoder(w) + return enc.Encode(rows) +} + +func marshalJSONL(w io.Writer, ds *dataset.DataSet) error { + rows := dataSetToRows(ds) + enc := json.NewEncoder(w) + for _, row := range rows { + if err := enc.Encode(row); err != nil { + return err + } + } + return nil +} + +func marshalCSV(w io.Writer, ds *dataset.DataSet) error { + cw := csv.NewWriter(w) + defer cw.Flush() + for _, result := range ds.Results { + for _, series := range result.SeriesList { + // header row + header := make([]string, 0, 1+len(series.Header.ValueFieldsList)) + header = append(header, series.Header.TimestampField.Name) + for _, fd := range series.Header.ValueFieldsList { + header = append(header, fd.Name) + } + if err := cw.Write(header); err != nil { + return err + } + // data rows + for _, pt := range series.Points { + record := make([]string, 0, 1+len(pt.Values)) + record = append(record, time.Unix(0, int64(pt.Epoch)).UTC().Format(time.RFC3339Nano)) + for _, v := range pt.Values { + record = append(record, formatValue(v)) + } + if err := cw.Write(record); err != nil { + return err + } + } + } + } + return nil +} + +func dataSetToRows(ds *dataset.DataSet) []map[string]any { + var rows []map[string]any + for _, result := range ds.Results { + for _, series := range result.SeriesList { + tsName := series.Header.TimestampField.Name + if tsName == "" { + tsName = DefaultTimestampField + } + for _, pt := range series.Points { + row := make(map[string]any, 1+len(pt.Values)) + row[tsName] = time.Unix(0, int64(pt.Epoch)).UTC().Format(time.RFC3339Nano) + for i, fd := range series.Header.ValueFieldsList { + if i < len(pt.Values) { + row[fd.Name] = pt.Values[i] + } + } + rows = append(rows, row) + } + } + } + if rows == nil { + return []map[string]any{} + } + return rows +} + +func formatValue(v any) string { + if v == nil { + return "" + } + switch t := v.(type) { + case string: + return t + case float64: + return fmt.Sprintf("%g", t) + case int64: + return strconv.FormatInt(t, 10) + case bool: + if t { + return "true" + } + return "false" + default: + return fmt.Sprintf("%v", t) + } +} diff --git a/pkg/backends/influxdb/sql/marshal_test.go b/pkg/backends/influxdb/sql/marshal_test.go new file mode 100644 index 000000000..b23a74487 --- /dev/null +++ b/pkg/backends/influxdb/sql/marshal_test.go @@ -0,0 +1,133 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "bytes" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + "github.com/trickstercache/trickster/v2/pkg/timeseries" + "github.com/trickstercache/trickster/v2/pkg/timeseries/dataset" + "github.com/trickstercache/trickster/v2/pkg/timeseries/epoch" +) + +func testDataSet() *dataset.DataSet { + t1 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + t2 := time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC) + return &dataset.DataSet{ + TimeRangeQuery: ×eries.TimeRangeQuery{ + Extent: timeseries.Extent{Start: t1, End: t2}, + }, + ExtentList: timeseries.ExtentList{{Start: t1, End: t2}}, + Results: dataset.Results{ + &dataset.Result{ + SeriesList: dataset.SeriesList{ + &dataset.Series{ + Header: dataset.SeriesHeader{ + Name: "default", + TimestampField: timeseries.FieldDefinition{ + Name: "time", + DataType: timeseries.DateTimeRFC3339Nano, + Role: timeseries.RoleTimestamp, + }, + ValueFieldsList: timeseries.FieldDefinitions{ + {Name: "temperature", DataType: timeseries.Float64, OutputPosition: 0, Role: timeseries.RoleValue}, + }, + Tags: map[string]string{}, + }, + Points: dataset.Points{ + {Epoch: epoch.Epoch(t1.UnixNano()), Values: []any{float64(72.5)}}, + {Epoch: epoch.Epoch(t2.UnixNano()), Values: []any{float64(73.2)}}, + }, + }, + }, + }, + }, + } +} + +func TestMarshalJSON(t *testing.T) { + ds := testDataSet() + rlo := ×eries.RequestOptions{OutputFormat: iofmt.V3OutputJSON} + data, err := MarshalTimeseries(ds, rlo, 200) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err := json.Unmarshal(data, &rows); err != nil { + t.Fatalf("invalid JSON: %v", err) + } + if len(rows) != 2 { + t.Fatalf("expected 2 rows, got %d", len(rows)) + } + if _, ok := rows[0]["time"]; !ok { + t.Error("missing time field") + } + if _, ok := rows[0]["temperature"]; !ok { + t.Error("missing temperature field") + } +} + +func TestMarshalJSONL(t *testing.T) { + ds := testDataSet() + rlo := ×eries.RequestOptions{OutputFormat: iofmt.V3OutputJSONL} + var buf bytes.Buffer + if err := MarshalTimeseriesWriter(ds, rlo, 200, &buf); err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(buf.String()), "\n") + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + for _, line := range lines { + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("invalid JSONL line: %v", err) + } + } +} + +func TestMarshalCSV(t *testing.T) { + ds := testDataSet() + rlo := ×eries.RequestOptions{OutputFormat: iofmt.V3OutputCSV} + data, err := MarshalTimeseries(ds, rlo, 200) + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + if len(lines) != 3 { // header + 2 data rows + t.Fatalf("expected 3 lines, got %d", len(lines)) + } + if !strings.Contains(lines[0], "time") { + t.Error("missing time in header") + } + if !strings.Contains(lines[0], "temperature") { + t.Error("missing temperature in header") + } +} + +func TestMarshalNilTimeseries(t *testing.T) { + rlo := ×eries.RequestOptions{OutputFormat: iofmt.V3OutputJSON} + _, err := MarshalTimeseries(nil, rlo, 200) + if err == nil { + t.Error("expected error for nil timeseries") + } +} diff --git a/pkg/backends/influxdb/sql/sql.go b/pkg/backends/influxdb/sql/sql.go new file mode 100644 index 000000000..e338efb35 --- /dev/null +++ b/pkg/backends/influxdb/sql/sql.go @@ -0,0 +1,551 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + "github.com/trickstercache/trickster/v2/pkg/parsing" + lsql "github.com/trickstercache/trickster/v2/pkg/parsing/lex/sql" + psql "github.com/trickstercache/trickster/v2/pkg/parsing/sql" + "github.com/trickstercache/trickster/v2/pkg/parsing/token" + pe "github.com/trickstercache/trickster/v2/pkg/proxy/errors" + "github.com/trickstercache/trickster/v2/pkg/proxy/methods" + "github.com/trickstercache/trickster/v2/pkg/proxy/request" + "github.com/trickstercache/trickster/v2/pkg/proxy/urls" + "github.com/trickstercache/trickster/v2/pkg/timeseries" + "github.com/trickstercache/trickster/v2/pkg/timeseries/sqlparser" + ts "github.com/trickstercache/trickster/v2/pkg/util/strings" +) + +// Query holds the parsed and tokenized SQL query +type Query struct { + TokenizedStatement string + // BaseTimestampFieldName is the underlying time column referenced inside a + // date_bin/date_trunc bucketing expression. When a SELECT alias is used + // (e.g. `date_bin(..., t) AS bucket`), the alias cannot appear in WHERE, + // so SetExtent must re-emit predicates against the base column instead. + BaseTimestampFieldName string +} + +// V3InfluxQLQuery marks a parsed InfluxQL query as originating from the v3 +// native endpoint (`/api/v3/query_influxql`). Serialization and SetExtent use +// this to route through the v3 JSON format while reusing the v1 InfluxQL parser. +type V3InfluxQLQuery struct { + Inner any // *influxql.Query from the v1 parser +} + +// Tokens for String Interpolation +const ( + tkRange = "<$RANGE$>" + tkTS1 = "<$TS1$>" + tkTS2 = "<$TS2$>" +) + +// Common URL Parameter Names +const ( + ParamQuery = "q" + ParamDB = "db" + ParamFormat = "format" +) + +// DefaultTimestampField is the default timestamp field name for v3 queries +const DefaultTimestampField = "time" + +// ExtractQuery returns the SQL query text from a v3 request, decoding the +// POST body based on Content-Type. Supports GET (?q=), POST application/json +// ({"q":"..."}), POST application/x-www-form-urlencoded (q=...), and falls +// back to treating the raw POST body as SQL. +func ExtractQuery(r *http.Request) (string, error) { + if !methods.HasBody(r.Method) { + return r.URL.Query().Get(ParamQuery), nil + } + b, err := request.GetBody(r) + if err != nil || len(b) == 0 { + return "", err + } + ct := r.Header.Get("Content-Type") + switch { + case strings.HasPrefix(ct, "application/json"): + var payload struct { + Q string `json:"q"` + } + if err := json.Unmarshal(b, &payload); err != nil { + return "", err + } + return payload.Q, nil + case strings.HasPrefix(ct, "application/x-www-form-urlencoded"): + vals, err := url.ParseQuery(string(b)) + if err != nil { + return "", err + } + return vals.Get(ParamQuery), nil + } + return string(b), nil +} + +// EncodeBody wraps a SQL statement in the body format matching the request's +// Content-Type. Used to preserve the inbound body shape when Trickster +// rewrites the upstream request (e.g. on SetExtent). +func EncodeBody(r *http.Request, sqlQuery string) []byte { + ct := r.Header.Get("Content-Type") + switch { + case strings.HasPrefix(ct, "application/json"): + b, _ := json.Marshal(map[string]string{ParamQuery: sqlQuery}) + return b + case strings.HasPrefix(ct, "application/x-www-form-urlencoded"): + return []byte(url.Values{ParamQuery: {sqlQuery}}.Encode()) + } + return []byte(sqlQuery) +} + +var ( + lexOpts = LexerOptions() + lexer = lsql.NewLexer(lexOpts) + dfParser = &dfSQLParser{ + Parser: sqlparser.New( + parsing.New(nil, lexer, lexOpts), + ).(*sqlparser.Parser), + } +) + +type dfSQLParser struct { + *sqlparser.Parser +} + +func parse(statement string) (*timeseries.TimeRangeQuery, *timeseries.RequestOptions, bool, error) { + trq := ×eries.TimeRangeQuery{Statement: statement} + ro := ×eries.RequestOptions{} + rs, err := dfParser.Run(sqlparser.NewRunContext(trq, ro), dfParser, trq.Statement) + results := rs.Results() + verb, ok := rs.GetResultsCollection("verb") + var canObjectCache bool + if !ok { + return nil, nil, false, sqlparser.ErrNotTimeRangeQuery + } + if vs, ok := verb.(string); ok { + canObjectCache = vs == lsql.TokenValSelect + } + + trq.CacheKeyElements = map[string]string{ + "query": trq.Statement, + } + + returnWithKey := func(e error) (*timeseries.TimeRangeQuery, *timeseries.RequestOptions, bool, error) { + if canObjectCache { + return trq, nil, canObjectCache, e + } + return nil, nil, canObjectCache, e + } + + if err != nil { + return returnWithKey(parsing.ParserError(err, rs.Current())) + } + var t *token.Token + if psql.HasLimitClause(results) { + return returnWithKey(pe.ErrNotTimeRangeQuery) + } + if t, err = parseGroupByTokens(results, trq); err != nil { + return returnWithKey(parsing.ParserError(err, t)) + } + if t, err = parseSelectTokens(results, trq, ro); err != nil { + return returnWithKey(parsing.ParserError(err, t)) + } + if t, err = parseWhereTokens(results, trq, ro); err != nil { + return returnWithKey(parsing.ParserError(err, t)) + } + return trq, ro, canObjectCache, nil +} + +// ParseTimeRangeQuery parses the key parts of a TimeRangeQuery from the inbound HTTP Request +func ParseTimeRangeQuery(r *http.Request, f iofmt.Format, +) (*timeseries.TimeRangeQuery, *timeseries.RequestOptions, bool, error) { + if r == nil || !f.IsV3SQL() { + return nil, nil, false, iofmt.ErrSupportedQueryLanguage + } + var qi url.Values + isBody := methods.HasBody(r.Method) + sqlQuery, err := ExtractQuery(r) + if err != nil { + return nil, nil, false, err + } + if !isBody { + qi = r.URL.Query() + } + if sqlQuery == "" { + return nil, nil, false, pe.MissingURLParam(ParamQuery) + } + + trq, ro, canOPC, err := parse(sqlQuery) + if err != nil { + return trq, ro, canOPC, err + } + ro.OutputFormat = iofmt.V3OutputFormat(r) + + if isBody && trq != nil { + trq.OriginalBody = []byte(sqlQuery) + } + if trq.BackfillTolerance == 0 { + bf := time.Minute + res := request.GetResources(r) + if res != nil { + bf = res.BackendOptions.BackfillTolerance + } + trq.BackfillTolerance = bf + } + trq.ParsedQuery = &Query{ + TokenizedStatement: trq.Statement, + BaseTimestampFieldName: ro.BaseTimestampFieldName, + } + trq.TemplateURL = urls.Clone(r.URL) + if isBody { + request.SetBody(r, EncodeBody(r, trq.Statement)) + } else { + qi.Set(ParamQuery, trq.Statement) + trq.TemplateURL.RawQuery = qi.Encode() + } + return trq, ro, canOPC, nil +} + +// interval duration lookups for DataFusion INTERVAL literals and date_trunc +// unit strings. Units with variable length (month, quarter, year) are omitted +// — queries with those bucketings fall through to proxy. +var intervalDurations = map[string]time.Duration{ + "second": time.Second, + "seconds": time.Second, + "minute": time.Minute, + "minutes": time.Minute, + "hour": time.Hour, + "hours": time.Hour, + "day": 24 * time.Hour, + "days": 24 * time.Hour, + "week": 7 * 24 * time.Hour, + "weeks": 7 * 24 * time.Hour, +} + +func parseSelectTokens(results ts.Lookup, + trq *timeseries.TimeRangeQuery, ro *timeseries.RequestOptions, +) (*token.Token, error) { + if results == nil { + return nil, sqlparser.ErrMissingTimeseries + } + v, ok := results["selectTokens"] + if !ok { + return nil, sqlparser.ErrMissingTimeseries + } + st, ok := v.([]token.Tokens) + if !ok { + return nil, sqlparser.ErrMissingTimeseries + } + if len(st) == 0 { + return nil, sqlparser.ErrMissingTimeseries + } + var foundTimeSeries bool + for _, fieldParts := range st { + if len(fieldParts) == 0 { + continue + } + var isDateBin, isDateTrunc, expectAlias bool + var intervalNum int + var intervalUnit string + for _, t := range fieldParts { + if t.Typ == token.LeftParen || t.Typ == token.RightParen || + t.Typ == token.Space || t.Typ == lsql.TokenComment { + continue + } + bucketing := isDateBin || isDateTrunc + if bucketing { + if t.Typ == lsql.TokenAs { + expectAlias = true + continue + } + if expectAlias { + trq.TimestampDefinition.Name = t.Val + break + } + if isDateBin { + // INTERVAL keyword, ignored + if t.Typ == tokenInterval { + continue + } + if t.Typ == token.String { + // e.g., '1 hour' — parse "N unit" + parts := strings.Fields(lsql.UnQuote(t.Val)) + if len(parts) == 2 { + n, err := strconv.Atoi(parts[0]) + if err == nil { + intervalNum = n + intervalUnit = strings.ToLower(parts[1]) + } + } + continue + } + if t.Typ == token.Number && intervalNum == 0 { + n, err := strconv.Atoi(t.Val) + if err == nil { + intervalNum = n + } + continue + } + if t.Typ == token.Identifier && intervalNum > 0 && intervalUnit == "" { + intervalUnit = strings.ToLower(t.Val) + continue + } + } + if isDateTrunc { + // date_trunc('unit', time) — string is the unit, multiplier is 1 + if t.Typ == token.String && intervalUnit == "" { + intervalUnit = strings.ToLower(lsql.UnQuote(t.Val)) + intervalNum = 1 + continue + } + } + // the time column reference + if t.Typ == token.Identifier && ro.BaseTimestampFieldName == "" { + ro.BaseTimestampFieldName = t.Val + continue + } + } + if t.Typ == tokenDateBin { + isDateBin = true + foundTimeSeries = true + continue + } + if t.Typ == tokenDateTrunc { + isDateTrunc = true + foundTimeSeries = true + continue + } + } + bucketing := isDateBin || isDateTrunc + if bucketing && intervalNum > 0 && intervalUnit != "" { + d, ok := intervalDurations[intervalUnit] + if ok { + trq.Step = d * time.Duration(intervalNum) + } + } + if bucketing && !expectAlias { + last := fieldParts[len(fieldParts)-1] + trq.TimestampDefinition.Name = trq.Statement[fieldParts[0].Pos : last.Pos+len(last.Val)] + } + } + if !foundTimeSeries { + return nil, sqlparser.ErrMissingTimeseries + } + return nil, nil +} + +func parseWhereTokens(results ts.Lookup, + trq *timeseries.TimeRangeQuery, ro *timeseries.RequestOptions, +) (*token.Token, error) { + if ro == nil { + return nil, nil + } + if results == nil { + return nil, sqlparser.ErrNotTimeRangeQuery + } + v, ok := results["whereTokens"] + if !ok { + return nil, sqlparser.ErrNotTimeRangeQuery + } + wt, ok := v.([]token.Tokens) + if !ok { + return nil, sqlparser.ErrNotTimeRangeQuery + } + l := len(wt) + if l&1 != 1 { + return nil, sqlparser.ErrTimerangeParse + } + var e timeseries.Extent + var s1, e1, s2, e2 int + var tsr1, tsr2 string + var isBetween bool + for n, fieldParts := range wt { + var atLowerBound bool + var state int + var i int + lfp := len(fieldParts) + for i = 0; i < lfp; i++ { + t := fieldParts[i] + if t.Typ == token.LeftParen || t.Typ == token.RightParen || + t.Typ == token.Space || t.Typ == lsql.TokenComment { + continue + } + if t.Typ.IsBreakable() { + break + } + sw: + switch state { + case 0: + if t.Val != ro.BaseTimestampFieldName && t.Val != trq.TimestampDefinition.Name { + goto nextSet + } + state++ + case 1: + isBetween = isBetween || t.Typ == lsql.TokenBetween + if !isBetween && !t.Typ.IsGreaterOrLessThan() { + return t, parsing.ErrUnexpectedToken + } + atLowerBound = isBetween || (t.Typ == token.GreaterThan || + t.Typ == token.GreaterThanOrEqual) + state++ + case 2: + ts, f, err := lsql.ParseTimeField(t) + if err != nil { + return t, err + } + trq.TimestampDefinition.ProviderData1 = byte(f) + val, j, err := solveMathExpression(fieldParts[i:], ts) + if err != nil { + return t, err + } + t2 := t.Clone() + t2.Val = strconv.FormatInt(val, 10) + t2.Typ = token.Number + if atLowerBound { + e.Start, _, _ = lsql.TokenToTime(t2) + tsr1 = t2.Val + atLowerBound = false + } else { + e.End, _, _ = lsql.TokenToTime(t2) + tsr2 = t2.Val + } + if s1 == 0 { + s1 = fieldParts[0].Pos + e1 = fieldParts[lfp-1].Pos + len(fieldParts[lfp-1].Val) + } else { + s2 = wt[n-1][0].Pos + e2 = fieldParts[lfp-1].Pos + len(fieldParts[lfp-1].Val) + } + i += j + state++ + case 3: + if t.Typ == token.LogicalAnd { + break sw + } + ts, _, err := lsql.ParseTimeField(t) + if err != nil { + return t, err + } + v, j, err := solveMathExpression(fieldParts[i:], ts) + if err != nil { + return t, err + } + e.End = time.Unix(v, 0) + tsr2 = t.Val + i += j + state++ + } + } + nextSet: + } + if e.Start.IsZero() { + return nil, sqlparser.ErrNoLowerBound + } + if isBetween && e.End.IsZero() { + return nil, sqlparser.ErrNoUpperBound + } + if e.End.IsZero() { + e.End = time.Now() + } + trq.Extent = e + var r1, r2 string + if s1 > 0 && e1 > s1 { + r1 = trq.Statement[s1:e1] + } + if s2 > 0 && e2 > s2 { + r2 = trq.Statement[s2:e2] + } + if r1 != "" { + trq.Statement = strings.ReplaceAll(trq.Statement, r1, tkRange) + } + if r2 != "" { + trq.Statement = strings.ReplaceAll(trq.Statement, r2, "") + } + if tsr1 != "" { + trq.Statement = strings.ReplaceAll(trq.Statement, tsr1, tkTS1) + } + if tsr2 != "" { + trq.Statement = strings.ReplaceAll(trq.Statement, tsr2, tkTS2) + } + return nil, nil +} + +func parseGroupByTokens(results ts.Lookup, + trq *timeseries.TimeRangeQuery, +) (*token.Token, error) { + v, ok := results["groupByTokens"] + if !ok { + return nil, lsql.ErrInvalidGroupByClause + } + gbt, ok := v.(token.Tokens) + if !ok || len(gbt) == 0 { + return nil, lsql.ErrInvalidGroupByClause + } + trq.TagFieldDefintions = make([]timeseries.FieldDefinition, len(gbt)) + for i, v := range gbt { + trq.TagFieldDefintions[i].Name = v.Val + } + return nil, nil +} + +// solveMathExpression handles simple integer math expressions like now() - 3600 +func solveMathExpression(fieldParts token.Tokens, startValue int64, +) (int64, int, error) { + var i, j int + var v int64 + var t *token.Token + prev := &token.Token{Typ: token.Plus} + for i, t = range fieldParts { + if i == 0 && startValue > 0 { + v = startValue + goto nextIteration + } + if t.Typ == token.LeftParen || t.Typ == token.RightParen || + t.Typ == token.Space || t.Typ == lsql.TokenComment { + continue + } + if t.Typ.IsBreakable() || token.IsLogicalOperator(t.Typ) { + i-- + break + } + if j%2 == 0 { + x, err := t.Int64() + if err != nil { + return -1, i, err + } + if prev.Typ == token.Minus { + x *= -1 + } + v += x + goto nextIteration + } + if !t.Typ.IsAddOrSubtract() { + return -1, i, parsing.ErrUnexpectedToken + } + nextIteration: + prev = t + j++ + } + return v, i, nil +} diff --git a/pkg/backends/influxdb/sql/sql_test.go b/pkg/backends/influxdb/sql/sql_test.go new file mode 100644 index 000000000..d2c2f1f48 --- /dev/null +++ b/pkg/backends/influxdb/sql/sql_test.go @@ -0,0 +1,196 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "bytes" + "io" + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" +) + +func TestParseTimeRangeQuery_SQL(t *testing.T) { + tests := []struct { + name string + query string + method string + path string + wantErr bool + wantStep time.Duration + }{ + { + name: "basic date_bin query with epoch", + query: "SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(temperature) AS temperature FROM weather WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: time.Hour, + }, + { + name: "date_bin with 5 minute interval", + query: "SELECT date_bin(INTERVAL '5 minutes', time) AS time, mean(cpu) AS cpu FROM metrics WHERE time >= 1704067200 AND time < 1704070800 GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: 5 * time.Minute, + }, + { + name: "date_bin with SQL datetime", + query: "SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(temp) FROM weather WHERE time >= '2024-01-01 00:00:00' AND time < '2024-01-02 00:00:00' GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: time.Hour, + }, + { + name: "non-select should fail", + query: "CREATE TABLE test (id INT)", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantErr: true, + }, + { + name: "date_trunc hour", + query: "SELECT date_trunc('hour', time) AS time, avg(temperature) AS temperature FROM weather WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: time.Hour, + }, + { + name: "date_trunc minute", + query: "SELECT date_trunc('minute', time) AS time, avg(cpu) AS cpu FROM metrics WHERE time >= 1704067200 AND time < 1704070800 GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: time.Minute, + }, + { + name: "date_trunc day", + query: "SELECT date_trunc('day', time) AS time, avg(val) FROM stats WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1", + method: http.MethodGet, + path: "/api/v3/query_sql", + wantStep: 24 * time.Hour, + }, + { + name: "wrong format flag", + query: "SELECT * FROM test", + method: http.MethodGet, + path: "/query", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := &url.URL{Path: tt.path, RawQuery: url.Values{ParamQuery: {tt.query}}.Encode()} + r := &http.Request{Method: tt.method, URL: u} + f := iofmt.Detect(r) + trq, _, _, err := ParseTimeRangeQuery(r, f) + if tt.wantErr { + if err == nil { + t.Error("expected error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if trq.Step != tt.wantStep { + t.Errorf("step: got %v, want %v", trq.Step, tt.wantStep) + } + }) + } +} + +func TestParseTimeRangeQuery_POST(t *testing.T) { + u := &url.URL{Path: "/api/v3/query_sql"} + r := &http.Request{ + Method: http.MethodPost, + URL: u, + Header: http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, + } + f := iofmt.Detect(r) + if !f.IsV3SQL() { + t.Fatal("expected V3SQL format") + } +} + +func TestParseTimeRangeQuery_POST_JSON(t *testing.T) { + sqlQuery := "SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(v) FROM m WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1" + body := []byte(`{"q":"` + sqlQuery + `"}`) + r := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/api/v3/query_sql"}, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(bytes.NewReader(body)), + ContentLength: int64(len(body)), + } + f := iofmt.Detect(r) + trq, _, _, err := ParseTimeRangeQuery(r, f) + if err != nil { + t.Fatalf("parse error: %v", err) + } + if trq.Step != time.Hour { + t.Errorf("step: got %v want 1h", trq.Step) + } + got, _ := io.ReadAll(r.Body) + if !strings.HasPrefix(string(got), "{") { + t.Errorf("expected JSON-wrapped body after parse, got: %s", got) + } +} + +func TestParseTimeRangeQuery_POST_Form(t *testing.T) { + sqlQuery := "SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(v) FROM m WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1" + body := []byte(url.Values{ParamQuery: {sqlQuery}}.Encode()) + r := &http.Request{ + Method: http.MethodPost, + URL: &url.URL{Path: "/api/v3/query_sql"}, + Header: http.Header{"Content-Type": {"application/x-www-form-urlencoded"}}, + Body: io.NopCloser(bytes.NewReader(body)), + ContentLength: int64(len(body)), + } + f := iofmt.Detect(r) + trq, _, _, err := ParseTimeRangeQuery(r, f) + if err != nil { + t.Fatalf("parse error: %v", err) + } + if trq.Step != time.Hour { + t.Errorf("step: got %v want 1h", trq.Step) + } + got, _ := io.ReadAll(r.Body) + if !strings.HasPrefix(string(got), "q=") { + t.Errorf("expected form-wrapped body after parse, got: %s", got) + } +} + +func TestParse_WhereTimeRange(t *testing.T) { + query := "SELECT date_bin(INTERVAL '1 hour', time) AS time, avg(temp) AS temp FROM weather WHERE time >= 1704067200 AND time < 1704153600 GROUP BY 1" + trq, _, _, err := parse(query) + if err != nil { + t.Fatalf("parse error: %v", err) + } + if trq.Extent.Start.IsZero() { + t.Error("expected non-zero start time") + } + if trq.Extent.End.IsZero() { + t.Error("expected non-zero end time") + } + if trq.Step != time.Hour { + t.Errorf("expected 1h step, got %v", trq.Step) + } +} diff --git a/pkg/backends/influxdb/sql/unmarshal.go b/pkg/backends/influxdb/sql/unmarshal.go new file mode 100644 index 000000000..7884684b8 --- /dev/null +++ b/pkg/backends/influxdb/sql/unmarshal.go @@ -0,0 +1,285 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "bufio" + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "strconv" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + "github.com/trickstercache/trickster/v2/pkg/timeseries" + "github.com/trickstercache/trickster/v2/pkg/timeseries/dataset" + "github.com/trickstercache/trickster/v2/pkg/timeseries/epoch" +) + +// UnmarshalTimeseries converts a v3 response body into a Timeseries +func UnmarshalTimeseries(data []byte, trq *timeseries.TimeRangeQuery, +) (timeseries.Timeseries, error) { + return UnmarshalTimeseriesReader(bytes.NewReader(data), trq) +} + +// UnmarshalTimeseriesReader converts a v3 response body into a Timeseries via io.Reader +func UnmarshalTimeseriesReader(reader io.Reader, trq *timeseries.TimeRangeQuery, +) (timeseries.Timeseries, error) { + // peek at first byte to determine format + br := bufio.NewReader(reader) + b, err := br.Peek(1) + if err != nil { + return nil, timeseries.ErrInvalidBody + } + var of byte + switch b[0] { + case '[': + of = iofmt.V3OutputJSON + case '{': + of = iofmt.V3OutputJSONL + default: + of = iofmt.V3OutputCSV + } + switch of { + case iofmt.V3OutputJSONL: + return unmarshalJSONL(br, trq) + case iofmt.V3OutputCSV: + return unmarshalCSV(br, trq) + default: + return unmarshalJSON(br, trq) + } +} + +func unmarshalJSON(r io.Reader, trq *timeseries.TimeRangeQuery, +) (timeseries.Timeseries, error) { + var rows []map[string]any + if err := json.NewDecoder(r).Decode(&rows); err != nil { + return nil, err + } + return rowsToDataSet(rows, trq) +} + +func unmarshalJSONL(r io.Reader, trq *timeseries.TimeRangeQuery, +) (timeseries.Timeseries, error) { + var rows []map[string]any + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + var row map[string]any + if err := json.Unmarshal(line, &row); err != nil { + continue + } + rows = append(rows, row) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return rowsToDataSet(rows, trq) +} + +func unmarshalCSV(r io.Reader, trq *timeseries.TimeRangeQuery, +) (timeseries.Timeseries, error) { + cr := csv.NewReader(r) + records, err := cr.ReadAll() + if err != nil { + return nil, err + } + if len(records) < 2 { + return nil, timeseries.ErrInvalidBody + } + headers := records[0] + var rows []map[string]any + for _, record := range records[1:] { + row := make(map[string]any, len(headers)) + for i, h := range headers { + if i < len(record) { + row[h] = record[i] + } + } + rows = append(rows, row) + } + return rowsToDataSet(rows, trq) +} + +func rowsToDataSet(rows []map[string]any, trq *timeseries.TimeRangeQuery, +) (*dataset.DataSet, error) { + if len(rows) == 0 { + ds := &dataset.DataSet{ + TimeRangeQuery: trq, + ExtentList: timeseries.ExtentList{trq.Extent}, + Results: dataset.Results{&dataset.Result{SeriesList: dataset.SeriesList{}}}, + } + return ds, nil + } + + tsName := trq.TimestampDefinition.Name + if tsName == "" { + tsName = DefaultTimestampField + } + + // determine field names from first row (stable ordering) + var fieldNames []string + for k := range rows[0] { + if k != tsName { + fieldNames = append(fieldNames, k) + } + } + + // build field definitions + tfd := timeseries.FieldDefinition{ + Name: tsName, + DataType: timeseries.DateTimeRFC3339Nano, + Role: timeseries.RoleTimestamp, + } + vfds := make(timeseries.FieldDefinitions, len(fieldNames)) + for i, name := range fieldNames { + vfds[i] = timeseries.FieldDefinition{ + Name: name, + DataType: detectFieldType(rows[0][name]), + OutputPosition: i, + Role: timeseries.RoleValue, + } + } + + points := make(dataset.Points, len(rows)) + for i, row := range rows { + ep, err := parseV3Timestamp(row[tsName]) + if err != nil { + continue + } + vals := make([]any, len(fieldNames)) + for j, name := range fieldNames { + vals[j] = coerceValue(row[name], vfds[j].DataType) + } + points[i] = dataset.Point{ + Epoch: ep, + Values: vals, + } + } + + sh := dataset.SeriesHeader{ + Name: "default", + TimestampField: tfd, + ValueFieldsList: vfds, + Tags: map[string]string{}, + } + series := &dataset.Series{ + Header: sh, + Points: points, + } + result := &dataset.Result{ + SeriesList: dataset.SeriesList{series}, + } + ds := &dataset.DataSet{ + TimeRangeQuery: trq, + ExtentList: timeseries.ExtentList{trq.Extent}, + Results: dataset.Results{result}, + } + return ds, nil +} + +func parseV3Timestamp(v any) (epoch.Epoch, error) { + switch t := v.(type) { + case string: + // try RFC3339 first (the common v3 format) + if ts, err := time.Parse(time.RFC3339Nano, t); err == nil { + return epoch.Epoch(ts.UnixNano()), nil + } + if ts, err := time.Parse(time.RFC3339, t); err == nil { + return epoch.Epoch(ts.UnixNano()), nil + } + // try epoch seconds + if n, err := strconv.ParseInt(t, 10, 64); err == nil { + return epoch.Epoch(n * 1e9), nil + } + return 0, timeseries.ErrInvalidTimeFormat + case float64: + return epoch.Epoch(int64(t) * 1e9), nil + case json.Number: + n, err := t.Int64() + if err != nil { + return 0, err + } + return epoch.Epoch(n * 1e9), nil + } + return 0, timeseries.ErrInvalidTimeFormat +} + +func detectFieldType(v any) timeseries.FieldDataType { + switch v := v.(type) { + case float64: + return timeseries.Float64 + case bool: + return timeseries.Bool + case string: + s := v + if _, err := strconv.ParseFloat(s, 64); err == nil { + return timeseries.Float64 + } + if _, err := strconv.ParseInt(s, 10, 64); err == nil { + return timeseries.Int64 + } + return timeseries.String + } + return timeseries.String +} + +func coerceValue(v any, dt timeseries.FieldDataType) any { + if v == nil { + return nil + } + switch dt { + case timeseries.Float64: + switch t := v.(type) { + case float64: + return t + case string: + if f, err := strconv.ParseFloat(t, 64); err == nil { + return f + } + } + case timeseries.Int64: + switch t := v.(type) { + case float64: + return int64(t) + case string: + if n, err := strconv.ParseInt(t, 10, 64); err == nil { + return n + } + } + case timeseries.Bool: + switch t := v.(type) { + case bool: + return t + case string: + if b, err := strconv.ParseBool(t); err == nil { + return b + } + } + case timeseries.String: + if s, ok := v.(string); ok { + return s + } + return fmt.Sprint(v) + } + return v +} diff --git a/pkg/backends/influxdb/sql/unmarshal_test.go b/pkg/backends/influxdb/sql/unmarshal_test.go new file mode 100644 index 000000000..b5fea7147 --- /dev/null +++ b/pkg/backends/influxdb/sql/unmarshal_test.go @@ -0,0 +1,153 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "testing" + "time" + + "github.com/trickstercache/trickster/v2/pkg/timeseries" + "github.com/trickstercache/trickster/v2/pkg/timeseries/dataset" +) + +func testTRQ() *timeseries.TimeRangeQuery { + t1 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + t2 := time.Date(2024, 1, 1, 1, 0, 0, 0, time.UTC) + return ×eries.TimeRangeQuery{ + Extent: timeseries.Extent{Start: t1, End: t2}, + TimestampDefinition: timeseries.FieldDefinition{ + Name: "time", + }, + ParsedQuery: &Query{}, + } +} + +func TestUnmarshalJSON(t *testing.T) { + data := []byte(`[{"time":"2024-01-01T00:00:00Z","temperature":72.5},{"time":"2024-01-01T01:00:00Z","temperature":73.2}]`) + trq := testTRQ() + ts, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatal(err) + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + if len(ds.Results) != 1 { + t.Fatalf("expected 1 result, got %d", len(ds.Results)) + } + if len(ds.Results[0].SeriesList) != 1 { + t.Fatalf("expected 1 series, got %d", len(ds.Results[0].SeriesList)) + } + if len(ds.Results[0].SeriesList[0].Points) != 2 { + t.Fatalf("expected 2 points, got %d", len(ds.Results[0].SeriesList[0].Points)) + } +} + +func TestUnmarshalJSONL(t *testing.T) { + data := []byte("{\"time\":\"2024-01-01T00:00:00Z\",\"temperature\":72.5}\n{\"time\":\"2024-01-01T01:00:00Z\",\"temperature\":73.2}\n") + trq := testTRQ() + ts, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatal(err) + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + if len(ds.Results[0].SeriesList[0].Points) != 2 { + t.Fatalf("expected 2 points, got %d", len(ds.Results[0].SeriesList[0].Points)) + } +} + +func TestUnmarshalCSV(t *testing.T) { + data := []byte("time,temperature\n2024-01-01T00:00:00Z,72.5\n2024-01-01T01:00:00Z,73.2\n") + trq := testTRQ() + ts, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatal(err) + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + if len(ds.Results[0].SeriesList[0].Points) != 2 { + t.Fatalf("expected 2 points, got %d", len(ds.Results[0].SeriesList[0].Points)) + } +} + +func TestUnmarshalEmpty(t *testing.T) { + data := []byte(`[]`) + trq := testTRQ() + ts, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatal(err) + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + if len(ds.Results) != 1 { + t.Fatalf("expected 1 result, got %d", len(ds.Results)) + } +} + +func TestUnmarshalNilData(t *testing.T) { + _, err := UnmarshalTimeseries(nil, testTRQ()) + if err == nil { + t.Error("expected error for nil data") + } +} + +func TestUnmarshalJSONNullThenNumeric(t *testing.T) { + data := []byte(`[{"time":"2024-01-01T00:00:00Z","temperature":null},{"time":"2024-01-01T01:00:00Z","temperature":72.5}]`) + trq := testTRQ() + ts, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatal(err) + } + ds, ok := ts.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + pts := ds.Results[0].SeriesList[0].Points + if len(pts) != 2 { + t.Fatalf("expected 2 points, got %d", len(pts)) + } +} + +func TestRoundTripJSON(t *testing.T) { + ds := testDataSet() + rlo := ×eries.RequestOptions{OutputFormat: 32} // V3OutputJSON + data, err := MarshalTimeseries(ds, rlo, 200) + if err != nil { + t.Fatal(err) + } + trq := testTRQ() + ts2, err := UnmarshalTimeseries(data, trq) + if err != nil { + t.Fatalf("round-trip unmarshal: %v", err) + } + ds2, ok := ts2.(*dataset.DataSet) + if !ok { + t.Fatal("expected *dataset.DataSet") + } + if len(ds2.Results[0].SeriesList[0].Points) != 2 { + t.Fatalf("expected 2 points after round-trip, got %d", + len(ds2.Results[0].SeriesList[0].Points)) + } +} diff --git a/pkg/backends/influxdb/sql/url.go b/pkg/backends/influxdb/sql/url.go new file mode 100644 index 000000000..7716bf7f9 --- /dev/null +++ b/pkg/backends/influxdb/sql/url.go @@ -0,0 +1,89 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/trickstercache/trickster/v2/pkg/proxy/methods" + "github.com/trickstercache/trickster/v2/pkg/proxy/request" + "github.com/trickstercache/trickster/v2/pkg/timeseries" +) + +// SetExtent interpolates the tokenized SQL query with concrete time bounds +func SetExtent(r *http.Request, trq *timeseries.TimeRangeQuery, + extent *timeseries.Extent, q *Query, +) { + if extent == nil || r == nil || trq == nil || q == nil { + return + } + stmt := interpolateTimeQuery(q, trq.TimestampDefinition, extent) + isBody := methods.HasBody(r.Method) + if isBody { + request.SetBody(r, EncodeBody(r, stmt)) + } else { + qi := r.URL.Query() + qi.Set(ParamQuery, stmt) + r.URL.RawQuery = qi.Encode() + } +} + +func interpolateTimeQuery(q *Query, tfd timeseries.FieldDefinition, + extent *timeseries.Extent, +) string { + start, end := formatTimestampValues(tfd, extent) + tStart, tEnd := formatTimestampValues( + timeseries.FieldDefinition{ProviderData1: tfd.ProviderData1}, extent) + tsName := q.BaseTimestampFieldName + if tsName == "" { + tsName = tfd.Name + } + if tsName == "" { + tsName = DefaultTimestampField + } + trange := fmt.Sprintf("%s >= %s AND %s < %s", tsName, start, tsName, end) + out := strings.NewReplacer( + tkRange, trange, + tkTS1, tStart, + tkTS2, tEnd, + ).Replace(q.TokenizedStatement) + return out +} + +func formatTimestampValues(tfd timeseries.FieldDefinition, + extent *timeseries.Extent, +) (string, string) { + dt := timeseries.FieldDataType(tfd.ProviderData1) + switch dt { + case timeseries.DateTimeUnixMilli: + return strconv.FormatInt(extent.Start.UnixMilli(), 10), + strconv.FormatInt(extent.End.UnixMilli(), 10) + case timeseries.DateTimeUnixNano: + return strconv.FormatInt(extent.Start.UnixNano(), 10), + strconv.FormatInt(extent.End.UnixNano(), 10) + case timeseries.DateTimeSQL: + return "'" + extent.Start.UTC().Format("2006-01-02 15:04:05") + "'", + "'" + extent.End.UTC().Format("2006-01-02 15:04:05") + "'" + default: + // RFC3339 — the default for InfluxDB v3 + return "'" + extent.Start.UTC().Format("2006-01-02T15:04:05Z") + "'", + "'" + extent.End.UTC().Format("2006-01-02T15:04:05Z") + "'" + } +} diff --git a/pkg/backends/influxdb/sql/url_test.go b/pkg/backends/influxdb/sql/url_test.go new file mode 100644 index 000000000..2b98093f0 --- /dev/null +++ b/pkg/backends/influxdb/sql/url_test.go @@ -0,0 +1,55 @@ +/* + * Copyright 2018 The Trickster Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package sql + +import ( + "net/http" + "net/url" + "strings" + "testing" + "time" + + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/iofmt" + "github.com/trickstercache/trickster/v2/pkg/timeseries" +) + +func TestSetExtent_AliasedTimeColumn(t *testing.T) { + query := "SELECT date_bin(INTERVAL '1 hour', t) AS bucket, avg(temperature) AS temperature FROM weather WHERE t >= 1704067200 AND t < 1704153600 GROUP BY 1" + u := &url.URL{Path: "/api/v3/query_sql", RawQuery: url.Values{ParamQuery: {query}}.Encode()} + r := &http.Request{Method: http.MethodGet, URL: u} + f := iofmt.Detect(r) + trq, _, _, err := ParseTimeRangeQuery(r, f) + if err != nil { + t.Fatalf("parse error: %v", err) + } + ext := ×eries.Extent{ + Start: time.Unix(1704067200, 0), + End: time.Unix(1704070800, 0), + } + q, ok := trq.ParsedQuery.(*Query) + if !ok { + t.Fatalf("expected *Query, got %T", trq.ParsedQuery) + } + SetExtent(r, trq, ext, q) + got := r.URL.Query().Get(ParamQuery) + if strings.Contains(got, "bucket >=") || strings.Contains(got, "bucket <") { + t.Errorf("WHERE must use base column `t`, not alias `bucket`; got:\n%s", got) + } + if !strings.Contains(got, "t >=") || !strings.Contains(got, "t <") { + t.Errorf("WHERE must reference base column `t`; got:\n%s", got) + } +} diff --git a/pkg/backends/influxdb/url.go b/pkg/backends/influxdb/url.go index 5bcbae1ef..4a072dd83 100644 --- a/pkg/backends/influxdb/url.go +++ b/pkg/backends/influxdb/url.go @@ -22,13 +22,16 @@ import ( "github.com/influxdata/influxql" "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/flux" ti "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/influxql" + isql "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/sql" "github.com/trickstercache/trickster/v2/pkg/timeseries" ) // Upstream Endpoints const ( - mnQuery = "query" - apiv2Query = "api/v2/query" + mnQuery = "query" + apiv2Query = "api/v2/query" + apiv3QuerySQL = "api/v3/query_sql" + apiv3QueryInfluxQL = "api/v3/query_influxql" ) // SetExtent will change the upstream request query to use the provided Extent @@ -43,6 +46,12 @@ func (c *Client) SetExtent(r *http.Request, trq *timeseries.TimeRangeQuery, trq.ParsedQuery = t2.ParsedQuery } switch q := trq.ParsedQuery.(type) { + case *isql.Query: + isql.SetExtent(r, trq, extent, q) + case *isql.V3InfluxQLQuery: + if inner, ok := q.Inner.(*influxql.Query); ok { + ti.SetExtent(r, trq, extent, inner) + } case *influxql.Query: ti.SetExtent(r, trq, extent, q) case *flux.Query: diff --git a/pkg/backends/options/options.go b/pkg/backends/options/options.go index 589861e99..9ed6e7270 100644 --- a/pkg/backends/options/options.go +++ b/pkg/backends/options/options.go @@ -60,6 +60,13 @@ type Options struct { // // Hosts identifies the frontend hostnames this backend should handle (virtual hosting) Hosts []string `yaml:"hosts,omitempty"` + // FlightPort enables an Apache Arrow Flight SQL listener on the given port + // when > 0 (InfluxDB 3.x backends only). The upstream Flight SQL address is + // derived from OriginURL (host:port). Leave 0 to disable. + FlightPort int `yaml:"flight_port,omitempty"` + // FlightUpstreamAddress overrides the upstream Flight SQL address when set. + // Defaults to the host:port from OriginURL. + FlightUpstreamAddress string `yaml:"flight_upstream_address,omitempty"` // Provider describes the type of backend (e.g., 'prometheus') Provider string `yaml:"provider,omitempty"` // OriginURL provides the base upstream URL for all proxied requests to this Backend. diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index d4ff69a78..8c1d80af2 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -28,6 +28,7 @@ import ( "github.com/trickstercache/trickster/v2/pkg/appinfo" "github.com/trickstercache/trickster/v2/pkg/appinfo/usage" + "github.com/trickstercache/trickster/v2/pkg/backends/influxdb/flight" "github.com/trickstercache/trickster/v2/pkg/config/reload" "github.com/trickstercache/trickster/v2/pkg/config/validate" "github.com/trickstercache/trickster/v2/pkg/daemon/instance" @@ -108,6 +109,11 @@ func Start(ctx context.Context, args ...string) error { si.Listeners.DrainAndClose("metricsListener", 0) si.Listeners.DrainAndClose("mgmtListener", 0) } + flightCtx, cancelFlight := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFlight() + if err := flight.ShutdownAll(flightCtx); err != nil { + logger.Warn("flight sql shutdown error", logging.Pairs{"error": err.Error()}) + } return nil }