From 4d07f7478b1e113c9562cc3a131f04b8364b5094 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 1 Jul 2026 11:31:59 -0700 Subject: [PATCH 1/5] Getting more window functions working --- server/analyzer/type_sanitizer.go | 38 ++++++++ testing/go/enginetest/doltgres_engine_test.go | 1 - testing/go/window_test.go | 94 +++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 testing/go/window_test.go diff --git a/server/analyzer/type_sanitizer.go b/server/analyzer/type_sanitizer.go index c9dd97369e..a168e8e75f 100644 --- a/server/analyzer/type_sanitizer.go +++ b/server/analyzer/type_sanitizer.go @@ -57,6 +57,19 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope return pgexprs.NewGMSCast(expr), transform.NewTree, nil } } + // Window function outputs carry GMS types in the Window schema; wrap with GMSCast + // so the type is properly converted to a Doltgres type for the wire protocol. + // A Sort node may sit between the Project and Window (for outer ORDER BY). + if isWindowOrWrapsWindow(child) { + if _, ok := expr.Type(ctx).(*pgtypes.DoltgresType); !ok { + // GMS ranking functions (rank, dense_rank, ntile) use Uint64, but Postgres + // returns bigint for these. ExplicitCast overrides the Numeric mapping. + if expr.Type(ctx).Type() == query.Type_UINT64 { + return pgexprs.NewExplicitCast(expr, pgtypes.Int64), transform.NewTree, nil + } + return pgexprs.NewGMSCast(expr), transform.NewTree, nil + } + } } return expr, transform.SameTree, nil case *expression.Literal: @@ -75,6 +88,12 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope case sql.FunctionExpression: // Compiled functions are Doltgres functions. We're only concerned with GMS functions. if _, ok := expr.(framework.Function); !ok { + // Window functions implement sql.WindowAdaptableExpression and must NOT be wrapped in + // GMSCast. Wrapping hides the interface from windowToIter, which then falls to the + // default "last value" path and calls Eval() directly, returning ErrWindowUnsupported. + if _, ok := expr.(sql.WindowAdaptableExpression); ok { + return expr, transform.SameTree, nil + } // Some aggregation functions cannot be wrapped due to expectations in the analyzer, so we exclude them here. switch expr.FunctionName() { case "Count", "CountDistinct", "group_concat", "JSONObjectAgg", "Sum": @@ -245,3 +264,22 @@ func typeSanitizerLiterals(ctx *sql.Context, gmsLiteral *expression.Literal) (sq return nil, transform.NewTree, errors.Errorf("SANITIZER: encountered a GMS type that cannot be handled: %s", gmsLiteral.Type(ctx).String()) } } + +// isWindowOrWrapsWindow reports whether n is a *plan.Window or is a transparent +// wrapper (Sort, Limit, Offset, Distinct, Filter) whose sole child is a *plan.Window. +// This handles cases like: Project → Sort → Window, which arises when the query +// has a top-level ORDER BY alongside a window function. +func isWindowOrWrapsWindow(n sql.Node) bool { + if _, ok := n.(*plan.Window); ok { + return true + } + switch n.(type) { + case *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter: + children := n.Children() + if len(children) == 1 { + _, ok := children[0].(*plan.Window) + return ok + } + } + return false +} diff --git a/testing/go/enginetest/doltgres_engine_test.go b/testing/go/enginetest/doltgres_engine_test.go index 0c5646a125..51d939beb3 100755 --- a/testing/go/enginetest/doltgres_engine_test.go +++ b/testing/go/enginetest/doltgres_engine_test.go @@ -834,7 +834,6 @@ func TestVersionedViews(t *testing.T) { } func TestWindowFunctions(t *testing.T) { - t.Skip() h := newDoltgresServerHarness(t) defer h.Close() enginetest.TestWindowFunctions(t, h) diff --git a/testing/go/window_test.go b/testing/go/window_test.go new file mode 100644 index 0000000000..00390e78d5 --- /dev/null +++ b/testing/go/window_test.go @@ -0,0 +1,94 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package _go + +import ( + "testing" + + "github.com/dolthub/go-mysql-server/sql" +) + +func TestWindowFunctions(t *testing.T) { + RunScripts(t, []ScriptTest{ + { + // https://github.com/dolthub/doltgresql/issues/1796 + Name: "basic window functions", + SetUpScript: []string{ + "CREATE TABLE c (c_id INT PRIMARY KEY, bill TEXT);", + "CREATE TABLE o (o_id INT PRIMARY KEY, c_id INT, ship TEXT);", + "INSERT INTO c VALUES (1, 'CA'), (2, 'TX'), (3, 'MA'), (4, 'TX'), (5, NULL), (6, 'FL');", + "INSERT INTO o VALUES (10, 1, 'CA'), (20, 1, 'CA'), (30, 1, 'CA'), (40, 2, 'CA'), (50, 2, 'TX'), (60, 2, NULL), (70, 4, 'WY'), (80, 4, NULL), (90, 6, 'WA');", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "SELECT row_number() OVER () AS rn FROM o WHERE c_id=-999", + Expected: []sql.Row{}, + }, + { + Query: "SELECT row_number() OVER () AS rn FROM o WHERE c_id=1", + Expected: []sql.Row{ + {int64(1)}, + {int64(2)}, + {int64(3)}, + }, + }, + { + Query: "SELECT rank() OVER () AS rnk FROM o WHERE c_id=-999", + Expected: []sql.Row{}, + }, + { + Query: "SELECT o_id, c_id, rank() OVER (ORDER BY o_id) AS rnk FROM o WHERE c_id=1", + Expected: []sql.Row{ + {10, 1, int64(1)}, + {20, 1, int64(2)}, + {30, 1, int64(3)}, + }, + }, + { + Query: "SELECT dense_rank() OVER () AS drnk FROM o WHERE c_id=-999", + Expected: []sql.Row{}, + }, + { + // Doltgres inherits GMS/MySQL null-first sort order, so NULLs sort before + // non-NULLs in window ORDER BY and outer ORDER BY. Postgres default is NULLS LAST. + // TODO: update expected values once Doltgres adopts Postgres NULLS LAST ordering. + Query: "SELECT ship, dense_rank() OVER (ORDER BY ship) AS drnk FROM o WHERE c_id IN (1, 2) ORDER BY ship", + Expected: []sql.Row{ + {nil, int64(1)}, + {"CA", int64(2)}, + {"CA", int64(2)}, + {"CA", int64(2)}, + {"CA", int64(2)}, + {"TX", int64(3)}, + }, + }, + { + Query: "SELECT * FROM (SELECT c_id AS c_c_id, bill FROM c) sq1, LATERAL (SELECT row_number() OVER () AS rownum FROM o WHERE c_id = c_c_id) sq2 ORDER BY c_c_id, bill, rownum", + Expected: []sql.Row{ + {1, "CA", int64(1)}, + {1, "CA", int64(2)}, + {1, "CA", int64(3)}, + {2, "TX", int64(1)}, + {2, "TX", int64(2)}, + {2, "TX", int64(3)}, + {4, "TX", int64(1)}, + {4, "TX", int64(2)}, + {6, "FL", int64(1)}, + }, + }, + }, + }, + }) +} From b7481b951b8f441d1b2153d77da8701349f5fd3e Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 Jul 2026 13:48:02 -0700 Subject: [PATCH 2/5] Additional tests and fixes --- server/analyzer/type_sanitizer.go | 139 +++++++++++++++--- server/expression/agg_cast.go | 132 +++++++++++++++++ server/expression/explicit_cast.go | 21 +++ .../go/enginetest/doltgres_harness_test.go | 3 + testing/go/window_test.go | 49 ++++++ 5 files changed, 327 insertions(+), 17 deletions(-) create mode 100644 server/expression/agg_cast.go diff --git a/server/analyzer/type_sanitizer.go b/server/analyzer/type_sanitizer.go index a168e8e75f..fd716342bd 100644 --- a/server/analyzer/type_sanitizer.go +++ b/server/analyzer/type_sanitizer.go @@ -48,7 +48,7 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope switch expr := expr.(type) { case *expression.GetField: switch n := n.(type) { - case *plan.Project, *plan.Filter, *plan.GroupBy: + case *plan.Project, *plan.Filter, *plan.GroupBy, *plan.Having: child := n.Children()[0] // Some dolt_ tables do not have doltgres types for their columns, so we convert them here if rt, ok := child.(*plan.ResolvedTable); ok && strings.HasPrefix(rt.Name(), "dolt_") { @@ -59,16 +59,66 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope } // Window function outputs carry GMS types in the Window schema; wrap with GMSCast // so the type is properly converted to a Doltgres type for the wire protocol. - // A Sort node may sit between the Project and Window (for outer ORDER BY). if isWindowOrWrapsWindow(child) { if _, ok := expr.Type(ctx).(*pgtypes.DoltgresType); !ok { - // GMS ranking functions (rank, dense_rank, ntile) use Uint64, but Postgres - // returns bigint for these. ExplicitCast overrides the Numeric mapping. + // GMS ranking functions (rank, dense_rank, ntile) use Uint64 internally, + // but Postgres returns bigint. ExplicitCast overrides the Numeric mapping. if expr.Type(ctx).Type() == query.Type_UINT64 { return pgexprs.NewExplicitCast(expr, pgtypes.Int64), transform.NewTree, nil } return pgexprs.NewGMSCast(expr), transform.NewTree, nil } + // After this point expr IS a DoltgresType. + doltType := expr.Type(ctx).(*pgtypes.DoltgresType) + // GMS window SUM propagates the child column's DoltgresType via Sum.Type(), + // but always accumulates float64 at runtime. Match by ColumnId in + // Window.SelectExprs; for integer aggregates apply Float64→Int64. + if winNode := findWindowNode(child); winNode != nil { + for _, selectExpr := range winNode.SelectExprs { + ide, ok := selectExpr.(sql.IdExpression) + if !ok || ide.Id() != expr.Id() { + continue + } + if _, isAgg := selectExpr.(sql.Aggregation); isAgg { + if doltType.Equals(pgtypes.Int32) || doltType.Equals(pgtypes.Int16) || doltType.Equals(pgtypes.Int64) { + return pgexprs.NewAssignmentCast(expr, pgtypes.Float64, pgtypes.Int64), transform.NewTree, nil + } + } + break + } + } + // An outer Project may carry a stale declared type; child.Schema() holds + // the corrected type after bottom-up transform. Re-annotate on mismatch. + if innerType := windowSchemaTypeByName(child, ctx, expr.Name()); innerType != nil { + if !doltType.Equals(innerType) { + return pgexprs.NewAssignmentCast(expr, innerType, innerType), transform.NewTree, nil + } + } + } + // If a GetField references a SubqueryAlias and has a GMS type but the SubqueryAlias + // schema already has a DoltgresType (e.g. from AggCast propagation through a CTE), + // rebuild the GetField with the correct type so downstream operators compile for the + // actual runtime type rather than the planbuilder-assigned GMS type. + if _, ok := child.(*plan.SubqueryAlias); ok { + if _, ok := expr.Type(ctx).(*pgtypes.DoltgresType); !ok { + if actualType := windowSchemaTypeByName(child, ctx, expr.Name()); actualType != nil { + return expression.NewGetField(expr.Index(), actualType, expr.Name(), expr.IsNullable(ctx)), transform.NewTree, nil + } + } + } + // When a GroupBy child (possibly through a Having wrapper) has AggCast applied + // (e.g. SUM over integers), its schema reports the corrected DoltgresType, but the + // GetField may have been constructed earlier with a stale type: either a GMS type + // (float64 from GMS's internal Sum type promotion), or the pre-AggCast DoltgresType + // (e.g. Int32, from Sum.Type() before AggCast overrode it to Int64). Re-annotate so + // rowToBytes uses the correct Doltgres type for wire serialization. The runtime value + // is already correct from AggCast. + if gb := findGroupByChild(child); gb != nil { + if doltType := groupBySchemaTypeById(gb, ctx, expr.Id()); doltType != nil { + if currentType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); !ok || !currentType.Equals(doltType) { + return pgexprs.NewAssignmentCast(expr, doltType, doltType), transform.NewTree, nil + } + } } } return expr, transform.SameTree, nil @@ -88,10 +138,21 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope case sql.FunctionExpression: // Compiled functions are Doltgres functions. We're only concerned with GMS functions. if _, ok := expr.(framework.Function); !ok { - // Window functions implement sql.WindowAdaptableExpression and must NOT be wrapped in - // GMSCast. Wrapping hides the interface from windowToIter, which then falls to the - // default "last value" path and calls Eval() directly, returning ErrWindowUnsupported. + // Window functions (and GMS Sum) implement sql.WindowAdaptableExpression and must + // NOT be wrapped in GMSCast — that hides the interface from windowToIter. if _, ok := expr.(sql.WindowAdaptableExpression); ok { + // GMS SUM always accumulates float64 internally, but Postgres SUM(int2/int4/int8) + // returns bigint. In GroupBy context, wrap with AggCast to convert the float64 + // result to int64 while preserving the sql.Aggregation interface. + if expr.FunctionName() == "Sum" { + if _, isGroupBy := n.(*plan.GroupBy); isGroupBy { + if doltType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); ok { + if doltType.Equals(pgtypes.Int32) || doltType.Equals(pgtypes.Int16) || doltType.Equals(pgtypes.Int64) { + return pgexprs.NewAggCast(expr.(sql.Aggregation), pgtypes.Int64), transform.NewTree, nil + } + } + } + } return expr, transform.SameTree, nil } // Some aggregation functions cannot be wrapped due to expectations in the analyzer, so we exclude them here. @@ -265,21 +326,65 @@ func typeSanitizerLiterals(ctx *sql.Context, gmsLiteral *expression.Literal) (sq } } -// isWindowOrWrapsWindow reports whether n is a *plan.Window or is a transparent -// wrapper (Sort, Limit, Offset, Distinct, Filter) whose sole child is a *plan.Window. -// This handles cases like: Project → Sort → Window, which arises when the query -// has a top-level ORDER BY alongside a window function. +// isWindowOrWrapsWindow reports whether n is, or transitively wraps, a *plan.Window. func isWindowOrWrapsWindow(n sql.Node) bool { - if _, ok := n.(*plan.Window); ok { - return true + return findWindowNode(n) != nil +} + +// findWindowNode traverses Sort/Limit/Offset/Distinct/Filter/Project wrappers to find +// the underlying *plan.Window, or nil if n does not wrap a Window. +func findWindowNode(n sql.Node) *plan.Window { + if w, ok := n.(*plan.Window); ok { + return w } switch n.(type) { - case *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter: + case *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter, *plan.Project: children := n.Children() if len(children) == 1 { - _, ok := children[0].(*plan.Window) - return ok + return findWindowNode(children[0]) + } + } + return nil +} + +// findGroupByChild returns the GroupBy if n is a GroupBy or transitively wraps one +// through Having nodes. Returns nil if n does not expose a GroupBy this way. +func findGroupByChild(n sql.Node) *plan.GroupBy { + switch typed := n.(type) { + case *plan.GroupBy: + return typed + case *plan.Having: + children := typed.Children() + if len(children) == 1 { + return findGroupByChild(children[0]) + } + } + return nil +} + +// windowSchemaTypeByName returns the DoltgresType for a named column in n's schema, +// or nil if no match is found. +func windowSchemaTypeByName(n sql.Node, ctx *sql.Context, name string) *pgtypes.DoltgresType { + for _, col := range n.Schema(ctx) { + if strings.EqualFold(col.Name, name) { + if t, ok := col.Type.(*pgtypes.DoltgresType); ok { + return t + } + } + } + return nil +} + +// groupBySchemaTypeById returns the DoltgresType of the SelectDeps expression in gb whose +// ColumnId matches id, or nil if no such expression exists or its type isn't a DoltgresType. +func groupBySchemaTypeById(gb *plan.GroupBy, ctx *sql.Context, id sql.ColumnId) *pgtypes.DoltgresType { + for _, e := range gb.SelectDeps { + ide, ok := e.(sql.IdExpression) + if !ok || ide.Id() != id { + continue } + t, _ := e.Type(ctx).(*pgtypes.DoltgresType) + return t } - return false + return nil } diff --git a/server/expression/agg_cast.go b/server/expression/agg_cast.go new file mode 100644 index 0000000000..a74f71432a --- /dev/null +++ b/server/expression/agg_cast.go @@ -0,0 +1,132 @@ +// Copyright 2026 Dolthub, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expression + +import ( + "math" + + "github.com/cockroachdb/errors" + "github.com/dolthub/go-mysql-server/sql" + + pgtypes "github.com/dolthub/doltgresql/server/types" +) + +// AggCast wraps a sql.Aggregation to override its declared return type and +// post-convert the buffer's Eval result. It preserves the sql.Aggregation +// interface so GroupBy aggregation machinery works correctly. +// +// GMS SUM over integer columns always accumulates as float64 internally, but +// Postgres specifies SUM(int2/int4/int8) → bigint. AggCast intercepts the +// float64 buffer result and converts it to int64 so the correct wire type is used. +type AggCast struct { + inner sql.Aggregation + targetType *pgtypes.DoltgresType +} + +var _ sql.Expression = (*AggCast)(nil) +var _ sql.Aggregation = (*AggCast)(nil) + +// NewAggCast wraps inner so that its declared type is targetType and its buffer +// Eval result is converted from float64 to int64. +func NewAggCast(inner sql.Aggregation, targetType *pgtypes.DoltgresType) *AggCast { + return &AggCast{inner: inner, targetType: targetType} +} + +// Type overrides the inner aggregation's declared type. +func (a *AggCast) Type(ctx *sql.Context) sql.Type { return a.targetType } + +// NewBuffer delegates to inner but wraps the result buffer. +func (a *AggCast) NewBuffer(ctx *sql.Context) (sql.AggregationBuffer, error) { + buf, err := a.inner.NewBuffer(ctx) + if err != nil { + return nil, err + } + return &aggCastBuffer{inner: buf}, nil +} + +// NewWindowFunction delegates to inner. +func (a *AggCast) NewWindowFunction(ctx *sql.Context) (sql.WindowFunction, error) { + return a.inner.NewWindowFunction(ctx) +} + +// WithWindow delegates to inner. +func (a *AggCast) WithWindow(ctx *sql.Context, w *sql.WindowDefinition) sql.WindowAdaptableExpression { + return a.inner.WithWindow(ctx, w) +} + +// Window delegates to inner. +func (a *AggCast) Window() *sql.WindowDefinition { return a.inner.Window() } + +// Id delegates to inner. +func (a *AggCast) Id() sql.ColumnId { + if ide, ok := a.inner.(sql.IdExpression); ok { + return ide.Id() + } + return 0 +} + +// WithId delegates to inner and rewraps. +func (a *AggCast) WithId(id sql.ColumnId) sql.IdExpression { + if ide, ok := a.inner.(sql.IdExpression); ok { + if agg, ok := ide.WithId(id).(sql.Aggregation); ok { + return NewAggCast(agg, a.targetType) + } + } + return a +} + +func (a *AggCast) Children() []sql.Expression { return a.inner.Children() } +func (a *AggCast) Eval(ctx *sql.Context, row sql.Row) (any, error) { + return a.inner.Eval(ctx, row) +} +func (a *AggCast) IsNullable(ctx *sql.Context) bool { return a.inner.IsNullable(ctx) } +func (a *AggCast) Resolved() bool { return a.inner.Resolved() } +func (a *AggCast) String() string { return a.inner.String() } + +func (a *AggCast) WithChildren(ctx *sql.Context, children ...sql.Expression) (sql.Expression, error) { + inner, err := a.inner.WithChildren(ctx, children...) + if err != nil { + return nil, err + } + agg, ok := inner.(sql.Aggregation) + if !ok { + return nil, errors.New("AggCast.WithChildren: rebuilt expression does not implement sql.Aggregation") + } + return NewAggCast(agg, a.targetType), nil +} + +// aggCastBuffer wraps an AggregationBuffer and post-converts float64 Eval results to int64. +type aggCastBuffer struct { + inner sql.AggregationBuffer +} + +func (b *aggCastBuffer) Update(ctx *sql.Context, row sql.Row) error { + return b.inner.Update(ctx, row) +} + +func (b *aggCastBuffer) Eval(ctx *sql.Context) (any, error) { + v, err := b.inner.Eval(ctx) + if err != nil || v == nil { + return v, err + } + if f, ok := v.(float64); ok { + return int64(math.RoundToEven(f)), nil + } + return v, nil +} + +func (b *aggCastBuffer) Dispose(ctx *sql.Context) { + b.inner.Dispose(ctx) +} diff --git a/server/expression/explicit_cast.go b/server/expression/explicit_cast.go index 280fbe952f..269d9b59ca 100644 --- a/server/expression/explicit_cast.go +++ b/server/expression/explicit_cast.go @@ -38,6 +38,7 @@ type ExplicitCast struct { var _ vitess.Injectable = (*ExplicitCast)(nil) var _ sql.Expression = (*ExplicitCast)(nil) +var _ sql.IdExpression = (*ExplicitCast)(nil) // NewExplicitCastInjectable returns an incomplete *ExplicitCast that must be resolved through the vitess.Injectable interface. func NewExplicitCastInjectable(castToType sql.Type) (*ExplicitCast, error) { @@ -60,6 +61,26 @@ func NewExplicitCast(expr sql.Expression, toType *pgtypes.DoltgresType) *Explici } } +// Id implements sql.IdExpression, forwarding the child's ColumnId so ExplicitCast +// is transparent to the GMS index-assignment pass. +func (c *ExplicitCast) Id() sql.ColumnId { + if ide, ok := c.sqlChild.(sql.IdExpression); ok { + return ide.Id() + } + return 0 +} + +// WithId implements sql.IdExpression. +func (c *ExplicitCast) WithId(id sql.ColumnId) sql.IdExpression { + if ide, ok := c.sqlChild.(sql.IdExpression); ok { + newChild, ok := ide.WithId(id).(sql.Expression) + if ok { + return NewExplicitCast(newChild, c.castToType) + } + } + return c +} + // Children implements the sql.Expression interface. func (c *ExplicitCast) Children() []sql.Expression { return []sql.Expression{c.sqlChild} diff --git a/testing/go/enginetest/doltgres_harness_test.go b/testing/go/enginetest/doltgres_harness_test.go index b5cc8a5a4a..597ca9caef 100644 --- a/testing/go/enginetest/doltgres_harness_test.go +++ b/testing/go/enginetest/doltgres_harness_test.go @@ -132,6 +132,9 @@ var defaultSkippedQueries = []string{ "<=>", // null-safe equality (PostgreSQL uses IS NOT DISTINCT FROM) "modify column", // MySQL ALTER TABLE MODIFY COLUMN syntax "column first", // MySQL ADD COLUMN ... FIRST positioning + + // GMS tests that expect MySQL SUM-returns-float64 semantics; Postgres SUM(int4) returns bigint (int64) + "sum(col1 * col2) > 0", } // Setup sets the setup scripts for this DoltHarness's engine diff --git a/testing/go/window_test.go b/testing/go/window_test.go index 00390e78d5..dc54663d46 100644 --- a/testing/go/window_test.go +++ b/testing/go/window_test.go @@ -88,6 +88,55 @@ func TestWindowFunctions(t *testing.T) { {6, "FL", int64(1)}, }, }, + // ORDER BY on rank alias with LIMIT (multi-hop Limit→Sort→Window chain) + { + Query: "SELECT c_id, rank() OVER (ORDER BY c_id) AS rnk FROM c ORDER BY rnk DESC LIMIT 3", + Expected: []sql.Row{ + {6, int64(6)}, + {5, int64(5)}, + {4, int64(4)}, + }, + }, + // ORDER BY on rank alias (Sort→Window chain) + { + Query: "SELECT c_id, rank() OVER (ORDER BY c_id) AS r FROM c ORDER BY r", + Expected: []sql.Row{ + {1, int64(1)}, + {2, int64(2)}, + {3, int64(3)}, + {4, int64(4)}, + {5, int64(5)}, + {6, int64(6)}, + }, + }, + // DISTINCT + ORDER BY on rank alias + { + Query: "SELECT DISTINCT c_id, rank() OVER (ORDER BY c_id) AS r FROM c ORDER BY r", + Expected: []sql.Row{ + {1, int64(1)}, + {2, int64(2)}, + {3, int64(3)}, + {4, int64(4)}, + {5, int64(5)}, + {6, int64(6)}, + }, + }, + // CASE expression over rank() subquery (int64 rank values flow into Numeric cast) + { + Query: "SELECT sum(CASE WHEN r > 0 THEN 1 ELSE 0 END) FROM (SELECT rank() OVER (ORDER BY c_id) AS r FROM c) t", + Expected: []sql.Row{ + {int64(6)}, + }, + }, + // window SUM with non-empty result (GMS SumAgg.Compute returns float64, not int32) + { + Query: "SELECT c_id, SUM(o_id) OVER (PARTITION BY c_id) AS s FROM o WHERE c_id = 1 ORDER BY o_id", + Expected: []sql.Row{ + {1, int64(60)}, + {1, int64(60)}, + {1, int64(60)}, + }, + }, }, }, }) From e18b1581485211c7613eaf2ff17fdeaa5a25e6f1 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Mon, 6 Jul 2026 15:08:25 -0700 Subject: [PATCH 3/5] More test fixes --- server/analyzer/init.go | 5 +- server/analyzer/resolve_values_types.go | 5 +- server/analyzer/type_sanitizer.go | 60 ++++++++++++++----- server/expression/agg_cast.go | 35 ++++++++--- testing/go/enginetest/doltgres_engine_test.go | 25 ++++---- testing/go/smoke_test.go | 4 +- testing/go/values_statement_test.go | 4 +- 7 files changed, 96 insertions(+), 42 deletions(-) diff --git a/server/analyzer/init.go b/server/analyzer/init.go index 12601ec6bc..bf240bea1d 100644 --- a/server/analyzer/init.go +++ b/server/analyzer/init.go @@ -68,8 +68,11 @@ func Init() { // because of custom batch set optimization in GMS skipping OnceBeforeDefault batch for some nodes. analyzer.Rule{Id: ruleId_ResolveType, Apply: ResolveType}, analyzer.Rule{Id: ruleId_SetRunner, Apply: SetRunner}, - analyzer.Rule{Id: ruleId_TypeSanitizer, Apply: TypeSanitizer}, + // ResolveValuesTypes must run before TypeSanitizer: it unifies VALUES-clause column + // types (e.g. VALUES(1),(2.01) -> numeric) which TypeSanitizer's aggregate-type + // corrections (e.g. SUM(int) -> bigint) depend on being already resolved. analyzer.Rule{Id: ruleId_ResolveValuesTypes, Apply: ResolveValuesTypes}, + analyzer.Rule{Id: ruleId_TypeSanitizer, Apply: TypeSanitizer}, analyzer.Rule{Id: ruleId_GenerateForeignKeyName, Apply: generateForeignKeyName}, analyzer.Rule{Id: ruleId_AddDomainConstraints, Apply: AddDomainConstraints}, analyzer.Rule{Id: ruleId_ValidateColumnDefaults, Apply: ValidateColumnDefaults}, diff --git a/server/analyzer/resolve_values_types.go b/server/analyzer/resolve_values_types.go index 95de83baa2..fc35a60a19 100644 --- a/server/analyzer/resolve_values_types.go +++ b/server/analyzer/resolve_values_types.go @@ -23,6 +23,7 @@ import ( "github.com/dolthub/go-mysql-server/sql/expression" "github.com/dolthub/go-mysql-server/sql/plan" "github.com/dolthub/go-mysql-server/sql/transform" + "github.com/dolthub/go-mysql-server/sql/types" pgexprs "github.com/dolthub/doltgresql/server/expression" "github.com/dolthub/doltgresql/server/functions/framework" @@ -189,7 +190,9 @@ func transformValuesNode(ctx *sql.Context, n sql.Node) (sql.Node, transform.Tree columnTypes[colIdx] = make([]*pgtypes.DoltgresType, len(values.ExpressionTuples)) for rowIdx, row := range values.ExpressionTuples { exprType := row[colIdx].Type(ctx) - if exprType == nil { + if exprType == nil || exprType == types.Null { + // A raw NULL literal reports the GMS sentinel Null type until TypeSanitizer + // converts it to a pgtypes.Unknown-typed literal. columnTypes[colIdx][rowIdx] = pgtypes.Unknown } else if pgType, ok := exprType.(*pgtypes.DoltgresType); ok { columnTypes[colIdx][rowIdx] = pgType diff --git a/server/analyzer/type_sanitizer.go b/server/analyzer/type_sanitizer.go index fd716342bd..bfe9c0858b 100644 --- a/server/analyzer/type_sanitizer.go +++ b/server/analyzer/type_sanitizer.go @@ -141,16 +141,39 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope // Window functions (and GMS Sum) implement sql.WindowAdaptableExpression and must // NOT be wrapped in GMSCast — that hides the interface from windowToIter. if _, ok := expr.(sql.WindowAdaptableExpression); ok { - // GMS SUM always accumulates float64 internally, but Postgres SUM(int2/int4/int8) - // returns bigint. In GroupBy context, wrap with AggCast to convert the float64 - // result to int64 while preserving the sql.Aggregation interface. - if expr.FunctionName() == "Sum" { - if _, isGroupBy := n.(*plan.GroupBy); isGroupBy { - if doltType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); ok { - if doltType.Equals(pgtypes.Int32) || doltType.Equals(pgtypes.Int16) || doltType.Equals(pgtypes.Int64) { - return pgexprs.NewAggCast(expr.(sql.Aggregation), pgtypes.Int64), transform.NewTree, nil - } + // GMS SUM and AVG always accumulate as float64 internally, regardless of the + // input column's width, but Postgres promotes their result types: + // SUM(smallint/integer) -> bigint + // SUM(bigint) -> numeric (a running sum of bigints can itself overflow bigint) + // SUM(real) -> real (needs narrowing back from the float64 accumulator) + // AVG(smallint/integer/bigint) -> numeric + // AVG(real) -> double precision + // In GroupBy context, wrap with AggCast to convert the float64 result to the + // correct target type while preserving the sql.Aggregation interface. + var aggTargetType *pgtypes.DoltgresType + if doltType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); ok { + switch expr.FunctionName() { + case "Sum": + switch { + case doltType.Equals(pgtypes.Int16), doltType.Equals(pgtypes.Int32): + aggTargetType = pgtypes.Int64 + case doltType.Equals(pgtypes.Int64): + aggTargetType = pgtypes.Numeric + case doltType.Equals(pgtypes.Float32): + aggTargetType = pgtypes.Float32 } + case "Avg": + switch { + case doltType.Equals(pgtypes.Int16), doltType.Equals(pgtypes.Int32), doltType.Equals(pgtypes.Int64): + aggTargetType = pgtypes.Numeric + case doltType.Equals(pgtypes.Float32): + aggTargetType = pgtypes.Float64 + } + } + } + if aggTargetType != nil { + if _, isGroupBy := n.(*plan.GroupBy); isGroupBy { + return pgexprs.NewAggCast(expr.(sql.Aggregation), aggTargetType), transform.NewTree, nil } } return expr, transform.SameTree, nil @@ -347,14 +370,19 @@ func findWindowNode(n sql.Node) *plan.Window { return nil } -// findGroupByChild returns the GroupBy if n is a GroupBy or transitively wraps one -// through Having nodes. Returns nil if n does not expose a GroupBy this way. +// findGroupByChild returns the GroupBy if n is a GroupBy or is a transparent wrapper +// (Having, Project, Sort, Limit, Offset, Distinct, Filter) whose sole child transitively +// wraps one. A HAVING clause's own condition, in particular, sits under a Having node whose +// child is a Project (not the GroupBy directly) — e.g. Having -> Project -> GroupBy — so +// this must traverse through Project to find it. Returns nil if n does not expose a GroupBy +// this way. func findGroupByChild(n sql.Node) *plan.GroupBy { - switch typed := n.(type) { - case *plan.GroupBy: - return typed - case *plan.Having: - children := typed.Children() + if gb, ok := n.(*plan.GroupBy); ok { + return gb + } + switch n.(type) { + case *plan.Having, *plan.Project, *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter: + children := n.Children() if len(children) == 1 { return findGroupByChild(children[0]) } diff --git a/server/expression/agg_cast.go b/server/expression/agg_cast.go index a74f71432a..e590408283 100644 --- a/server/expression/agg_cast.go +++ b/server/expression/agg_cast.go @@ -16,7 +16,9 @@ package expression import ( "math" + "strconv" + "github.com/cockroachdb/apd/v3" "github.com/cockroachdb/errors" "github.com/dolthub/go-mysql-server/sql" @@ -27,9 +29,10 @@ import ( // post-convert the buffer's Eval result. It preserves the sql.Aggregation // interface so GroupBy aggregation machinery works correctly. // -// GMS SUM over integer columns always accumulates as float64 internally, but -// Postgres specifies SUM(int2/int4/int8) → bigint. AggCast intercepts the -// float64 buffer result and converts it to int64 so the correct wire type is used. +// GMS SUM and AVG over integer columns always accumulate as float64 internally, but +// Postgres specifies SUM(int2/int4/int8) → bigint and AVG(int2/int4/int8) → numeric. +// AggCast intercepts the float64 buffer result and converts it to the target type so +// the correct wire type is used. type AggCast struct { inner sql.Aggregation targetType *pgtypes.DoltgresType @@ -53,7 +56,7 @@ func (a *AggCast) NewBuffer(ctx *sql.Context) (sql.AggregationBuffer, error) { if err != nil { return nil, err } - return &aggCastBuffer{inner: buf}, nil + return &aggCastBuffer{inner: buf, targetType: a.targetType}, nil } // NewWindowFunction delegates to inner. @@ -107,9 +110,11 @@ func (a *AggCast) WithChildren(ctx *sql.Context, children ...sql.Expression) (sq return NewAggCast(agg, a.targetType), nil } -// aggCastBuffer wraps an AggregationBuffer and post-converts float64 Eval results to int64. +// aggCastBuffer wraps an AggregationBuffer and post-converts its float64 Eval result to +// match targetType (int64 for bigint, numeric, or float32 for real; float64 is a no-op). type aggCastBuffer struct { - inner sql.AggregationBuffer + inner sql.AggregationBuffer + targetType *pgtypes.DoltgresType } func (b *aggCastBuffer) Update(ctx *sql.Context, row sql.Row) error { @@ -121,10 +126,24 @@ func (b *aggCastBuffer) Eval(ctx *sql.Context) (any, error) { if err != nil || v == nil { return v, err } - if f, ok := v.(float64); ok { + f, ok := v.(float64) + if !ok { + return v, nil + } + switch { + case b.targetType.Equals(pgtypes.Numeric): + d, _, err := apd.NewFromString(strconv.FormatFloat(f, 'f', -1, 64)) + if err != nil { + return nil, err + } + return d, nil + case b.targetType.Equals(pgtypes.Float32): + return float32(f), nil + case b.targetType.Equals(pgtypes.Float64): + return f, nil + default: return int64(math.RoundToEven(f)), nil } - return v, nil } func (b *aggCastBuffer) Dispose(ctx *sql.Context) { diff --git a/testing/go/enginetest/doltgres_engine_test.go b/testing/go/enginetest/doltgres_engine_test.go index 51d939beb3..5c2f116a7a 100755 --- a/testing/go/enginetest/doltgres_engine_test.go +++ b/testing/go/enginetest/doltgres_engine_test.go @@ -547,18 +547,19 @@ func TestScripts(t *testing.T) { "preserve now()", // harness error "binary type primary key", // ERROR: blob/text column 'b' used in key specification without a key length "varbinary primary key", // ERROR: blob/text column 'b' used in key specification without a key length - "insert into t1 (a, b) values ('1234567890', '12345')", // different error message - "insert into t2 (a, b) values ('1234567890', '12345')", // different error message - "invalid utf8 encoding strings", // need to investigate why some strings aren't giving errors, might be a harness error - "mismatched collation using hash in tuples", // ERROR: plan is not resolved because of node '*plan.Project' - "validate_password_strength and validate_password.length", // unsupported - "validate_password_strength and validate_password.number_count", // unsupported - "validate_password_strength and validate_password.mixed_case_count", // unsupported - "validate_password_strength and validate_password.special_char_count", // unsupported - "coalesce with system types", // unsupported - "histogram bucket merging error for implementor buckets", // unsupported "with recursive" syntax - "varchar primary key", // literal values longer than the key length returns incorrect results for some queries - "can't create view with same name as existing table", // different error message + "insert into t1 (a, b) values ('1234567890', '12345')", // different error message + "insert into t2 (a, b) values ('1234567890', '12345')", // different error message + "invalid utf8 encoding strings", // need to investigate why some strings aren't giving errors, might be a harness error + "mismatched collation using hash in tuples", // ERROR: plan is not resolved because of node '*plan.Project' + "validate_password_strength and validate_password.length", // unsupported + "validate_password_strength and validate_password.number_count", // unsupported + "validate_password_strength and validate_password.mixed_case_count", // unsupported + "validate_password_strength and validate_password.special_char_count", // unsupported + "coalesce with system types", // unsupported + "histogram bucket merging error for implementor buckets", // unsupported "with recursive" syntax + "varchar primary key", // literal values longer than the key length returns incorrect results for some queries + "can't create view with same name as existing table", // different error message + "sum() and avg() on non-DECIMAL type column returns the DOUBLE type result", // expects MySQL SUM/AVG(int)-returns-DOUBLE semantics; Postgres SUM(int4) returns bigint and AVG(int4) returns numeric }) defer h.Close() enginetest.TestScripts(t, h) diff --git a/testing/go/smoke_test.go b/testing/go/smoke_test.go index d86fca7ed4..54acced277 100644 --- a/testing/go/smoke_test.go +++ b/testing/go/smoke_test.go @@ -648,7 +648,7 @@ func TestSmokeTests(t *testing.T) { { Query: "SELECT SUM(v1) FROM test WHERE v1 BETWEEN 3 AND 5;", Expected: []sql.Row{ - {12.0}, + {int64(12)}, }, }, { @@ -658,7 +658,7 @@ func TestSmokeTests(t *testing.T) { { Query: "SELECT SUM(v1) FROM test WHERE v1 BETWEEN 3 AND 5;", Expected: []sql.Row{ - {12.0}, + {int64(12)}, }, }, }, diff --git a/testing/go/values_statement_test.go b/testing/go/values_statement_test.go index 9f8590df61..9e0fcc8c55 100644 --- a/testing/go/values_statement_test.go +++ b/testing/go/values_statement_test.go @@ -240,9 +240,9 @@ var ValuesStatementTests = []ScriptTest{ Name: "VALUES with aggregate functions", Assertions: []ScriptTestAssertion{ { - // AVG on mixed types + // AVG on mixed types: Postgres AVG(integer) returns numeric, not double precision Query: `SELECT AVG(n) FROM (VALUES(1),(2),(3),(4)) v(n);`, - Expected: []sql.Row{{2.5}}, + Expected: []sql.Row{{Numeric("2.5")}}, }, { // MIN/MAX on mixed types From f256286b8177b55c770aafeddfc4239d26987d84 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Tue, 7 Jul 2026 12:30:09 -0700 Subject: [PATCH 4/5] Fixing double-cast panic on window SUM/AVG in subqueries --- server/analyzer/type_sanitizer.go | 41 ++++++++---------- server/expression/agg_cast.go | 69 ++++++++++++++++++++++++------- testing/go/window_test.go | 25 +++++++++++ 3 files changed, 98 insertions(+), 37 deletions(-) diff --git a/server/analyzer/type_sanitizer.go b/server/analyzer/type_sanitizer.go index bfe9c0858b..5de429c501 100644 --- a/server/analyzer/type_sanitizer.go +++ b/server/analyzer/type_sanitizer.go @@ -70,25 +70,12 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope } // After this point expr IS a DoltgresType. doltType := expr.Type(ctx).(*pgtypes.DoltgresType) - // GMS window SUM propagates the child column's DoltgresType via Sum.Type(), - // but always accumulates float64 at runtime. Match by ColumnId in - // Window.SelectExprs; for integer aggregates apply Float64→Int64. - if winNode := findWindowNode(child); winNode != nil { - for _, selectExpr := range winNode.SelectExprs { - ide, ok := selectExpr.(sql.IdExpression) - if !ok || ide.Id() != expr.Id() { - continue - } - if _, isAgg := selectExpr.(sql.Aggregation); isAgg { - if doltType.Equals(pgtypes.Int32) || doltType.Equals(pgtypes.Int16) || doltType.Equals(pgtypes.Int64) { - return pgexprs.NewAssignmentCast(expr, pgtypes.Float64, pgtypes.Int64), transform.NewTree, nil - } - } - break - } - } - // An outer Project may carry a stale declared type; child.Schema() holds - // the corrected type after bottom-up transform. Re-annotate on mismatch. + // Window SUM/AVG over integer or real columns are fixed at the source (see the + // AggCast wrap for *plan.Window below), so the Window's own schema already + // reports the corrected type. An outer Project (possibly reached through a + // SubqueryAlias) may still carry a stale declared type from before that fix, or + // from GMS's own type propagation; child.Schema() holds the corrected type after + // the bottom-up transform. Re-annotate on mismatch. if innerType := windowSchemaTypeByName(child, ctx, expr.Name()); innerType != nil { if !doltType.Equals(innerType) { return pgexprs.NewAssignmentCast(expr, innerType, innerType), transform.NewTree, nil @@ -148,8 +135,15 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope // SUM(real) -> real (needs narrowing back from the float64 accumulator) // AVG(smallint/integer/bigint) -> numeric // AVG(real) -> double precision - // In GroupBy context, wrap with AggCast to convert the float64 result to the - // correct target type while preserving the sql.Aggregation interface. + // In GroupBy or Window context, wrap with AggCast to convert the float64 result + // to the correct target type while preserving the sql.Aggregation and + // sql.WindowAdaptableExpression interfaces. Fixing this at the source (the + // aggregate itself) rather than at each outer reference to it is what makes this + // idempotent: AggCast doesn't implement sql.FunctionExpression, so once applied it + // no longer matches this case on a later revisit (e.g. via the opaque traversal + // that re-walks an already-analyzed subquery), unlike wrapping an outer GetField + // reference, whose declared type never changes and so looks like it "still needs + // fixing" on every subsequent visit. var aggTargetType *pgtypes.DoltgresType if doltType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); ok { switch expr.FunctionName() { @@ -172,7 +166,8 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope } } if aggTargetType != nil { - if _, isGroupBy := n.(*plan.GroupBy); isGroupBy { + switch n.(type) { + case *plan.GroupBy, *plan.Window: return pgexprs.NewAggCast(expr.(sql.Aggregation), aggTargetType), transform.NewTree, nil } } @@ -361,7 +356,7 @@ func findWindowNode(n sql.Node) *plan.Window { return w } switch n.(type) { - case *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter, *plan.Project: + case *plan.Sort, *plan.Limit, *plan.Offset, *plan.Distinct, *plan.Filter, *plan.Project, *plan.SubqueryAlias: children := n.Children() if len(children) == 1 { return findWindowNode(children[0]) diff --git a/server/expression/agg_cast.go b/server/expression/agg_cast.go index e590408283..ef0a428354 100644 --- a/server/expression/agg_cast.go +++ b/server/expression/agg_cast.go @@ -25,14 +25,15 @@ import ( pgtypes "github.com/dolthub/doltgresql/server/types" ) -// AggCast wraps a sql.Aggregation to override its declared return type and -// post-convert the buffer's Eval result. It preserves the sql.Aggregation -// interface so GroupBy aggregation machinery works correctly. +// AggCast wraps a sql.Aggregation to override its declared return type and post-convert +// its result, whether reached through the GroupBy buffer path (NewBuffer/Eval) or the +// window function path (NewWindowFunction/Compute). It preserves both the sql.Aggregation +// and sql.WindowAdaptableExpression interfaces so GroupBy and window execution both work. // // GMS SUM and AVG over integer columns always accumulate as float64 internally, but // Postgres specifies SUM(int2/int4/int8) → bigint and AVG(int2/int4/int8) → numeric. -// AggCast intercepts the float64 buffer result and converts it to the target type so -// the correct wire type is used. +// AggCast intercepts the float64 result and converts it to the target type so the +// correct wire type is used. type AggCast struct { inner sql.Aggregation targetType *pgtypes.DoltgresType @@ -40,6 +41,7 @@ type AggCast struct { var _ sql.Expression = (*AggCast)(nil) var _ sql.Aggregation = (*AggCast)(nil) +var _ sql.WindowFunction = (*aggCastWindowFunction)(nil) // NewAggCast wraps inner so that its declared type is targetType and its buffer // Eval result is converted from float64 to int64. @@ -59,9 +61,14 @@ func (a *AggCast) NewBuffer(ctx *sql.Context) (sql.AggregationBuffer, error) { return &aggCastBuffer{inner: buf, targetType: a.targetType}, nil } -// NewWindowFunction delegates to inner. +// NewWindowFunction delegates to inner but wraps the result so Compute's float64 output is +// converted to match targetType, the same way NewBuffer wraps the AggregationBuffer path. func (a *AggCast) NewWindowFunction(ctx *sql.Context) (sql.WindowFunction, error) { - return a.inner.NewWindowFunction(ctx) + fn, err := a.inner.NewWindowFunction(ctx) + if err != nil { + return nil, err + } + return &aggCastWindowFunction{inner: fn, targetType: a.targetType}, nil } // WithWindow delegates to inner. @@ -126,26 +133,60 @@ func (b *aggCastBuffer) Eval(ctx *sql.Context) (any, error) { if err != nil || v == nil { return v, err } + return convertAggResult(v, b.targetType) +} + +func (b *aggCastBuffer) Dispose(ctx *sql.Context) { + b.inner.Dispose(ctx) +} + +// aggCastWindowFunction wraps a sql.WindowFunction and post-converts its float64 Compute +// result the same way aggCastBuffer does for the sql.AggregationBuffer (GroupBy) path. +type aggCastWindowFunction struct { + inner sql.WindowFunction + targetType *pgtypes.DoltgresType +} + +func (w *aggCastWindowFunction) StartPartition(ctx *sql.Context, interval sql.WindowInterval, buffer sql.WindowBuffer) error { + return w.inner.StartPartition(ctx, interval, buffer) +} + +func (w *aggCastWindowFunction) DefaultFramer() sql.WindowFramer { + return w.inner.DefaultFramer() +} + +func (w *aggCastWindowFunction) Compute(ctx *sql.Context, interval sql.WindowInterval, buffer sql.WindowBuffer) (any, error) { + v, err := w.inner.Compute(ctx, interval, buffer) + if err != nil || v == nil { + return v, err + } + return convertAggResult(v, w.targetType) +} + +func (w *aggCastWindowFunction) Dispose(ctx *sql.Context) { + w.inner.Dispose(ctx) +} + +// convertAggResult converts a raw aggregation result (always float64 from GMS's SUM/AVG +// implementations, regardless of the input column's width) to match targetType: numeric, +// float32, float64 (a no-op), or int64 for everything else (bigint). +func convertAggResult(v any, targetType *pgtypes.DoltgresType) (any, error) { f, ok := v.(float64) if !ok { return v, nil } switch { - case b.targetType.Equals(pgtypes.Numeric): + case targetType.Equals(pgtypes.Numeric): d, _, err := apd.NewFromString(strconv.FormatFloat(f, 'f', -1, 64)) if err != nil { return nil, err } return d, nil - case b.targetType.Equals(pgtypes.Float32): + case targetType.Equals(pgtypes.Float32): return float32(f), nil - case b.targetType.Equals(pgtypes.Float64): + case targetType.Equals(pgtypes.Float64): return f, nil default: return int64(math.RoundToEven(f)), nil } } - -func (b *aggCastBuffer) Dispose(ctx *sql.Context) { - b.inner.Dispose(ctx) -} diff --git a/testing/go/window_test.go b/testing/go/window_test.go index dc54663d46..1eab1f03b6 100644 --- a/testing/go/window_test.go +++ b/testing/go/window_test.go @@ -139,5 +139,30 @@ func TestWindowFunctions(t *testing.T) { }, }, }, + { + Name: "window SUM/AVG wrapped in a subquery projection", + SetUpScript: []string{ + "CREATE TABLE wrapper_probe (grp INT, val INT);", + "INSERT INTO wrapper_probe VALUES (1, 10), (1, 20), (2, 5);", + }, + Assertions: []ScriptTestAssertion{ + { + Query: "SELECT grp, val, grp_total FROM (SELECT grp, val, SUM(val) OVER (PARTITION BY grp) AS grp_total FROM wrapper_probe) sub ORDER BY grp, val;", + Expected: []sql.Row{ + {1, 10, int64(30)}, + {1, 20, int64(30)}, + {2, 5, int64(5)}, + }, + }, + { + Query: "SELECT grp, val, grp_avg FROM (SELECT grp, val, AVG(val) OVER (PARTITION BY grp) AS grp_avg FROM wrapper_probe) sub ORDER BY grp, val;", + Expected: []sql.Row{ + {1, 10, Numeric("15")}, + {1, 20, Numeric("15")}, + {2, 5, Numeric("5")}, + }, + }, + }, + }, }) } From ec3569873e373b817e55ae08901ebb6c57f0d561 Mon Sep 17 00:00:00 2001 From: Jason Fulghum Date: Wed, 8 Jul 2026 10:25:50 -0700 Subject: [PATCH 5/5] Testing some ideas for improving performance --- server/analyzer/type_sanitizer.go | 31 ++++++++++---- server/expression/agg_cast.go | 68 ++++++++++++++++++++++--------- 2 files changed, 72 insertions(+), 27 deletions(-) diff --git a/server/analyzer/type_sanitizer.go b/server/analyzer/type_sanitizer.go index 5de429c501..5753e73dfb 100644 --- a/server/analyzer/type_sanitizer.go +++ b/server/analyzer/type_sanitizer.go @@ -42,6 +42,11 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope // TODO: this probably should not be opaque, we should let the analyzer dig into subqueries and analyze them when // it chooses. Doing all type transformations upfront like this masks bugs where certain tyupe conversion errors // only manifest in a subquery + // groupByTypeCache memoizes each GroupBy's ColumnId->DoltgresType map (see + // groupBySchemaTypesById) for the duration of this single TypeSanitizer call, so a + // GroupBy with M output columns pays the cost of scanning its SelectDeps once, not once + // per column reference to it. + groupByTypeCache := make(map[*plan.GroupBy]map[sql.ColumnId]*pgtypes.DoltgresType) return pgtransform.NodeExprsWithNodeWithOpaque(ctx, node, func(ctx *sql.Context, n sql.Node, expr sql.Expression) (sql.Expression, transform.TreeIdentity, error) { // This can be updated if we find more expressions that return GMS types. // These should eventually be replaced with Doltgres-equivalents over time, rendering this function unnecessary. @@ -101,7 +106,7 @@ func TypeSanitizer(ctx *sql.Context, a *analyzer.Analyzer, node sql.Node, scope // rowToBytes uses the correct Doltgres type for wire serialization. The runtime value // is already correct from AggCast. if gb := findGroupByChild(child); gb != nil { - if doltType := groupBySchemaTypeById(gb, ctx, expr.Id()); doltType != nil { + if doltType := groupBySchemaTypesById(groupByTypeCache, gb, ctx)[expr.Id()]; doltType != nil { if currentType, ok := expr.Type(ctx).(*pgtypes.DoltgresType); !ok || !currentType.Equals(doltType) { return pgexprs.NewAssignmentCast(expr, doltType, doltType), transform.NewTree, nil } @@ -398,16 +403,26 @@ func windowSchemaTypeByName(n sql.Node, ctx *sql.Context, name string) *pgtypes. return nil } -// groupBySchemaTypeById returns the DoltgresType of the SelectDeps expression in gb whose -// ColumnId matches id, or nil if no such expression exists or its type isn't a DoltgresType. -func groupBySchemaTypeById(gb *plan.GroupBy, ctx *sql.Context, id sql.ColumnId) *pgtypes.DoltgresType { +// groupBySchemaTypesById returns a map from ColumnId to DoltgresType for every SelectDeps +// expression in gb that has one, building it once per gb and caching the result in cache for +// the remainder of the current TypeSanitizer call. Without this, resolving a single GetField +// against gb required a linear scan of all of gb's SelectDeps; since this lookup runs once per +// column reference to gb, a GroupBy with M output columns paid O(M) per column — O(M^2) total +// — even when nothing needed correcting. Caching drops that to O(M) total per GroupBy. +func groupBySchemaTypesById(cache map[*plan.GroupBy]map[sql.ColumnId]*pgtypes.DoltgresType, gb *plan.GroupBy, ctx *sql.Context) map[sql.ColumnId]*pgtypes.DoltgresType { + if m, ok := cache[gb]; ok { + return m + } + m := make(map[sql.ColumnId]*pgtypes.DoltgresType, len(gb.SelectDeps)) for _, e := range gb.SelectDeps { ide, ok := e.(sql.IdExpression) - if !ok || ide.Id() != id { + if !ok { continue } - t, _ := e.Type(ctx).(*pgtypes.DoltgresType) - return t + if t, ok := e.Type(ctx).(*pgtypes.DoltgresType); ok { + m[ide.Id()] = t + } } - return nil + cache[gb] = m + return m } diff --git a/server/expression/agg_cast.go b/server/expression/agg_cast.go index ef0a428354..ff49ca2f75 100644 --- a/server/expression/agg_cast.go +++ b/server/expression/agg_cast.go @@ -16,7 +16,6 @@ package expression import ( "math" - "strconv" "github.com/cockroachdb/apd/v3" "github.com/cockroachdb/errors" @@ -37,16 +36,46 @@ import ( type AggCast struct { inner sql.Aggregation targetType *pgtypes.DoltgresType + convKind aggConvKind } var _ sql.Expression = (*AggCast)(nil) var _ sql.Aggregation = (*AggCast)(nil) var _ sql.WindowFunction = (*aggCastWindowFunction)(nil) +// aggConvKind identifies which conversion convertAggResult should apply. It's derived from +// targetType once, at AggCast construction time (query-analysis time), rather than by +// calling DoltgresType.Equals on every row-group's Eval/Compute call (execution time, +// happening once per group per query execution): DoltgresType.Equals is not a cheap identity +// check — it compares two types by fully serializing both to byte buffers and comparing the +// bytes. Precomputing the target once and comparing a small int on the hot path avoids +// paying that serialization cost repeatedly. +type aggConvKind byte + +const ( + aggConvInt64 aggConvKind = iota + aggConvNumeric + aggConvFloat32 + aggConvFloat64 +) + +func aggConvKindFor(targetType *pgtypes.DoltgresType) aggConvKind { + switch { + case targetType.Equals(pgtypes.Numeric): + return aggConvNumeric + case targetType.Equals(pgtypes.Float32): + return aggConvFloat32 + case targetType.Equals(pgtypes.Float64): + return aggConvFloat64 + default: + return aggConvInt64 + } +} + // NewAggCast wraps inner so that its declared type is targetType and its buffer -// Eval result is converted from float64 to int64. +// Eval result is converted from float64 to match targetType. func NewAggCast(inner sql.Aggregation, targetType *pgtypes.DoltgresType) *AggCast { - return &AggCast{inner: inner, targetType: targetType} + return &AggCast{inner: inner, targetType: targetType, convKind: aggConvKindFor(targetType)} } // Type overrides the inner aggregation's declared type. @@ -58,7 +87,7 @@ func (a *AggCast) NewBuffer(ctx *sql.Context) (sql.AggregationBuffer, error) { if err != nil { return nil, err } - return &aggCastBuffer{inner: buf, targetType: a.targetType}, nil + return &aggCastBuffer{inner: buf, convKind: a.convKind}, nil } // NewWindowFunction delegates to inner but wraps the result so Compute's float64 output is @@ -68,7 +97,7 @@ func (a *AggCast) NewWindowFunction(ctx *sql.Context) (sql.WindowFunction, error if err != nil { return nil, err } - return &aggCastWindowFunction{inner: fn, targetType: a.targetType}, nil + return &aggCastWindowFunction{inner: fn, convKind: a.convKind}, nil } // WithWindow delegates to inner. @@ -118,10 +147,11 @@ func (a *AggCast) WithChildren(ctx *sql.Context, children ...sql.Expression) (sq } // aggCastBuffer wraps an AggregationBuffer and post-converts its float64 Eval result to -// match targetType (int64 for bigint, numeric, or float32 for real; float64 is a no-op). +// match convKind's target type (int64 for bigint, numeric, or float32 for real; float64 is +// a no-op). type aggCastBuffer struct { - inner sql.AggregationBuffer - targetType *pgtypes.DoltgresType + inner sql.AggregationBuffer + convKind aggConvKind } func (b *aggCastBuffer) Update(ctx *sql.Context, row sql.Row) error { @@ -133,7 +163,7 @@ func (b *aggCastBuffer) Eval(ctx *sql.Context) (any, error) { if err != nil || v == nil { return v, err } - return convertAggResult(v, b.targetType) + return convertAggResult(v, b.convKind) } func (b *aggCastBuffer) Dispose(ctx *sql.Context) { @@ -143,8 +173,8 @@ func (b *aggCastBuffer) Dispose(ctx *sql.Context) { // aggCastWindowFunction wraps a sql.WindowFunction and post-converts its float64 Compute // result the same way aggCastBuffer does for the sql.AggregationBuffer (GroupBy) path. type aggCastWindowFunction struct { - inner sql.WindowFunction - targetType *pgtypes.DoltgresType + inner sql.WindowFunction + convKind aggConvKind } func (w *aggCastWindowFunction) StartPartition(ctx *sql.Context, interval sql.WindowInterval, buffer sql.WindowBuffer) error { @@ -160,7 +190,7 @@ func (w *aggCastWindowFunction) Compute(ctx *sql.Context, interval sql.WindowInt if err != nil || v == nil { return v, err } - return convertAggResult(v, w.targetType) + return convertAggResult(v, w.convKind) } func (w *aggCastWindowFunction) Dispose(ctx *sql.Context) { @@ -168,23 +198,23 @@ func (w *aggCastWindowFunction) Dispose(ctx *sql.Context) { } // convertAggResult converts a raw aggregation result (always float64 from GMS's SUM/AVG -// implementations, regardless of the input column's width) to match targetType: numeric, +// implementations, regardless of the input column's width) to match convKind: numeric, // float32, float64 (a no-op), or int64 for everything else (bigint). -func convertAggResult(v any, targetType *pgtypes.DoltgresType) (any, error) { +func convertAggResult(v any, convKind aggConvKind) (any, error) { f, ok := v.(float64) if !ok { return v, nil } - switch { - case targetType.Equals(pgtypes.Numeric): - d, _, err := apd.NewFromString(strconv.FormatFloat(f, 'f', -1, 64)) + switch convKind { + case aggConvNumeric: + d, err := new(apd.Decimal).SetFloat64(f) if err != nil { return nil, err } return d, nil - case targetType.Equals(pgtypes.Float32): + case aggConvFloat32: return float32(f), nil - case targetType.Equals(pgtypes.Float64): + case aggConvFloat64: return f, nil default: return int64(math.RoundToEven(f)), nil