From d294477db7a94e2c63bdabe5d5df263ca0341baf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 13:38:47 +0000 Subject: [PATCH] fix(validator): accept any order among rows tied on the ORDER BY keys An ORDER BY only constrains relative order between rows whose sort keys differ; rows that tie on the keys may come out in any order depending on how the engine groups/scans. The validator compared rows strictly positionally whenever the solution had an ORDER BY, so a correct answer could be rejected just because its tied rows happened to land in a different order than the solution query's. Level 20 hit this hardest: every customer in the result has exactly 2 accounts, so ORDER BY account_count DESC constrains nothing, yet only one specific permutation was accepted. The fix parses the solution's outer ORDER BY terms into result-column indices and compares each run of key-tied expected rows as a multiset. When a term can't be resolved to a result column, comparison falls back to the previous strict behavior. https://claude.ai/code/session_01GJY9mTyQU4xL6eVnfYg2aH --- src/lib/validator.ts | Bin 6106 -> 9086 bytes tests/levels.test.ts | 28 +++++++++++++++++-- tests/validator.test.ts | 60 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/src/lib/validator.ts b/src/lib/validator.ts index ce3dd3f156a2d06a233e639cf6f99f4d6df6338e..4b7ac2af5a9300c1e977fe7bf8b916d093933447 100644 GIT binary patch delta 2752 zcmai0&5j#I5SCy`z!CWo0;DJsg%qu4*52{1#Dz_qD3E|q2rD)~1h$j3$31py+ucrg z&pIZK%sZH;h=c@+L|%atH!d7FaODBG0ae{I{>g&yVRvS_y6UU1s=n@D8-IQ9{dl9V zO^=(Kd}5&q{`nSjjvHG6{9Oh9&>5Dx{le-a)e|YVk<~JHLaR@6ZYFiK8u8Te-T}9{ zbV-Vo*SIy)OY@ zhG?Tfgvj{c?=U_R6p@k&Y{3C{xlz`KgYkeXU~1wWK+06rL6-w=)AES-$RnE-+acmf zKLOz|JX7Qf^Fd3C34+qDjA^;VEI|tE;yQgC4@5dZ=a%CiDUmg>1D(qrWcUhQmL8Hf zjAfk>xOJqhAljJ3(8H4>W`xyhe(l5?Z~h0-bN0f8i$#`u-i_+FI+d7M9d@~Cbzm9@ z{_W)_vs5eVV8|T8YJzI|g?sc~hiVlgEc_HzTtVN2 zR)^wb+CUV788-e5%MLn&5I3YX|x&uKK8!9te0a(MvY&z|6{DUM) z<~UN&9@nPSlNO(&Oc?8?l3`5|d81E|p&D8jU18mZk>@C|)VSO3)h;#?$E}Mf`~neT zntA#~?;&FP#FrK8Xz;p`qc%`}MW64$D4UQGTZwv#FjB-054+SVs zg%c|0mqPxJeV|4IZX@;)FmS9Zd4oy8EIi2ZeCu#+MxSs`^8tlff=M4iBzU3Ns zTi3R~V%uNe>l6<=+wb1%Y?TC4OeFZZmMjqkqDu+Lgqrny%qU7@XkLLdar$FD=H`8z z&eE|Dq@{5n9W8y3l;B7_r{9+j7LnIO@rLdmDv7?f5L!ugMt&)=#et$vXXrmHJz z5wz~?EGGKmmqPy?9jt@z^UbAz^FyH3ik_1Dc;MhL)n^?(VU%pZNuM|9-V%M zwJnt8-%uUkY%fu)zb!GXjz=AK5ZB{MCCgPxmyl!GVgpdZn+40jEQL{?3^{UI}(kgZYKC UyYny3{xq+i`|jy4=UzJT4?7EvDF6Tf delta 66 zcmez8c1wRlIm_f1ERLH^SpTt1{>=Y+vH+*(WFDc=%?(18%#)2J*G;}F^>*?#X}!ry VL { return; // seed scaffold doesn't even parse — that's fine } const orderMatters = level.orderMatters ?? solutionRequiresOrder(level.solutionQuery); - expect(compareResults(seedResult, expected, orderMatters)).not.toBeNull(); + const orderKeys = orderMatters + ? extractOrderKeyIndices(level.solutionQuery, expected.columns) + : null; + expect(compareResults(seedResult, expected, orderMatters, orderKeys)).not.toBeNull(); + }); +}); + +describe('order-only ties accept any tie order (level 20 regression)', () => { + it('accepts a correct query whose tied rows come out in a different order', () => { + const level = levels.find((l) => l.id === 20)!; + const userQuery = `SELECT + customer_name, + count(distinct account_id) as account_count + FROM customers c + JOIN accounts a ON c.customer_id = a.customer_id + GROUP BY customer_name +HAVING account_count > 1 + ORDER BY account_count desc`; + const expected = runReadOnlyQuery(db, level.solutionQuery); + const user = runReadOnlyQuery(db, userQuery); + const orderMatters = level.orderMatters ?? solutionRequiresOrder(level.solutionQuery); + const orderKeys = extractOrderKeyIndices(level.solutionQuery, expected.columns); + expect(orderMatters).toBe(true); + expect(orderKeys).toEqual([1]); + expect(compareResults(user, expected, orderMatters, orderKeys)).toBeNull(); }); }); diff --git a/tests/validator.test.ts b/tests/validator.test.ts index b5ebeca..ffc536a 100644 --- a/tests/validator.test.ts +++ b/tests/validator.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { compareResults, solutionRequiresOrder, valuesEqual } from '@/lib/validator'; +import { + compareResults, + extractOrderKeyIndices, + solutionRequiresOrder, + valuesEqual, +} from '@/lib/validator'; import type { QueryResult } from '@/types'; const result = (columns: string[], values: unknown[][]): QueryResult => ({ columns, values }); @@ -75,6 +80,59 @@ describe('compareResults', () => { }); }); +describe('compareResults with ORDER BY tie groups', () => { + // ORDER BY cnt DESC: 'Ann'/'Bob' tie on cnt, 'Cy' does not. + const expected = result(['name', 'cnt'], [['Bob', 2], ['Ann', 2], ['Cy', 1]]); + + it('accepts any order among rows tied on the ORDER BY keys', () => { + const user = result(['name', 'cnt'], [['Ann', 2], ['Bob', 2], ['Cy', 1]]); + expect(compareResults(user, expected, true)).not.toBeNull(); + expect(compareResults(user, expected, true, [1])).toBeNull(); + }); + + it('rejects rows placed outside their tie group (wrong ORDER BY)', () => { + const user = result(['name', 'cnt'], [['Cy', 1], ['Ann', 2], ['Bob', 2]]); + expect(compareResults(user, expected, true, [1])).toContain('ORDER BY'); + }); + + it('rejects wrong values within a tie group', () => { + const user = result(['name', 'cnt'], [['Bob', 2], ['Zed', 2], ['Cy', 1]]); + expect(compareResults(user, expected, true, [1])).not.toBeNull(); + }); +}); + +describe('extractOrderKeyIndices', () => { + const columns = ['customer_name', 'account_count']; + + it('maps plain and qualified terms (with ASC/DESC) to column indices', () => { + expect( + extractOrderKeyIndices('SELECT ... ORDER BY account_count DESC', columns) + ).toEqual([1]); + expect( + extractOrderKeyIndices('SELECT ... ORDER BY c.customer_name, account_count ASC', columns) + ).toEqual([0, 1]); + }); + + it('supports positional terms and ignores a trailing LIMIT', () => { + expect(extractOrderKeyIndices('SELECT ... ORDER BY 2 DESC LIMIT 5', columns)).toEqual([1]); + }); + + it('ignores ORDER BY inside parentheses', () => { + expect( + extractOrderKeyIndices( + 'SELECT RANK() OVER (ORDER BY balance) AS r FROM t ORDER BY account_count', + columns + ) + ).toEqual([1]); + }); + + it('returns null when a term cannot be resolved to a column', () => { + expect(extractOrderKeyIndices('SELECT ... ORDER BY COUNT(*) DESC', columns)).toBeNull(); + expect(extractOrderKeyIndices('SELECT ... ORDER BY 3', columns)).toBeNull(); + expect(extractOrderKeyIndices('SELECT a FROM t', columns)).toBeNull(); + }); +}); + describe('solutionRequiresOrder', () => { it('detects a top-level ORDER BY', () => { expect(solutionRequiresOrder('SELECT a FROM t ORDER BY a')).toBe(true);