Add formatExplainJSON exporter for ClickHouse EXPLAIN AST json = 1 with vendored reference suite

docker-compose.yml

@@ -1,6 +1,9 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:26.2.4
+    # Override CLICKHOUSE_IMAGE to test against a custom build, e.g. one
+    # containing the `EXPLAIN AST json = 1` PR (see
+    # tests/clickhouse-reference-ast-json/README.md).
+    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:26.2.4}
     environment:
       - CLICKHOUSE_USER=default
       - CLICKHOUSE_PASSWORD=clickhouse

package.json

@@ -39,13 +39,15 @@
     "prepublishOnly": "npm run typecheck && npm run lint && npm run test && npm run build",
     "release": "npm publish --access public",
     "generate:expected-outputs": "npx tsx scripts/generate-expected-outputs.ts",
+    "generate:ast-json-fixtures": "tsx scripts/generate-ast-json-fixtures.ts",
     "parse": "tsx scripts/parse.ts",
     "format": "tsx scripts/format.ts",
     "explain": "tsx scripts/explain.ts",
     "explain:ch": "node scripts/clickhouse-explain.js",
     "diff:ast": "tsx scripts/diff-ast.ts",
     "diff:format": "tsx scripts/diff-format.ts",
-    "diff:explain": "tsx scripts/diff-explain.ts"
+    "diff:explain": "tsx scripts/diff-explain.ts",
+    "diff:ast-json": "tsx scripts/diff-ast-json.ts"
   },
   "keywords": [
     "clickhouse",

scripts/clickhouse-explain.js

@@ -2,23 +2,28 @@
 'use strict';
 
 // Run EXPLAIN AST against a local ClickHouse server (default http://localhost:8125).
-// Usage: npm run explain:ch "<SQL>"
+// Usage: npm run explain:ch -- [--json] "<SQL>"
+//
+// With --json, runs `EXPLAIN AST json = 1` (requires a ClickHouse build that
+// contains the JSON AST PR — see tests/clickhouse-reference-ast-json/README.md).
 
 const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL || 'http://localhost:8125';
 const USER = process.env.CLICKHOUSE_USER || 'default';
 const PASSWORD = process.env.CLICKHOUSE_PASSWORD || 'clickhouse';
 
-const sql = process.argv.slice(2).join(' ');
+const args = process.argv.slice(2);
+const jsonMode = args.includes('--json');
+const sql = args.filter((a) => a !== '--json').join(' ');
 if (!sql) {
-  process.stderr.write('Usage: npm run explain:ch "<SQL>"\n');
+  process.stderr.write('Usage: npm run explain:ch -- [--json] "<SQL>"\n');
   process.exit(1);
 }
 
 const url = `${CLICKHOUSE_URL}/?user=${encodeURIComponent(USER)}&password=${encodeURIComponent(PASSWORD)}&default_format=Raw&union_default_mode=ALL`;
 
 fetch(url, {
   method: 'POST',
-  body: `EXPLAIN AST ${sql}`,
+  body: `EXPLAIN AST ${jsonMode ? 'json = 1 ' : ''}${sql}`,
   signal: AbortSignal.timeout(30_000),
 })
   .then(async (res) => {

scripts/diff-ast-json.ts

@@ -0,0 +1,63 @@
+#!/usr/bin/env tsx
+
+/**
+ * Shows the `EXPLAIN AST json = 1` output produced by explainJSON() for one or
+ * more vendored reference cases (tests/clickhouse-reference-ast-json/cases),
+ * the expected JSON generated by the reference ClickHouse build, and the diff
+ * between them.
+ *
+ * Both sides are re-serialized with sorted object keys so key order never
+ * causes spurious diffs (the comparison is structural, like the test suite).
+ *
+ * Run via: npm run diff:ast-json -- <reference> [options]   (see --help)
+ */
+
+import { explainJSON, parse } from '../src/index.js';
+import { parseCli, run } from './diff-lib.js';
+
+const CASES_DIR = new URL('../tests/clickhouse-reference-ast-json/cases', import.meta.url)
+  .pathname;
+
+function sortKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortKeys);
+  if (value !== null && typeof value === 'object') {
+    const result: Record<string, unknown> = {};
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      result[key] = sortKeys((value as Record<string, unknown>)[key]);
+    }
+    return result;
+  }
+  return value;
+}
+
+function canonicalize(json: unknown): string {
+  return JSON.stringify(sortKeys(json), null, 2);
+}
+
+const opts = parseCli(
+  process.argv,
+  'diff:ast-json',
+  'explainJSON() output vs. vendored EXPLAIN AST json = 1 output',
+);
+
+run(
+  opts,
+  '.json',
+  (sql) => {
+    const statements = parse(sql);
+    if (statements.length !== 1) {
+      throw new Error(`expected exactly one statement, got ${statements.length}`);
+    }
+    return canonicalize(explainJSON(statements[0]).ast);
+  },
+  {
+    dir: CASES_DIR,
+    expectedPathFor: (sqlPath) => sqlPath.replace(/\.sql$/, '.json'),
+    normalizeExpected: (expected) => {
+      const parsed = JSON.parse(expected) as Record<string, unknown>;
+      // Fixtures may be the bare AST node or the {version, ast} document.
+      return canonicalize('ast' in parsed ? parsed.ast : parsed);
+    },
+  },
+);
+

scripts/diff-lib.ts

@@ -24,13 +24,13 @@ const EXPLAIN_ERROR = '<Explain Error>';
 
 // ── Output computation (shared by the plain and diff scripts) ───────────────────
 
-/** Recursively removes `location` and `parent` keys, matching tests/helpers.ts. */
+/** Recursively removes `location`, `parent`, and `isOperator` keys, matching tests/helpers.ts. */
 export function stripMeta(value: unknown): unknown {
   if (value === null || value === undefined || typeof value !== 'object') return value;
   if (Array.isArray(value)) return value.map(stripMeta);
   const result: Record<string, unknown> = {};
   for (const [k, v] of Object.entries(value as Record<string, unknown>)) {
-    if (k === 'location' || k === 'parent') continue;
+    if (k === 'location' || k === 'parent' || k === 'isOperator') continue;
     result[k] = stripMeta(v);
   }
   return result;
@@ -85,13 +85,13 @@ export type Compute = (sql: string, expected: string | null) => string;
  * pattern (matched against the `.sql` filenames). The `.sql` suffix is optional
  * for exact-name matches.
  */
-export function resolveReferences(selector: string): string[] {
-  if (!fs.existsSync(CLICKHOUSE_DIR)) {
-    throw new Error(`Reference directory not found: ${CLICKHOUSE_DIR}`);
+export function resolveReferences(selector: string, dir: string = CLICKHOUSE_DIR): string[] {
+  if (!fs.existsSync(dir)) {
+    throw new Error(`Reference directory not found: ${dir}`);
   }
 
   const allCases = fs
-    .readdirSync(CLICKHOUSE_DIR)
+    .readdirSync(dir)
     .filter((f) => f.endsWith('.sql') && !f.includes('.expected.'));
   const caseSet = new Set(allCases);
 
@@ -291,8 +291,24 @@ function renderCase(result: CaseResult, opts: CliOptions): boolean {
  * Top-level runner for the diff scripts. Resolves references, computes each case via
  * `compute`, renders them, and prints a summary / sets exit code.
  */
-export function run(opts: CliOptions, expectedSuffix: string, compute: Compute): void {
-  const files = resolveReferences(opts.selector);
+/** Optional corpus layout overrides for {@link run}. */
+export interface RunLayout {
+  /** Directory holding the `.sql` cases. Defaults to {@link CLICKHOUSE_DIR}. */
+  dir?: string;
+  /** Maps a case's `.sql` path to its expected-output path. Defaults to appending the suffix. */
+  expectedPathFor?: (sqlPath: string) => string;
+  /** Normalizes the raw expected file contents before comparison/diffing. */
+  normalizeExpected?: (expected: string) => string;
+}
+
+export function run(
+  opts: CliOptions,
+  expectedSuffix: string,
+  compute: Compute,
+  layout: RunLayout = {},
+): void {
+  const dir = layout.dir ?? CLICKHOUSE_DIR;
+  const files = resolveReferences(opts.selector, dir);
 
   if (files.length === 0) {
     process.stderr.write(pc.red('No reference cases matched the given selector.\n'));
@@ -303,9 +319,12 @@ export function run(opts: CliOptions, expectedSuffix: string, compute: Compute):
   let shown = 0;
 
   for (const fileName of files) {
-    const sqlPath = path.join(CLICKHOUSE_DIR, fileName);
-    const expectedPath = `${sqlPath}${expectedSuffix}`;
-    const expected = fs.existsSync(expectedPath) ? fs.readFileSync(expectedPath, 'utf-8') : null;
+    const sqlPath = path.join(dir, fileName);
+    const expectedPath = layout.expectedPathFor
+      ? layout.expectedPathFor(sqlPath)
+      : `${sqlPath}${expectedSuffix}`;
+    let expected = fs.existsSync(expectedPath) ? fs.readFileSync(expectedPath, 'utf-8') : null;
+    if (expected !== null && layout.normalizeExpected) expected = layout.normalizeExpected(expected);
 
     let actual: string;
     try {

scripts/generate-ast-json-fixtures.ts

@@ -0,0 +1,70 @@
+#!/usr/bin/env tsx
+
+/**
+ * Regenerates the vendored `EXPLAIN AST json = 1` fixtures in
+ * tests/clickhouse-reference-ast-json/cases from their `.sql` inputs.
+ *
+ * Requires a ClickHouse build that contains the JSON AST PR (see the corpus
+ * README for the source PR and pinned commit SHA). Point docker-compose at
+ * such a build and start it first:
+ *
+ *   CLICKHOUSE_IMAGE=<image-with-json-ast-pr> docker compose up -d
+ *   npm run generate:ast-json-fixtures
+ *
+ * Until such an image is available the vendored copies are the source of
+ * truth — this script will fail against a stock ClickHouse server because it
+ * does not understand the `json` EXPLAIN AST option.
+ */
+
+import { readFileSync, readdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+
+const CASES_DIR = new URL('../tests/clickhouse-reference-ast-json/cases', import.meta.url)
+  .pathname;
+
+const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL || 'http://localhost:8125';
+const USER = process.env.CLICKHOUSE_USER || 'default';
+const PASSWORD = process.env.CLICKHOUSE_PASSWORD || 'clickhouse';
+
+async function explainASTJSON(sql: string): Promise<string> {
+  const url =
+    `${CLICKHOUSE_URL}/?user=${encodeURIComponent(USER)}&password=${encodeURIComponent(PASSWORD)}&default_format=Raw&union_default_mode=ALL`;
+  const res = await fetch(url, {
+    method: 'POST',
+    body: `EXPLAIN AST json = 1 ${sql}`,
+    signal: AbortSignal.timeout(30_000),
+  });
+  const text = await res.text();
+  if (!res.ok) {
+    throw new Error(`HTTP ${res.status}: ${text}`);
+  }
+  return text;
+}
+
+async function main(): Promise<void> {
+  const cases = readdirSync(CASES_DIR)
+    .filter((f) => f.endsWith('.sql'))
+    .sort();
+  let failures = 0;
+  for (const fileName of cases) {
+    const sql = readFileSync(join(CASES_DIR, fileName), 'utf-8').trim();
+    const jsonPath = join(CASES_DIR, fileName.replace(/\.sql$/, '.json'));
+    try {
+      const output = await explainASTJSON(sql);
+      writeFileSync(jsonPath, output.trimEnd() + '\n', 'utf-8');
+      process.stdout.write(`✓ ${fileName}\n`);
+    } catch (e) {
+      failures++;
+      process.stderr.write(`✗ ${fileName}: ${e instanceof Error ? e.message : String(e)}\n`);
+    }
+  }
+  if (failures > 0) {
+    process.stderr.write(
+      `\n${failures} case(s) failed. Is a ClickHouse server with the JSON AST PR running ` +
+        `at ${CLICKHOUSE_URL}? See tests/clickhouse-reference-ast-json/README.md.\n`,
+    );
+    process.exit(1);
+  }
+}
+
+void main();

scripts/generate-expected-outputs.ts

@@ -257,7 +257,7 @@ async function processFile(file: string): Promise<void> {
 
   try {
     ast = parse(content);
-    astOutput = JSON.stringify(ast, (key, value) => (key === 'location' || key === 'parent' ? undefined : value), 2);
+    astOutput = JSON.stringify(ast, (key, value) => (key === 'location' || key === 'parent' || key === 'isOperator' ? undefined : value), 2);
     try {
       formattedOutput = format(ast);
     } catch (e) {