From 8690cd37f0dc8c4957f9af6fab1d8ee0effd694f Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Sat, 27 Jun 2026 19:25:02 +0100 Subject: [PATCH 1/2] fix(parser): skip interpolated templates in disabled-feature recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the parser rejects a disabled traditional `for (;;)` loop, it records the warning and skips the loop's header and body via the error-recovery routines (SkipBalancedParens, SkipBlock, SkipUntilSemicolon). Those routines counted braces token by token, but the `}` that closes a `${ ... }` template substitution is lexed as a plain RightBrace under the default goal — the TemplateMiddle/TemplateTail span is only produced when the lexer is invoked with the template-tail goal *after* that `}`. So the skip miscounted the substitution's `}` as the structural brace ending the skipped region, stopped early, and the trailing backtick was re-scanned as a fresh, unterminated template literal running to EOF. The result was a misleading "Unterminated template literal" error at EOF instead of the real "enable --compat-traditional-for-loop" diagnostic at the `for` keyword. Add a shared SkipTemplateLiteral helper that skips a complete template literal as a unit — recursing through nested substitutions and templates, balancing inner object/block braces, and re-lexing each continuation span with the template-tail goal (as ParseTemplateLiteral does). Route SkipBlock, SkipBalancedParens, and SkipUntilSemicolon through it, which fixes the whole class of disabled-feature recovery skips (for-loop body/header/non-block body, and the sibling while / do-while constructs), not just the reported case. Tests (both verified red without the parser change): - tests/language/statements/unsupported-features.js: a disabled for-loop with an interpolated template (block body, non-block body, header, nested) is skipped as a clean no-op and subsequent code runs. - scripts/test-cli-parser.ts: recovery succeeds in interpreter and bytecode mode, never regresses to "Unterminated template literal", and the human-readable diagnostic references --compat-traditional-for-loop. Co-Authored-By: Claude Opus 4.8 --- scripts/test-cli-parser.ts | 76 +++++++++++++++++ source/units/Goccia.Parser.pas | 83 +++++++++++++++++-- .../statements/unsupported-features.js | 26 ++++++ 3 files changed, 179 insertions(+), 6 deletions(-) diff --git a/scripts/test-cli-parser.ts b/scripts/test-cli-parser.ts index cad841328..2e3f0bd69 100644 --- a/scripts/test-cli-parser.ts +++ b/scripts/test-cli-parser.ts @@ -178,4 +178,80 @@ console.log("Unsupported var recovery (ASI and compat-var flags)..."); if (compatVarNoAsiRes.json.error?.type !== "SyntaxError") throw new Error(`Expected SyntaxError without ASI, got: ${compatVarNoAsiRes.json.error?.type}`); } +// -- Disabled traditional for-loop with interpolated template literal ----------- +// Regression: error recovery for a disabled `for (;;)` loop must skip a +// `${ ... }` substitution as part of its template literal. Previously the +// substitution's closing brace was miscounted as the structural brace that ends +// the skipped region, and the trailing backtick was then re-scanned as a fresh, +// unterminated template literal -- surfacing "Unterminated template literal" at +// EOF instead of the real "enable --compat-traditional-for-loop" diagnostic. + +console.log("Disabled traditional for-loop with interpolated template literal..."); +{ + const recoveryCases = [ + { + desc: "interpolated template in block body", + source: [ + "const obj = {};", + "for (let level = 0; level < 2; level++) {", + " obj[`u${level}`] = level;", + "}", + "console.log(JSON.stringify(obj));", + "", + ].join("\n"), + expected: "{}\n", + }, + { + desc: "interpolated template in non-block body", + source: [ + "const obj = {};", + "for (let i = 0; i < 2; i++) obj[`u${i}`] = i;", + 'console.log("after");', + "", + ].join("\n"), + expected: "after\n", + }, + { + desc: "interpolated template in loop header", + source: [ + "let x = 1;", + "for (let i = `s${0}`.length; i < 2; i++) { x = 99; }", + "console.log(x);", + "", + ].join("\n"), + expected: "1\n", + }, + ]; + + for (const { desc, source, expected } of recoveryCases) { + for (const args of [[] as string[], ["--mode=bytecode"]]) { + const label = args.length ? `${desc} (bytecode)` : desc; + const { exitCode, json } = runLoaderJson(source, args); + if (json.ok !== true) { + if (json.error?.message === "Unterminated template literal") + throw new Error( + `${label}: regressed -- a disabled traditional for-loop with a template literal surfaced "Unterminated template literal" instead of recovering`, + ); + throw new Error(`${label}: should recover, got ok=${json.ok} error=${JSON.stringify(json.error)}`); + } + if (exitCode !== 0) throw new Error(`${label}: should exit 0, got ${exitCode}`); + if (normalizeLineEndings(json.output) !== expected) + throw new Error(`${label}: expected output ${JSON.stringify(expected)}, got ${JSON.stringify(json.output)}`); + } + } + + // The human-readable diagnostic must name the compat flag, not the lexer's + // unterminated-template error. + const diag = Bun.spawnSync([LOADER], { + stdin: new TextEncoder().encode(recoveryCases[0].source), + stdout: "pipe", + stderr: "pipe", + }); + const diagOut = `${diag.stdout.toString()}${diag.stderr.toString()}`; + if (!diagOut.includes("--compat-traditional-for-loop")) + throw new Error(`Expected the diagnostic to reference --compat-traditional-for-loop, got: ${diagOut}`); + if (diagOut.includes("Unterminated template literal")) + throw new Error(`Diagnostic should not mention an unterminated template literal, got: ${diagOut}`); +} + console.log("\nAll test-cli-parser.ts tests passed."); diff --git a/source/units/Goccia.Parser.pas b/source/units/Goccia.Parser.pas index f9e98b37f..47fb80170 100644 --- a/source/units/Goccia.Parser.pas +++ b/source/units/Goccia.Parser.pas @@ -232,6 +232,7 @@ TGocciaParser = class procedure SkipBlock; procedure SkipBalancedParens; procedure SkipStatementOrBlock; + procedure SkipTemplateLiteral; procedure SkipUnsupportedFunctionSignature; // Centralized 'async function' check: enforces same-line constraint, // handles disabled-mode fallback. Call after consuming the 'async' token. @@ -4723,6 +4724,58 @@ function TGocciaParser.IfStatement: TGocciaStatement; Line, Column); end; +// Skips a complete template literal whose leading token (gttTemplate or +// gttTemplateHead) is at the cursor, leaving the cursor just past the closing +// backtick. The error-recovery skips below (SkipBlock, SkipBalancedParens, +// SkipUntilSemicolon) must route templates through here: the '}' that closes a +// '${ ... }' substitution is lexed as a plain gttRightBrace under the default +// goal, so naive brace/paren counting would miscount it as a structural brace, +// stop early, and then re-scan the trailing span as a fresh, unterminated +// template literal. After consuming that '}', the continuation span must be +// re-lexed with the template-tail goal, exactly as ParseTemplateLiteral does. +procedure TGocciaParser.SkipTemplateLiteral; +var + Depth: Integer; +begin + // A template with no substitutions is a single token. + if Check(gttTemplate) then + begin + Advance; + Exit; + end; + + Advance; // gttTemplateHead: `...${ + repeat + // Skip the substitution expression up to the '}' that closes it, treating + // nested templates as units and balancing any inner object/block braces. + Depth := 0; + while not IsAtEnd do + begin + if Check(gttTemplateHead) or Check(gttTemplate) then + SkipTemplateLiteral + else if Check(gttLeftBrace) then + begin + Inc(Depth); + Advance; + end + else if Check(gttRightBrace) then + begin + if Depth = 0 then + Break; + Dec(Depth); + Advance; + end + else + Advance; + end; + if IsAtEnd then + Exit; + Advance; // gttRightBrace closing the substitution + // Re-lex the following span with the template-tail goal so it becomes a + // TemplateMiddle/TemplateTail token instead of a stray template start. + until ConsumeTemplateContinuation.TokenType = gttTemplateTail; +end; + procedure TGocciaParser.SkipBlock; var Depth: Integer; @@ -4732,9 +4785,14 @@ procedure TGocciaParser.SkipBlock; Depth := 1; while not IsAtEnd and (Depth > 0) do begin - if Check(gttLeftBrace) then Inc(Depth) - else if Check(gttRightBrace) then Dec(Depth); - if Depth > 0 then Advance; + if Check(gttTemplateHead) or Check(gttTemplate) then + SkipTemplateLiteral + else + begin + if Check(gttLeftBrace) then Inc(Depth) + else if Check(gttRightBrace) then Dec(Depth); + if Depth > 0 then Advance; + end; end; Consume(gttRightBrace, 'Expected "}"', SSuggestCloseBlock); @@ -4749,9 +4807,14 @@ procedure TGocciaParser.SkipBalancedParens; Depth := 1; while not IsAtEnd and (Depth > 0) do begin - if Check(gttLeftParen) then Inc(Depth) - else if Check(gttRightParen) then Dec(Depth); - if Depth > 0 then Advance; + if Check(gttTemplateHead) or Check(gttTemplate) then + SkipTemplateLiteral + else + begin + if Check(gttLeftParen) then Inc(Depth) + else if Check(gttRightParen) then Dec(Depth); + if Depth > 0 then Advance; + end; end; Consume(gttRightParen, 'Expected ")"', SSuggestCloseParenExpression); @@ -7610,6 +7673,14 @@ procedure TGocciaParser.SkipUntilSemicolon; (Previous.Line < Peek.Line) then Exit; + // Skip template literals whole so a substitution's closing '}' is not + // miscounted as a structural brace (see SkipTemplateLiteral). + if Check(gttTemplateHead) or Check(gttTemplate) then + begin + SkipTemplateLiteral; + Continue; + end; + case Peek.TokenType of gttLeftParen, gttLeftBracket, gttLeftBrace, gttLess: Inc(Depth); gttRightParen, gttRightBracket, gttRightBrace, gttGreater: Dec(Depth); diff --git a/tests/language/statements/unsupported-features.js b/tests/language/statements/unsupported-features.js index b121a418d..2d43bb3e0 100644 --- a/tests/language/statements/unsupported-features.js +++ b/tests/language/statements/unsupported-features.js @@ -43,6 +43,32 @@ describe.runIf(hasGoccia)("unsupported features are skipped", () => { expect(x).toBe(1); }); + test("for loop with interpolated template literal in block body is skipped", () => { + const obj = {}; + for (let level = 0; level < 2; level++) { + obj[`u${level}`] = level; + } + expect(Object.keys(obj).length).toBe(0); + }); + + test("for loop with interpolated template literal in non-block body is skipped", () => { + const obj = {}; + for (let i = 0; i < 2; i++) obj[`u${i}`] = i; + expect(Object.keys(obj).length).toBe(0); + }); + + test("for loop with interpolated template literal in its header is skipped", () => { + let x = 1; + for (let i = `s${0}`.length; i < 2; i++) { x = 99; } + expect(x).toBe(1); + }); + + test("for loop with nested interpolated template literals is skipped", () => { + const parts = []; + for (let i = 0; i < 2; i++) { parts.push(`a${`b${i}c`}d`); } + expect(parts.length).toBe(0); + }); + test("code after skipped while loop executes", () => { let x = 1; while (x < 10) { x = 99; } From 4e156c6afde1d9238711841378fd07cb3a026e9e Mon Sep 17 00:00:00 2001 From: Johannes Stein Date: Sat, 27 Jun 2026 20:21:07 +0100 Subject: [PATCH 2/2] test(parser): cover while/do-while template-literal recovery The disabled while and do...while recovery paths share the same SkipBalancedParens / SkipBlock + SkipTemplateLiteral flow as the traditional for-loop, so extend the CLI parser recovery cases with one regression each: a while loop with an interpolated template in its condition and a do...while loop with one in its body. Each case now carries its own compat flag and the human-readable diagnostic assertion runs per case, so json.ok, exit code, expected output, and the compat-flag diagnostic stay covered across the whole shared parser contract. Co-Authored-By: Claude Opus 4.8 --- scripts/test-cli-parser.ts | 79 +++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 26 deletions(-) diff --git a/scripts/test-cli-parser.ts b/scripts/test-cli-parser.ts index 2e3f0bd69..16a23f17f 100644 --- a/scripts/test-cli-parser.ts +++ b/scripts/test-cli-parser.ts @@ -178,19 +178,22 @@ console.log("Unsupported var recovery (ASI and compat-var flags)..."); if (compatVarNoAsiRes.json.error?.type !== "SyntaxError") throw new Error(`Expected SyntaxError without ASI, got: ${compatVarNoAsiRes.json.error?.type}`); } -// -- Disabled traditional for-loop with interpolated template literal ----------- -// Regression: error recovery for a disabled `for (;;)` loop must skip a -// `${ ... }` substitution as part of its template literal. Previously the -// substitution's closing brace was miscounted as the structural brace that ends -// the skipped region, and the trailing backtick was then re-scanned as a fresh, -// unterminated template literal -- surfacing "Unterminated template literal" at -// EOF instead of the real "enable --compat-traditional-for-loop" diagnostic. +// -- Disabled-feature recovery with interpolated template literals -------------- +// Regression: error recovery for a disabled construct must skip a `${ ... }` +// substitution as part of its template literal. Previously the substitution's +// closing brace was miscounted as the structural brace that ends the skipped +// region, and the trailing backtick was then re-scanned as a fresh, unterminated +// template literal -- surfacing "Unterminated template literal" at EOF instead of +// the real compat-flag diagnostic. The for, while, and do...while recovery paths +// all share the SkipBalancedParens/SkipBlock/SkipUntilSemicolon + SkipTemplate- +// Literal flow, so each is covered here. -console.log("Disabled traditional for-loop with interpolated template literal..."); +console.log("Disabled-feature recovery with interpolated template literals..."); { const recoveryCases = [ { - desc: "interpolated template in block body", + desc: "for-loop, interpolated template in block body", + compatFlag: "--compat-traditional-for-loop", source: [ "const obj = {};", "for (let level = 0; level < 2; level++) {", @@ -202,7 +205,8 @@ console.log("Disabled traditional for-loop with interpolated template literal... expected: "{}\n", }, { - desc: "interpolated template in non-block body", + desc: "for-loop, interpolated template in non-block body", + compatFlag: "--compat-traditional-for-loop", source: [ "const obj = {};", "for (let i = 0; i < 2; i++) obj[`u${i}`] = i;", @@ -212,7 +216,8 @@ console.log("Disabled traditional for-loop with interpolated template literal... expected: "after\n", }, { - desc: "interpolated template in loop header", + desc: "for-loop, interpolated template in loop header", + compatFlag: "--compat-traditional-for-loop", source: [ "let x = 1;", "for (let i = `s${0}`.length; i < 2; i++) { x = 99; }", @@ -221,16 +226,38 @@ console.log("Disabled traditional for-loop with interpolated template literal... ].join("\n"), expected: "1\n", }, + { + desc: "while-loop, interpolated template in condition", + compatFlag: "--compat-while-loops", + source: [ + "let n = 0;", + "while (`v${n}`.length > 99) { n = 99; }", + 'console.log("while-after");', + "", + ].join("\n"), + expected: "while-after\n", + }, + { + desc: "do...while loop, interpolated template in body", + compatFlag: "--compat-while-loops", + source: [ + "let n = 0;", + "do { const s = `d${n}`; } while (n > 99);", + 'console.log("do-after");', + "", + ].join("\n"), + expected: "do-after\n", + }, ]; - for (const { desc, source, expected } of recoveryCases) { + for (const { desc, compatFlag, source, expected } of recoveryCases) { for (const args of [[] as string[], ["--mode=bytecode"]]) { const label = args.length ? `${desc} (bytecode)` : desc; const { exitCode, json } = runLoaderJson(source, args); if (json.ok !== true) { if (json.error?.message === "Unterminated template literal") throw new Error( - `${label}: regressed -- a disabled traditional for-loop with a template literal surfaced "Unterminated template literal" instead of recovering`, + `${label}: regressed -- a disabled construct with a template literal surfaced "Unterminated template literal" instead of recovering`, ); throw new Error(`${label}: should recover, got ok=${json.ok} error=${JSON.stringify(json.error)}`); } @@ -238,20 +265,20 @@ console.log("Disabled traditional for-loop with interpolated template literal... if (normalizeLineEndings(json.output) !== expected) throw new Error(`${label}: expected output ${JSON.stringify(expected)}, got ${JSON.stringify(json.output)}`); } - } - // The human-readable diagnostic must name the compat flag, not the lexer's - // unterminated-template error. - const diag = Bun.spawnSync([LOADER], { - stdin: new TextEncoder().encode(recoveryCases[0].source), - stdout: "pipe", - stderr: "pipe", - }); - const diagOut = `${diag.stdout.toString()}${diag.stderr.toString()}`; - if (!diagOut.includes("--compat-traditional-for-loop")) - throw new Error(`Expected the diagnostic to reference --compat-traditional-for-loop, got: ${diagOut}`); - if (diagOut.includes("Unterminated template literal")) - throw new Error(`Diagnostic should not mention an unterminated template literal, got: ${diagOut}`); + // The human-readable diagnostic must name the compat flag, not the lexer's + // unterminated-template error. Parsing is shared across modes, so check once. + const diag = Bun.spawnSync([LOADER], { + stdin: new TextEncoder().encode(source), + stdout: "pipe", + stderr: "pipe", + }); + const diagOut = `${diag.stdout.toString()}${diag.stderr.toString()}`; + if (!diagOut.includes(compatFlag)) + throw new Error(`${desc}: expected the diagnostic to reference ${compatFlag}, got: ${diagOut}`); + if (diagOut.includes("Unterminated template literal")) + throw new Error(`${desc}: diagnostic should not mention an unterminated template literal, got: ${diagOut}`); + } } console.log("\nAll test-cli-parser.ts tests passed.");