diff --git a/Parsing/GeneratorArrowLifter.cs b/Parsing/GeneratorArrowLifter.cs index 28cd8f5d..fc740bde 100644 --- a/Parsing/GeneratorArrowLifter.cs +++ b/Parsing/GeneratorArrowLifter.cs @@ -2,7 +2,7 @@ namespace SharpTS.Parsing; /// /// Lifts generator function EXPRESSIONS (represented as with -/// IsGenerator = true) into top-level declarations. +/// IsGenerator = true) into declarations. /// /// The IL compiler has a mature generator-declaration pipeline (GeneratorStateMachineBuilder /// + GeneratorMoveNextEmitter) but no arrow-expression counterpart. Rather than duplicate @@ -23,21 +23,20 @@ namespace SharpTS.Parsing; /// } /// /// -/// Lifted declarations are APPENDED to the end of the module body, not prepended. Function -/// declarations hoist, so placing them last is runtime-equivalent in both the interpreter and the -/// compiler (verified: a declaration is usable before its textual position). The end position -/// matters for the type checker, which checks function bodies in source order: appending means the -/// lifted body is checked AFTER the surrounding module-level let/const bindings have -/// been defined, so a generator expression closing over such a binding type-checks rather than -/// failing with "Undefined variable" (issue #522). Prepending placed the body before those -/// declarations and broke that closure. +/// Lift target. A generator expression that does NOT close over an enclosing function's +/// locals is lifted to the END of the module body. Function declarations hoist, so the trailing +/// position is runtime-equivalent in both the interpreter and the compiler, and it lets the +/// source-order type checker see any module-level let/const the body closes over before +/// that body is checked (#522). A generator expression that DOES close over an enclosing function's +/// local is instead lifted to the end of that enclosing function's body, so the lifted declaration +/// keeps the local in lexical scope (#534). The interpreter runs such nested generator declarations +/// natively; the compiler's nested-generator lowering is still incomplete (#501/#532) and reports a +/// clear "Yield not supported in this context" compile error for that case. /// -/// Limitation: only module/global-scope bindings (and this, which binds from the call -/// site) survive the lift. A generator expression nested inside another function that closes over -/// that function's LOCALS is still moved out of its lexical scope (#534); lifting it into the -/// enclosing function instead is not viable because the compiler's nested-generator-declaration -/// path is itself incomplete (#501) and the type checker infers a nested generator's yield type -/// as void (#532). +/// Traversal. The rewriter descends through every expression- and statement-bearing AST +/// position so a generator expression is found wherever it is legal to write one — call/IIFE position +/// (#488), for/for-of/for-in/do-while/try/switch/labeled +/// statement bodies (#634), ternaries, array/object literals, etc. /// /// The rewriter returns the original AST node whenever no change was required, which keeps /// downstream passes (e.g. the type checker) from seeing fresh reference identities for untouched @@ -45,9 +44,23 @@ namespace SharpTS.Parsing; /// internal sealed class GeneratorArrowLifter { - private readonly List _liftedFunctions = new(); + /// Generator expressions that close over no enclosing-function local — appended to the module body. + private readonly List _moduleLifted = new(); private int _counter; + /// + /// Stack of enclosing function/arrow scopes (innermost last). Each frame records the local + /// binding names visible in that scope and collects generator declarations that must be lifted + /// into it (because they close over one of those locals). + /// + private readonly List _frames = new(); + + private sealed class FunctionFrame + { + public required HashSet Locals { get; init; } + public List Lifted { get; } = new(); + } + public static List Lift(List body) { // Cheap pre-scan: if there are no generator function expressions anywhere, return the @@ -62,14 +75,16 @@ public static List Lift(List body) { rewritten.Add(lifter.RewriteStmt(stmt)); } - if (lifter._liftedFunctions.Count == 0) return body; + // `rewritten` already carries any frame-local lifts (injected into the rewritten function + // bodies); only the module-scope lifts still need to be appended here. + if (lifter._moduleLifted.Count == 0) return rewritten; - // Append (not prepend) the lifted declarations: function declarations hoist, so the trailing + // Append (not prepend) the module-scope lifts: function declarations hoist, so the trailing // position is runtime-equivalent, and it lets the source-order type checker see any // module-level let/const the generator body closes over before that body is checked (#522). - var result = new List(lifter._liftedFunctions.Count + rewritten.Count); + var result = new List(rewritten.Count + lifter._moduleLifted.Count); result.AddRange(rewritten); - result.AddRange(lifter._liftedFunctions); + result.AddRange(lifter._moduleLifted); return result; } @@ -100,42 +115,52 @@ protected override void VisitArrowFunction(Expr.ArrowFunction expr) } } + // --------------------------------------------------------------------------------------------- + // Statement rewriting + // --------------------------------------------------------------------------------------------- + private Stmt RewriteStmt(Stmt stmt) { return stmt switch { - Stmt.Expression e => ReplaceIfChanged(e, e.Expr, RewriteExpr(e.Expr), (s, expr) => new Stmt.Expression(expr)), - Stmt.Return r => r.Value is null ? r : ReplaceIfChanged(r, r.Value, RewriteExpr(r.Value), (s, expr) => new Stmt.Return(s.Keyword, expr)), - Stmt.Var v => v.Initializer is null ? v : ReplaceIfChanged(v, v.Initializer, RewriteExpr(v.Initializer), (s, expr) => s with { Initializer = expr }), - Stmt.Const c => ReplaceIfChanged(c, c.Initializer, RewriteExpr(c.Initializer), (s, expr) => s with { Initializer = expr }), + Stmt.Expression e => ReplaceIfChanged(e, e.Expr, RewriteExpr(e.Expr), (_, x) => new Stmt.Expression(x)), + Stmt.Return r => r.Value is null ? r : ReplaceIfChanged(r, r.Value, RewriteExpr(r.Value), (s, x) => new Stmt.Return(s.Keyword, x)), + Stmt.Var v => v.Initializer is null ? v : ReplaceIfChanged(v, v.Initializer, RewriteExpr(v.Initializer), (s, x) => s with { Initializer = x }), + Stmt.Const c => ReplaceIfChanged(c, c.Initializer, RewriteExpr(c.Initializer), (s, x) => s with { Initializer = x }), + Stmt.Throw t => ReplaceIfChanged(t, t.Value, RewriteExpr(t.Value), (s, x) => new Stmt.Throw(s.Keyword, x)), + Stmt.Print p => ReplaceIfChanged(p, p.Expr, RewriteExpr(p.Expr), (_, x) => new Stmt.Print(x)), Stmt.Block b => RewriteBlock(b), + Stmt.Sequence sq => RewriteSequence(sq), Stmt.If i => RewriteIf(i), Stmt.While w => RewriteWhile(w), - Stmt.Function f => RewriteFunction(f), + Stmt.DoWhile d => RewriteDoWhile(d), + Stmt.For f => RewriteFor(f), + Stmt.ForOf fo => RewriteForOf(fo), + Stmt.ForIn fi => RewriteForIn(fi), + Stmt.LabeledStatement ls => RewriteLabeled(ls), + Stmt.Switch sw => RewriteSwitch(sw), + Stmt.TryCatch tc => RewriteTryCatch(tc), + Stmt.Function fn => RewriteFunction(fn), + Stmt.Namespace ns => RewriteNamespace(ns), + Stmt.Export ex => RewriteExport(ex), _ => stmt, }; } private static Stmt ReplaceIfChanged(T original, Expr originalChild, Expr newChild, Func factory) where T : Stmt + => ReferenceEquals(originalChild, newChild) ? original : factory(original, newChild); + + private Stmt RewriteBlock(Stmt.Block b) { - return ReferenceEquals(originalChild, newChild) ? original : factory(original, newChild); + var rewritten = RewriteListIfChanged(b.Statements, RewriteStmt); + return ReferenceEquals(rewritten, b.Statements) ? b : new Stmt.Block(rewritten); } - private Stmt RewriteBlock(Stmt.Block b) + private Stmt RewriteSequence(Stmt.Sequence sq) { - List? rewritten = null; - for (int i = 0; i < b.Statements.Count; i++) - { - var original = b.Statements[i]; - var rewrittenStmt = RewriteStmt(original); - if (!ReferenceEquals(original, rewrittenStmt)) - { - rewritten ??= new List(b.Statements); - rewritten[i] = rewrittenStmt; - } - } - return rewritten is null ? b : new Stmt.Block(rewritten); + var rewritten = RewriteListIfChanged(sq.Statements, RewriteStmt); + return ReferenceEquals(rewritten, sq.Statements) ? sq : new Stmt.Sequence(rewritten); } private Stmt RewriteIf(Stmt.If i) @@ -158,23 +183,115 @@ private Stmt RewriteWhile(Stmt.While w) return new Stmt.While(newCond, newBody); } - private Stmt RewriteFunction(Stmt.Function f) + private Stmt RewriteDoWhile(Stmt.DoWhile d) { - if (f.Body is null) return f; - List? rewritten = null; - for (int i = 0; i < f.Body.Count; i++) + var newBody = RewriteStmt(d.Body); + var newCond = RewriteExpr(d.Condition); + if (ReferenceEquals(newBody, d.Body) && ReferenceEquals(newCond, d.Condition)) return d; + return new Stmt.DoWhile(newBody, newCond); + } + + private Stmt RewriteFor(Stmt.For f) + { + var newInit = f.Initializer is null ? null : RewriteStmt(f.Initializer); + var newCond = f.Condition is null ? null : RewriteExpr(f.Condition); + var newIncr = f.Increment is null ? null : RewriteExpr(f.Increment); + var newBody = RewriteStmt(f.Body); + if ((f.Initializer is null || ReferenceEquals(newInit, f.Initializer)) + && (f.Condition is null || ReferenceEquals(newCond, f.Condition)) + && (f.Increment is null || ReferenceEquals(newIncr, f.Increment)) + && ReferenceEquals(newBody, f.Body)) + return f; + return new Stmt.For(newInit, newCond, newIncr, newBody); + } + + private Stmt RewriteForOf(Stmt.ForOf fo) + { + var newIter = RewriteExpr(fo.Iterable); + var newBody = RewriteStmt(fo.Body); + if (ReferenceEquals(newIter, fo.Iterable) && ReferenceEquals(newBody, fo.Body)) return fo; + return fo with { Iterable = newIter, Body = newBody }; + } + + private Stmt RewriteForIn(Stmt.ForIn fi) + { + var newObj = RewriteExpr(fi.Object); + var newBody = RewriteStmt(fi.Body); + if (ReferenceEquals(newObj, fi.Object) && ReferenceEquals(newBody, fi.Body)) return fi; + return fi with { Object = newObj, Body = newBody }; + } + + private Stmt RewriteLabeled(Stmt.LabeledStatement ls) + { + var newInner = RewriteStmt(ls.Statement); + return ReferenceEquals(newInner, ls.Statement) ? ls : new Stmt.LabeledStatement(ls.Label, newInner); + } + + private Stmt RewriteSwitch(Stmt.Switch sw) + { + var newSubject = RewriteExpr(sw.Subject); + List? newCases = null; + for (int i = 0; i < sw.Cases.Count; i++) { - var original = f.Body[i]; - var rewrittenStmt = RewriteStmt(original); - if (!ReferenceEquals(original, rewrittenStmt)) + var c = sw.Cases[i]; + var newValue = RewriteExpr(c.Value); + var newBody = RewriteListIfChanged(c.Body, RewriteStmt); + if (!ReferenceEquals(newValue, c.Value) || !ReferenceEquals(newBody, c.Body)) { - rewritten ??= new List(f.Body); - rewritten[i] = rewrittenStmt; + newCases ??= new List(sw.Cases); + newCases[i] = new Stmt.SwitchCase(newValue, newBody); } } - return rewritten is null ? f : f with { Body = rewritten }; + var newDefault = sw.DefaultBody is null ? null : RewriteListIfChanged(sw.DefaultBody, RewriteStmt); + if (ReferenceEquals(newSubject, sw.Subject) && newCases is null + && (sw.DefaultBody is null || ReferenceEquals(newDefault, sw.DefaultBody))) + return sw; + return new Stmt.Switch(newSubject, newCases ?? sw.Cases, newDefault); + } + + private Stmt RewriteTryCatch(Stmt.TryCatch tc) + { + var newTry = RewriteListIfChanged(tc.TryBlock, RewriteStmt); + var newCatch = tc.CatchBlock is null ? null : RewriteListIfChanged(tc.CatchBlock, RewriteStmt); + var newFinally = tc.FinallyBlock is null ? null : RewriteListIfChanged(tc.FinallyBlock, RewriteStmt); + if (ReferenceEquals(newTry, tc.TryBlock) + && (tc.CatchBlock is null || ReferenceEquals(newCatch, tc.CatchBlock)) + && (tc.FinallyBlock is null || ReferenceEquals(newFinally, tc.FinallyBlock))) + return tc; + return tc with { TryBlock = newTry, CatchBlock = newCatch, FinallyBlock = newFinally }; + } + + private Stmt RewriteFunction(Stmt.Function f) + { + if (f.Body is null) return f; + var newParams = RewriteParameters(f.Parameters); + var rewrittenBody = RewriteFunctionBody(f.Parameters, f.Body); + if (ReferenceEquals(newParams, f.Parameters) && ReferenceEquals(rewrittenBody, f.Body)) return f; + return f with { Parameters = newParams, Body = rewrittenBody }; + } + + private Stmt RewriteNamespace(Stmt.Namespace ns) + { + var rewritten = RewriteListIfChanged(ns.Members, RewriteStmt); + return ReferenceEquals(rewritten, ns.Members) ? ns : ns with { Members = rewritten }; + } + + private Stmt RewriteExport(Stmt.Export ex) + { + var newDecl = ex.Declaration is null ? null : RewriteStmt(ex.Declaration); + var newDefault = ex.DefaultExpr is null ? null : RewriteExpr(ex.DefaultExpr); + var newAssign = ex.ExportAssignment is null ? null : RewriteExpr(ex.ExportAssignment); + if ((ex.Declaration is null || ReferenceEquals(newDecl, ex.Declaration)) + && (ex.DefaultExpr is null || ReferenceEquals(newDefault, ex.DefaultExpr)) + && (ex.ExportAssignment is null || ReferenceEquals(newAssign, ex.ExportAssignment))) + return ex; + return ex with { Declaration = newDecl, DefaultExpr = newDefault, ExportAssignment = newAssign }; } + // --------------------------------------------------------------------------------------------- + // Expression rewriting + // --------------------------------------------------------------------------------------------- + private Expr RewriteExpr(Expr expr) { switch (expr) @@ -182,42 +299,111 @@ private Expr RewriteExpr(Expr expr) case Expr.ArrowFunction af when af.IsGenerator && af.BlockBody is not null: return LiftGeneratorArrow(af); case Expr.ArrowFunction af: + return RewriteNonGeneratorArrow(af); + + case Expr.Comma c: + { + var l = RewriteExpr(c.Left); var r = RewriteExpr(c.Right); + return ReferenceEquals(l, c.Left) && ReferenceEquals(r, c.Right) ? c : new Expr.Comma(l, r); + } + case Expr.Binary b: + { + var l = RewriteExpr(b.Left); var r = RewriteExpr(b.Right); + return ReferenceEquals(l, b.Left) && ReferenceEquals(r, b.Right) ? b : new Expr.Binary(l, b.Operator, r); + } + case Expr.Logical lg: + { + var l = RewriteExpr(lg.Left); var r = RewriteExpr(lg.Right); + return ReferenceEquals(l, lg.Left) && ReferenceEquals(r, lg.Right) ? lg : new Expr.Logical(l, lg.Operator, r); + } + case Expr.NullishCoalescing nc: + { + var l = RewriteExpr(nc.Left); var r = RewriteExpr(nc.Right); + return ReferenceEquals(l, nc.Left) && ReferenceEquals(r, nc.Right) ? nc : new Expr.NullishCoalescing(l, r); + } + case Expr.Ternary t: + { + var c = RewriteExpr(t.Condition); var th = RewriteExpr(t.ThenBranch); var el = RewriteExpr(t.ElseBranch); + return ReferenceEquals(c, t.Condition) && ReferenceEquals(th, t.ThenBranch) && ReferenceEquals(el, t.ElseBranch) + ? t : new Expr.Ternary(c, th, el); + } + case Expr.Grouping g: + { + var inner = RewriteExpr(g.Expression); + return ReferenceEquals(inner, g.Expression) ? g : new Expr.Grouping(inner); + } + case Expr.Unary u: + { + var r = RewriteExpr(u.Right); + return ReferenceEquals(r, u.Right) ? u : new Expr.Unary(u.Operator, r); + } + case Expr.Delete d: { - List? newBody = af.BlockBody is null ? null : RewriteListIfChanged(af.BlockBody, RewriteStmt); - var newExpr = af.ExpressionBody is null ? null : RewriteExpr(af.ExpressionBody); - bool bodyChanged = !ReferenceEquals(newBody, af.BlockBody); - bool exprChanged = !ReferenceEquals(newExpr, af.ExpressionBody); - if (!bodyChanged && !exprChanged) return af; - return af with { BlockBody = newBody, ExpressionBody = newExpr }; + var o = RewriteExpr(d.Operand); + return ReferenceEquals(o, d.Operand) ? d : new Expr.Delete(d.Keyword, o); } case Expr.Assign a: { var v = RewriteExpr(a.Value); - return ReferenceEquals(v, a.Value) ? a : new Expr.Assign(a.Name, v); + return ReferenceEquals(v, a.Value) ? a : a with { Value = v }; } case Expr.Set s: { - var o = RewriteExpr(s.Object); - var v = RewriteExpr(s.Value); + var o = RewriteExpr(s.Object); var v = RewriteExpr(s.Value); return ReferenceEquals(o, s.Object) && ReferenceEquals(v, s.Value) ? s : new Expr.Set(o, s.Name, v); } case Expr.SetIndex si: { - var o = RewriteExpr(si.Object); - var idx = RewriteExpr(si.Index); - var v = RewriteExpr(si.Value); + var o = RewriteExpr(si.Object); var idx = RewriteExpr(si.Index); var v = RewriteExpr(si.Value); return ReferenceEquals(o, si.Object) && ReferenceEquals(idx, si.Index) && ReferenceEquals(v, si.Value) ? si : new Expr.SetIndex(o, idx, v); } - case Expr.Binary b: + case Expr.SetPrivate sp: { - var l = RewriteExpr(b.Left); var r = RewriteExpr(b.Right); - return ReferenceEquals(l, b.Left) && ReferenceEquals(r, b.Right) ? b : new Expr.Binary(l, b.Operator, r); + var o = RewriteExpr(sp.Object); var v = RewriteExpr(sp.Value); + return ReferenceEquals(o, sp.Object) && ReferenceEquals(v, sp.Value) ? sp : new Expr.SetPrivate(o, sp.Name, v); } - case Expr.Logical lg: + case Expr.CompoundAssign ca: { - var l = RewriteExpr(lg.Left); var r = RewriteExpr(lg.Right); - return ReferenceEquals(l, lg.Left) && ReferenceEquals(r, lg.Right) ? lg : new Expr.Logical(l, lg.Operator, r); + var v = RewriteExpr(ca.Value); + return ReferenceEquals(v, ca.Value) ? ca : new Expr.CompoundAssign(ca.Name, ca.Operator, v); + } + case Expr.CompoundSet cs: + { + var o = RewriteExpr(cs.Object); var v = RewriteExpr(cs.Value); + return ReferenceEquals(o, cs.Object) && ReferenceEquals(v, cs.Value) ? cs : new Expr.CompoundSet(o, cs.Name, cs.Operator, v); + } + case Expr.CompoundSetIndex csi: + { + var o = RewriteExpr(csi.Object); var idx = RewriteExpr(csi.Index); var v = RewriteExpr(csi.Value); + return ReferenceEquals(o, csi.Object) && ReferenceEquals(idx, csi.Index) && ReferenceEquals(v, csi.Value) + ? csi : new Expr.CompoundSetIndex(o, idx, csi.Operator, v); + } + case Expr.LogicalAssign la: + { + var v = RewriteExpr(la.Value); + return ReferenceEquals(v, la.Value) ? la : new Expr.LogicalAssign(la.Name, la.Operator, v); + } + case Expr.LogicalSet lset: + { + var o = RewriteExpr(lset.Object); var v = RewriteExpr(lset.Value); + return ReferenceEquals(o, lset.Object) && ReferenceEquals(v, lset.Value) ? lset : new Expr.LogicalSet(o, lset.Name, lset.Operator, v); + } + case Expr.LogicalSetIndex lsi: + { + var o = RewriteExpr(lsi.Object); var idx = RewriteExpr(lsi.Index); var v = RewriteExpr(lsi.Value); + return ReferenceEquals(o, lsi.Object) && ReferenceEquals(idx, lsi.Index) && ReferenceEquals(v, lsi.Value) + ? lsi : new Expr.LogicalSetIndex(o, idx, lsi.Operator, v); + } + case Expr.PrefixIncrement pi: + { + var o = RewriteExpr(pi.Operand); + return ReferenceEquals(o, pi.Operand) ? pi : new Expr.PrefixIncrement(pi.Operator, o); + } + case Expr.PostfixIncrement po: + { + var o = RewriteExpr(po.Operand); + return ReferenceEquals(o, po.Operand) ? po : new Expr.PostfixIncrement(o, po.Operator); } case Expr.Call c: { @@ -226,23 +412,190 @@ private Expr RewriteExpr(Expr expr) return ReferenceEquals(callee, c.Callee) && ReferenceEquals(args, c.Arguments) ? c : new Expr.Call(callee, c.Paren, c.TypeArgs, args, c.Optional); } + case Expr.CallPrivate cp: + { + var o = RewriteExpr(cp.Object); + var args = RewriteListIfChanged(cp.Arguments, RewriteExpr); + return ReferenceEquals(o, cp.Object) && ReferenceEquals(args, cp.Arguments) + ? cp : new Expr.CallPrivate(o, cp.Name, args); + } + case Expr.New n: + { + var callee = RewriteExpr(n.Callee); + var args = RewriteListIfChanged(n.Arguments, RewriteExpr); + return ReferenceEquals(callee, n.Callee) && ReferenceEquals(args, n.Arguments) + ? n : new Expr.New(callee, n.TypeArgs, args); + } case Expr.Get g: { var o = RewriteExpr(g.Object); return ReferenceEquals(o, g.Object) ? g : new Expr.Get(o, g.Name, g.Optional); } + case Expr.GetPrivate gp: + { + var o = RewriteExpr(gp.Object); + return ReferenceEquals(o, gp.Object) ? gp : new Expr.GetPrivate(o, gp.Name); + } case Expr.GetIndex gi: { - var o = RewriteExpr(gi.Object); - var idx = RewriteExpr(gi.Index); + var o = RewriteExpr(gi.Object); var idx = RewriteExpr(gi.Index); return ReferenceEquals(o, gi.Object) && ReferenceEquals(idx, gi.Index) ? gi : new Expr.GetIndex(o, idx, gi.Optional); } + case Expr.ArrayLiteral arr: + { + var elems = RewriteListIfChanged(arr.Elements, RewriteExpr); + return ReferenceEquals(elems, arr.Elements) ? arr : new Expr.ArrayLiteral(elems, arr.HoleIndices); + } + case Expr.ObjectLiteral obj: + return RewriteObjectLiteral(obj); + case Expr.TemplateLiteral tl: + { + var exprs = RewriteListIfChanged(tl.Expressions, RewriteExpr); + return ReferenceEquals(exprs, tl.Expressions) ? tl : new Expr.TemplateLiteral(tl.Strings, exprs); + } + case Expr.TaggedTemplateLiteral ttl: + { + var tag = RewriteExpr(ttl.Tag); + var exprs = RewriteListIfChanged(ttl.Expressions, RewriteExpr); + return ReferenceEquals(tag, ttl.Tag) && ReferenceEquals(exprs, ttl.Expressions) + ? ttl : new Expr.TaggedTemplateLiteral(tag, ttl.CookedStrings, ttl.RawStrings, exprs); + } + case Expr.Spread sp2: + { + var inner = RewriteExpr(sp2.Expression); + return ReferenceEquals(inner, sp2.Expression) ? sp2 : new Expr.Spread(inner); + } + case Expr.TypeAssertion ta: + { + var inner = RewriteExpr(ta.Expression); + return ReferenceEquals(inner, ta.Expression) ? ta : ta with { Expression = inner }; + } + case Expr.Satisfies sat: + { + var inner = RewriteExpr(sat.Expression); + return ReferenceEquals(inner, sat.Expression) ? sat : sat with { Expression = inner }; + } + case Expr.NonNullAssertion nn: + { + var inner = RewriteExpr(nn.Expression); + return ReferenceEquals(inner, nn.Expression) ? nn : new Expr.NonNullAssertion(inner); + } + case Expr.Await aw: + { + var inner = RewriteExpr(aw.Expression); + return ReferenceEquals(inner, aw.Expression) ? aw : new Expr.Await(aw.Keyword, inner); + } + case Expr.DynamicImport di: + { + var inner = RewriteExpr(di.PathExpression); + return ReferenceEquals(inner, di.PathExpression) ? di : new Expr.DynamicImport(di.Keyword, inner); + } + case Expr.Yield y: + { + if (y.Value is null) return y; + var inner = RewriteExpr(y.Value); + return ReferenceEquals(inner, y.Value) ? y : new Expr.Yield(y.Keyword, inner, y.IsDelegating); + } + default: + // Leaf / type-only / declaration-only nodes carry no nested generator expression: + // Literal, Variable, This, Super, RegexLiteral, ImportMeta, ClassExpr (handled + // structurally elsewhere). Returned unchanged. return expr; } } + private Expr RewriteObjectLiteral(Expr.ObjectLiteral obj) + { + List? rewritten = null; + for (int i = 0; i < obj.Properties.Count; i++) + { + var prop = obj.Properties[i]; + var newKey = prop.Key is Expr.ComputedKey ck ? RewriteComputedKey(ck) : prop.Key; + var newValue = RewriteExpr(prop.Value); + if (!ReferenceEquals(newKey, prop.Key) || !ReferenceEquals(newValue, prop.Value)) + { + rewritten ??= new List(obj.Properties); + rewritten[i] = prop with { Key = newKey, Value = newValue }; + } + } + return rewritten is null ? obj : new Expr.ObjectLiteral(rewritten) { IsFresh = obj.IsFresh }; + } + + private Expr.PropertyKey RewriteComputedKey(Expr.ComputedKey ck) + { + var inner = RewriteExpr(ck.Expression); + return ReferenceEquals(inner, ck.Expression) ? ck : new Expr.ComputedKey(inner); + } + + /// Rewrites a non-generator arrow / function expression, descending into its body and + /// parameter defaults. Function expressions and arrows establish a new lexical scope, so they + /// push a frame for the #534 enclosing-local capture analysis. + private Expr RewriteNonGeneratorArrow(Expr.ArrowFunction af) + { + var newParams = RewriteParameters(af.Parameters); + List? newBody; + if (af.BlockBody is not null) + { + newBody = RewriteFunctionBody(af.Parameters, af.BlockBody, selfName: af.Name?.Lexeme); + } + else + { + newBody = null; + } + var newExpr = af.ExpressionBody is null ? null : RewriteExpr(af.ExpressionBody); + bool changed = !ReferenceEquals(newParams, af.Parameters) + || !ReferenceEquals(newBody, af.BlockBody) + || !ReferenceEquals(newExpr, af.ExpressionBody); + return changed ? af with { Parameters = newParams, BlockBody = newBody, ExpressionBody = newExpr } : af; + } + + private List RewriteParameters(List parameters) + { + List? rewritten = null; + for (int i = 0; i < parameters.Count; i++) + { + var p = parameters[i]; + if (p.DefaultValue is null) continue; + var newDefault = RewriteExpr(p.DefaultValue); + if (!ReferenceEquals(newDefault, p.DefaultValue)) + { + rewritten ??= new List(parameters); + rewritten[i] = p with { DefaultValue = newDefault }; + } + } + return rewritten ?? parameters; + } + + /// + /// Rewrites a function/arrow body within a fresh so that any + /// generator expression closing over one of this body's locals is lifted into this body + /// (#534) rather than the module. Lifts collected for this frame are appended to the body. + /// + private List RewriteFunctionBody(List parameters, List body, string? selfName = null) + { + var locals = LocalBindingCollector.Collect(parameters, body, selfName); + var frame = new FunctionFrame { Locals = locals }; + _frames.Add(frame); + try + { + var rewritten = RewriteListIfChanged(body, RewriteStmt); + if (frame.Lifted.Count == 0) return rewritten; + + // Append the frame's lifts (declarations hoist; trailing keeps the source-order type + // checker happy with locals declared earlier in the body — mirrors the module case). + var result = new List(rewritten.Count + frame.Lifted.Count); + result.AddRange(rewritten); + result.AddRange(frame.Lifted); + return result; + } + finally + { + _frames.RemoveAt(_frames.Count - 1); + } + } + private static List RewriteListIfChanged(List source, Func rewrite) { List? rewritten = null; @@ -264,20 +617,181 @@ private Expr LiftGeneratorArrow(Expr.ArrowFunction af) var name = $"__genArrow_{_counter++}"; var nameToken = new Token(TokenType.IDENTIFIER, name, null, af.Parameters.FirstOrDefault()?.Name.Line ?? 1); - // Recurse into the body first so nested generator arrows are also lifted. - var rewrittenBody = RewriteListIfChanged(af.BlockBody!, RewriteStmt); + // Recurse into the body first so nested generator arrows are also lifted. The body is its + // own function scope, so run it through the frame machinery too. + var rewrittenBody = RewriteFunctionBody(af.Parameters, af.BlockBody!, selfName: af.Name?.Lexeme); + var rewrittenParams = RewriteParameters(af.Parameters); var funcStmt = new Stmt.Function( Name: nameToken, TypeParams: af.TypeParams, ThisType: af.ThisType, - Parameters: af.Parameters, + Parameters: rewrittenParams, Body: rewrittenBody, ReturnType: af.ReturnType, IsAsync: af.IsAsync, IsGenerator: true); - _liftedFunctions.Add(funcStmt); + // Lift into the nearest enclosing function if the body closes over one of the enclosing + // functions' locals (#534); otherwise to the module body (#522). Lifting into the enclosing + // function keeps the captured local in lexical scope. Determining capture conservatively + // over-approximates the BOUND set, so a free-variable name is only attributed to an + // enclosing local when it truly escapes the generator body — never a false positive that + // would relocate a module-closing generator into a function (which the compiler can't lower). + if (_frames.Count > 0 && ClosesOverEnclosingLocal(af)) + { + _frames[^1].Lifted.Add(funcStmt); + } + else + { + _moduleLifted.Add(funcStmt); + } + return new Expr.Variable(nameToken); } + + /// + /// True when the generator arrow references a free variable that is bound as a local by one of + /// the enclosing function scopes (so the lift must stay inside that scope, #534). + /// + private bool ClosesOverEnclosingLocal(Expr.ArrowFunction af) + { + var freeVars = FreeVariableCollector.Collect(af); + if (freeVars.Count == 0) return false; + foreach (var frame in _frames) + { + foreach (var name in freeVars) + { + if (frame.Locals.Contains(name)) return true; + } + } + return false; + } +} + +/// +/// Collects the names declared at the top level of a function/arrow body (its directly-visible +/// locals): parameters, the optional self-reference name, and body-level var/let/ +/// const/function/class declarations. Block-, loop-, and catch-scoped bindings +/// are intentionally excluded: a generator lifted to the end of this body would sit outside those +/// inner scopes, so a generator closing over one of them must NOT be attributed to this frame (it +/// falls through to a module-scope lift instead — the same behavior as before #534). +/// +internal static class LocalBindingCollector +{ + public static HashSet Collect(List parameters, List body, string? selfName) + { + var locals = new HashSet(StringComparer.Ordinal); + foreach (var p in parameters) + locals.Add(p.Name.Lexeme); + if (selfName is not null) + locals.Add(selfName); + foreach (var stmt in body) + { + switch (stmt) + { + case Stmt.Var v: locals.Add(v.Name.Lexeme); break; // covers `let` and `var` + case Stmt.Const c: locals.Add(c.Name.Lexeme); break; + case Stmt.Function f: locals.Add(f.Name.Lexeme); break; + case Stmt.Class cl: locals.Add(cl.Name.Lexeme); break; + } + } + return locals; + } +} + +/// +/// Collects the free variables of a generator function expression: identifier names referenced in +/// value position but not bound inside the function itself. The BOUND set is deliberately +/// over-approximated — every binding found anywhere in the body (including nested functions and +/// blocks) is treated as bound — which can only shrink the free set. That bias guarantees no false +/// positive: a name is reported free only when it genuinely escapes the generator, so a generator +/// that actually closes over a module-scope binding is never mis-attributed to an enclosing function +/// (whose nested-generator lowering the compiler can't perform, #501). The cost is occasional false +/// negatives (a capture also shadowed in a nested scope is missed), which merely fall back to the +/// pre-#534 module-scope lift — never a regression. +/// +internal sealed class FreeVariableCollector : Visitors.AstVisitorBase +{ + private readonly HashSet _referenced = new(StringComparer.Ordinal); + private readonly HashSet _bound = new(StringComparer.Ordinal); + + public static HashSet Collect(Expr.ArrowFunction af) + { + var c = new FreeVariableCollector(); + c.SeedFunctionBindings(af.Parameters, af.Name?.Lexeme); + foreach (var p in af.Parameters) + if (p.DefaultValue is not null) c.Visit(p.DefaultValue); + if (af.ExpressionBody is not null) c.Visit(af.ExpressionBody); + if (af.BlockBody is not null) + foreach (var s in af.BlockBody) c.Visit(s); + c._referenced.ExceptWith(c._bound); + return c._referenced; + } + + private void SeedFunctionBindings(List parameters, string? selfName) + { + foreach (var p in parameters) + _bound.Add(p.Name.Lexeme); + if (selfName is not null) + _bound.Add(selfName); + _bound.Add("arguments"); // function expressions bind their own `arguments` + } + + protected override void VisitVariable(Expr.Variable expr) => _referenced.Add(expr.Name.Lexeme); + + protected override void VisitVar(Stmt.Var stmt) + { + _bound.Add(stmt.Name.Lexeme); + base.VisitVar(stmt); + } + + protected override void VisitConst(Stmt.Const stmt) + { + _bound.Add(stmt.Name.Lexeme); + base.VisitConst(stmt); + } + + protected override void VisitFunction(Stmt.Function stmt) + { + _bound.Add(stmt.Name.Lexeme); + SeedFunctionBindings(stmt.Parameters, stmt.Name.Lexeme); + base.VisitFunction(stmt); + } + + protected override void VisitArrowFunction(Expr.ArrowFunction expr) + { + SeedFunctionBindings(expr.Parameters, expr.Name?.Lexeme); + base.VisitArrowFunction(expr); + } + + protected override void VisitClass(Stmt.Class stmt) + { + _bound.Add(stmt.Name.Lexeme); + base.VisitClass(stmt); + } + + protected override void VisitClassExpr(Expr.ClassExpr expr) + { + if (expr.Name is not null) _bound.Add(expr.Name.Lexeme); + base.VisitClassExpr(expr); + } + + protected override void VisitForOf(Stmt.ForOf stmt) + { + _bound.Add(stmt.Variable.Lexeme); + base.VisitForOf(stmt); + } + + protected override void VisitForIn(Stmt.ForIn stmt) + { + _bound.Add(stmt.Variable.Lexeme); + base.VisitForIn(stmt); + } + + protected override void VisitTryCatch(Stmt.TryCatch stmt) + { + if (stmt.CatchParam is not null) _bound.Add(stmt.CatchParam.Lexeme); + base.VisitTryCatch(stmt); + } } diff --git a/Parsing/Parser.Expressions.cs b/Parsing/Parser.Expressions.cs index 89433e4c..5f70b7fc 100644 --- a/Parsing/Parser.Expressions.cs +++ b/Parsing/Parser.Expressions.cs @@ -973,9 +973,18 @@ private Expr Primary() return new Expr.ObjectLiteral(properties); } - // async arrow function: async () => {} or async (x) => x + // async function expression: async function [name]() {} or async function*() {} + // async arrow function: async () => {} or async (x) => x if (Match(TokenType.ASYNC)) { + // `async function ...` is an async function expression — defer to the shared + // FunctionExpression parser (which also handles the `*` for async generators), + // mirroring the statement-level `async function` declaration path. + if (Match(TokenType.FUNCTION)) + { + return FunctionExpression(isAsync: true); + } + if (Check(TokenType.LESS)) { Expr? genericArrow = TryParseGenericArrowFunction(isAsync: true); @@ -1448,9 +1457,9 @@ private Expr ParseTaggedTemplateLiteral(Expr tag) /// Supports optional name (for named function expressions), generator syntax (function*), /// this parameter, and type annotations. /// - private Expr FunctionExpression() + private Expr FunctionExpression(bool isAsync = false) { - // Check for generator function: function* () { } + // Check for generator function: function* () { } (or async function* () {}) bool isGenerator = Match(TokenType.STAR); // Optional function name (for named function expressions) @@ -1585,6 +1594,7 @@ private Expr FunctionExpression() BlockBody: body, ReturnType: returnType, HasOwnThis: true, + IsAsync: isAsync, IsGenerator: isGenerator ); } diff --git a/SharpTS.Tests/SharedTests/AsyncFunctionExpressionTests.cs b/SharpTS.Tests/SharedTests/AsyncFunctionExpressionTests.cs new file mode 100644 index 00000000..273043ef --- /dev/null +++ b/SharpTS.Tests/SharedTests/AsyncFunctionExpressionTests.cs @@ -0,0 +1,113 @@ +using SharpTS.Tests.Infrastructure; +using Xunit; + +namespace SharpTS.Tests.SharedTests; + +/// +/// Tests for async function EXPRESSIONS (`async function () {}` / `async function name() {}`), +/// as opposed to async arrow functions. Issue #635: the parser previously only attempted an async +/// arrow after `async`, so an async function expression failed to parse. +/// +public class AsyncFunctionExpressionTests +{ + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_Anonymous_ResolvesValue(ExecutionMode mode) + { + // The exact repro from issue #635. + var source = """ + const af = async function () { return 9; }; + af().then((v: number) => console.log(v)); + """; + + Assert.Equal("9\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_Named_ResolvesValue(ExecutionMode mode) + { + var source = """ + const compute = async function doubler(x: number) { return x * 2; }; + compute(21).then((v: number) => console.log(v)); + """; + + Assert.Equal("42\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_Awaited(ExecutionMode mode) + { + var source = """ + async function main(): Promise { + const af = async function () { return 7; }; + const v = await af(); + console.log(v + 1); + } + main(); + """; + + Assert.Equal("8\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_ImmediatelyInvoked(ExecutionMode mode) + { + var source = """ + (async function () { console.log("ran"); return 1; })(); + """; + + Assert.Equal("ran\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_AwaitInsideBody(ExecutionMode mode) + { + var source = """ + async function main(): Promise { + const af = async function (x: number) { + const y = await Promise.resolve(x + 1); + return y * 10; + }; + console.log(await af(2)); + } + main(); + """; + + Assert.Equal("30\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunctionExpression_AsCallbackArgument(ExecutionMode mode) + { + // Async function expression passed directly as an argument (a common position that the + // async-arrow-only parser path could not reach). + var source = """ + function run(fn: () => Promise): Promise { return fn(); } + run(async function () { return 5; }).then((v: number) => console.log(v)); + """; + + Assert.Equal("5\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncGeneratorFunctionExpression_ForAwaitOf(ExecutionMode mode) + { + // `async function*` expression — the `function` keyword path also accepts the `*`, giving an + // async generator expression. (Compiled `for await` over async generators landed in #430/#645.) + var source = """ + const ag = async function* () { yield 1; yield 2; }; + async function main(): Promise { + for await (const v of ag()) console.log(v); + } + main(); + """; + + Assert.Equal("1\n2\n", TestHarness.Run(source, mode)); + } +} diff --git a/SharpTS.Tests/SharedTests/GeneratorTests.cs b/SharpTS.Tests/SharedTests/GeneratorTests.cs index 3cbc8851..8d38483b 100644 --- a/SharpTS.Tests/SharedTests/GeneratorTests.cs +++ b/SharpTS.Tests/SharedTests/GeneratorTests.cs @@ -1311,6 +1311,240 @@ function make() { #endregion + #region Generator function EXPRESSION in call/IIFE position — issue #488 + + // A generator function expression invoked inline as an IIFE establishes a generator context. + // The GeneratorArrowLifter must descend through the grouped callee so the expression is lifted + // (otherwise the type checker rejects its `yield` because the generator context is never set). + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_IifeCallPosition(ExecutionMode mode) + { + // The exact repro from issue #488. + var source = """ + const it = (function* () { yield 1; })(); + console.log(it.next().value); + """; + + Assert.Equal("1\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_IifeSpreadIntoArray(ExecutionMode mode) + { + var source = """ + const arr = [...(function* () { yield 1; yield 2; yield 3; })()]; + console.log(arr.join(",")); + """; + + Assert.Equal("1,2,3\n", TestHarness.Run(source, mode)); + } + + #endregion + + #region Generator function EXPRESSION inside loop/try/switch/labeled bodies — issue #634 + + // GeneratorArrowLifter.RewriteStmt must descend into for / for-of / for-in / do-while / + // try-catch-finally / switch / labeled statement bodies, or a generator function expression + // declared there is never lifted and its `yield` is rejected at type-check time. + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideForLoop(ExecutionMode mode) + { + // The exact repro from issue #634. + var source = """ + for (let k = 0; k < 1; k++) { + const g = function* () { yield 99; }; + console.log(g().next().value); + } + """; + + Assert.Equal("99\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideForOf(ExecutionMode mode) + { + // The generator yields a constant (not the loop variable): #634 is about the lifter reaching + // the for-of body, not closure capture. Capturing the block-scoped loop variable is a + // separate, unsupported case (the lift would move the body outside the loop's scope). + var source = """ + for (const n of [10, 20]) { + const g = function* () { yield 1; }; + console.log(g().next().value); + } + """; + + Assert.Equal("1\n1\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideForIn(ExecutionMode mode) + { + var source = """ + const obj = { a: 1 }; + for (const k in obj) { + const g = function* () { yield 2; }; + console.log(g().next().value); + } + """; + + Assert.Equal("2\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideLoopClosesOverModuleScope(ExecutionMode mode) + { + // A generator expression inside a loop body may still close over a MODULE-scope binding; + // the lift carries it to module scope, where `factor` is visible (both modes). + var source = """ + const factor = 7; + for (const n of [1, 2]) { + const g = function* () { yield factor; }; + console.log(g().next().value); + } + """; + + Assert.Equal("7\n7\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideDoWhile(ExecutionMode mode) + { + var source = """ + let i = 0; + do { + const g = function* () { yield 7; }; + console.log(g().next().value); + i++; + } while (i < 1); + """; + + Assert.Equal("7\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideTryCatchFinally(ExecutionMode mode) + { + var source = """ + try { + const g = function* () { yield 1; }; + console.log(g().next().value); + } catch (e) { + const g = function* () { yield 2; }; + console.log(g().next().value); + } finally { + const g = function* () { yield 3; }; + console.log(g().next().value); + } + """; + + Assert.Equal("1\n3\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideSwitchCase(ExecutionMode mode) + { + var source = """ + switch (1) { + case 1: { + const g = function* () { yield 42; }; + console.log(g().next().value); + break; + } + default: + console.log("none"); + } + """; + + Assert.Equal("42\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_InsideLabeledStatement(ExecutionMode mode) + { + var source = """ + outer: { + const g = function* () { yield 5; }; + console.log(g().next().value); + } + """; + + Assert.Equal("5\n", TestHarness.Run(source, mode)); + } + + #endregion + + #region Generator function EXPRESSION closing over an enclosing function's local — issue #534 + + // The interpreter handles a generator expression that closes over an enclosing FUNCTION local: + // the lifter relocates it to the end of that function's body (keeping the local in lexical + // scope) rather than to module scope. The compiler's nested-generator lowering is incomplete + // (#501), so these run interpreted only — the compiler reports a clear "Yield not supported in + // this context" error for the same source (verified separately). + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_ClosesOverFunctionLocal(ExecutionMode mode) + { + // The exact repro from issue #534. + var source = """ + function outer() { + let y = 5; + const g = function*() { yield y; }; + return [...g()]; + } + console.log(outer()); + """; + + Assert.Equal("[5]\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_ClosesOverFunctionParameter(ExecutionMode mode) + { + var source = """ + function make(seed: number) { + const g = function*() { yield seed; yield seed * 2; }; + return [...g()]; + } + console.log(make(3)); + """; + + Assert.Equal("[3, 6]\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void GeneratorExpression_ClosesOverLocalCapturesByReference(ExecutionMode mode) + { + // Closures bind the variable, so a mutation before iteration is observed (not a snapshot). + var source = """ + function outer() { + let c = 1; + const g = function*() { yield c; }; + c = 99; + return [...g()]; + } + console.log(outer()); + """; + + Assert.Equal("[99]\n", TestHarness.Run(source, mode)); + } + + #endregion + #region Re-entrant next()/return()/throw() — "already running" (ECMA-262 §27.5.3.3) — issues #515, #521 // ECMA-262 §27.5.3.3 (GeneratorValidate): calling next/return/throw on a generator whose state