Fix #435, #669, #625, #641: arrow/closure capture in generator & async-arrow state machines

Compilation/AsyncArrowMoveNextEmitter.ArrowFunctions.cs

@@ -111,18 +111,55 @@ private void EmitNestedAsyncArrow(Expr.ArrowFunction af)
         // stub's arg0, clobbering the first real parameter. (#615)
         if (nestedBuilder.IsStandalone)
         {
-            // A capturing standalone nested arrow would need its capture fields populated from the
-            // enclosing frame, which this path does not wire up — fail loudly rather than read
-            // uninitialized captures (it did not compile at all before #615; tracked in #641).
-            if (nestedBuilder.StandaloneCaptureFields.Count > 0)
+            // A capturing standalone nested arrow's stub takes its single captured value as a leading
+            // argument. $TSFunction's "static method with target" mechanism prepends the target before
+            // the call args, so we pass the capture AS the target, read from THIS (enclosing) arrow's
+            // frame. Mirrors the module-level standalone path in ILEmitter.EmitAsyncArrowFunction.
+            // Before #641 every capturing standalone nested arrow threw "not yet supported"; now the
+            // single read-only capture (the #641 repros) is supported, and the two harder shapes below
+            // fail loudly rather than miscompiling.
+            var captureOrder = nestedBuilder.StandaloneCaptureFields.Keys
+                .OrderBy(k => k, System.StringComparer.Ordinal).ToList();
+
+            if (captureOrder.Count > 0)
             {
-                throw new CompileException(
-                    "A nested async arrow that captures variables inside a top-level async arrow is " +
-                    "not yet supported in compiled mode; hoist it to a named async arrow or async " +
-                    "function, or run in interpreted mode.");
+                // A standalone arrow captures BY VALUE (the values are copied into its own state
+                // machine). A write to a capture therefore cannot propagate back to the enclosing
+                // binding, so reject it instead of silently dropping the write (#684).
+                var written = CapturedWriteAnalysis.CollectImmediateWrites(af);
+                written.IntersectWith(captureOrder);
+                if (written.Count > 0)
+                {
+                    throw new CompileException(
+                        "A nested async arrow inside a top-level async arrow that WRITES a captured " +
+                        $"variable ({string.Join(", ", written.OrderBy(n => n, System.StringComparer.Ordinal))}) " +
+                        "is not yet supported in compiled mode (#684): the standalone arrow captures by " +
+                        "value, so the write would be lost. Hoist it to a named async function, or run " +
+                        "in interpreted mode.");
+                }
+
+                // Multiple captures would need to ride in $TSFunction's single prepended target slot
+                // (an object[]), but the standalone stub expects each capture as a separate arg — a
+                // mismatch also broken for module-level standalone async arrows (#684).
+                if (captureOrder.Count > 1)
+                {
+                    throw new CompileException(
+                        "A nested async arrow inside a top-level async arrow that captures more than " +
+                        "one variable is not yet supported in compiled mode (#684); reduce it to a " +
+                        "single capture, hoist it to a named async function, or run in interpreted mode.");
+                }
+            }
+
+            if (captureOrder.Count == 1)
+            {
+                LoadVariableForCapture(captureOrder[0]);
+                EnsureBoxed();
+            }
+            else
+            {
+                _il.Emit(OpCodes.Ldnull);
             }
 
-            _il.Emit(OpCodes.Ldnull);
             _il.Emit(OpCodes.Ldtoken, nestedBuilder.StubMethod);
             _il.Emit(OpCodes.Call, Types.MethodBaseGetMethodFromHandle);
             _il.Emit(OpCodes.Castclass, typeof(MethodInfo));

Compilation/AsyncArrowMoveNextEmitter.Variables.cs

@@ -10,6 +10,18 @@ protected override void EmitVariable(Expr.Variable v)
     {
         string name = v.Name.Lexeme;
 
+        // A capture promoted into the enclosing function's display class (#625) is read through
+        // `outer.functionDC.field`, NOT the boxed state-machine field. Checked before the resolver:
+        // the variable may still have a (now-unused) hoisted SM field that the resolver would load
+        // a stale value from.
+        if (TryGetOuterFunctionDCField(name, out var dcReadField))
+        {
+            EmitLoadOuterFunctionDC();
+            _il.Emit(OpCodes.Ldfld, dcReadField);
+            SetStackUnknown();
+            return;
+        }
+
         // Try resolver first (params, locals, hoisted, captured)
         var stackType = _resolver!.TryLoadVariable(name);
         if (stackType != null)
@@ -57,6 +69,20 @@ protected override void EmitVariable(Expr.Variable v)
             return;
         }
 
+        // Standalone capture (#641): a value the enclosing arrow's frame passed as a leading stub
+        // argument, which the stub copied into a field on THIS arrow's state machine. Checked LAST,
+        // below the module-level globals above: a top-level variable a standalone arrow closes over
+        // is ALSO registered as a standalone capture but must be read LIVE from its static field
+        // (the standalone copy is a stale snapshot) — handling it earlier broke compound/logical
+        // assignment to such a variable. Reaches here only for genuine enclosing-arrow locals.
+        if (_builder.StandaloneCaptureFields.TryGetValue(name, out var standaloneField))
+        {
+            _il.Emit(OpCodes.Ldarg_0);
+            _il.Emit(OpCodes.Ldfld, standaloneField);
+            SetStackUnknown();
+            return;
+        }
+
         // Not found - push null
         _il.Emit(OpCodes.Ldnull);
         SetStackType(StackType.Null);
@@ -70,6 +96,20 @@ protected override void EmitAssign(Expr.Assign a)
         EnsureBoxed();
         _il.Emit(OpCodes.Dup);
 
+        // A capture promoted into the enclosing function's display class (#625) is written through
+        // `outer.functionDC.field = value` (the DC is a reference type, so the store is verifiable),
+        // not by mutating the boxed value-type state machine in place. Checked first.
+        if (TryGetOuterFunctionDCField(name, out var dcAssignField))
+        {
+            var temp = _il.DeclareLocal(_types.Object);
+            _il.Emit(OpCodes.Stloc, temp);     // consume the duplicated value
+            EmitLoadOuterFunctionDC();
+            _il.Emit(OpCodes.Ldloc, temp);
+            _il.Emit(OpCodes.Stfld, dcAssignField);
+            SetStackUnknown();                 // remaining copy is the assignment's value
+            return;
+        }
+
         // Check if it's a captured top-level variable in entry-point display class
         if (_ctx?.CapturedTopLevelVars?.Contains(name) == true &&
             _ctx.EntryPointDisplayClassFields?.TryGetValue(name, out var entryPointField) == true &&
@@ -185,6 +225,18 @@ private void LoadVariable(string name)
 
     private void StoreVariable(string name)
     {
+        // A capture promoted into the enclosing function's display class (#625) is stored through
+        // `outer.functionDC.field`. Checked first so it wins over any (now-unused) hoisted SM field.
+        if (TryGetOuterFunctionDCField(name, out var dcStoreField))
+        {
+            var temp = _il.DeclareLocal(_types.Object);
+            _il.Emit(OpCodes.Stloc, temp);
+            EmitLoadOuterFunctionDC();
+            _il.Emit(OpCodes.Ldloc, temp);
+            _il.Emit(OpCodes.Stfld, dcStoreField);
+            return;
+        }
+
         // Check if it's a parameter of this arrow
         if (_builder.ParameterFields.TryGetValue(name, out var paramField))
         {
@@ -343,4 +395,34 @@ private void LoadVariableForCapture(string name)
         _il.Emit(OpCodes.Ldnull);
         SetStackType(StackType.Null);
     }
+
+    /// <summary>
+    /// True when <paramref name="name"/> is a captured variable the enclosing async function placed
+    /// in its (reference-type) display class (#625). Such a variable must be read/written through
+    /// <c>outer.functionDC.field</c> rather than the boxed state-machine field. Requires the outer
+    /// reference plumbing — present only for non-standalone arrows nested directly in an async
+    /// function — so standalone/top-level async arrows fall through to the existing paths.
+    /// </summary>
+    private bool TryGetOuterFunctionDCField(string name, out FieldBuilder dcField)
+    {
+        dcField = null!;
+        return _ctx?.OuterFunctionDCField != null
+            && _builder.OuterStateMachineField != null
+            && _builder.OuterStateMachineType != null
+            && _ctx.FunctionDisplayClassFields?.TryGetValue(name, out dcField!) == true;
+    }
+
+    /// <summary>
+    /// Pushes the enclosing async function's display-class instance: <c>outer.functionDC</c>.
+    /// Reading the DC reference field through the <c>unbox</c>'d (readonly) outer pointer is
+    /// verifiable; the resulting reference is an ordinary class instance, so the caller's
+    /// subsequent <c>ldfld</c>/<c>stfld</c> on it verifies (unlike storing into the boxed struct).
+    /// </summary>
+    private void EmitLoadOuterFunctionDC()
+    {
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldfld, _builder.OuterStateMachineField!);
+        _il.Emit(OpCodes.Unbox, _builder.OuterStateMachineType!);
+        _il.Emit(OpCodes.Ldfld, _ctx!.OuterFunctionDCField!);
+    }
 }

Compilation/AsyncArrowStateMachineBuilder.cs

@@ -329,7 +329,8 @@ public void DefineStubMethod(TypeBuilder programType)
 
         // For standalone arrows with captures, add capture parameters first
         // Order: [captures...], [arrow params...]
-        var captureOrder = StandaloneCaptureFields.Keys.OrderBy(k => k).ToList();
+        // Ordinal ordering must match the call sites (ILEmitter / AsyncArrowMoveNextEmitter).
+        var captureOrder = StandaloneCaptureFields.Keys.OrderBy(k => k, System.StringComparer.Ordinal).ToList();
         if (IsStandalone)
         {
             foreach (var _ in captureOrder)

Compilation/AsyncArrowVariableResolver.cs

@@ -71,6 +71,10 @@ public AsyncArrowVariableResolver(
             return StackType.Unknown;
         }
 
+        // NOTE: standalone captures (#641) are resolved by the EMITTER (AsyncArrowMoveNextEmitter)
+        // at LOWER priority than module-level globals, because a top-level variable a standalone
+        // arrow closes over is ALSO registered as a standalone capture but must be read LIVE from
+        // its static field, not from the by-value snapshot. Handling it here would shadow that.
         return null; // Not found - caller handles fallback
     }
 

Compilation/CapturedWriteAnalysis.cs

@@ -0,0 +1,69 @@
+using SharpTS.Parsing;
+using SharpTS.Parsing.Visitors;
+
+namespace SharpTS.Compilation;
+
+/// <summary>
+/// Determines which variables an arrow/function body assigns to within its <em>own</em> scope —
+/// descending through blocks, loops, conditionals and call arguments, but NOT into nested arrows
+/// or function declarations (those introduce their own bindings, so a write there is not a write
+/// to this scope's captures). Intersecting the result with a closure's capture set identifies the
+/// captures it mutates, which the compiler must back with shared (reference-type) storage rather
+/// than a by-value snapshot.
+/// </summary>
+internal static class CapturedWriteAnalysis
+{
+    /// <summary>
+    /// Returns the names assigned within <paramref name="arrow"/>'s own scope.
+    /// </summary>
+    public static HashSet<string> CollectImmediateWrites(Expr.ArrowFunction arrow)
+    {
+        var collector = new WrittenNameCollector();
+        if (arrow.ExpressionBody != null)
+            collector.Visit(arrow.ExpressionBody);
+        if (arrow.BlockBody != null)
+            foreach (var stmt in arrow.BlockBody)
+                collector.Visit(stmt);
+        return collector.Names;
+    }
+
+    private sealed class WrittenNameCollector : AstVisitorBase
+    {
+        public readonly HashSet<string> Names = [];
+
+        protected override void VisitAssign(Expr.Assign expr)
+        {
+            Names.Add(expr.Name.Lexeme);
+            base.VisitAssign(expr);
+        }
+
+        protected override void VisitCompoundAssign(Expr.CompoundAssign expr)
+        {
+            Names.Add(expr.Name.Lexeme);
+            base.VisitCompoundAssign(expr);
+        }
+
+        protected override void VisitLogicalAssign(Expr.LogicalAssign expr)
+        {
+            Names.Add(expr.Name.Lexeme);
+            base.VisitLogicalAssign(expr);
+        }
+
+        protected override void VisitPrefixIncrement(Expr.PrefixIncrement expr)
+        {
+            if (expr.Operand is Expr.Variable v) Names.Add(v.Name.Lexeme);
+            base.VisitPrefixIncrement(expr);
+        }
+
+        protected override void VisitPostfixIncrement(Expr.PostfixIncrement expr)
+        {
+            if (expr.Operand is Expr.Variable v) Names.Add(v.Name.Lexeme);
+            base.VisitPostfixIncrement(expr);
+        }
+
+        // Stop at nested closure boundaries: their assignments target their own bindings (or
+        // deeper captures), so they must not be attributed to the enclosing scope's captures.
+        protected override void VisitArrowFunction(Expr.ArrowFunction expr) { }
+        protected override void VisitFunction(Stmt.Function stmt) { }
+    }
+}

Compilation/CompilationContext.Closures.cs

@@ -144,6 +144,15 @@ public partial class CompilationContext
     /// </summary>
     public FieldBuilder? CurrentArrowFunctionDCField { get; set; }
 
+    /// <summary>
+    /// For an async arrow's MoveNext: the <c>&lt;&gt;__functionDC</c> field on the enclosing async
+    /// function's state machine. Combined with <see cref="FunctionDisplayClassFields"/>, it lets the
+    /// arrow read/write captured locals that were promoted into the (reference-type) function display
+    /// class — `outer.functionDC.field` — instead of mutating a field of the boxed value-type state
+    /// machine in place, which is unverifiable (`unbox` yields a readonly managed pointer; #625).
+    /// </summary>
+    public FieldBuilder? OuterFunctionDCField { get; set; }
+
     // ============================================
     // Arrow Scope Display Class (for arrow-local vars captured by nested arrows)
     // ============================================

Compilation/ExpressionEmitterBase.cs

@@ -1502,6 +1502,22 @@ protected virtual void EmitArrowFunction(Expr.ArrowFunction af)
             return;
         }
 
+        // Capturing arrow: its body is emitted as an instance method on a display class, so the
+        // $TSFunction must be bound to a freshly-constructed display-class instance whose fields
+        // hold the captured values. Emitting a null target here (the old behaviour) produced
+        // "Non-static method requires a target" when the arrow was later invoked (#435/#669,
+        // sync generator bodies). Captured values are read through the GetHoistedVariableField /
+        // GetThisField hooks plus Ctx.Locals, which every emitter that reaches this base default
+        // already implements. GeneratorMoveNextEmitter is currently the only such emitter — all
+        // others (ILEmitter, Async/AsyncArrow/AsyncGenerator MoveNext) override EmitArrowFunction.
+        if (Ctx.DisplayClasses?.ContainsKey(af) == true &&
+            Ctx.DisplayClassConstructors?.TryGetValue(af, out var displayCtor) == true)
+        {
+            EmitCapturingArrowViaHooks(af, method, displayCtor);
+            return;
+        }
+
+        // Non-capturing arrow: static method, null target.
         IL.Emit(OpCodes.Ldnull);
         IL.Emit(OpCodes.Ldtoken, method);
         IL.Emit(OpCodes.Call, Types.MethodBaseGetMethodFromHandle);
@@ -1515,6 +1531,59 @@ protected virtual void EmitArrowFunction(Expr.ArrowFunction af)
         SetStackUnknown();
     }
 
+    /// <summary>
+    /// Emits a capturing arrow as a $TSFunction bound to a new display-class instance, populating
+    /// the display class's captured fields from the emitter's storage (hoisted state-machine
+    /// fields, <c>this</c>, or IL locals). Mirrors the per-emitter overrides in
+    /// AsyncMoveNextEmitter / AsyncGeneratorMoveNextEmitter, but resolves storage through the
+    /// shared GetHoistedVariableField / GetThisField hooks so a single implementation serves any
+    /// emitter that uses the base <see cref="EmitArrowFunction"/>.
+    /// </summary>
+    private void EmitCapturingArrowViaHooks(Expr.ArrowFunction af, MethodBuilder method, ConstructorBuilder displayCtor)
+    {
+        IL.Emit(OpCodes.Newobj, displayCtor);
+
+        if (Ctx.DisplayClassFields?.TryGetValue(af, out var fieldMap) == true)
+        {
+            foreach (var (capturedVar, field) in fieldMap)
+            {
+                IL.Emit(OpCodes.Dup);
+
+                // `this` is checked before the hoisted-variable map: the generator's
+                // hoisting analyzer can mint a state-machine field keyed "this" that the
+                // stub never populates, so resolving it via the map would snapshot null
+                // (NRE when the arrow dereferences `this`). The real receiver lives in the
+                // builder's dedicated ThisField (set by the instance-method stub).
+                if (capturedVar == "this" && GetThisField() is FieldBuilder thisField)
+                {
+                    IL.Emit(OpCodes.Ldarg_0);
+                    IL.Emit(OpCodes.Ldfld, thisField);
+                }
+                else if (GetHoistedVariableField(capturedVar) is FieldBuilder hoistedField)
+                {
+                    IL.Emit(OpCodes.Ldarg_0);
+                    IL.Emit(OpCodes.Ldfld, hoistedField);
+                }
+                else if (Ctx.Locals.TryGetLocal(capturedVar, out var local))
+                {
+                    IL.Emit(OpCodes.Ldloc, local);
+                }
+                else
+                {
+                    IL.Emit(OpCodes.Ldnull);
+                }
+
+                IL.Emit(OpCodes.Stfld, field);
+            }
+        }
+
+        IL.Emit(OpCodes.Ldtoken, method);
+        IL.Emit(OpCodes.Call, Types.MethodBaseGetMethodFromHandle);
+        IL.Emit(OpCodes.Castclass, typeof(MethodInfo));
+        IL.Emit(OpCodes.Newobj, Ctx.Runtime!.TSFunctionCtor);
+        SetStackUnknown();
+    }
+
     // EmitCall is virtual with a default implementation in ExpressionEmitterBase.CallHelpers.cs
     // EmitCompoundAssign, EmitLogicalAssign, EmitPrefixIncrement, EmitPostfixIncrement
     // are implemented in ExpressionEmitterBase.Operators.cs as virtual methods.