Compiled generator/async-generator fixes: #547, #631, #542, #583

Compilation/AsyncMoveNextEmitter.Expressions.cs

@@ -10,15 +10,27 @@ public partial class AsyncMoveNextEmitter
 
     protected override void EmitAwait(Expr.Await a)
     {
-        int stateNumber = _currentAwaitState++;
-        var resumeLabel = _stateLabels[stateNumber];
-        var continueLabel = _il.DefineLabel();
-        var awaiterField = _builder.AwaiterFields[stateNumber];
-
         // 1. Emit the awaited expression (should produce Task<object> or $Promise)
         EmitExpression(a.Expression);
         EnsureBoxed();
 
+        // 2+. Coerce to Task<object>, suspend/resume, and leave the awaited result on the stack.
+        EmitAwaitFromValueOnStack(_currentAwaitState++);
+    }
+
+    /// <summary>
+    /// Emits the await of a value already on the evaluation stack (boxed): coerces it to
+    /// <c>Task&lt;object&gt;</c> (unwrapping $Promise / adopting thenables / wrapping plain values),
+    /// suspends the state machine until it settles, and leaves the awaited result on the stack.
+    /// Shared by <see cref="EmitAwait"/> and the <c>for await…of</c> loop's implicit next()/return()
+    /// awaits (#631); <paramref name="stateNumber"/> is the reserved suspension state for this await.
+    /// </summary>
+    internal void EmitAwaitFromValueOnStack(int stateNumber)
+    {
+        var resumeLabel = _stateLabels[stateNumber];
+        var continueLabel = _il.DefineLabel();
+        var awaiterField = _builder.AwaiterFields[stateNumber];
+
         // 2. Convert to Task<object> - handle $Promise, Task<object>, or non-Task values
         var taskLocal = _il.DeclareLocal(typeof(Task<object>));
         var isPromiseLabel = _il.DefineLabel();

Compilation/AsyncMoveNextEmitter.Statements.TryCatch.cs

@@ -382,7 +382,10 @@ private bool ContainsAwaitInStmt(Stmt stmt)
                        (f.Increment != null && ContainsAwaitInExpr(f.Increment)) ||
                        ContainsAwaitInStmt(f.Body);
             case Stmt.ForOf fo:
-                return ContainsAwaitInExpr(fo.Iterable) || ContainsAwaitInStmt(fo.Body);
+                // `for await…of` always suspends (it awaits iterator.next()/return()), even when the
+                // iterable and body contain no explicit await — so a try enclosing one must take the
+                // flag-based path, not a real IL try (whose resume labels would be branched into) (#631).
+                return fo.IsAsync || ContainsAwaitInExpr(fo.Iterable) || ContainsAwaitInStmt(fo.Body);
             case Stmt.ForIn fi:
                 return ContainsAwaitInExpr(fi.Object) || ContainsAwaitInStmt(fi.Body);
             case Stmt.Block b:

Compilation/AsyncStateAnalyzer.Expressions.cs

@@ -11,7 +11,16 @@ protected override void VisitAwait(Expr.Await expr)
         // Visit await expression BEFORE marking _seenAwait, so variables in the await
         // expression are not incorrectly marked as "used after await"
         base.VisitAwait(expr);
+        RecordAwaitPoint(expr);
+    }
 
+    /// <summary>
+    /// Records an await point (real or synthetic) and the surrounding try-region/await bookkeeping.
+    /// <paramref name="expr"/> is null for synthetic awaits — the implicit next()/return() awaits a
+    /// <c>for await…of</c> loop performs (#631), which have no <see cref="Expr.Await"/> node.
+    /// </summary>
+    internal void RecordAwaitPoint(Expr.Await? expr)
+    {
         // Record this await point with try block context
         var liveVars = new HashSet<string>(_declaredVariables);
         _awaitPoints.Add(new AwaitPoint(

Compilation/AsyncStateAnalyzer.Statements.cs

@@ -33,6 +33,20 @@ protected override void VisitForOf(Stmt.ForOf stmt)
         if (!_seenAwait)
             _variablesDeclaredBeforeAwait.Add(stmt.Variable.Lexeme);
 
+        if (stmt.IsAsync)
+        {
+            // `for await…of` suspends on the iterator protocol: it awaits iterator.next() each
+            // iteration and iterator.return() on early exit. Each needs a reserved state (resume
+            // label + awaiter field), matching the two inline awaits EmitForAwaitOf emits (#631).
+            // Allocation order mirrors emission order: the iterable is evaluated before the loop, so
+            // its awaits come first; the next() await before the body; the return() await after.
+            Visit(stmt.Iterable);
+            RecordAwaitPoint(null);   // iterator.next() — awaited at the loop head
+            Visit(stmt.Body);
+            RecordAwaitPoint(null);   // iterator.return() — awaited in the break cleanup
+            return;
+        }
+
         base.VisitForOf(stmt);
     }
 

Compilation/AsyncStateAnalyzer.cs

@@ -15,7 +15,7 @@ public partial class AsyncStateAnalyzer : AstVisitorBase
     /// </summary>
     public record AwaitPoint(
         int StateNumber,
-        Expr.Await AwaitExpr,
+        Expr.Await? AwaitExpr,  // null for synthetic await points (e.g. for await…of's implicit next()/return() awaits, #631)
         HashSet<string> LiveVariables,
         int TryBlockDepth = 0,  // 0 = not in try, 1+ = nested try depth
         int? EnclosingTryId = null  // ID of the innermost try block containing this await

Compilation/EmittedRuntime.cs

@@ -1393,6 +1393,10 @@ public class EmittedRuntime
     // Async Generator await continuation helper
     public MethodBuilder AsyncGeneratorAwaitContinue { get; set; } = null!;
 
+    // Async Generator next-result builder: awaits a MoveNextAsync ValueTask<bool> and produces the
+    // { value, done } Task<object> for next(), so next() never blocks the event-loop thread (#631/#542).
+    public MethodBuilder AsyncGeneratorBuildResult { get; set; } = null!;
+
     // Iterator helper methods (ES2025 Iterator Helpers)
     public MethodBuilder NormalizeToEnumerator { get; set; } = null!;
     public MethodBuilder IteratorMap { get; set; } = null!;

Compilation/GeneratorMoveNextEmitter.Statements.cs

@@ -244,10 +244,95 @@ protected override void EmitForOf(Stmt.ForOf f)
 
     #endregion
 
+    #region For...In Loop Override (Hoisted Key-List/Index Support)
+
+    /// <summary>
+    /// Emits a for...in loop with hoisted key-list and index when the body contains a yield.
+    /// The base emitter keeps the enumerated key list and current index in IL locals, which a
+    /// MoveNext re-entry across a yield wipes — so the loop would restart from (or stop after) the
+    /// first key (#547). Storing both in state-machine fields makes the iteration position survive
+    /// the suspension, mirroring the hoisted-enumerator treatment <see cref="EmitForOf"/> gives
+    /// for...of.
+    /// </summary>
+    protected override void EmitForIn(Stmt.ForIn f)
+    {
+        var keysField = _builder.GetForInKeysField(f);
+        if (keysField == null)
+        {
+            // No yield inside this loop - use base implementation with local key list/index.
+            base.EmitForIn(f);
+            return;
+        }
+        var indexField = _builder.GetForInIndexField(f)!;
+
+        var startLabel = _il.DefineLabel();
+        var endLabel = _il.DefineLabel();
+        var continueLabel = _il.DefineLabel();
+
+        // Get keys from the object and stash them in the hoisted field (need a temp for the swap).
+        EmitExpression(f.Object);
+        EnsureBoxed();
+        _il.Emit(OpCodes.Call, _ctx!.Runtime!.GetKeys);
+        var keysTemp = _il.DeclareLocal(_types.ListOfObject);
+        _il.Emit(OpCodes.Stloc, keysTemp);
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldloc, keysTemp);
+        _il.Emit(OpCodes.Stfld, keysField);
+
+        // index = 0
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldc_I4_0);
+        _il.Emit(OpCodes.Stfld, indexField);
+
+        EnterLoop(endLabel, continueLabel);
+
+        var loopVarLocal = DeclareLoopVariable(f.Variable.Lexeme);
+
+        _il.MarkLabel(startLabel);
+        EmitCancellationCheck();
+
+        // if (index < keys.Count) else goto end — both read from hoisted fields.
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldfld, indexField);
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldfld, keysField);
+        _il.Emit(OpCodes.Call, _ctx.Runtime!.GetLength);
+        _il.Emit(OpCodes.Clt);
+        _il.Emit(OpCodes.Brfalse, endLabel);
+
+        // loopVar = keys[index]
+        EmitStoreLoopVariable(loopVarLocal, f.Variable.Lexeme, () =>
+        {
+            _il.Emit(OpCodes.Ldarg_0);
+            _il.Emit(OpCodes.Ldfld, keysField);
+            _il.Emit(OpCodes.Ldarg_0);
+            _il.Emit(OpCodes.Ldfld, indexField);
+            _il.Emit(OpCodes.Call, _ctx.Runtime!.GetElement);
+        });
+
+        EmitStatement(f.Body);
+
+        _il.MarkLabel(continueLabel);
+
+        // index = index + 1 (store back to the hoisted field)
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldarg_0);
+        _il.Emit(OpCodes.Ldfld, indexField);
+        _il.Emit(OpCodes.Ldc_I4_1);
+        _il.Emit(OpCodes.Add);
+        _il.Emit(OpCodes.Stfld, indexField);
+
+        _il.Emit(OpCodes.Br, startLabel);
+
+        _il.MarkLabel(endLabel);
+        ExitLoop();
+    }
+
+    #endregion
+
     // Note: The following methods are inherited from StatementEmitterBase:
     // - EmitStatement (dispatch)
     // - EmitIf, EmitWhile, EmitDoWhile (control flow)
-    // - EmitForIn (loops with DeclareLoopVariable/EmitStoreLoopVariable overrides)
     // - EmitBlock, EmitLabeledStatement, EmitSwitch, EmitPrint
     //
     // EmitBreak, EmitContinue and EmitThrow are overridden in

Compilation/GeneratorStateAnalyzer.cs

@@ -28,7 +28,8 @@ public record GeneratorFunctionAnalysis(
         HashSet<string> HoistedParameters,
         bool UsesThis,
         bool HasYieldStar,
-        List<Stmt.ForOf> ForOfLoopsWithYield  // for...of loops containing yields that need enumerator hoisting
+        List<Stmt.ForOf> ForOfLoopsWithYield,  // for...of loops containing yields that need enumerator hoisting
+        List<Stmt.ForIn> ForInLoopsWithYield   // for...in loops containing yields that need key-list/index hoisting (#547)
     );
 
     // State during analysis
@@ -37,6 +38,7 @@ public record GeneratorFunctionAnalysis(
     private readonly HashSet<string> _variablesUsedAfterYield = [];
     private readonly HashSet<string> _variablesDeclaredBeforeYield = [];
     private readonly List<Stmt.ForOf> _forOfLoopsWithYield = [];  // for...of loops containing yields (enumerator hoisting)
+    private readonly List<Stmt.ForIn> _forInLoopsWithYield = [];  // for...in loops containing yields (key-list/index hoisting, #547)
     // Loop bodies currently being analyzed (innermost on top). A loop whose body contains a
     // yield re-executes after the yield resumes, so every local used anywhere in it is live
     // across the suspension and must be hoisted to a state-machine field — otherwise the IL
@@ -91,7 +93,8 @@ public GeneratorFunctionAnalysis Analyze(Stmt.Function func)
             HoistedParameters: parameters,
             UsesThis: _usesThis,
             HasYieldStar: _hasYieldStar,
-            ForOfLoopsWithYield: [.. _forOfLoopsWithYield]
+            ForOfLoopsWithYield: [.. _forOfLoopsWithYield],
+            ForInLoopsWithYield: [.. _forInLoopsWithYield]
         );
     }
 
@@ -102,6 +105,7 @@ private void Reset()
         _variablesUsedAfterYield.Clear();
         _variablesDeclaredBeforeYield.Clear();
         _forOfLoopsWithYield.Clear();
+        _forInLoopsWithYield.Clear();
         _loopStack.Clear();
         _yieldCounter = 0;
         _seenYield = false;
@@ -114,14 +118,15 @@ private void Reset()
     /// yield occurs anywhere inside it. <see cref="ForOf"/> is non-null only for for...of loops,
     /// which additionally need their enumerator hoisted to a field.
     /// </summary>
-    private sealed class LoopScope(Stmt.ForOf? forOf)
+    private sealed class LoopScope(Stmt.ForOf? forOf, Stmt.ForIn? forIn)
     {
         public readonly HashSet<string> UsedVariables = [];
         public bool ContainsYield;
         public readonly Stmt.ForOf? ForOf = forOf;
+        public readonly Stmt.ForIn? ForIn = forIn;
     }
 
-    private void EnterLoop(Stmt.ForOf? forOf = null) => _loopStack.Push(new LoopScope(forOf));
+    private void EnterLoop(Stmt.ForOf? forOf = null, Stmt.ForIn? forIn = null) => _loopStack.Push(new LoopScope(forOf, forIn));
 
     // On leaving a loop body that contained a yield, hoist every local it used: the body
     // re-executes after the yield resumes, so those values must survive the suspension (#497).
@@ -185,7 +190,9 @@ protected override void VisitForIn(Stmt.ForIn stmt)
         _declaredVariables.Add(stmt.Variable.Lexeme);
         if (!_seenYield)
             _variablesDeclaredBeforeYield.Add(stmt.Variable.Lexeme);
-        EnterLoop();
+
+        // Pass the loop node so a yield inside also records it for key-list/index hoisting (#547).
+        EnterLoop(forIn: stmt);
         base.VisitForIn(stmt);  // object + body
         ExitLoop();
     }
@@ -263,6 +270,8 @@ protected override void VisitYield(Expr.Yield expr)
             scope.ContainsYield = true;
             if (scope.ForOf != null && !_forOfLoopsWithYield.Contains(scope.ForOf))
                 _forOfLoopsWithYield.Add(scope.ForOf);
+            if (scope.ForIn != null && !_forInLoopsWithYield.Contains(scope.ForIn))
+                _forInLoopsWithYield.Add(scope.ForIn);
         }
     }
 

Compilation/GeneratorStateMachineBuilder.cs

@@ -160,6 +160,7 @@ public void DefineStateMachine(
         _hoisting.DefineHoistedParameters(analysis.HoistedParameters);
         _hoisting.DefineHoistedLocals(analysis.HoistedLocals);
         _hoisting.DefineHoistedEnumerators(analysis.ForOfLoopsWithYield, _types.IEnumerator);
+        _hoisting.DefineHoistedForInState(analysis.ForInLoopsWithYield, _types.ListOfObject, _types.Int32);
 
         // Define 'this' field for instance methods that use 'this'
         if (isInstanceMethod && analysis.UsesThis)
@@ -415,6 +416,12 @@ private void DefineGetEnumeratorMethods()
     /// </summary>
     public FieldBuilder? GetEnumeratorField(Parsing.Stmt.ForOf loop) => _hoisting.GetEnumeratorField(loop);
 
+    /// <summary>
+    /// Gets the hoisted key-list / index fields for a for...in loop containing yields, or null if not hoisted (#547).
+    /// </summary>
+    public FieldBuilder? GetForInKeysField(Parsing.Stmt.ForIn loop) => _hoisting.GetForInKeysField(loop);
+    public FieldBuilder? GetForInIndexField(Parsing.Stmt.ForIn loop) => _hoisting.GetForInIndexField(loop);
+
     /// <summary>
     /// Finalizes the type after MoveNext body has been emitted.
     /// </summary>

Compilation/HoistingManager.cs

@@ -32,6 +32,15 @@ public class HoistingManager
     /// </summary>
     public Dictionary<Stmt.ForOf, FieldBuilder> HoistedEnumerators { get; } = [];
 
+    /// <summary>
+    /// Key-list and index fields for for...in loops that contain yield/await. The base for...in
+    /// emitter keeps the enumerated key list and the current index in IL locals, which a state-machine
+    /// MoveNext re-entry wipes — so a yield in the loop body would restart from the first key (#547).
+    /// Hoisting both to fields lets the iteration position survive the suspension.
+    /// </summary>
+    public Dictionary<Stmt.ForIn, FieldBuilder> HoistedForInKeys { get; } = [];
+    public Dictionary<Stmt.ForIn, FieldBuilder> HoistedForInIndex { get; } = [];
+
     public HoistingManager(TypeBuilder typeBuilder, Type objectType)
     {
         _typeBuilder = typeBuilder;
@@ -105,4 +114,27 @@ public void DefineHoistedEnumerators(IEnumerable<Stmt.ForOf> forOfLoops, Type en
     /// </summary>
     public FieldBuilder? GetEnumeratorField(Stmt.ForOf loop) =>
         HoistedEnumerators.TryGetValue(loop, out var field) ? field : null;
+
+    /// <summary>
+    /// Defines the key-list and index fields for for...in loops that contain yields/awaits (#547).
+    /// <paramref name="keysListType"/> is <c>List&lt;object&gt;</c>; <paramref name="indexType"/> is <c>int</c>.
+    /// </summary>
+    public void DefineHoistedForInState(IEnumerable<Stmt.ForIn> forInLoops, Type keysListType, Type indexType)
+    {
+        int index = 0;
+        foreach (var loop in forInLoops)
+        {
+            HoistedForInKeys[loop] = _typeBuilder.DefineField($"<>7__inKeys{index}", keysListType, FieldAttributes.Private);
+            HoistedForInIndex[loop] = _typeBuilder.DefineField($"<>7__inIdx{index}", indexType, FieldAttributes.Private);
+            index++;
+        }
+    }
+
+    /// <summary>Gets the hoisted key-list field for a for...in loop, or null if not hoisted.</summary>
+    public FieldBuilder? GetForInKeysField(Stmt.ForIn loop) =>
+        HoistedForInKeys.TryGetValue(loop, out var field) ? field : null;
+
+    /// <summary>Gets the hoisted index field for a for...in loop, or null if not hoisted.</summary>
+    public FieldBuilder? GetForInIndexField(Stmt.ForIn loop) =>
+        HoistedForInIndex.TryGetValue(loop, out var field) ? field : null;
 }