From e24562b461897caa281924eef412d266614c3d80 Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Mon, 15 Jun 2026 20:17:26 -0700 Subject: [PATCH 1/4] Fix #547: compiled generator for-in with yield stops after first key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator MoveNext emitter hoists the for-of enumerator across yields but kept the for-in key list and current index in IL locals, which a MoveNext re-entry wipes — so a yielding for-in restarted from (or stopped after) the first key. Mirror the for-of treatment: GeneratorStateAnalyzer records for-in loops whose body yields (ForInLoopsWithYield), Hoisting- Manager allocates a key-list field + index field per such loop, and GeneratorMoveNextEmitter.EmitForIn reads/writes both from those fields. Interpreter was already correct; compiled-mode only. --- .../GeneratorMoveNextEmitter.Statements.cs | 87 ++++++++++++++++++- Compilation/GeneratorStateAnalyzer.cs | 19 ++-- Compilation/GeneratorStateMachineBuilder.cs | 7 ++ Compilation/HoistingManager.cs | 32 +++++++ SharpTS.Tests/SharedTests/GeneratorTests.cs | 57 ++++++++++++ 5 files changed, 196 insertions(+), 6 deletions(-) diff --git a/Compilation/GeneratorMoveNextEmitter.Statements.cs b/Compilation/GeneratorMoveNextEmitter.Statements.cs index 4d8f142c..c1af3003 100644 --- a/Compilation/GeneratorMoveNextEmitter.Statements.cs +++ b/Compilation/GeneratorMoveNextEmitter.Statements.cs @@ -244,10 +244,95 @@ protected override void EmitForOf(Stmt.ForOf f) #endregion + #region For...In Loop Override (Hoisted Key-List/Index Support) + + /// + /// Emits a for...in loop with hoisted key-list and index when the body contains a yield. + /// The base emitter keeps the enumerated key list and current index in IL locals, which a + /// MoveNext re-entry across a yield wipes — so the loop would restart from (or stop after) the + /// first key (#547). Storing both in state-machine fields makes the iteration position survive + /// the suspension, mirroring the hoisted-enumerator treatment gives + /// for...of. + /// + protected override void EmitForIn(Stmt.ForIn f) + { + var keysField = _builder.GetForInKeysField(f); + if (keysField == null) + { + // No yield inside this loop - use base implementation with local key list/index. + base.EmitForIn(f); + return; + } + var indexField = _builder.GetForInIndexField(f)!; + + var startLabel = _il.DefineLabel(); + var endLabel = _il.DefineLabel(); + var continueLabel = _il.DefineLabel(); + + // Get keys from the object and stash them in the hoisted field (need a temp for the swap). + EmitExpression(f.Object); + EnsureBoxed(); + _il.Emit(OpCodes.Call, _ctx!.Runtime!.GetKeys); + var keysTemp = _il.DeclareLocal(_types.ListOfObject); + _il.Emit(OpCodes.Stloc, keysTemp); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldloc, keysTemp); + _il.Emit(OpCodes.Stfld, keysField); + + // index = 0 + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldc_I4_0); + _il.Emit(OpCodes.Stfld, indexField); + + EnterLoop(endLabel, continueLabel); + + var loopVarLocal = DeclareLoopVariable(f.Variable.Lexeme); + + _il.MarkLabel(startLabel); + EmitCancellationCheck(); + + // if (index < keys.Count) else goto end — both read from hoisted fields. + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, indexField); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, keysField); + _il.Emit(OpCodes.Call, _ctx.Runtime!.GetLength); + _il.Emit(OpCodes.Clt); + _il.Emit(OpCodes.Brfalse, endLabel); + + // loopVar = keys[index] + EmitStoreLoopVariable(loopVarLocal, f.Variable.Lexeme, () => + { + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, keysField); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, indexField); + _il.Emit(OpCodes.Call, _ctx.Runtime!.GetElement); + }); + + EmitStatement(f.Body); + + _il.MarkLabel(continueLabel); + + // index = index + 1 (store back to the hoisted field) + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, indexField); + _il.Emit(OpCodes.Ldc_I4_1); + _il.Emit(OpCodes.Add); + _il.Emit(OpCodes.Stfld, indexField); + + _il.Emit(OpCodes.Br, startLabel); + + _il.MarkLabel(endLabel); + ExitLoop(); + } + + #endregion + // Note: The following methods are inherited from StatementEmitterBase: // - EmitStatement (dispatch) // - EmitIf, EmitWhile, EmitDoWhile (control flow) - // - EmitForIn (loops with DeclareLoopVariable/EmitStoreLoopVariable overrides) // - EmitBlock, EmitLabeledStatement, EmitSwitch, EmitPrint // // EmitBreak, EmitContinue and EmitThrow are overridden in diff --git a/Compilation/GeneratorStateAnalyzer.cs b/Compilation/GeneratorStateAnalyzer.cs index 4e046176..93360101 100644 --- a/Compilation/GeneratorStateAnalyzer.cs +++ b/Compilation/GeneratorStateAnalyzer.cs @@ -28,7 +28,8 @@ public record GeneratorFunctionAnalysis( HashSet HoistedParameters, bool UsesThis, bool HasYieldStar, - List ForOfLoopsWithYield // for...of loops containing yields that need enumerator hoisting + List ForOfLoopsWithYield, // for...of loops containing yields that need enumerator hoisting + List ForInLoopsWithYield // for...in loops containing yields that need key-list/index hoisting (#547) ); // State during analysis @@ -37,6 +38,7 @@ public record GeneratorFunctionAnalysis( private readonly HashSet _variablesUsedAfterYield = []; private readonly HashSet _variablesDeclaredBeforeYield = []; private readonly List _forOfLoopsWithYield = []; // for...of loops containing yields (enumerator hoisting) + private readonly List _forInLoopsWithYield = []; // for...in loops containing yields (key-list/index hoisting, #547) // Loop bodies currently being analyzed (innermost on top). A loop whose body contains a // yield re-executes after the yield resumes, so every local used anywhere in it is live // across the suspension and must be hoisted to a state-machine field — otherwise the IL @@ -91,7 +93,8 @@ public GeneratorFunctionAnalysis Analyze(Stmt.Function func) HoistedParameters: parameters, UsesThis: _usesThis, HasYieldStar: _hasYieldStar, - ForOfLoopsWithYield: [.. _forOfLoopsWithYield] + ForOfLoopsWithYield: [.. _forOfLoopsWithYield], + ForInLoopsWithYield: [.. _forInLoopsWithYield] ); } @@ -102,6 +105,7 @@ private void Reset() _variablesUsedAfterYield.Clear(); _variablesDeclaredBeforeYield.Clear(); _forOfLoopsWithYield.Clear(); + _forInLoopsWithYield.Clear(); _loopStack.Clear(); _yieldCounter = 0; _seenYield = false; @@ -114,14 +118,15 @@ private void Reset() /// yield occurs anywhere inside it. is non-null only for for...of loops, /// which additionally need their enumerator hoisted to a field. /// - private sealed class LoopScope(Stmt.ForOf? forOf) + private sealed class LoopScope(Stmt.ForOf? forOf, Stmt.ForIn? forIn) { public readonly HashSet UsedVariables = []; public bool ContainsYield; public readonly Stmt.ForOf? ForOf = forOf; + public readonly Stmt.ForIn? ForIn = forIn; } - private void EnterLoop(Stmt.ForOf? forOf = null) => _loopStack.Push(new LoopScope(forOf)); + private void EnterLoop(Stmt.ForOf? forOf = null, Stmt.ForIn? forIn = null) => _loopStack.Push(new LoopScope(forOf, forIn)); // On leaving a loop body that contained a yield, hoist every local it used: the body // re-executes after the yield resumes, so those values must survive the suspension (#497). @@ -185,7 +190,9 @@ protected override void VisitForIn(Stmt.ForIn stmt) _declaredVariables.Add(stmt.Variable.Lexeme); if (!_seenYield) _variablesDeclaredBeforeYield.Add(stmt.Variable.Lexeme); - EnterLoop(); + + // Pass the loop node so a yield inside also records it for key-list/index hoisting (#547). + EnterLoop(forIn: stmt); base.VisitForIn(stmt); // object + body ExitLoop(); } @@ -263,6 +270,8 @@ protected override void VisitYield(Expr.Yield expr) scope.ContainsYield = true; if (scope.ForOf != null && !_forOfLoopsWithYield.Contains(scope.ForOf)) _forOfLoopsWithYield.Add(scope.ForOf); + if (scope.ForIn != null && !_forInLoopsWithYield.Contains(scope.ForIn)) + _forInLoopsWithYield.Add(scope.ForIn); } } diff --git a/Compilation/GeneratorStateMachineBuilder.cs b/Compilation/GeneratorStateMachineBuilder.cs index e21b8219..e597e981 100644 --- a/Compilation/GeneratorStateMachineBuilder.cs +++ b/Compilation/GeneratorStateMachineBuilder.cs @@ -160,6 +160,7 @@ public void DefineStateMachine( _hoisting.DefineHoistedParameters(analysis.HoistedParameters); _hoisting.DefineHoistedLocals(analysis.HoistedLocals); _hoisting.DefineHoistedEnumerators(analysis.ForOfLoopsWithYield, _types.IEnumerator); + _hoisting.DefineHoistedForInState(analysis.ForInLoopsWithYield, _types.ListOfObject, _types.Int32); // Define 'this' field for instance methods that use 'this' if (isInstanceMethod && analysis.UsesThis) @@ -415,6 +416,12 @@ private void DefineGetEnumeratorMethods() /// public FieldBuilder? GetEnumeratorField(Parsing.Stmt.ForOf loop) => _hoisting.GetEnumeratorField(loop); + /// + /// Gets the hoisted key-list / index fields for a for...in loop containing yields, or null if not hoisted (#547). + /// + public FieldBuilder? GetForInKeysField(Parsing.Stmt.ForIn loop) => _hoisting.GetForInKeysField(loop); + public FieldBuilder? GetForInIndexField(Parsing.Stmt.ForIn loop) => _hoisting.GetForInIndexField(loop); + /// /// Finalizes the type after MoveNext body has been emitted. /// diff --git a/Compilation/HoistingManager.cs b/Compilation/HoistingManager.cs index 2f1cc726..369b257b 100644 --- a/Compilation/HoistingManager.cs +++ b/Compilation/HoistingManager.cs @@ -32,6 +32,15 @@ public class HoistingManager /// public Dictionary HoistedEnumerators { get; } = []; + /// + /// Key-list and index fields for for...in loops that contain yield/await. The base for...in + /// emitter keeps the enumerated key list and the current index in IL locals, which a state-machine + /// MoveNext re-entry wipes — so a yield in the loop body would restart from the first key (#547). + /// Hoisting both to fields lets the iteration position survive the suspension. + /// + public Dictionary HoistedForInKeys { get; } = []; + public Dictionary HoistedForInIndex { get; } = []; + public HoistingManager(TypeBuilder typeBuilder, Type objectType) { _typeBuilder = typeBuilder; @@ -105,4 +114,27 @@ public void DefineHoistedEnumerators(IEnumerable forOfLoops, Type en /// public FieldBuilder? GetEnumeratorField(Stmt.ForOf loop) => HoistedEnumerators.TryGetValue(loop, out var field) ? field : null; + + /// + /// Defines the key-list and index fields for for...in loops that contain yields/awaits (#547). + /// is List<object>; is int. + /// + public void DefineHoistedForInState(IEnumerable forInLoops, Type keysListType, Type indexType) + { + int index = 0; + foreach (var loop in forInLoops) + { + HoistedForInKeys[loop] = _typeBuilder.DefineField($"<>7__inKeys{index}", keysListType, FieldAttributes.Private); + HoistedForInIndex[loop] = _typeBuilder.DefineField($"<>7__inIdx{index}", indexType, FieldAttributes.Private); + index++; + } + } + + /// Gets the hoisted key-list field for a for...in loop, or null if not hoisted. + public FieldBuilder? GetForInKeysField(Stmt.ForIn loop) => + HoistedForInKeys.TryGetValue(loop, out var field) ? field : null; + + /// Gets the hoisted index field for a for...in loop, or null if not hoisted. + public FieldBuilder? GetForInIndexField(Stmt.ForIn loop) => + HoistedForInIndex.TryGetValue(loop, out var field) ? field : null; } diff --git a/SharpTS.Tests/SharedTests/GeneratorTests.cs b/SharpTS.Tests/SharedTests/GeneratorTests.cs index e9d38d4e..20b311c7 100644 --- a/SharpTS.Tests/SharedTests/GeneratorTests.cs +++ b/SharpTS.Tests/SharedTests/GeneratorTests.cs @@ -53,6 +53,63 @@ public void Generator_WithParameters_UsesParameters(ExecutionMode mode) #endregion + #region For...In Integration (#547) + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_ForInLoop_YieldsAllKeys(ExecutionMode mode) + { + // A for...in whose body yields must continue past the first key: the key list and index + // are hoisted to state-machine fields so they survive the MoveNext re-entry (#547). + var source = """ + function* inGen() { + const obj = { a: 1, b: 2, c: 3 }; + for (const k in obj) { yield k; } + } + console.log([...inGen()].join(",")); + """; + + var output = TestHarness.Run(source, mode); + Assert.Equal("a,b,c\n", output); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_ForInLoop_TwoYieldsPerKey_AccumulatesAcrossSuspension(ExecutionMode mode) + { + // Two yields per iteration force the loop to re-enter mid-body; the index must persist (#547). + var source = """ + function* kv() { + const obj: any = { x: 10, y: 20 }; + for (const k in obj) { yield k; yield obj[k]; } + } + console.log([...kv()].join(",")); + """; + + var output = TestHarness.Run(source, mode); + Assert.Equal("x,10,y,20\n", output); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_ForInLoop_Nested_YieldsCartesianProduct(ExecutionMode mode) + { + // Nested for...in loops, each with its own hoisted key-list/index fields. + var source = """ + function* nested() { + const a = { p: 0, q: 0 }; + const b = { m: 0, n: 0 }; + for (const i in a) { for (const j in b) { yield i + j; } } + } + console.log([...nested()].join(",")); + """; + + var output = TestHarness.Run(source, mode); + Assert.Equal("pm,pn,qm,qn\n", output); + } + + #endregion + #region Yield* Delegation [Theory] From 328306c27f48f86487eb2ded5e623bd2d0fbeb30 Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Mon, 15 Jun 2026 20:17:42 -0700 Subject: [PATCH 2/4] Fix #631, #542: truly-async, serialized compiled async-generator next() Compiled async generators drove the body synchronously (next() did MoveNextAsync().AsTask().GetResult(); for await...of likewise blocked on GetResult), so a not-yet-settled await deadlocked the event-loop thread (#631) and concurrent/re-entrant next() recursed into MoveNextAsync, only kept from overflowing by a reject-on-busy guard, never queued (#542). - next() is now truly asynchronous: it drives one step and returns the Task built by an emitted AsyncGeneratorBuildResult helper. - Requests serialize on a per-generator _pendingTail task chain (the ECMA-262 27.6.3 AsyncGeneratorQueue modeled as a Task chain) so overlapping next() calls run FIFO. A synchronously re-entrant next() still rejects with a TypeError (the self-reentrant case would deadlock under a real queue anyway). - for await...of suspends on the iterator protocol via the async state machine instead of blocking. The for-await lowering is shared in StatementEmitterBase (main's #430/#645); AsyncMoveNextEmitter overrides EmitForAwaitOf to SUSPEND on each next()/return() await (the shared base still blocks for async arrows / async generators -- async-gen tracked by #697). AsyncStateAnalyzer reserves a suspension state per implicit await; a try enclosing a for-await takes the flag-based path (ContainsAwait treats for-await as containing awaits) so resume labels aren't branched into. - AsyncGeneratorAwaitContinue no longer short-circuits on the awaited task's fault; it resumes MoveNextAsync so a pending rejection reaches the body's resume point and its try/catch (unblocks the pending sub-case of #617). Interpreter async generators use a separate eager-drain model and are unaffected; compiled-mode only. --- .../AsyncGeneratorStateMachineBuilder.cs | 328 ++++++++++++------ .../AsyncMoveNextEmitter.Expressions.cs | 22 +- .../AsyncMoveNextEmitter.Statements.Loops.cs | 325 ++++++++++++++++- ...syncMoveNextEmitter.Statements.TryCatch.cs | 5 +- Compilation/AsyncStateAnalyzer.Expressions.cs | 9 + Compilation/AsyncStateAnalyzer.Statements.cs | 14 + Compilation/AsyncStateAnalyzer.cs | 2 +- Compilation/EmittedRuntime.cs | 4 + Compilation/RuntimeEmitter.AsyncGenerator.cs | 198 ++++++++++- .../SharedTests/AsyncGeneratorTests.cs | 94 +++++ 10 files changed, 883 insertions(+), 118 deletions(-) diff --git a/Compilation/AsyncGeneratorStateMachineBuilder.cs b/Compilation/AsyncGeneratorStateMachineBuilder.cs index 5580458e..64f3aabe 100644 --- a/Compilation/AsyncGeneratorStateMachineBuilder.cs +++ b/Compilation/AsyncGeneratorStateMachineBuilder.cs @@ -47,11 +47,25 @@ public class AsyncGeneratorStateMachineBuilder // Flag set by return() to trigger finally blocks during MoveNextAsync resume public FieldBuilder ReturnRequestedField { get; private set; } = null!; - // Re-entrancy flag: true only while next()/return() synchronously drives MoveNextAsync. A guest - // next()/return()/throw() observing it means the generator body is advancing itself; without the - // guard the synchronous drive recurses into MoveNextAsync until the stack overflows (#542). + // Re-entrancy flag: true only while the generator body is synchronously advancing (the window of a + // MoveNextAsync call before it suspends or completes). A guest next()/return()/throw() observing it + // means the body is advancing itself; without the guard that synchronous re-entry recurses into + // MoveNextAsync until the stack overflows (#542). Set/cleared by and return()'s drive. public FieldBuilder ExecutingField { get; private set; } = null!; + // Tail of the request chain (ECMA-262 §27.6.3 AsyncGeneratorQueue, modeled as a Task chain): the + // Task of the most recently enqueued next(). A new next() awaits this before driving, so + // overlapping next() calls run in FIFO order instead of re-entering MoveNextAsync concurrently and + // corrupting state. Lets next() be truly asynchronous — it never blocks the event-loop thread on a + // pending await (#631) — and serializes concurrent next() requests (#542). Null until the first next(). + public FieldBuilder PendingTailField { get; private set; } = null!; + + // (): drives one MoveNextAsync step (under the executing guard) and returns its + // { value, done } Task; (antecedent, state): static trampoline that + // calls on a queued next() once the prior request settles. + private MethodBuilder _driveOnceMethod = null!; + private MethodBuilder _driveContinuationMethod = null!; + // Constructor public ConstructorBuilder Constructor { get; private set; } = null!; @@ -147,6 +161,13 @@ public void DefineStateMachine( FieldAttributes.Private ); + // Tail of the request chain for truly-async, serialized next() (#631/#542). + PendingTailField = _stateMachineType.DefineField( + "<>5__pendingTail", + _types.Task, + FieldAttributes.Private + ); + // Define delegated enumerator field for yield* expressions (typed as object to hold either sync or async enumerators) if (analysis.HasYieldStar) { @@ -336,6 +357,22 @@ private void DefineGetAsyncEnumeratorMethod() /// private void DefineAsyncGeneratorMethods() { + // Internal drive helpers, defined before next() so its body can reference them. + // () : Task — drive one step under the executing guard. + _driveOnceMethod = _stateMachineType.DefineMethod( + "", + MethodAttributes.Private | MethodAttributes.HideBySig, + _types.TaskOfObject, + Type.EmptyTypes + ); + // static (Task antecedent, object state) : Task — queue trampoline. + _driveContinuationMethod = _stateMachineType.DefineMethod( + "", + MethodAttributes.Private | MethodAttributes.Static | MethodAttributes.HideBySig, + _types.TaskOfObject, + [_types.Task, _types.Object] + ); + // next() method - returns Task with { value, done } NextMethod = _stateMachineType.DefineMethod( "next", @@ -345,6 +382,8 @@ private void DefineAsyncGeneratorMethods() ); EmitNextMethodBody(); + EmitDriveOnceBody(); + EmitDriveContinuationBody(); _stateMachineType.DefineMethodOverride(NextMethod, _runtime!.AsyncGeneratorNextMethod); // return(value) method - returns Task with { value, done: true } @@ -373,118 +412,160 @@ private void DefineAsyncGeneratorMethods() private void EmitNextMethodBody() { var il = NextMethod.GetILGenerator(); - var doneLabel = il.DefineLabel(); - var endLabel = il.DefineLabel(); - // Reject a re-entrant next() — the body advancing itself — before driving MoveNextAsync (#542). + // Reject only a *synchronously* re-entrant next() — the body calling next() on itself before it + // suspends, which would recurse into MoveNextAsync and overflow the stack (#542). A next() issued + // while the body is suspended is NOT rejected: it queues on the request chain below (and, for the + // self-reentrant case, that queued request can never run before the suspended body it sits behind — + // a deadlock, matching Node, rather than the old stack overflow). EmitThrowIfExecutingAsync(il); - // Drive the body with the executing flag set so a re-entrant next()/return()/throw() from inside - // it hits the guard instead of recursing into MoveNextAsync (which overflowed the stack). An - // outer try/finally clears the flag on suspension/completion AND on an uncaught body throw; - // an inner try/catch captures that throw into exLocal so next() returns a *rejected* Task - // rather than throwing synchronously out of the call — `await it.next()` must be catchable - // (ECMA-262 §27.6.1.2: next() returns a promise that rejects on an uncaught body throw) (#566). - var movedLocal = il.DeclareLocal(_types.Boolean); + var prevLocal = il.DeclareLocal(_types.Task); + var resultLocal = il.DeclareLocal(_types.TaskOfObject); + var fastPath = il.DefineLabel(); + var setTail = il.DefineLabel(); + + // prev = this._pendingTail + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, PendingTailField); + il.Emit(OpCodes.Stloc, prevLocal); + + // if (prev == null || prev.IsCompleted) drive immediately; else queue behind it. + il.Emit(OpCodes.Ldloc, prevLocal); + il.Emit(OpCodes.Brfalse, fastPath); + il.Emit(OpCodes.Ldloc, prevLocal); + il.Emit(OpCodes.Callvirt, _types.GetPropertyGetter(_types.Task, "IsCompleted")); + il.Emit(OpCodes.Brtrue, fastPath); + + // Slow path — a prior request is still in flight: + // result = prev.ContinueWith>(, this, ExecuteSynchronously).Unwrap(); + // ExecuteSynchronously runs the continuation inline on whichever (event-loop) thread settles prev, + // so the body is never advanced on a thread-pool thread. The continuation fires regardless of how + // prev settled (a faulted prior next() does not stall the queue). + il.Emit(OpCodes.Ldloc, prevLocal); + il.Emit(OpCodes.Ldnull); + il.Emit(OpCodes.Ldftn, _driveContinuationMethod); + var funcType = _types.MakeGenericType(typeof(Func<,,>), _types.Task, _types.Object, _types.TaskOfObject); + il.Emit(OpCodes.Newobj, funcType.GetConstructor([_types.Object, typeof(IntPtr)])!); + il.Emit(OpCodes.Ldarg_0); // state = this + il.Emit(OpCodes.Ldc_I4, (int)System.Threading.Tasks.TaskContinuationOptions.ExecuteSynchronously); + il.Emit(OpCodes.Callvirt, ResolveContinueWithFuncState()); + il.Emit(OpCodes.Call, ResolveTaskUnwrap()); + il.Emit(OpCodes.Stloc, resultLocal); + il.Emit(OpCodes.Br, setTail); + + // Fast path — no in-flight request: drive one step now. + il.MarkLabel(fastPath); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, _driveOnceMethod); + il.Emit(OpCodes.Stloc, resultLocal); + + il.MarkLabel(setTail); + // this._pendingTail = result; return result; + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldloc, resultLocal); + il.Emit(OpCodes.Stfld, PendingTailField); + il.Emit(OpCodes.Ldloc, resultLocal); + il.Emit(OpCodes.Ret); + } + + /// + /// Emits <DriveOnce>(): sets the executing guard, calls MoveNextAsync() (which runs + /// the body's synchronous segment up to the next suspension), clears the guard, then hands the resulting + /// ValueTask to AsyncGeneratorBuildResult to produce the { value, done } Task<object>. + /// The executing flag is therefore set only for the synchronous-advance window — once the body suspends + /// it is cleared, so a concurrent next() observes "not executing" and queues rather than rejecting. + /// An uncaught synchronous throw from the body becomes a faulted Task (a rejected next() promise, #566). + /// + private void EmitDriveOnceBody() + { + var il = _driveOnceMethod.GetILGenerator(); + var vtLocal = il.DeclareLocal(_types.ValueTaskOfBool); + var resultLocal = il.DeclareLocal(_types.TaskOfObject); var exLocal = il.DeclareLocal(_types.Exception); + var doneLabel = il.DefineLabel(); + + il.BeginExceptionBlock(); + // executing = true il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldc_I4_1); il.Emit(OpCodes.Stfld, ExecutingField); - il.BeginExceptionBlock(); // outer: finally clears the executing flag - il.BeginExceptionBlock(); // inner: catch captures a body throw into exLocal - - // Call MoveNextAsync(), convert ValueTask → Task, then block for the result. - // (This works for already-completed awaits — the common case; a pending await blocks the calling - // thread until the continuation drives MoveNextAsync to completion.) + // vt = this.MoveNextAsync() — runs the synchronous segment; may throw synchronously il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Call, MoveNextAsyncMethod); - var vtLocal = il.DeclareLocal(_types.ValueTaskOfBool); il.Emit(OpCodes.Stloc, vtLocal); - il.Emit(OpCodes.Ldloca, vtLocal); - var asTaskMethod = _types.GetMethodNoParams(_types.ValueTaskOfBool, "AsTask"); - il.Emit(OpCodes.Call, asTaskMethod); - var taskBoolLocal = il.DeclareLocal(_types.MakeGenericType(_types.TaskOpen, _types.Boolean)); - il.Emit(OpCodes.Stloc, taskBoolLocal); - - // Get the result: task.GetAwaiter().GetResult() - il.Emit(OpCodes.Ldloc, taskBoolLocal); - var getAwaiterMethod = _types.GetMethodNoParams(_types.MakeGenericType(_types.TaskOpen, _types.Boolean), "GetAwaiter"); - il.Emit(OpCodes.Call, getAwaiterMethod); - var awaiterType = _types.MakeGenericType(_types.TaskAwaiterOpen, _types.Boolean); - var awaiterLocal = il.DeclareLocal(awaiterType); - il.Emit(OpCodes.Stloc, awaiterLocal); - il.Emit(OpCodes.Ldloca, awaiterLocal); - var getResultMethod = _types.GetMethodNoParams(awaiterType, "GetResult"); - il.Emit(OpCodes.Call, getResultMethod); - il.Emit(OpCodes.Stloc, movedLocal); + // executing = false (normal path: body suspended or completed) + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldc_I4_0); + il.Emit(OpCodes.Stfld, ExecutingField); + // result = AsyncGeneratorBuildResult(vt, (IAsyncEnumerator)this) + il.Emit(OpCodes.Ldloc, vtLocal); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Castclass, _types.IAsyncEnumeratorOfObject); + il.Emit(OpCodes.Call, _runtime!.AsyncGeneratorBuildResult); + il.Emit(OpCodes.Stloc, resultLocal); + il.Emit(OpCodes.Leave, doneLabel); - // inner catch: stash the exception. The outer finally still clears the flag; the conversion to - // a faulted Task happens after both blocks close, so the throw never escapes synchronously (#566). + // catch (Exception e) { executing = false; result = Task.FromException(e); } il.BeginCatchBlock(_types.Exception); il.Emit(OpCodes.Stloc, exLocal); - il.EndExceptionBlock(); // end inner try/catch - - il.BeginFinallyBlock(); il.Emit(OpCodes.Ldarg_0); il.Emit(OpCodes.Ldc_I4_0); il.Emit(OpCodes.Stfld, ExecutingField); - il.EndExceptionBlock(); // end outer try/finally - - // A body throw was captured → return Task.FromException(ex), i.e. a rejected promise. - var buildResultLabel = il.DefineLabel(); il.Emit(OpCodes.Ldloc, exLocal); - il.Emit(OpCodes.Brfalse, buildResultLabel); - il.Emit(OpCodes.Ldloc, exLocal); - var fromExceptionNext = typeof(Task).GetMethod("FromException", 1, [typeof(Exception)])!.MakeGenericMethod(_types.Object); - il.Emit(OpCodes.Call, fromExceptionNext); - il.Emit(OpCodes.Ret); - - il.MarkLabel(buildResultLabel); - // movedLocal: true = has value, false = done - il.Emit(OpCodes.Ldloc, movedLocal); - il.Emit(OpCodes.Brfalse, doneLabel); - - // Not done: create { value: Current, done: false } - il.Emit(OpCodes.Newobj, _types.GetDefaultConstructor(_types.DictionaryStringObject)); - il.Emit(OpCodes.Dup); - il.Emit(OpCodes.Ldstr, "value"); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldfld, CurrentField); - il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); - il.Emit(OpCodes.Dup); - il.Emit(OpCodes.Ldstr, "done"); - il.Emit(OpCodes.Ldc_I4_0); - il.Emit(OpCodes.Box, _types.Boolean); - il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); - il.Emit(OpCodes.Br, endLabel); + var fromException = typeof(Task).GetMethod("FromException", 1, [typeof(Exception)])!.MakeGenericMethod(_types.Object); + il.Emit(OpCodes.Call, fromException); + il.Emit(OpCodes.Stloc, resultLocal); + il.Emit(OpCodes.Leave, doneLabel); + il.EndExceptionBlock(); - // Done: create { value: CurrentField (return value), done: true } il.MarkLabel(doneLabel); - il.Emit(OpCodes.Newobj, _types.GetDefaultConstructor(_types.DictionaryStringObject)); - il.Emit(OpCodes.Dup); - il.Emit(OpCodes.Ldstr, "value"); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldfld, CurrentField); - il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); - il.Emit(OpCodes.Dup); - il.Emit(OpCodes.Ldstr, "done"); - il.Emit(OpCodes.Ldc_I4_1); - il.Emit(OpCodes.Box, _types.Boolean); - il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); + il.Emit(OpCodes.Ldloc, resultLocal); + il.Emit(OpCodes.Ret); + } - il.MarkLabel(endLabel); - // Wrap in Task.FromResult - var fromResultMethod = typeof(Task).GetMethod("FromResult")!.MakeGenericMethod(_types.Object); - il.Emit(OpCodes.Call, fromResultMethod); + /// + /// Emits the static trampoline <DriveContinuation>(Task antecedent, object state) used as the + /// ContinueWith body for a queued next(): once the prior request settles, it drives the next step by + /// calling the private instance <DriveOnce> on the generator passed via . + /// + private void EmitDriveContinuationBody() + { + var il = _driveContinuationMethod.GetILGenerator(); + // return (()state).(); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Castclass, _stateMachineType); + il.Emit(OpCodes.Call, _driveOnceMethod); il.Emit(OpCodes.Ret); } + /// Resolves Task.ContinueWith<Task<object>>(Func<Task,object,Task<object>>, object, TaskContinuationOptions). + private MethodInfo ResolveContinueWithFuncState() => + typeof(Task).GetMethods(BindingFlags.Public | BindingFlags.Instance) + .First(m => m.Name == "ContinueWith" && m.IsGenericMethodDefinition + && m.GetParameters() is { Length: 3 } p + && p[0].ParameterType.IsGenericType + && p[0].ParameterType.GetGenericTypeDefinition() == typeof(Func<,,>) + && p[1].ParameterType == typeof(object) + && p[2].ParameterType == typeof(System.Threading.Tasks.TaskContinuationOptions)) + .MakeGenericMethod(_types.TaskOfObject); + + /// Resolves TaskExtensions.Unwrap<object>(Task<Task<object>>). + private MethodInfo ResolveTaskUnwrap() => + typeof(System.Threading.Tasks.TaskExtensions).GetMethods(BindingFlags.Public | BindingFlags.Static) + .First(m => m.Name == "Unwrap" && m.IsGenericMethodDefinition) + .MakeGenericMethod(_types.Object); + private void EmitReturnMethodBody() { var il = ReturnMethod.GetILGenerator(); - // Reject a re-entrant return() before mutating state (#542; see EmitThrowIfExecutingAsync). - EmitThrowIfExecutingAsync(il); + // Reject return() while a request is in flight (the body executing, or a next() still pending on + // the request chain): return() drives MoveNextAsync synchronously to run finallys, which would + // re-enter a suspended/advancing state machine and corrupt it. Full per-spec queuing of return() + // behind pending next()s is a follow-up; in a `for await…of` early-exit (the common return() path) + // each next() is awaited before return() is called, so this fast-rejects only pathological overlap (#542). + EmitThrowIfBusyAsync(il); // If state >= 0 (suspended at a yield point), we need to trigger finally blocks // by setting __returnRequested and calling MoveNextAsync @@ -568,9 +649,10 @@ private void EmitThrowMethodBody() { var il = ThrowMethod.GetILGenerator(); - // Reject a re-entrant throw() before mutating state (#542). throw() does not itself drive - // MoveNextAsync, so it only needs the entry guard, not the executing flag. - EmitThrowIfExecutingAsync(il); + // Reject throw() while a request is in flight (the body executing, or a next() still pending): + // throw() completes the generator immediately, so letting it run concurrently with a pending + // next() would settle that next() against an already-closed generator. (#542) + EmitThrowIfBusyAsync(il); // Set state to -2 (completed) il.Emit(OpCodes.Ldarg_0); @@ -600,15 +682,13 @@ private void EmitThrowMethodBody() } /// - /// Emits, at the head of next()/return()/throw(), a guard that rejects a re-entrant call — the - /// generator body advancing itself — by returning a faulted Task<object> carrying a TypeError. - /// ECMA-262 §27.6.3 (AsyncGeneratorEnqueue) actually *queues* such a request, but this - /// synchronous-drive state machine (next() blocks on MoveNextAsync via GetResult) cannot queue - /// without deadlocking the only case re-entrancy is reachable in — the body awaiting its own - /// request. So it rejects rather than recurse into MoveNextAsync until the stack overflows (#542). - /// The error is wrapped via CreateException so the guest observes a catchable TypeError, and is - /// surfaced as a rejected promise (not a synchronous throw) since next() returns a Task. _runtime - /// is non-null here — DefineAsyncGeneratorMethods only runs when the runtime is present. + /// Emits, at the head of next(), a guard that rejects a *synchronously* re-entrant call — the body + /// calling next() on itself before it suspends — by returning a faulted Task<object> carrying a + /// TypeError. Without it that call would recurse into MoveNextAsync and overflow the stack (#542). A + /// next() issued while the body is *suspended* is not rejected here; it queues on the request chain. + /// The error is wrapped via CreateException so the guest observes a catchable TypeError, surfaced as a + /// rejected promise (not a synchronous throw) since next() returns a Task. _runtime is non-null here — + /// DefineAsyncGeneratorMethods only runs when the runtime is present. /// private void EmitThrowIfExecutingAsync(ILGenerator il) { @@ -617,15 +697,57 @@ private void EmitThrowIfExecutingAsync(ILGenerator il) il.Emit(OpCodes.Ldfld, ExecutingField); il.Emit(OpCodes.Brfalse, okLabel); - // return Task.FromException(CreateException(new TypeError("..."))); + EmitRejectAlreadyRunning(il); + + il.MarkLabel(okLabel); + } + + /// + /// Emits, at the head of return()/throw(), a guard that rejects when ANY request is in flight: the + /// body is synchronously advancing (executing), OR a next() is still pending on the request chain + /// ( non-null and not yet completed). Unlike next(), return()/throw() + /// do not queue — they mutate/close the generator immediately, so they must not overlap a pending + /// next(). Reuses the same catchable "already running" TypeError rejection. + /// + private void EmitThrowIfBusyAsync(ILGenerator il) + { + var okLabel = il.DefineLabel(); + var rejectLabel = il.DefineLabel(); + + // if (executing) reject; + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, ExecutingField); + il.Emit(OpCodes.Brtrue, rejectLabel); + + // if (pendingTail == null) ok; + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, PendingTailField); + il.Emit(OpCodes.Brfalse, okLabel); + + // if (pendingTail.IsCompleted) ok; else reject; + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, PendingTailField); + il.Emit(OpCodes.Callvirt, _types.GetPropertyGetter(_types.Task, "IsCompleted")); + il.Emit(OpCodes.Brtrue, okLabel); + + il.MarkLabel(rejectLabel); + EmitRejectAlreadyRunning(il); + + il.MarkLabel(okLabel); + } + + /// + /// Emits return Task.FromException<object>(CreateException(new TypeError("Async generator is + /// already running"))); — the shared rejection used by both re-entrancy guards. + /// + private void EmitRejectAlreadyRunning(ILGenerator il) + { il.Emit(OpCodes.Ldstr, "Async generator is already running"); il.Emit(OpCodes.Newobj, _runtime!.TSTypeErrorCtor); il.Emit(OpCodes.Call, _runtime!.CreateException); var fromException = typeof(Task).GetMethod("FromException", 1, [typeof(Exception)])!.MakeGenericMethod(_types.Object); il.Emit(OpCodes.Call, fromException); il.Emit(OpCodes.Ret); - - il.MarkLabel(okLabel); } /// diff --git a/Compilation/AsyncMoveNextEmitter.Expressions.cs b/Compilation/AsyncMoveNextEmitter.Expressions.cs index f2bc3752..eda2ce6d 100644 --- a/Compilation/AsyncMoveNextEmitter.Expressions.cs +++ b/Compilation/AsyncMoveNextEmitter.Expressions.cs @@ -10,15 +10,27 @@ public partial class AsyncMoveNextEmitter protected override void EmitAwait(Expr.Await a) { - int stateNumber = _currentAwaitState++; - var resumeLabel = _stateLabels[stateNumber]; - var continueLabel = _il.DefineLabel(); - var awaiterField = _builder.AwaiterFields[stateNumber]; - // 1. Emit the awaited expression (should produce Task or $Promise) EmitExpression(a.Expression); EnsureBoxed(); + // 2+. Coerce to Task, suspend/resume, and leave the awaited result on the stack. + EmitAwaitFromValueOnStack(_currentAwaitState++); + } + + /// + /// Emits the await of a value already on the evaluation stack (boxed): coerces it to + /// Task<object> (unwrapping $Promise / adopting thenables / wrapping plain values), + /// suspends the state machine until it settles, and leaves the awaited result on the stack. + /// Shared by and the for await…of loop's implicit next()/return() + /// awaits (#631); is the reserved suspension state for this await. + /// + internal void EmitAwaitFromValueOnStack(int stateNumber) + { + var resumeLabel = _stateLabels[stateNumber]; + var continueLabel = _il.DefineLabel(); + var awaiterField = _builder.AwaiterFields[stateNumber]; + // 2. Convert to Task - handle $Promise, Task, or non-Task values var taskLocal = _il.DeclareLocal(typeof(Task)); var isPromiseLabel = _il.DefineLabel(); diff --git a/Compilation/AsyncMoveNextEmitter.Statements.Loops.cs b/Compilation/AsyncMoveNextEmitter.Statements.Loops.cs index 52927c95..b0ac3f2e 100644 --- a/Compilation/AsyncMoveNextEmitter.Statements.Loops.cs +++ b/Compilation/AsyncMoveNextEmitter.Statements.Loops.cs @@ -1,16 +1,21 @@ +using System.Reflection; +using System.Reflection.Emit; using SharpTS.Parsing; namespace SharpTS.Compilation; public partial class AsyncMoveNextEmitter { + // Per-method counter giving each for await…of loop unique iterator/protocol field names. + private int _forAwaitCounter; + // EmitWhile: inherited from StatementEmitterBase (identical logic) protected override void EmitForOf(Stmt.ForOf f) { if (f.IsAsync) { - // for await...of uses the shared async-iterator lowering in StatementEmitterBase. + // for await...of: EmitForAwaitOf is overridden below to suspend (vs the blocking shared base). EmitForAwaitOf(f); return; } @@ -20,8 +25,322 @@ protected override void EmitForOf(Stmt.ForOf f) base.EmitForOf(f); } - // EmitForAwaitOf: inherited from StatementEmitterBase. The async-function, async-arrow, - // and async-generator emitters now share one async-iterator lowering (#430/#645). + /// + /// Async-function override of the shared (#430/#645). + /// The shared base drives the async iterator with a blocking GetResult; this override instead SUSPENDS + /// the state machine on each next()/return() await, so a genuinely-async step (e.g. a setTimeout-backed + /// await inside the iterator) doesn't deadlock the event-loop thread (#631). The analyzer reserved one + /// suspension state for each await (AsyncStateAnalyzer.VisitForOf), consumed below in the same order. + /// The base lowering still serves async arrows and async generators (the latter tracked by #697). + /// + protected override void EmitForAwaitOf(Stmt.ForOf f) + { + // for await…of drives an async iterator: resolve it (Symbol.asyncIterator, else assume the + // value is itself an async iterator / $IAsyncGenerator), then each iteration await + // iterator.next() and, on an early `break`, await iterator.return(). Both awaits SUSPEND the + // enclosing async function via its state machine rather than blocking on GetResult — so a + // genuinely-async step (e.g. a setTimeout-backed await inside the iterator) no longer deadlocks + // the event-loop thread (#631). The analyzer reserved one suspension state for each await + // (AsyncStateAnalyzer.VisitForOf), consumed below in the same order. + + string varName = f.Variable.Lexeme; + var varField = _builder.GetVariableField(varName); + + var iterableLocal = _il.DeclareLocal(_types.Object); + var asyncIterFnLocal = _il.DeclareLocal(_types.Object); + + // The iterator and its protocol kind must survive the per-iteration suspensions (a MoveNext + // re-entry wipes IL locals), so store them in state-machine fields. Unique per loop so nested / + // sequential for-await loops don't collide. The type is still open here (CreateType runs after + // MoveNext), so defining fields now is valid. + int loopId = _forAwaitCounter++; + var iteratorField = _builder.StateMachineType.DefineField($"<>7__aiter{loopId}", _types.Object, FieldAttributes.Private); + var isCustomField = _builder.StateMachineType.DefineField($"<>7__aiterCustom{loopId}", _types.Boolean, FieldAttributes.Private); + + // Synchronous prelude: evaluate the iterable and resolve the iterator + protocol kind: + // isCustom == true → iterable[Symbol.asyncIterator]() ; step via InvokeIteratorNext / "return". + // isCustom == false → the value is itself the iterator ; step via the $IAsyncGenerator interface. + // Evaluating the iterable, or calling its [Symbol.asyncIterator], can throw synchronously (e.g. a + // pre-aborted signal) — guarded below so that throw reaches an enclosing try's catch. + void EmitPrelude() + { + EmitExpression(f.Iterable); + EnsureBoxed(); + _il.Emit(OpCodes.Stloc, iterableLocal); + + _il.Emit(OpCodes.Ldloc, iterableLocal); + _il.Emit(OpCodes.Ldsfld, _ctx!.Runtime!.SymbolAsyncIterator); + _il.Emit(OpCodes.Call, _ctx.Runtime.GetIteratorFunction); + _il.Emit(OpCodes.Stloc, asyncIterFnLocal); + + var customSetup = _il.DefineLabel(); + var afterSetup = _il.DefineLabel(); + _il.Emit(OpCodes.Ldloc, asyncIterFnLocal); + _il.Emit(OpCodes.Brtrue, customSetup); + + // iterator = iterable; isCustom = false + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldloc, iterableLocal); + _il.Emit(OpCodes.Stfld, iteratorField); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldc_I4_0); + _il.Emit(OpCodes.Stfld, isCustomField); + _il.Emit(OpCodes.Br, afterSetup); + + // iterator = iterable[Symbol.asyncIterator](); isCustom = true + _il.MarkLabel(customSetup); + var iteratorTemp = _il.DeclareLocal(_types.Object); + _il.Emit(OpCodes.Ldloc, iterableLocal); + _il.Emit(OpCodes.Ldloc, asyncIterFnLocal); + _il.Emit(OpCodes.Ldc_I4_0); + _il.Emit(OpCodes.Newarr, _types.Object); + _il.Emit(OpCodes.Call, _ctx.Runtime.InvokeMethodValue); + _il.Emit(OpCodes.Stloc, iteratorTemp); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldloc, iteratorTemp); + _il.Emit(OpCodes.Stfld, iteratorField); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldc_I4_1); + _il.Emit(OpCodes.Stfld, isCustomField); + + _il.MarkLabel(afterSetup); + } + + // The prelude is await-free unless the iterable expression itself awaits; only then can it not + // sit in a real IL try (its resume label would be branched into) — in that case the iterable's + // own await handling routes rejections, and the (rare) synchronous resolution throw is unguarded. + if (ContainsAwaitInExpr(f.Iterable)) + EmitPrelude(); + else + EmitGuardedSyncSegment(EmitPrelude); + + int nextState = _currentAwaitState++; // iterator.next() await (reused each iteration) + + var startLabel = _il.DefineLabel(); + var endLabel = _il.DefineLabel(); + var cleanupLabel = _il.DefineLabel(); + var continueLabel = _il.DefineLabel(); + + // break → cleanup (await iterator.return()); natural done → endLabel (no return() per spec). + EnterLoop(cleanupLabel, continueLabel); + + _il.MarkLabel(startLabel); + + // result = await iterator.next() + var stepLocal = _il.DeclareLocal(_types.Object); + EmitAsyncStep(stepLocal, iteratorField, isCustomField, isReturn: false); + _il.Emit(OpCodes.Ldloc, stepLocal); + SetStackUnknown(); + EmitAwaitFromValueOnStack(nextState); + var resultLocal = _il.DeclareLocal(_types.Object); + _il.Emit(OpCodes.Stloc, resultLocal); + + // if (result.done) exit without calling return() + _il.Emit(OpCodes.Ldloc, resultLocal); + _il.Emit(OpCodes.Call, _ctx!.Runtime!.GetIteratorDone); + _il.Emit(OpCodes.Brtrue, endLabel); + + // loopVar = result.value + _il.Emit(OpCodes.Ldloc, resultLocal); + _il.Emit(OpCodes.Call, _ctx.Runtime.GetIteratorValue); + if (varField != null) + { + var valueTemp = _il.DeclareLocal(_types.Object); + _il.Emit(OpCodes.Stloc, valueTemp); + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldloc, valueTemp); + _il.Emit(OpCodes.Stfld, varField); + } + else + { + var varLocal = _il.DeclareLocal(_types.Object); + _ctx.Locals.RegisterLocal(varName, varLocal); + _il.Emit(OpCodes.Stloc, varLocal); + } + + EmitForAwaitBody(f.Body); + + _il.MarkLabel(continueLabel); + _il.Emit(OpCodes.Br, startLabel); + + // Cleanup on break: await iterator.return() (runs the iterator's finally blocks), discard result. + int returnState = _currentAwaitState++; // allocated after the body, matching the analyzer + _il.MarkLabel(cleanupLabel); + var returnStepLocal = _il.DeclareLocal(_types.Object); + EmitAsyncStep(returnStepLocal, iteratorField, isCustomField, isReturn: true); + _il.Emit(OpCodes.Ldloc, returnStepLocal); + SetStackUnknown(); + EmitAwaitFromValueOnStack(returnState); + _il.Emit(OpCodes.Pop); + + _il.MarkLabel(endLabel); + ExitLoop(); + } + + /// + /// Emits the for-await loop body. When the loop sits inside a try-with-awaits (flag-based, so there is + /// no real IL try around it), a synchronous throw in the body must still reach that try's catch. + /// The body can't sit inside a real IL try when it contains awaits (their resume labels would be + /// branched into) or top-level break/continue (which the async emitter branches to with Br, not + /// Leave); for an await-free, jump-free body we wrap it in a real try that captures the throw + /// into the try's exception local and exits the loop to the catch dispatch. Other bodies use the plain + /// path — a rejected next()/return() is still caught (that path is suspension-based), only a *synchronous* + /// throw inside such a body escapes (a narrow, pre-existing-style limitation). (#631) + /// + private void EmitForAwaitBody(Stmt body) + { + // An await-free, jump-free body can sit in a real IL try (see EmitGuardedSyncSegment); a body + // with awaits (resume labels would be branched into) or break/continue (Br out of the try is + // invalid) keeps the plain path — only a *synchronous* throw inside such a body escapes the + // enclosing try, a narrow limitation. + if (!ContainsAwaitInStmt(body) && !ContainsBreakOrContinue(body)) + EmitGuardedSyncSegment(() => EmitStatement(body)); + else + EmitStatement(body); + } + + /// + /// Runs (a synchronous, suspension-free IL span that leaves the eval stack + /// balanced) under the enclosing try-with-awaits, if any: a throw from the span is captured into the + /// try's exception local and control jumps to the catch dispatch. Outside such a try it just runs the + /// span. Lets the for-await prelude / step / body route synchronous throws to a guest catch even though + /// the loop as a whole can't sit in a real IL try (its awaits would be branched into). (#631) + /// + private void EmitGuardedSyncSegment(System.Action emit) + { + if (_currentTryCatchExceptionLocal == null || _currentTryCatchSkipLabel == null) + { + emit(); + return; + } + var exLocal = _currentTryCatchExceptionLocal; + _il.BeginExceptionBlock(); + emit(); + _il.BeginCatchBlock(typeof(Exception)); + _il.Emit(OpCodes.Call, _ctx!.Runtime!.WrapException); + _il.Emit(OpCodes.Stloc, exLocal); + _il.EndExceptionBlock(); + _il.Emit(OpCodes.Ldloc, exLocal); + _il.Emit(OpCodes.Brtrue, _currentTryCatchSkipLabel.Value); + } + + /// + /// True if contains a break/continue anywhere (a conservative + /// over-approximation — it does not exclude jumps that target a loop/switch nested in the body). Used + /// to keep such bodies off the real-IL-try path in , where a Br + /// out of the try would be invalid IL. + /// + private static bool ContainsBreakOrContinue(Stmt stmt) => stmt switch + { + Stmt.Break => true, + Stmt.Continue => true, + Stmt.Block b => b.Statements.Any(ContainsBreakOrContinue), + Stmt.Sequence s => s.Statements.Any(ContainsBreakOrContinue), + Stmt.If i => ContainsBreakOrContinue(i.ThenBranch) || (i.ElseBranch != null && ContainsBreakOrContinue(i.ElseBranch)), + Stmt.While w => ContainsBreakOrContinue(w.Body), + Stmt.DoWhile d => ContainsBreakOrContinue(d.Body), + Stmt.For f => ContainsBreakOrContinue(f.Body), + Stmt.ForOf fo => ContainsBreakOrContinue(fo.Body), + Stmt.ForIn fi => ContainsBreakOrContinue(fi.Body), + Stmt.LabeledStatement l => ContainsBreakOrContinue(l.Statement), + Stmt.Switch sw => sw.Cases.Any(c => c.Body.Any(ContainsBreakOrContinue)) || (sw.DefaultBody?.Any(ContainsBreakOrContinue) ?? false), + Stmt.TryCatch t => t.TryBlock.Any(ContainsBreakOrContinue) + || (t.CatchBlock?.Any(ContainsBreakOrContinue) ?? false) + || (t.FinallyBlock?.Any(ContainsBreakOrContinue) ?? false), + _ => false + }; + + /// + /// Produces one async-iterator protocol step into , guarding the call + /// itself. The protocol call (iterator.next()/return()) can throw synchronously — e.g. a + /// pre-aborted signal makes next() throw rather than reject — and that throw happens before the await, + /// so when inside a try-with-awaits it must be captured into the try's exception local and routed to + /// the catch (otherwise it escapes the state machine). Outside a try, it propagates normally. (#631) + /// + private void EmitAsyncStep(LocalBuilder resultLocal, FieldBuilder iteratorField, FieldBuilder isCustomField, bool isReturn) + => EmitGuardedSyncSegment(() => EmitProduceAsyncStep(resultLocal, iteratorField, isCustomField, isReturn)); + + /// + /// Stores into the result of one async-iterator protocol call — + /// iterator.next() when is false, else iterator.return() — + /// selecting the call by the protocol kind in . The value (a + /// Task<object>, $Promise, or plain value) is later coerced and awaited by + /// . A custom iterator with no return method stores + /// undefined (awaited as an already-resolved value), so the reserved return-await state is + /// still consumed and its resume label stays marked. + /// + private void EmitProduceAsyncStep(LocalBuilder resultLocal, FieldBuilder iteratorField, FieldBuilder isCustomField, bool isReturn) + { + var stepLocal = resultLocal; + var customLabel = _il.DefineLabel(); + var doneLabel = _il.DefineLabel(); + + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, isCustomField); + _il.Emit(OpCodes.Brtrue, customLabel); + + // ----- $IAsyncGenerator path: invoke the interface method (returns Task) ----- + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, iteratorField); + _il.Emit(OpCodes.Castclass, _ctx!.Runtime!.AsyncGeneratorInterfaceType); + if (isReturn) + { + _il.Emit(OpCodes.Ldnull); + _il.Emit(OpCodes.Callvirt, _ctx.Runtime.AsyncGeneratorReturnMethod); + } + else + { + _il.Emit(OpCodes.Callvirt, _ctx.Runtime.AsyncGeneratorNextMethod); + } + _il.Emit(OpCodes.Stloc, stepLocal); + _il.Emit(OpCodes.Br, doneLabel); + + // ----- custom Symbol.asyncIterator path ----- + _il.MarkLabel(customLabel); + if (isReturn) + { + // fn = GetProperty(iterator, "return"); absent/undefined → no-op close (push undefined). + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, iteratorField); + _il.Emit(OpCodes.Ldstr, "return"); + _il.Emit(OpCodes.Call, _ctx.Runtime.GetProperty); + var fnLocal = _il.DeclareLocal(_types.Object); + _il.Emit(OpCodes.Stloc, fnLocal); + + var noFnLabel = _il.DefineLabel(); + var haveFnLabel = _il.DefineLabel(); + _il.Emit(OpCodes.Ldloc, fnLocal); + _il.Emit(OpCodes.Brfalse, noFnLabel); + _il.Emit(OpCodes.Ldloc, fnLocal); + _il.Emit(OpCodes.Isinst, _ctx.Runtime.UndefinedType); + _il.Emit(OpCodes.Brtrue, noFnLabel); + + // InvokeMethodValue(iterator, fn, []) + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, iteratorField); + _il.Emit(OpCodes.Ldloc, fnLocal); + _il.Emit(OpCodes.Ldc_I4_0); + _il.Emit(OpCodes.Newarr, _types.Object); + _il.Emit(OpCodes.Call, _ctx.Runtime.InvokeMethodValue); + _il.Emit(OpCodes.Stloc, stepLocal); + _il.Emit(OpCodes.Br, haveFnLabel); + + _il.MarkLabel(noFnLabel); + _il.Emit(OpCodes.Ldsfld, _ctx.Runtime.UndefinedInstance); + _il.Emit(OpCodes.Stloc, stepLocal); + _il.MarkLabel(haveFnLabel); + } + else + { + _il.Emit(OpCodes.Ldarg_0); + _il.Emit(OpCodes.Ldfld, iteratorField); + _il.Emit(OpCodes.Call, _ctx.Runtime.InvokeIteratorNext); + _il.Emit(OpCodes.Stloc, stepLocal); + } + + _il.MarkLabel(doneLabel); + // Result is left in resultLocal (== stepLocal); the caller loads and awaits it. + } // EmitDoWhile: inherited from StatementEmitterBase (identical logic) // EmitForIn: inherited from StatementEmitterBase (uses DeclareLoopVariable/EmitStoreLoopVariable diff --git a/Compilation/AsyncMoveNextEmitter.Statements.TryCatch.cs b/Compilation/AsyncMoveNextEmitter.Statements.TryCatch.cs index 36ff7a2c..4cab499b 100644 --- a/Compilation/AsyncMoveNextEmitter.Statements.TryCatch.cs +++ b/Compilation/AsyncMoveNextEmitter.Statements.TryCatch.cs @@ -382,7 +382,10 @@ private bool ContainsAwaitInStmt(Stmt stmt) (f.Increment != null && ContainsAwaitInExpr(f.Increment)) || ContainsAwaitInStmt(f.Body); case Stmt.ForOf fo: - return ContainsAwaitInExpr(fo.Iterable) || ContainsAwaitInStmt(fo.Body); + // `for await…of` always suspends (it awaits iterator.next()/return()), even when the + // iterable and body contain no explicit await — so a try enclosing one must take the + // flag-based path, not a real IL try (whose resume labels would be branched into) (#631). + return fo.IsAsync || ContainsAwaitInExpr(fo.Iterable) || ContainsAwaitInStmt(fo.Body); case Stmt.ForIn fi: return ContainsAwaitInExpr(fi.Object) || ContainsAwaitInStmt(fi.Body); case Stmt.Block b: diff --git a/Compilation/AsyncStateAnalyzer.Expressions.cs b/Compilation/AsyncStateAnalyzer.Expressions.cs index 954fa07a..66e2f09f 100644 --- a/Compilation/AsyncStateAnalyzer.Expressions.cs +++ b/Compilation/AsyncStateAnalyzer.Expressions.cs @@ -11,7 +11,16 @@ protected override void VisitAwait(Expr.Await expr) // Visit await expression BEFORE marking _seenAwait, so variables in the await // expression are not incorrectly marked as "used after await" base.VisitAwait(expr); + RecordAwaitPoint(expr); + } + /// + /// Records an await point (real or synthetic) and the surrounding try-region/await bookkeeping. + /// is null for synthetic awaits — the implicit next()/return() awaits a + /// for await…of loop performs (#631), which have no node. + /// + internal void RecordAwaitPoint(Expr.Await? expr) + { // Record this await point with try block context var liveVars = new HashSet(_declaredVariables); _awaitPoints.Add(new AwaitPoint( diff --git a/Compilation/AsyncStateAnalyzer.Statements.cs b/Compilation/AsyncStateAnalyzer.Statements.cs index 50d75d98..d8e4b743 100644 --- a/Compilation/AsyncStateAnalyzer.Statements.cs +++ b/Compilation/AsyncStateAnalyzer.Statements.cs @@ -33,6 +33,20 @@ protected override void VisitForOf(Stmt.ForOf stmt) if (!_seenAwait) _variablesDeclaredBeforeAwait.Add(stmt.Variable.Lexeme); + if (stmt.IsAsync) + { + // `for await…of` suspends on the iterator protocol: it awaits iterator.next() each + // iteration and iterator.return() on early exit. Each needs a reserved state (resume + // label + awaiter field), matching the two inline awaits EmitForAwaitOf emits (#631). + // Allocation order mirrors emission order: the iterable is evaluated before the loop, so + // its awaits come first; the next() await before the body; the return() await after. + Visit(stmt.Iterable); + RecordAwaitPoint(null); // iterator.next() — awaited at the loop head + Visit(stmt.Body); + RecordAwaitPoint(null); // iterator.return() — awaited in the break cleanup + return; + } + base.VisitForOf(stmt); } diff --git a/Compilation/AsyncStateAnalyzer.cs b/Compilation/AsyncStateAnalyzer.cs index b07656ba..1e4b7528 100644 --- a/Compilation/AsyncStateAnalyzer.cs +++ b/Compilation/AsyncStateAnalyzer.cs @@ -15,7 +15,7 @@ public partial class AsyncStateAnalyzer : AstVisitorBase /// public record AwaitPoint( int StateNumber, - Expr.Await AwaitExpr, + Expr.Await? AwaitExpr, // null for synthetic await points (e.g. for await…of's implicit next()/return() awaits, #631) HashSet LiveVariables, int TryBlockDepth = 0, // 0 = not in try, 1+ = nested try depth int? EnclosingTryId = null // ID of the innermost try block containing this await diff --git a/Compilation/EmittedRuntime.cs b/Compilation/EmittedRuntime.cs index 3d018531..f76946dd 100644 --- a/Compilation/EmittedRuntime.cs +++ b/Compilation/EmittedRuntime.cs @@ -1393,6 +1393,10 @@ public class EmittedRuntime // Async Generator await continuation helper public MethodBuilder AsyncGeneratorAwaitContinue { get; set; } = null!; + // Async Generator next-result builder: awaits a MoveNextAsync ValueTask and produces the + // { value, done } Task for next(), so next() never blocks the event-loop thread (#631/#542). + public MethodBuilder AsyncGeneratorBuildResult { get; set; } = null!; + // Iterator helper methods (ES2025 Iterator Helpers) public MethodBuilder NormalizeToEnumerator { get; set; } = null!; public MethodBuilder IteratorMap { get; set; } = null!; diff --git a/Compilation/RuntimeEmitter.AsyncGenerator.cs b/Compilation/RuntimeEmitter.AsyncGenerator.cs index b27a475a..0d4fd181 100644 --- a/Compilation/RuntimeEmitter.AsyncGenerator.cs +++ b/Compilation/RuntimeEmitter.AsyncGenerator.cs @@ -99,6 +99,192 @@ private void EmitAsyncGeneratorAwaitContinueMethods(TypeBuilder typeBuilder, Mod // Create the state machine type sm.Type.CreateType(); + + // Emit the next()-result builder used by truly-async next() (#631/#542). + EmitAsyncGeneratorBuildResultMethod(typeBuilder, moduleBuilder, runtime); + } + + /// + /// Emits static Task<object> AsyncGeneratorBuildResult(ValueTask<bool> moveNext, + /// IAsyncEnumerator<object> gen) — an async helper that awaits a MoveNextAsync result and + /// produces the { value: gen.Current, done: !moved } iterator-result dictionary. + /// + /// This is what lets the emitted async-generator next() be truly asynchronous: instead of + /// blocking the event-loop thread on MoveNextAsync().AsTask().GetResult() (which deadlocks a + /// genuinely-async await — the continuation needs the very thread that is blocked, #631), next() + /// drives one step and hands the (possibly pending) ValueTask here, returning the Task this produces. + /// A faulted MoveNext (uncaught body throw) surfaces as a faulted Task — i.e. a rejected next() + /// promise — exactly as ECMA-262 §27.6.1.2 requires. + /// + /// + private void EmitAsyncGeneratorBuildResultMethod(TypeBuilder typeBuilder, ModuleBuilder moduleBuilder, EmittedRuntime runtime) + { + var builderType = typeof(System.Runtime.CompilerServices.AsyncTaskMethodBuilder<>).MakeGenericType(_types.Object); + var valueTaskAwaiterType = _types.ValueTaskAwaiterOfBool; + + // --- State machine type --- + var smType = moduleBuilder.DefineType( + "$AsyncGeneratorBuildResult_StateMachine", + TypeAttributes.Public | TypeAttributes.Sealed | TypeAttributes.BeforeFieldInit, + typeof(ValueType), + [typeof(IAsyncStateMachine)]); + + var stateField = smType.DefineField("<>1__state", typeof(int), FieldAttributes.Public); + var builderField = smType.DefineField("<>t__builder", builderType, FieldAttributes.Public); + var genField = smType.DefineField("gen", _types.IAsyncEnumeratorOfObject, FieldAttributes.Public); + var awaiterField = smType.DefineField("<>u__1", valueTaskAwaiterType, FieldAttributes.Public); + + var moveNext = smType.DefineMethod( + "MoveNext", + MethodAttributes.Private | MethodAttributes.Virtual | MethodAttributes.Final | MethodAttributes.HideBySig | MethodAttributes.NewSlot, + typeof(void), Type.EmptyTypes); + smType.DefineMethodOverride(moveNext, _types.AsyncStateMachineMoveNext); + + var setStateMachine = smType.DefineMethod( + "SetStateMachine", + MethodAttributes.Private | MethodAttributes.Virtual | MethodAttributes.Final | MethodAttributes.HideBySig | MethodAttributes.NewSlot, + typeof(void), [typeof(IAsyncStateMachine)]); + smType.DefineMethodOverride(setStateMachine, _types.AsyncStateMachineSetStateMachine); + setStateMachine.GetILGenerator().Emit(OpCodes.Ret); + + // --- Wrapper: AsyncGeneratorBuildResult(ValueTask moveNext, IAsyncEnumerator gen) --- + var method = typeBuilder.DefineMethod( + "AsyncGeneratorBuildResult", + MethodAttributes.Public | MethodAttributes.Static, + _types.TaskOfObject, + [_types.ValueTaskOfBool, _types.IAsyncEnumeratorOfObject]); + runtime.AsyncGeneratorBuildResult = method; + + { + var il = method.GetILGenerator(); + var smLocal = il.DeclareLocal(smType); + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Initobj, smType); + // sm.<>1__state = -1 + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Ldc_I4_M1); + il.Emit(OpCodes.Stfld, stateField); + // sm.<>u__1 = moveNext.GetAwaiter() + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Ldarga_S, (byte)0); + il.Emit(OpCodes.Call, _types.GetMethodNoParams(_types.ValueTaskOfBool, "GetAwaiter")); + il.Emit(OpCodes.Stfld, awaiterField); + // sm.gen = arg1 + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Ldarg_1); + il.Emit(OpCodes.Stfld, genField); + // sm.<>t__builder = AsyncTaskMethodBuilder.Create() + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Call, builderType.GetMethod("Create", BindingFlags.Public | BindingFlags.Static)!); + il.Emit(OpCodes.Stfld, builderField); + // sm.<>t__builder.Start(ref sm) + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Ldflda, builderField); + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Call, builderType.GetMethods(BindingFlags.Public | BindingFlags.Instance) + .First(m => m.Name == "Start" && m.IsGenericMethod).MakeGenericMethod(smType)); + // return sm.<>t__builder.Task + il.Emit(OpCodes.Ldloca, smLocal); + il.Emit(OpCodes.Ldflda, builderField); + il.Emit(OpCodes.Call, builderType.GetProperty("Task", BindingFlags.Public | BindingFlags.Instance)!.GetGetMethod()!); + il.Emit(OpCodes.Ret); + } + + // --- MoveNext --- + { + var il = moveNext.GetILGenerator(); + var exLocal = il.DeclareLocal(typeof(Exception)); + var movedLocal = il.DeclareLocal(_types.Boolean); + var resultLocal = il.DeclareLocal(_types.Object); + var resumeLabel = il.DefineLabel(); + var continueLabel = il.DefineLabel(); + var returnLabel = il.DefineLabel(); + + il.BeginExceptionBlock(); + + // switch (state) { case 0: goto resume; default: fall through (-1) } + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, stateField); + il.Emit(OpCodes.Switch, [resumeLabel]); + + // state -1: if (awaiter.IsCompleted) goto continue; + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, awaiterField); + il.Emit(OpCodes.Call, valueTaskAwaiterType.GetProperty("IsCompleted")!.GetGetMethod()!); + il.Emit(OpCodes.Brtrue, continueLabel); + + // not completed: suspend + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldc_I4_0); + il.Emit(OpCodes.Stfld, stateField); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, builderField); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, awaiterField); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, builderType.GetMethods(BindingFlags.Public | BindingFlags.Instance) + .First(m => m.Name == "AwaitUnsafeOnCompleted" && m.IsGenericMethod) + .MakeGenericMethod(valueTaskAwaiterType, smType)); + il.Emit(OpCodes.Leave, returnLabel); + + // state 0 resume: + il.MarkLabel(resumeLabel); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldc_I4_M1); + il.Emit(OpCodes.Stfld, stateField); + + il.MarkLabel(continueLabel); + // moved = awaiter.GetResult() (throws if the body faulted → faults this Task) + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, awaiterField); + il.Emit(OpCodes.Call, _types.GetMethodNoParams(valueTaskAwaiterType, "GetResult")); + il.Emit(OpCodes.Stloc, movedLocal); + + // result = new Dictionary { ["value"] = gen.Current, ["done"] = !moved } + il.Emit(OpCodes.Newobj, _types.GetDefaultConstructor(_types.DictionaryStringObject)); + il.Emit(OpCodes.Dup); + il.Emit(OpCodes.Ldstr, "value"); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldfld, genField); + il.Emit(OpCodes.Callvirt, _types.GetPropertyGetter(_types.IAsyncEnumeratorOfObject, "Current")); + il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); + il.Emit(OpCodes.Dup); + il.Emit(OpCodes.Ldstr, "done"); + il.Emit(OpCodes.Ldloc, movedLocal); + il.Emit(OpCodes.Ldc_I4_0); + il.Emit(OpCodes.Ceq); // !moved + il.Emit(OpCodes.Box, _types.Boolean); + il.Emit(OpCodes.Callvirt, _types.GetMethod(_types.DictionaryStringObject, "set_Item", _types.String, _types.Object)); + il.Emit(OpCodes.Stloc, resultLocal); + + // state = -2; builder.SetResult(result) + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldc_I4, -2); + il.Emit(OpCodes.Stfld, stateField); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, builderField); + il.Emit(OpCodes.Ldloc, resultLocal); + il.Emit(OpCodes.Call, builderType.GetMethod("SetResult")!); + il.Emit(OpCodes.Leave, returnLabel); + + // catch (Exception e) { state=-2; builder.SetException(e); } + il.BeginCatchBlock(typeof(Exception)); + il.Emit(OpCodes.Stloc, exLocal); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldc_I4, -2); + il.Emit(OpCodes.Stfld, stateField); + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Ldflda, builderField); + il.Emit(OpCodes.Ldloc, exLocal); + il.Emit(OpCodes.Call, builderType.GetMethod("SetException")!); + il.Emit(OpCodes.Leave, returnLabel); + il.EndExceptionBlock(); + + il.MarkLabel(returnLabel); + il.Emit(OpCodes.Ret); + } + + smType.CreateType(); } /// @@ -284,11 +470,13 @@ private void EmitAsyncGeneratorAwaitContinueMoveNext(AsyncGeneratorAwaitContinue // ========== Continue after task await ========== il.MarkLabel(continueAfterTaskAwait); - // Get result (we don't use it, just need to trigger any exception) - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldflda, sm.TaskAwaiterField); - il.Emit(OpCodes.Call, typeof(TaskAwaiter).GetMethod("GetResult")!); - il.Emit(OpCodes.Pop); // Discard result + // Deliberately do NOT call awaiter.GetResult() here. A *rejected* awaited task must reach the + // generator body's own resume point (which reads its AwaiterField.GetResult and re-throws into + // the body's try/catch), not be re-thrown here — calling GetResult would fault this helper's + // ValueTask and bypass the body's resume entirely, so a pending rejection inside a guest + // try/catch could never reach its catch (#631, #617). We only needed the await above to wait + // for completion; resuming MoveNextAsync regardless of fault lets the body observe it (mirrors + // the ContinueWith-based RuntimeTypes.AsyncGeneratorAwaitContinue). // Call generator.MoveNextAsync() il.Emit(OpCodes.Ldarg_0); diff --git a/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs b/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs index 8c047286..ad0ed02c 100644 --- a/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs +++ b/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs @@ -1181,4 +1181,98 @@ async function main() { } #endregion + + #region Genuinely-async awaits and request queuing (#631 / #542) + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_PendingAwait_ForAwaitOf_YieldsAllValues(ExecutionMode mode) + { + // A not-yet-settled (setTimeout-backed) await inside an async generator consumed by for await…of + // previously hung the compiled program: next() drove MoveNextAsync synchronously via GetResult, + // blocking the event-loop thread the continuation needed. next() is now truly asynchronous (#631). + var source = """ + function later(n: number): Promise { + return new Promise(res => setTimeout(() => res(n), 5)); + } + async function* g() { yield await later(1); yield await later(2); } + async function main() { + for await (const v of g()) console.log("v" + v); + } + main(); + """; + + Assert.Equal("v1\nv2\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_PendingAwait_DirectNext_ResolvesInOrder(ExecutionMode mode) + { + // Driving a pending-await async generator by awaiting next() directly (compiled mode; the + // interpreter eagerly drains the body, a separate non-conformant model). #631. + var source = """ + function later(n: number): Promise { + return new Promise(res => setTimeout(() => res(n), 5)); + } + async function* g() { yield await later(7); yield await later(8); } + async function main() { + const it = g(); + const a = await it.next(); + const b = await it.next(); + const c = await it.next(); + console.log(a.value + " " + a.done + " " + b.value + " " + b.done + " " + c.value + " " + c.done); + } + main(); + """; + + Assert.Equal("7 false 8 false undefined true\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_PendingRejection_InTry_ReachesCatch(ExecutionMode mode) + { + // A pending (genuinely-async) rejected await must reach the consumer's catch through for await…of. + // The emitted AsyncGeneratorAwaitContinue no longer short-circuits on the faulted task; it resumes + // the body so its own resume point re-throws into place (#631, unblocks the pending sub-case of #617). + var source = """ + function fail(): Promise { + return new Promise((_res, rej) => setTimeout(() => rej("boom"), 5)); + } + async function* g() { yield await fail(); } + async function main() { + try { for await (const v of g()) console.log("v" + v); } + catch (e) { console.log("caught " + e); } + } + main(); + """; + + Assert.Equal("caught boom\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_ConcurrentNext_QueuesInOrder(ExecutionMode mode) + { + // Two next() calls issued before the first settles must be serviced FIFO (ECMA-262 §27.6.3 + // AsyncGeneratorQueue), modeled as a task chain — not rejected as "already running" (#542). + // Compiled-only: the interpreter's eager-drain async generator can't service concurrent next(). + var source = """ + function later(n: number): Promise { + return new Promise(res => setTimeout(() => res(n), 5)); + } + async function* g() { yield await later(1); yield await later(2); } + async function main() { + const it: any = g(); + const [a, b] = await Promise.all([it.next(), it.next()]); + console.log(a.value + " " + a.done + " " + b.value + " " + b.done); + } + main(); + """; + + Assert.Equal("1 false 2 false\n", TestHarness.Run(source, mode)); + } + + #endregion } From 9ea23b746b33020409fba02c32804fa583fa002a Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Mon, 15 Jun 2026 20:17:58 -0700 Subject: [PATCH 3/4] Fix #583 (part 1): lift capturing nested generator/async functions NestedFunctionLifter refused to relocate any declaration capturing an enclosing function scope, so a nested generator/async/plain function reading an outer local failed with "Yield not supported in this context". Extend the existing lambda-lift (previously module-block only) to inside- function captures: the captured function-scope bindings become leading parameters of the relocated top-level declaration, forwarded by an in-place arrow that closes over them. Relies on capturing arrows in generator/state-machine bodies binding their display instance (landed on main as #674) so the arrow reads its captures live. Self-recursive nested declarations decline cleanly (left nested): a compiled arrow snapshots captures by value and its own let binding is in its TDZ at creation, so a forwarded self would be null. #583 part 2 (relocated identity shared across enclosing-call instances) remains. --- Compilation/NestedFunctionLifter.cs | 55 +++++++++++++- .../GeneratorClosureCaptureTests.cs | 74 +++++++++++++++++++ 2 files changed, 125 insertions(+), 4 deletions(-) diff --git a/Compilation/NestedFunctionLifter.cs b/Compilation/NestedFunctionLifter.cs index e69d9656..f3f03933 100644 --- a/Compilation/NestedFunctionLifter.cs +++ b/Compilation/NestedFunctionLifter.cs @@ -108,12 +108,23 @@ public static List Lift(List module) var lambdaForwards = new Dictionary>(ReferenceEqualityComparer.Instance); foreach (var f in scan.Candidates) { - // A reference into an intermediate FUNCTION scope is a real closure capture (#583 §1) - // that neither plain relocation nor lambda-lifting can perform — leave it nested. - if (!IsNonCapturing(analyzer, f)) continue; - bool isModuleBlock = scan.ModuleBlockEnclosingBindings.TryGetValue(f, out var blockBindings); + // A reference into an intermediate FUNCTION scope is a real closure capture (#583 §1). Only + // an INSIDE-FUNCTION candidate can have one (a module-block candidate has no enclosing + // function). Lambda-lift it: the captured function-scope bindings (and the function's own + // name when it recurses) become leading parameters of the relocated top-level declaration, + // forwarded by an in-place arrow that closes over them. The arrow may sit inside a generator/ + // async body, which now binds captured display instances correctly (see + // GeneratorMoveNextEmitter.EmitArrowFunction). Declines (leaves nested — a clean failure) for + // bodies using this/arguments or rest/default params, which the forwarding arrow can't carry. + if (!IsNonCapturing(analyzer, f)) + { + if (!isModuleBlock && TryComputeFunctionCaptureForward(analyzer, f, out var fnForwarded)) + lambdaForwards[f] = fnForwarded; + continue; + } + // A module-block candidate that captures an enclosing block/loop binding (e.g. a // generator in a `for` reading the loop variable) can't move to module top level as-is — // that name doesn't exist there. Lambda-lift it: the captured bindings become leading @@ -228,6 +239,42 @@ private static bool TryComputeLambdaForward( return forwarded.Count > 0; } + /// + /// The inside-function analogue of (#583 §1): produces the + /// ordered list of captures to forward as leading parameters when relocating a nested declaration + /// that captures an enclosing FUNCTION scope. The forwarded set is every free variable resolving to + /// such a scope — including the function's own name when it recurses, so the relocated body's + /// self-calls resolve to the forwarded arrow (which closes over its own let binding). Declines + /// (returns false → stays nested, a clean failure) on rest/default parameters or a body using + /// this/arguments, exactly as the module-block path does. + /// + private static bool TryComputeFunctionCaptureForward( + ClosureAnalyzer analyzer, Stmt.Function f, out List forwarded) + { + forwarded = []; + + foreach (var p in f.Parameters) + if (p.IsRest || p.DefaultValue != null) + return false; + + if (UsesThisOrArguments(f.Body)) + return false; + + // Self-recursion can't be lambda-lifted here: the relocated body's self-calls must resolve to the + // forwarding arrow, but a compiled arrow snapshots its captures by value — and the arrow's own + // `let` binding is still in its temporal dead zone when the arrow is created, so it would capture + // an unassigned (null) self and crash on the first recursive call. Leave such a declaration nested + // (a clean "not supported" failure, never a miscompile). Non-recursive captures lift fine. + if (analyzer.GetCaptures(f).Contains(f.Name.Lexeme)) + return false; + + forwarded = analyzer.GetCaptures(f) + .Where(c => analyzer.GetCaptureSource(f, c) != null) + .OrderBy(c => c, System.StringComparer.Ordinal) + .ToList(); + return forwarded.Count > 0; + } + /// /// True if any statement in reads this or arguments. /// Deliberately over-approximates: it descends through nested function/arrow boundaries (which diff --git a/SharpTS.Tests/SharedTests/GeneratorClosureCaptureTests.cs b/SharpTS.Tests/SharedTests/GeneratorClosureCaptureTests.cs index de8c4fe7..8be8be22 100644 --- a/SharpTS.Tests/SharedTests/GeneratorClosureCaptureTests.cs +++ b/SharpTS.Tests/SharedTests/GeneratorClosureCaptureTests.cs @@ -171,4 +171,78 @@ public void Generator_ReferencesTopLevelFunctionAsValue(ExecutionMode mode) Assert.Equal("H\n", TestHarness.Run(source, mode)); } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_CapturingArrowCalledInsideBody(ExecutionMode mode) + { + // Calling a *capturing* arrow inside a generator body previously failed in compiled mode with + // "Non-static method requires a target" — the arrow's display instance was not bound as the + // $TSFunction target. Foundational for lifting capturing nested function-likes (#583). + var source = """ + function* outer() { + const x = 10; + const f = () => x; + yield f(); + } + console.log([...outer()].join(",")); + """; + + Assert.Equal("10\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_NestedGeneratorCapturingLocal_IsLifted(ExecutionMode mode) + { + // The #583 §1 repro: a nested generator that captures an enclosing local is lambda-lifted (the + // capture becomes a leading parameter forwarded by an in-place arrow), instead of failing with + // "Yield not supported in this context" in compiled mode. + var source = """ + function* outer(): Generator { + const x = 10; + function* inner(): Generator { yield x; } + yield* inner(); + } + for (const v of outer()) console.log(v); + """; + + Assert.Equal("10\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncFunction_NestedAsyncCapturingLocal_IsLifted(ExecutionMode mode) + { + // The async analogue of #583 §1: a nested async function capturing an enclosing local lifts too. + var source = """ + async function outer(): Promise { + const x = 10; + async function inner(): Promise { return x + 1; } + return await inner(); + } + outer().then(v => console.log(v)); + """; + + Assert.Equal("11\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void Generator_PlainNestedFunctionCapturingLocal_IsLifted(ExecutionMode mode) + { + // A plain function nested in a generator, capturing an enclosing local (#583 §1, case-B analogue): + // lambda-lifted with the captured local forwarded as a leading parameter. + var source = """ + function* outer(): Generator { + const base = 100; + function add(d: number): number { return base + d; } + yield add(1); + yield add(2); + } + console.log([...outer()].join(",")); + """; + + Assert.Equal("101,102\n", TestHarness.Run(source, mode)); + } } From 2d3b358490929783d4c3f00406a7b4a64611c785 Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Mon, 15 Jun 2026 20:17:58 -0700 Subject: [PATCH 4/4] test: register new state-machine emitter overrides in the emitter-sync allowlist GeneratorMoveNextEmitter.EmitForIn (#547) and AsyncMoveNextEmitter.Emit- ForAwaitOf (#631, override of the shared base to suspend) are new overrides; add them to EmitterSyncTests.StateMachineEmitters_OnlyOverride- AllowedMethods. --- SharpTS.Tests/Compilation/EmitterSyncTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SharpTS.Tests/Compilation/EmitterSyncTests.cs b/SharpTS.Tests/Compilation/EmitterSyncTests.cs index ffb41059..4224c7f8 100644 --- a/SharpTS.Tests/Compilation/EmitterSyncTests.cs +++ b/SharpTS.Tests/Compilation/EmitterSyncTests.cs @@ -44,6 +44,7 @@ public class EmitterSyncTests "EmitReturn", // Async return: store result + leave to SetResult label "EmitTryCatch", // Await-aware exception handling with flag-based tracking "EmitForOf", // for-await-of protocol dispatch + "EmitForAwaitOf", // #631: override the shared base to SUSPEND on next()/return() (vs blocking GetResult) "EmitLabeledStatement", // Labeled continue for for loops (base doesn't handle correctly) "EmitAwait", // Core: suspend/resume state machine "EmitArrowFunction", // Display class in state machine context @@ -103,6 +104,7 @@ public class EmitterSyncTests "EmitReturn", // Generator return: set state -2, return false "EmitTryCatch", // Generator exception handling "EmitForOf", // Hoisted enumerator for yield across loop boundaries + "EmitForIn", // #547: hoisted key-list/index for yield across for-in iterations "EmitYield", // Core: yield value + suspend "EmitSuper", // This field indirection "EmitDynamicImport", // Dynamic import fallback