From 766e5f709d189f298d36da5a7705b2eda29bd888 Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Tue, 16 Jun 2026 01:24:44 -0700 Subject: [PATCH 1/2] Interpreter: rewrite async generator as a lazy coroutine (#690/#717/#752/#734) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the interpreter's eager-drain SharpTSAsyncGenerator with a lazy, main-thread C# async coroutine plus an ECMA-262 §27.6.3 request queue. The body now runs as an ordinary interpreter async execution (ExecuteBlockAsync) on the single event-loop thread and suspends at each yield by handing the value to the driving request and awaiting the next one. A worker thread (as the synchronous generator uses) is impossible here: the interpreter is single-threaded and a worker would race the event loop on the shared interpreter environment. Reusing the real async execution path fixes several issues at once: - #690: two next() calls before the first settles are serviced FIFO via the request queue instead of racing a half-populated drain. - #717: a for await...of INSIDE an async generator drives the async-iterator protocol natively instead of throwing "Cannot iterate over non-iterable". - #752: an await nested in a yielded expression (yield f(await x)) preserves the ambient environment like any async function, so the caller's bindings no longer read back undefined. - #734: an async generator function expression that closes over a block-scoped binding runs natively (new SharpTSAsyncArrowGeneratorFunction; the GeneratorArrowLifter leaves it in place). The compiler still reports a clean "Yield not supported in this context" error. Mechanism: new Interpreter.CurrentAsyncGenerator identifies the active generator; Expr.Yield in the async path routes through EvaluateYieldAsync -> OnYieldAsync. AwaitPreservingEnvironment now also saves/restores CurrentAsyncGenerator so interleaved generators never see each other's binding. for await...of early exit (break / return / throw) calls CloseAsyncIteratorOnEarlyExit (AsyncIteratorClose) so a suspended generator runs its finally blocks. return()/throw() reuse the synchronous generator's GeneratorResume / GeneratorReturnException injection, so finally blocks run. A re-entrant next() during the body's initial synchronous segment rejects with a catchable TypeError instead of deadlocking (#542 analog). Promotes 7 previously CompiledOnly async-generator tests to run in the interpreter too, and adds #734 coverage. Full suite: 13200 passing. Gaps filed: #770 (labeled loop containing await/yield has no async handler -- pre-existing, exposed here), #771 (deep re-entrancy stall). --- Execution/Interpreter.Async.cs | 56 +- Execution/Interpreter.Expressions.cs | 40 +- Execution/Interpreter.cs | 10 + Parsing/GeneratorArrowLifter.cs | 24 +- Runtime/BuiltIns/AsyncGeneratorBuiltIns.cs | 7 +- Runtime/Types/SharpTSAsyncGenerator.cs | 798 ++++++------------ .../Types/SharpTSAsyncGeneratorFunction.cs | 67 +- .../SharedTests/AsyncGeneratorTests.cs | 136 +-- SharpTS.Tests/SharedTests/GeneratorTests.cs | 95 +++ 9 files changed, 632 insertions(+), 601 deletions(-) diff --git a/Execution/Interpreter.Async.cs b/Execution/Interpreter.Async.cs index 1bb1dc61..d95d4fe2 100644 --- a/Execution/Interpreter.Async.cs +++ b/Execution/Interpreter.Async.cs @@ -35,6 +35,10 @@ public partial class Interpreter internal async Task AwaitPreservingEnvironment(Task task) { var saved = _environment; + // Also preserve the active async generator: a generator body whose yielded expression awaits + // must resume with its own generator (and scope) restored, even if interleaved event-loop work + // ran another generator in the meantime (#752). + var savedGen = CurrentAsyncGenerator; try { return await task; @@ -42,6 +46,7 @@ public partial class Interpreter finally { _environment = saved; + CurrentAsyncGenerator = savedGen; } } @@ -261,9 +266,22 @@ private async Task IterateAsyncIterator(object asyncIterator, S var result = await ExecuteLoopBodyAsync(forOf.Variable.Lexeme, value, forOf.Body); var (shouldBreak, shouldContinue, abruptResult) = HandleLoopResult(result, labels); - if (shouldBreak) return ExecutionResult.Success(); + // An early exit (break, or a return/throw out of the loop body) closes the iterator before + // leaving: ECMA-262 AsyncIteratorClose calls return() and awaits it, so a suspended async + // generator runs its finally blocks (#697 / cleanup). A lazy generator is otherwise simply + // abandoned at its yield and its finally never runs. The labels (#728) match a labeled + // break/continue that targets this for-await loop rather than escaping it. + if (shouldBreak) + { + await CloseAsyncIteratorOnEarlyExit(asyncIterator); + return ExecutionResult.Success(); + } if (shouldContinue) continue; - if (abruptResult.HasValue) return abruptResult.Value; + if (abruptResult.HasValue) + { + await CloseAsyncIteratorOnEarlyExit(asyncIterator); + return abruptResult.Value; + } // Process any pending timer callbacks ProcessPendingCallbacks(); @@ -272,6 +290,40 @@ private async Task IterateAsyncIterator(object asyncIterator, S return ExecutionResult.Success(); } + /// + /// Closes an async iterator when a for await…of exits early (break, or a return/throw out of + /// the loop body) — ECMA-262 AsyncIteratorClose. Calls return() if the iterator provides one + /// and awaits it, so a suspended async generator runs its finally blocks before the loop + /// leaves. Cleanup is best-effort: a missing return() is skipped, and a rejection from the + /// return() itself is swallowed so the loop's own completion (the break / return / throw that + /// triggered the close) takes precedence. + /// + private async Task CloseAsyncIteratorOnEarlyExit(object asyncIterator) + { + object? result; + try + { + result = CallMethodOnObject(asyncIterator, "return", []); + } + catch + { + // No return() method (or it threw synchronously) — nothing to close. + return; + } + + try + { + if (result is SharpTSPromise promise) + await AwaitPreservingEnvironment(promise.Task); + else if (result is Task task) + await AwaitPreservingEnvironment(task); + } + catch + { + // The iterator's return() rejected during cleanup; the loop's own completion wins. + } + } + /// /// Calls a method on an object by name. /// diff --git a/Execution/Interpreter.Expressions.cs b/Execution/Interpreter.Expressions.cs index ce478f56..2f153b91 100644 --- a/Execution/Interpreter.Expressions.cs +++ b/Execution/Interpreter.Expressions.cs @@ -170,7 +170,7 @@ internal async Task EvaluateAsync(Expr expr) case Expr.Await awaitExpr: return await EvaluateAwaitAsync(awaitExpr); case Expr.DynamicImport di: return EvaluateDynamicImport(di); case Expr.ImportMeta im: return EvaluateImportMeta(im); - case Expr.Yield yieldExpr: return EvaluateYield(yieldExpr); + case Expr.Yield yieldExpr: return await EvaluateYieldAsync(yieldExpr); case Expr.RegexLiteral regex: return RuntimeValue.FromObject(new SharpTSRegExp(regex.Pattern, regex.Flags)); case Expr.ClassExpr classExpr: return EvaluateClassExpression(classExpr); default: throw new InvalidOperationException($"Runtime Error: Unhandled expression type in async Interpreter: {expr.GetType().Name}"); @@ -210,6 +210,34 @@ private RuntimeValue EvaluateYield(Expr.Yield yieldExpr) throw new YieldException(value, yieldExpr.IsDelegating); } + /// + /// Evaluates a yield/yield* inside an async generator body. The body runs as an + /// ordinary interpreter async execution, so the yielded expression is evaluated asynchronously + /// (supporting yield await x) and the suspension is delegated to the active async generator + /// (), which hands the value to the + /// driving next() and awaits the resume. Falls back to the synchronous yield path when + /// no async generator is active (a misplaced yield reached via the async dispatcher). + /// + private async Task EvaluateYieldAsync(Expr.Yield yieldExpr) + { + var activeGenerator = CurrentAsyncGenerator; + if (activeGenerator == null) + return EvaluateYield(yieldExpr); + + object? value = yieldExpr.Value != null + ? (await EvaluateAsync(yieldExpr.Value)).ToObject() + : SharpTSUndefined.Instance; + + object? result = await activeGenerator.OnYieldAsync(value, yieldExpr.IsDelegating); + + // For a plain `yield`, the resume value is delivered verbatim (so `next(null)` yields null and a + // bare next() yields undefined). For `yield*`, a non-generator delegate's completion value is + // undefined; coalesce null → undefined to preserve that. + if (yieldExpr.IsDelegating) + return RuntimeValue.FromBoxed(result ?? SharpTSUndefined.Instance); + return RuntimeValue.FromBoxed(result); + } + /// /// Evaluates an await expression, unwrapping the Promise value. /// @@ -449,7 +477,15 @@ private RuntimeValue EvaluateArrowFunction(Expr.ArrowFunction arrow) RuntimeEnvironment closure = _environment; ISharpTSCallable func; - if (arrow.IsAsync) + if (arrow.IsAsync && arrow.IsGenerator) + { + // Async generator function expressions (async function*) — wrap in an async-generator- + // creating function. Dispatched before the plain-async branch so an async generator + // expression that closes over a block-scoped binding (left in place by GeneratorArrowLifter) + // runs natively rather than being mishandled as a plain async function (#734). + func = new SharpTSAsyncArrowGeneratorFunction(arrow, closure, arrow.HasOwnThis); + } + else if (arrow.IsAsync) { func = new SharpTSAsyncArrowFunction(arrow, closure, arrow.HasOwnThis); } diff --git a/Execution/Interpreter.cs b/Execution/Interpreter.cs index ed09a6e5..d021fcd8 100644 --- a/Execution/Interpreter.cs +++ b/Execution/Interpreter.cs @@ -474,6 +474,16 @@ public override void Send(SendOrPostCallback d, object? state) /// internal Func? YieldCallback { get; set; } + /// + /// The async generator whose body is currently executing, if any. An async generator body runs as + /// an ordinary interpreter async execution on the single event-loop thread; this binding lets the + /// async yield evaluator () suspend through the right + /// generator. It is re-asserted across every suspension the body crosses — guest awaits restore it + /// via , and the generator re-asserts it itself at each + /// yield resume — so interleaved async generators never observe each other's binding. + /// + internal Runtime.Types.SharpTSAsyncGenerator? CurrentAsyncGenerator { get; set; } + /// /// Registers a timer for tracking. Called by TimerBuiltIns when creating setTimeout/setInterval. /// Enables proper cleanup of all pending timers when the interpreter is disposed. diff --git a/Parsing/GeneratorArrowLifter.cs b/Parsing/GeneratorArrowLifter.cs index 76e41782..7ada14ba 100644 --- a/Parsing/GeneratorArrowLifter.cs +++ b/Parsing/GeneratorArrowLifter.cs @@ -760,19 +760,17 @@ private Expr LiftGeneratorArrow(Expr.ArrowFunction af) { var freeVars = FreeVariableCollector.Collect(af); - // #678: a generator expression that closes over a block-scoped binding (loop variable, catch - // parameter, or a let/const/class declared in a nested block) cannot be lifted — the module - // body and every enclosing function body sit outside that block, so the lift would unbind the - // reference. Leave it in place as a generator EXPRESSION: the interpreter runs generator - // expressions natively and the type checker establishes the generator context directly. The - // body is still rewritten so any nested generator expressions inside it are handled, and its - // own name (if any) stays bound natively — no #679 self-binding rewrite is needed in place. - // - // ASYNC generator expressions are excluded: the interpreter's native arrow path does not yet - // build an async generator instance (EvaluateArrowFunction treats an async arrow as a plain - // async function), so an in-place async generator would be mishandled. They keep their prior - // lift behavior — a block-capturing async generator still reports "Undefined variable" (#734). - if (!af.IsAsync && CapturesBlockScopedBinding(freeVars)) + // #678/#734: a generator expression (sync or async) that closes over a block-scoped binding + // (loop variable, catch parameter, or a let/const/class declared in a nested block) cannot be + // lifted — the module body and every enclosing function body sit outside that block, so the lift + // would unbind the reference. Leave it in place as a generator EXPRESSION: the interpreter runs + // both sync (SharpTSArrowGeneratorFunction) and async (SharpTSAsyncArrowGeneratorFunction) + // generator expressions natively, and the type checker establishes the generator context + // directly (CheckArrowFunction handles arrow.IsAsync). The body is still rewritten so any nested + // generator expressions inside it are handled, and its own name (if any) stays bound natively — + // no #679 self-binding rewrite is needed in place. The compiler has no generator-expression IL + // path and reports a clear "Yield not supported in this context" error for the capturing case. + if (CapturesBlockScopedBinding(freeVars)) { var inPlaceBody = RewriteFunctionBody(af.Parameters, af.BlockBody!, selfName: af.Name?.Lexeme); var inPlaceParams = RewriteParameters(af.Parameters); diff --git a/Runtime/BuiltIns/AsyncGeneratorBuiltIns.cs b/Runtime/BuiltIns/AsyncGeneratorBuiltIns.cs index 2daac505..2edeae13 100644 --- a/Runtime/BuiltIns/AsyncGeneratorBuiltIns.cs +++ b/Runtime/BuiltIns/AsyncGeneratorBuiltIns.cs @@ -17,11 +17,14 @@ public static class AsyncGeneratorBuiltIns { return name switch { - "next" => new BuiltInAsyncMethod("next", 0, 0, async (_, receiver, _) => + "next" => new BuiltInAsyncMethod("next", 0, 1, async (_, receiver, args) => { if (receiver is SharpTSAsyncGenerator gen) { - return await gen.Next(); + // A resumed `yield` evaluates to the value sent via next(v); an omitted argument is + // undefined, not null (ECMA-262 §27.6.3.6). + object? sent = args.Count > 0 ? args[0] : SharpTSUndefined.Instance; + return await gen.Next(sent); } throw new Exception("Runtime Error: next() called on non-async-generator."); }), diff --git a/Runtime/Types/SharpTSAsyncGenerator.cs b/Runtime/Types/SharpTSAsyncGenerator.cs index 96130fee..be2174f6 100644 --- a/Runtime/Types/SharpTSAsyncGenerator.cs +++ b/Runtime/Types/SharpTSAsyncGenerator.cs @@ -10,622 +10,378 @@ namespace SharpTS.Runtime.Types; /// Runtime object representing an active async generator instance. /// /// -/// Created by when called. -/// Combines async execution (await) with generator semantics (yield). -/// Each call to next() returns a Promise that resolves to { value, done }. +/// Created by (declarations) and +/// (function expressions) when called — both drive +/// the same body-statement list, so a single type serves both. Each next()/return()/ +/// throw() returns a that resolves to a . +/// +/// Lazy coroutine model. Unlike a fully synchronous generator (which suspends a worker +/// thread at each yield), an async generator can await mid-body, so it cannot use a +/// background thread: the interpreter is single-threaded (a custom +/// routes every async continuation back to the one event-loop thread), and a worker would race the +/// event loop on the shared interpreter environment. Instead the body runs as an ordinary +/// interpreter async execution () on that event-loop +/// thread and suspends at each yield by handing the value to the driving request and awaiting +/// the next one. Reusing the real async execution means an await nested inside a yielded +/// expression preserves the ambient environment like any async function (no closure leak, #752) and a +/// for await…of inside the body drives the async-iterator protocol natively (#717). +/// +/// Request queue. Pending next()/return()/throw() calls are serviced +/// FIFO (ECMA-262 §27.6.3 AsyncGenerator request queue), so two next() issued before the first +/// settles resolve in call order instead of racing (#690). A re-entrant resume (the body advancing +/// itself synchronously) is rejected with a catchable TypeError rather than deadlocking (#542, +/// mirrors compiled mode). /// /// +/// /// public class SharpTSAsyncGenerator : ITypeCategorized { /// public TypeCategory RuntimeCategory => TypeCategory.AsyncGenerator; - private readonly Stmt.Function _declaration; + private readonly List _body; private readonly RuntimeEnvironment _environment; private readonly Interpreter _interpreter; - private List? _values = null; // Collected yielded values (null = not yet executed) - // The single eager body drain, shared by every next() caller. Started by the first next() and - // awaited by all subsequent calls so overlapping requests are serviced in call order (ECMA-262 - // §27.6.3 services the async-generator request queue FIFO) rather than racing the half-populated - // _values list. _values alone can't gate this: it is set to an empty list at drain start, so a - // second next() issued before the first settles would see it non-null and wrongly report completion - // before any value was collected (#690). - private Task? _drainTask = null; - private int _index = 0; - // The generator's completion value. Defaults to undefined so a body that falls off the end (or a - // no-arg return) reports { value: undefined, done: true }, not C# null (#540). - private object? _returnValue = SharpTSUndefined.Instance; - private bool _closed = false; - // Whether the one-time completion result has already been handed out. Once the body is drained - // (or the generator is closed), the completion value is reported exactly once; every later next() - // reports undefined (ECMA-262 §27.6.1.2 → CreateIterResultObject(undefined, true), #540). - private bool _completionDelivered = false; - // A throw (an uncaught guest `throw` or a rejected `await`) raised by the eagerly-drained body. - // The body runs to completion on the first next(), but the throw must be observed from the next() - // that follows the values yielded before it — so it is buffered here and rethrown by Next() once - // those values are exhausted, rejecting that call's promise (#566). Null when the body did not throw. - private Exception? _pendingException = null; - - // The interpreter's ambient environment captured when the eager body drain begins — i.e. the scope - // active in the caller that triggered the first next(). The drain temporarily repoints the shared - // interpreter environment at the generator's own closure (_environment); because the body suspends - // at `await` points while that closure is installed, control can return to the caller — or to - // interleaved event-loop work — with the generator's closure still in place. Every guest-suspending - // await restores this caller environment before suspending and re-asserts the generator's - // environment on resume, so the generator never leaks its closure into the ambient environment - // (otherwise an outer binding read after the generator is driven resolves against the wrong scope: - // "Undefined variable" in a for-await body (#689), or a member access on the wrong value (#690)). - private RuntimeEnvironment? _callerEnv; - - public SharpTSAsyncGenerator(Stmt.Function declaration, RuntimeEnvironment environment, Interpreter interpreter) + private enum State { - _declaration = declaration; - _environment = environment; - _interpreter = interpreter; + /// Created but body not started; the first next() starts it. + SuspendedStart, + /// Body suspended at a yield, awaiting the next request. + SuspendedYield, + /// Body running or suspended at an await (started, not at a yield, not finished). + Executing, + /// Body finished (ran off the end, returned, or threw). + Completed, } + private State _state = State.SuspendedStart; + /// - /// Advances the async generator to the next yield point. - /// Returns a Promise that resolves to { value, done } result object. + /// A queued next/return/throw: how it resumes the body, the carried value, and + /// the promise handed back to the caller (resolved/rejected when the body produces the matching + /// result). Continuations run asynchronously so a resolved request never re-enters the body inline. /// - public async Task Next() + private sealed class Request(GeneratorResumeKind kind, object? value) { - // A finished/disposed generator (closed via return()/throw()) reports undefined; its completion - // value was already delivered/consumed by that return()/throw() call (ECMA-262 §27.6.1.2, #540). - if (_closed) - { - return new SharpTSIteratorResult(SharpTSUndefined.Instance, done: true); - } + public GeneratorResumeKind Kind { get; } = kind; + public object? Value { get; } = value; + public TaskCompletionSource Completion { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); + } - // Execute the generator body on the first call; serialize concurrent/overlapping next() calls by - // awaiting the same drain task. A second next() issued before the first settles must wait for the - // shared drain and then hand out the next collected value in call order — not race ahead and read - // the not-yet-populated _values list as completion (#690). - _drainTask ??= ExecuteBodyAsync(); - await _drainTask; + // Requests received while the body is Executing (mid-await), drained in arrival order (§27.6.3). + private readonly Queue _queue = new(); + // The request the body's next yield/completion will fulfill. + private Request? _currentRequest; + // Set while the body is SuspendedYield: completed by the resuming request to wake the body. + private TaskCompletionSource? _pendingResume; - // Defensive check: ExecuteBodyAsync should always initialize _values, - // but verify to provide a clear error message if something goes wrong. - if (_values == null) - { - throw new InvalidOperationException( - "Internal error: Async generator body did not initialize values collection. " + - "This indicates a bug in ExecuteBodyAsync."); - } + // True only while the body runs its initial synchronous segment (between the first next() and the + // first suspension). A next/return/throw observing this is the body advancing itself synchronously + // (re-entrancy), which a real queue can't serve here without deadlocking the body it blocks (#542). + private bool _running; - if (_index < _values.Count) - { - return new SharpTSIteratorResult(_values[_index++], done: false); - } + public SharpTSAsyncGenerator(List body, RuntimeEnvironment environment, Interpreter interpreter) + { + _body = body; + _environment = environment; + _interpreter = interpreter; + } - // Body fully drained. If it threw after the values above, surface that now so this next()'s - // promise rejects — the throw is delivered after the preceding values, then the generator is - // done (#566). Reported exactly once. - if (_pendingException != null) - { - var ex = _pendingException; - _pendingException = null; - _closed = true; - _completionDelivered = true; - System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(ex).Throw(); - } + /// Advances the generator, resuming a suspended yield with undefined. + public Task Next() => Resume(GeneratorResumeKind.Next, SharpTSUndefined.Instance); - // Deliver the completion value once (the body's `return X`, or undefined when it ran off the - // end), then undefined on every later call — the stale completion / last yielded value must - // not replay forever (#540; mirrors the sync generator's done semantics). - if (_completionDelivered) - { - return new SharpTSIteratorResult(SharpTSUndefined.Instance, done: true); - } - _completionDelivered = true; - return new SharpTSIteratorResult(_returnValue, done: true); - } + /// Advances the generator, resuming a suspended yield with . + public Task Next(object? sentValue) => Resume(GeneratorResumeKind.Next, sentValue); /// - /// Closes the async generator and returns a Promise resolving to { value, done: true }. + /// Resumes the generator with a return completion. A suspended body runs its enclosing + /// finally blocks before settling as { value, done: true } (a yielding finally + /// suspends here instead, ECMA-262 §27.6.1.3 / §14.4.14); a not-yet-started or finished one simply + /// reports the value. /// - public Task Return(object? value = null) - { - // return(v) reports { value: v, done: true } (echoing the argument, per ECMA-262 §27.6.1.3) and - // closes the generator; a later next() then reports undefined via the `_closed` guard in Next() - // rather than replaying v (#540). _completionDelivered is set so the once-only accounting stays - // consistent even if the body had already run off the end before this return(). - _closed = true; - _completionDelivered = true; - // return() closes the generator and wins over a not-yet-observed body throw: the code after - // the last yield never "runs" observably, so discard any buffered exception (#566). - _pendingException = null; - return Task.FromResult(new SharpTSIteratorResult(value, done: true)); - } + public Task Return(object? value = null) => Resume(GeneratorResumeKind.Return, value); /// - /// Throws an exception at the current yield point. - /// Returns a Promise that rejects with the error. + /// Resumes the generator with a throw completion at the suspended yield, running any + /// active catch/finally. If a catch handles it the body continues; otherwise the + /// returned promise rejects. /// - public Task Throw(object? error = null) - { - _closed = true; - string message = error?.ToString() ?? "AsyncGenerator.throw() called"; - throw new ThrowException(error ?? message); - } + public Task Throw(object? error = null) => Resume(GeneratorResumeKind.Throw, error); /// - /// Executes the async generator body, collecting all yielded values. - /// Handles both yield and await expressions. + /// Core of next()/return()/throw(): enqueues the request and, depending on the + /// generator's state, starts the body, resumes it from a yield, or queues behind an in-flight run. /// - private async Task ExecuteBodyAsync() + private Task Resume(GeneratorResumeKind kind, object? value) { - _values = []; + // Re-entrancy (the body advancing itself during its synchronous segment): reject rather than + // deadlock — the queued request could only be served by the body that is blocking on it (#542). + if (_running) + return Task.FromException( + new ThrowException(new SharpTSTypeError("Async generator is already running"))); - if (_declaration.Body == null || _declaration.Body.Count == 0) + switch (_state) { - return; + case State.Completed: + return SettleCompleted(kind, value); + + case State.SuspendedStart: + if (kind == GeneratorResumeKind.Return) + { + // return() before the body starts closes it without running the body. + _state = State.Completed; + return Task.FromResult(new SharpTSIteratorResult(value, done: true)); + } + if (kind == GeneratorResumeKind.Throw) + { + // throw() before the body starts completes it abnormally. + _state = State.Completed; + return Task.FromException(ThrowException.FromResult(value)); + } + return StartBody(new Request(kind, value)); + + case State.SuspendedYield: + var resumeReq = new Request(kind, value); + _currentRequest = resumeReq; + _state = State.Executing; + var pending = _pendingResume!; + _pendingResume = null; + pending.SetResult(resumeReq); // wakes the body on a later turn (RunContinuationsAsynchronously) + return resumeReq.Completion.Task; + + case State.Executing: + default: + // Body is mid-await (concurrent next): queue and let the body pick it up at its next yield. + var queued = new Request(kind, value); + _queue.Enqueue(queued); + return queued.Completion.Task; } + } - // Save and set the interpreter environment. _callerEnv is the scope the drain must restore - // whenever it suspends at an await, so the shared interpreter environment is never left pointing - // at this generator's closure while control is elsewhere (#689, #690). - RuntimeEnvironment previousEnv = _interpreter.Environment; - _callerEnv = previousEnv; - _interpreter.SetEnvironment(_environment); + /// + /// Starts the body for the first next(). The body runs synchronously up to its first + /// suspension (matching ECMA-262, where the first resume runs the body until its first + /// await/yield); the surrounding save/restore keeps that synchronous prologue from leaking the + /// generator's environment / active-generator binding into the caller that issued next(). + /// + private Task StartBody(Request first) + { + _currentRequest = first; + _state = State.Executing; + RuntimeEnvironment savedEnv = _interpreter.Environment; + SharpTSAsyncGenerator? savedGen = _interpreter.CurrentAsyncGenerator; + _running = true; try { - var result = await ExecuteStatementsAsync(_declaration.Body); - if (result.Type == ExecutionResult.ResultType.Return) - { - _returnValue = result.Value.ToObject(); - } - else if (result.Type == ExecutionResult.ResultType.Throw) - { - // Buffer rather than throw now: the body drains eagerly, so a throw after some yields - // must surface from the next() that follows those values, not the first one (#566). - // The original throw value is preserved through ThrowException (see SharpTSFunction.Call). - _pendingException = ThrowException.FromResult(result.Value.ToObject()); - } - } - catch (Exception ex) when (ex is not YieldException) - { - // A host exception escaped the body — a rejected `await`, or a `throw` surfaced as a C# - // exception. Buffer it so the post-drain next() rejects its promise with it (#566). - _pendingException = ex; + _ = RunBodyAsync(); } finally { - _interpreter.SetEnvironment(previousEnv); + _running = false; + _interpreter.CurrentAsyncGenerator = savedGen; + _interpreter.SetEnvironment(savedEnv); } + return first.Completion.Task; } /// - /// Awaits a host task at a point where the eager body drain suspends, keeping the interpreter's - /// shared ambient environment consistent across the suspension. The drain runs with that environment - /// repointed at this generator's closure; this restores the caller's environment () - /// before suspending — so control returning to the caller, or interleaved event-loop work, sees the - /// correct scope — and re-asserts the generator's environment on resume so the drain continues in the - /// right scope. Without it, an outer binding read after the generator is driven resolves against the - /// wrong scope: "Undefined variable" in a for-await body (#689) or a member access on the wrong value - /// (#690). Mirrors the caller-state save/restore the synchronous generator performs at each yield. + /// Runs the generator body as an ordinary interpreter async execution, installing this generator as + /// the interpreter's active async-generator (so yield expressions suspend through it) and the + /// generator closure as the ambient environment. Settles the driving request with the completion + /// value, a return value, or a thrown error. /// - private async Task AwaitDetached(Task task) + private async Task RunBodyAsync() { - RuntimeEnvironment generatorEnv = _interpreter.Environment; - _interpreter.SetEnvironment(_callerEnv ?? generatorEnv); + RuntimeEnvironment prevEnv = _interpreter.Environment; + SharpTSAsyncGenerator? prevGen = _interpreter.CurrentAsyncGenerator; + _interpreter.SetEnvironment(_environment); + _interpreter.CurrentAsyncGenerator = this; + + object? completionValue = SharpTSUndefined.Instance; + Exception? pendingException = null; try { - return await task; + ExecutionResult result = await _interpreter.ExecuteBlockAsync(_body, _environment); + if (result.Type == ExecutionResult.ResultType.Return) + completionValue = result.Value.ToObject(); + else if (result.Type == ExecutionResult.ResultType.Throw) + pendingException = ThrowException.FromResult(result.Value.ToObject()); + } + catch (GeneratorReturnException grex) + { + // A return() injected at a yield with no enclosing try unwinds to here: settle as a return. + completionValue = grex.Value; + } + catch (Exception ex) when (ex is not YieldException) + { + // A rejected await, or a guest throw surfaced as a host exception: reject the driving request. + pendingException = ex; } finally { - _interpreter.SetEnvironment(generatorEnv); + _interpreter.CurrentAsyncGenerator = prevGen; + _interpreter.SetEnvironment(prevEnv); } + + CompleteBody(completionValue, pendingException); } /// - /// Recursively executes statements asynchronously, collecting yields. + /// Suspends the body at a plain yield: hands the yielded value to the driving request and + /// awaits the next one. Invoked by the interpreter when it evaluates a yield inside this + /// generator's body (see ). /// - private async Task ExecuteStatementsAsync(List statements) + internal Task OnYieldAsync(object? value, bool isDelegating) + => isDelegating ? DelegateYieldStarAsync(value) : PlainYieldAsync(value); + + private async Task PlainYieldAsync(object? value) { - foreach (var stmt in statements) - { - var result = await ExecuteStatementAsync(stmt); - if (result.IsAbrupt) return result; - } - return ExecutionResult.Success(); + GeneratorResume resume = await SuspendAtYieldAsync(value); + // The resumed yield evaluates to the sent value; an abrupt resume (return/throw) is realized as + // the control-flow exception that unwinds the body's own try/finally blocks (§27.5.3.4). + return resume.Realize(); } /// - /// Executes a single statement asynchronously, handling yield and await expressions. + /// The core suspend: delivers as { value, done: false } to the + /// driving request, awaits the next request, and re-asserts the generator's scope on resume (the + /// await may have run unrelated event-loop work in between). /// - private async Task ExecuteStatementAsync(Stmt stmt) + private async Task SuspendAtYieldAsync(object? value) { - switch (stmt) - { - case Stmt.Expression exprStmt: - try - { - await EvaluateAsync(exprStmt.Expr); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - } - return ExecutionResult.Success(); - - case Stmt.Block block: - if (block.Statements != null) - { - var blockEnv = new RuntimeEnvironment(_interpreter.Environment); - RuntimeEnvironment prevEnv = _interpreter.Environment; - _interpreter.SetEnvironment(blockEnv); - try - { - return await ExecuteStatementsAsync(block.Statements); - } - finally - { - _interpreter.SetEnvironment(prevEnv); - } - } - return ExecutionResult.Success(); - - case Stmt.Var varStmt: - object? value = null; - try - { - if (varStmt.Initializer != null) - { - value = await EvaluateAsync(varStmt.Initializer); - } - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - value = null; - } - _environment.Define(varStmt.Name.Lexeme, value); - return ExecutionResult.Success(); - - case Stmt.If ifStmt: - object? condition; - try - { - condition = await EvaluateAsync(ifStmt.Condition); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - condition = false; - } - - if (IsTruthy(condition)) - { - return await ExecuteStatementAsync(ifStmt.ThenBranch); - } - else if (ifStmt.ElseBranch != null) - { - return await ExecuteStatementAsync(ifStmt.ElseBranch); - } - return ExecutionResult.Success(); + RuntimeEnvironment genEnv = _interpreter.Environment; + Request req = _currentRequest!; + _currentRequest = null; + _state = State.SuspendedYield; + req.Completion.SetResult(new SharpTSIteratorResult(value, done: false)); - case Stmt.While whileStmt: - while (true) - { - object? whileCond; - try - { - whileCond = await EvaluateAsync(whileStmt.Condition); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - whileCond = false; - } - - if (!IsTruthy(whileCond)) break; - - var result = await ExecuteStatementAsync(whileStmt.Body); - if (result.Type == ExecutionResult.ResultType.Break && result.TargetLabel == null) break; - if (result.Type == ExecutionResult.ResultType.Continue && result.TargetLabel == null) continue; - if (result.IsAbrupt) return result; - } - return ExecutionResult.Success(); - - case Stmt.For forStmt: - // Execute initializer once - if (forStmt.Initializer != null) - { - var initResult = await ExecuteStatementAsync(forStmt.Initializer); - if (initResult.IsAbrupt) return initResult; - } - - // Loop - while (true) - { - // Check condition - if (forStmt.Condition != null) - { - object? forCond; - try - { - forCond = await EvaluateAsync(forStmt.Condition); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - forCond = false; - } - - if (!IsTruthy(forCond)) break; - } - - // Execute body - var result = await ExecuteStatementAsync(forStmt.Body); - - if (result.Type == ExecutionResult.ResultType.Break && result.TargetLabel == null) - break; - - // On continue OR normal completion, execute increment - if (result.Type == ExecutionResult.ResultType.Continue && result.TargetLabel == null) - { - // Execute increment before continuing - if (forStmt.Increment != null) - { - try - { - await EvaluateAsync(forStmt.Increment); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - } - } - continue; - } - - if (result.IsAbrupt) return result; - - // Normal completion: execute increment - if (forStmt.Increment != null) - { - try - { - await EvaluateAsync(forStmt.Increment); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - } - } - } - return ExecutionResult.Success(); - - case Stmt.ForOf forOf: - object? iterable; - try - { - iterable = await EvaluateAsync(forOf.Iterable); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - iterable = new SharpTSArray([]); - } + Request next = await TakeNextRequestAsync(); - IEnumerable elements = GetIterableElements(iterable); - foreach (var element in elements) - { - var loopEnv = new RuntimeEnvironment(_interpreter.Environment); - loopEnv.Define(forOf.Variable.Lexeme, element); - - RuntimeEnvironment prevEnv = _interpreter.Environment; - _interpreter.SetEnvironment(loopEnv); - try - { - var result = await ExecuteStatementAsync(forOf.Body); - if (result.Type == ExecutionResult.ResultType.Break && result.TargetLabel == null) break; - if (result.Type == ExecutionResult.ResultType.Continue && result.TargetLabel == null) continue; - if (result.IsAbrupt) return result; - } - finally - { - _interpreter.SetEnvironment(prevEnv); - } - } - return ExecutionResult.Success(); - - case Stmt.Return returnStmt: - // A value-less `return;` (or running off the end) completes with undefined, not C# - // null; only an explicit `return null;` reports null (#540, mirrors the sync gen). - object? returnValue = SharpTSUndefined.Instance; - if (returnStmt.Value != null) - { - try - { - returnValue = await EvaluateAsync(returnStmt.Value); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - } - } - return ExecutionResult.Return(returnValue); - - case Stmt.TryCatch tryCatch: - ExecutionResult tryResult; - try - { - tryResult = await ExecuteStatementsAsync(tryCatch.TryBlock); - } - catch (Exception ex) when (ex is not YieldException) - { - // A host exception escaped the try body — most often a rejected `await` - // (SharpTSPromiseRejectedException) or a guest `throw` surfaced as a C# - // exception. Convert it to a guest Throw so this try's catch/finally run (#617). - tryResult = ExecutionResult.Throw(_interpreter.TranslateException(ex)); - } - - if (tryResult.Type == ExecutionResult.ResultType.Throw && tryCatch.CatchBlock != null) - { - var catchEnv = new RuntimeEnvironment(_interpreter.Environment); - // `catch {}` (no binding) is valid — only define the param when present, and bind - // the unwrapped guest value rather than the boxed RuntimeValue struct. - if (tryCatch.CatchParam != null) - catchEnv.Define(tryCatch.CatchParam.Lexeme, - _interpreter.CoerceCaughtValueForBinding(tryResult.Value.ToObject())); - RuntimeEnvironment prevEnv = _interpreter.Environment; - _interpreter.SetEnvironment(catchEnv); - try - { - tryResult = await ExecuteStatementsAsync(tryCatch.CatchBlock); - } - catch (Exception ex) when (ex is not YieldException) - { - tryResult = ExecutionResult.Throw(_interpreter.TranslateException(ex)); - } - finally - { - _interpreter.SetEnvironment(prevEnv); - } - } - - if (tryCatch.FinallyBlock != null) - { - ExecutionResult finallyResult; - try - { - finallyResult = await ExecuteStatementsAsync(tryCatch.FinallyBlock); - } - catch (Exception ex) when (ex is not YieldException) - { - finallyResult = ExecutionResult.Throw(_interpreter.TranslateException(ex)); - } - if (finallyResult.IsAbrupt) return finallyResult; - } - return tryResult; - - default: - // For other statements, delegate to the interpreter's async handler - try - { - return await _interpreter.ExecuteBlockAsync([stmt], _environment); - } - catch (YieldException yield) - { - await HandleYieldAsync(yield); - return ExecutionResult.Success(); - } - } + _interpreter.SetEnvironment(genEnv); + _interpreter.CurrentAsyncGenerator = this; + return new GeneratorResume(next.Kind, next.Value); } /// - /// Evaluates an expression asynchronously, handling await and yield expressions. + /// Returns the next request to resume with: a concurrently-queued one immediately, otherwise a task + /// completed by the resuming next()/return()/throw(). /// - private async Task EvaluateAsync(Expr expr) + private Task TakeNextRequestAsync() { - // Check for await expression - if (expr is Expr.Await awaitExpr) - { - // Recursively evaluate the inner expression (may contain await) - var value = await EvaluateAsync(awaitExpr.Expression); - // Handle SharpTSPromise (wraps Task). Await through AwaitDetached so the suspension - // does not leak this generator's closure into the shared interpreter environment (#689, #690). - if (value is SharpTSPromise promise) - { - return await AwaitDetached(promise.Task); - } - if (value is Task task) - { - return await AwaitDetached(task); - } - return value; - } - - // Check for yield expression - evaluate its value asynchronously - if (expr is Expr.Yield yieldExpr) + if (_queue.Count > 0) { - object? value = null; - if (yieldExpr.Value != null) - { - value = await EvaluateAsync(yieldExpr.Value); - } - throw new YieldException(value, yieldExpr.IsDelegating); + Request req = _queue.Dequeue(); + _currentRequest = req; + _state = State.Executing; + return Task.FromResult(req); } - - // For other expressions, evaluate asynchronously to support nested await - return (await _interpreter.EvaluateAsync(expr)).ToObject(); + // _state stays SuspendedYield; Resume() sets _currentRequest + Executing when it wakes us. + _pendingResume = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + return _pendingResume.Task; } /// - /// Handles a yield exception by collecting the value. + /// yield* delegation (ECMA-262 §14.4.14, async path). A delegated async generator is driven + /// via next/return/throw, forwarding the outer's resume completion into it; a sync iterable + /// is iterated lazily with the outer suspended for each element. /// - private async Task HandleYieldAsync(YieldException yield) + private async Task DelegateYieldStarAsync(object? iterable) { - if (yield.IsDelegating) + if (iterable is SharpTSAsyncGenerator inner) { - // yield* - delegate to another iterable - var value = yield.Value; - - // If delegating to an async iterable, await each value - if (value is SharpTSAsyncGenerator asyncGen) + var received = new GeneratorResume(GeneratorResumeKind.Next, SharpTSUndefined.Instance); + while (true) { - while (true) + // Drive the delegate with the completion the outer was resumed with, preserving the + // outer's environment across the inner's own suspensions. + Task step = received.Kind switch { - var result = await AwaitDetached(asyncGen.Next()); - if (result is SharpTSIteratorResult ir) - { - if (ir.Done) break; - _values!.Add(ir.Value); - } - else - { - _values!.Add(result); - } - } - } - else - { - var elements = GetIterableElements(value); - foreach (var element in elements) + GeneratorResumeKind.Return => inner.Return(received.Value), + GeneratorResumeKind.Throw => inner.Throw(received.Value), + _ => inner.Next(received.Value), + }; + object? innerResult = await _interpreter.AwaitPreservingEnvironment(step); + (bool done, object? innerValue) = ReadIteratorResult(innerResult); + + if (done) { - _values!.Add(element); + // return → the outer generator itself returns the delegate's value (step c.viii); + // next/throw(handled) → yield* evaluates to it and the outer continues (steps a.v/b.5). + if (received.Kind == GeneratorResumeKind.Return) + throw new GeneratorReturnException(innerValue); + return innerValue; } + + received = await SuspendAtYieldAsync(innerValue); } } - else + + // Sync iterable (array, Map, Set, string, custom iterator): iterate lazily, suspending the outer + // for each element. An abrupt resume realizes the control-flow exception to unwind the outer. + foreach (object? element in _interpreter.GetIterableElements(iterable)) { - // If yielding a promise, await it first (through AwaitDetached so the suspension does not - // leak this generator's closure into the shared interpreter environment — #689, #690). - var value = yield.Value; - if (value is Task task) - { - value = await AwaitDetached(task); - } - _values!.Add(value); + GeneratorResume resume = await SuspendAtYieldAsync(element); + resume.Realize(); } + return SharpTSUndefined.Instance; } /// - /// Gets elements from an iterable value. + /// Settles the body's completion: resolves the driving request once with the completion value + /// (a return value or undefined when it ran off the end), or rejects it with a thrown error. + /// Any requests queued behind it report { undefined, done: true } (the value is delivered + /// once, §27.6.1.2). /// - private static IEnumerable GetIterableElements(object? value) + private void CompleteBody(object? completionValue, Exception? pendingException) + { + _state = State.Completed; + Request? req = _currentRequest; + _currentRequest = null; + if (req != null) + { + if (pendingException != null) + req.Completion.SetException(pendingException); + else + req.Completion.SetResult(new SharpTSIteratorResult(completionValue, done: true)); + } + + while (_queue.Count > 0) + SettleCompletedInto(_queue.Dequeue()); + } + + /// Resolves a next/return/throw issued after the generator has finished (§27.6.1.2). + private static Task SettleCompleted(GeneratorResumeKind kind, object? value) => kind switch { - return value switch + GeneratorResumeKind.Throw => Task.FromException(ThrowException.FromResult(value)), + GeneratorResumeKind.Return => Task.FromResult(new SharpTSIteratorResult(value, done: true)), + _ => Task.FromResult(new SharpTSIteratorResult(SharpTSUndefined.Instance, done: true)), + }; + + private static void SettleCompletedInto(Request req) + { + switch (req.Kind) { - SharpTSArray array => array, - SharpTSGenerator gen => gen, - SharpTSIterator iter => iter.Elements, - SharpTSMap map => map.Entries().Elements, - SharpTSSet set => set.Values().Elements, - string s => s.Select(c => (object?)c.ToString()), - IEnumerable enumerable => enumerable, - null => [], - _ => throw new Exception($"Runtime Error: Cannot iterate over non-iterable value.") - }; + case GeneratorResumeKind.Throw: + req.Completion.SetException(ThrowException.FromResult(req.Value)); + break; + case GeneratorResumeKind.Return: + req.Completion.SetResult(new SharpTSIteratorResult(req.Value, done: true)); + break; + default: + req.Completion.SetResult(new SharpTSIteratorResult(SharpTSUndefined.Instance, done: true)); + break; + } } - private static bool IsTruthy(object? obj) => RuntimeTypes.IsTruthy(obj); + private static (bool Done, object? Value) ReadIteratorResult(object? result) => result switch + { + SharpTSIteratorResult ir => (ir.Done, ir.Value), + SharpTSObject obj => (RuntimeTypes.IsTruthy(obj.GetProperty("done")), obj.GetProperty("value")), + _ => (true, result), + }; public override string ToString() => "[object AsyncGenerator]"; } diff --git a/Runtime/Types/SharpTSAsyncGeneratorFunction.cs b/Runtime/Types/SharpTSAsyncGeneratorFunction.cs index 3ec96ea3..5bdddfa0 100644 --- a/Runtime/Types/SharpTSAsyncGeneratorFunction.cs +++ b/Runtime/Types/SharpTSAsyncGeneratorFunction.cs @@ -36,8 +36,9 @@ public SharpTSAsyncGeneratorFunction(Stmt.Function declaration, RuntimeEnvironme // Bind parameters to arguments ParameterBinder.Bind(_declaration.Parameters ?? [], arguments, environment, interpreter); - // Return the async generator object (not yet started) - return new SharpTSAsyncGenerator(_declaration, environment, interpreter); + // Return the async generator object (not yet started). It drives the same SharpTSAsyncGenerator + // as a function expression — only the body and captured environment differ. + return new SharpTSAsyncGenerator(_declaration.Body ?? [], environment, interpreter); } /// @@ -61,3 +62,65 @@ public SharpTSAsyncGeneratorFunction BindStatic(SharpTSClass klass) public override string ToString() => $"[async function* {_declaration.Name?.Lexeme ?? "anonymous"}]"; } + +/// +/// Runtime wrapper for async generator function EXPRESSIONS (async function*() { } as an +/// expression). The async analogue of . +/// +/// +/// Wraps an with IsAsync = true and IsGenerator = true +/// instead of a . lifts most async +/// generator expressions to declarations, but leaves in place those that close over a block-scoped +/// binding (loop variable, catch parameter, nested-block let/const); this native path runs +/// those (#734). Drives the same as a declaration. +/// +/// +/// +public class SharpTSAsyncArrowGeneratorFunction : ISharpTSCallable +{ + private readonly Expr.ArrowFunction _declaration; + private readonly RuntimeEnvironment _closure; + private readonly int _arity; + + /// + /// Whether this has its own this binding (a function* expression) rather than + /// capturing this from the enclosing scope (an arrow — which cannot be a generator anyway). + /// + public bool HasOwnThis { get; } + + public SharpTSAsyncArrowGeneratorFunction(Expr.ArrowFunction declaration, RuntimeEnvironment closure, bool hasOwnThis = false) + { + _declaration = declaration; + _closure = closure; + HasOwnThis = hasOwnThis; + _arity = declaration.Parameters.Count(p => p.DefaultValue == null && !p.IsRest && !p.IsOptional); + } + + public int Arity() => _arity; + + /// Creates a new async generator instance. Does NOT execute the function body. + public object? Call(Interpreter interpreter, List arguments) + { + RuntimeEnvironment environment = new(_closure); + // Named async generator function expression: bind self-reference alongside params. + if (_declaration.Name != null) + { + environment.Define(_declaration.Name.Lexeme, this); + } + ParameterBinder.Bind(_declaration.Parameters, arguments, environment, interpreter); + + // Generator expressions always have a block body (the parser never produces an + // expression-bodied generator). + return new SharpTSAsyncGenerator(_declaration.BlockBody ?? [], environment, interpreter); + } + + /// Binds this. Only applicable for function expressions with HasOwnThis=true. + public SharpTSAsyncArrowGeneratorFunction Bind(object thisObject) + { + RuntimeEnvironment boundEnv = new(_closure); + boundEnv.Define("this", thisObject); + return new SharpTSAsyncArrowGeneratorFunction(_declaration, boundEnv, hasOwnThis: true); + } + + public override string ToString() => $"[async function* {_declaration.Name?.Lexeme ?? "anonymous"}]"; +} diff --git a/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs b/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs index 2e9487e8..affa1991 100644 --- a/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs +++ b/SharpTS.Tests/SharedTests/AsyncGeneratorTests.cs @@ -1105,15 +1105,12 @@ async function main() { #region Completion / resume value semantics — #481, #540 [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_ResumedYield_EvaluatesToUndefined(ExecutionMode mode) { // The resumed `yield` expression evaluates to undefined (no value sent), not null (#481, the - // async analog of #443). Previously the compiled emitter loaded CLR null here. - // - // COMPILED-ONLY: #481 is a compiled-emitter bug. The interpreter eagerly drains the body and - // does not bind a variable whose initializer is a yield (`const r = yield 1`) at all — it - // throws "Undefined variable 'r'" — a separate pre-existing eager-drain gap, not what #481 fixes. + // async analog of #443). The interpreter's lazy async-generator coroutine binds `const r = yield 1` + // (an earlier eager-drain model threw "Undefined variable 'r'" — fixed by the coroutine rewrite). var source = """ async function* ag() { const r = yield 1; console.log("r=" + r); } async function main() { for await (const v of ag()) {} } @@ -1124,12 +1121,11 @@ public void AsyncGenerator_ResumedYield_EvaluatesToUndefined(ExecutionMode mode) } [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_YieldStarCompletion_EvaluatesToUndefined(ExecutionMode mode) { // The completion value of `yield* inner()` (when the delegate has no explicit return value) is - // undefined, not null (#481). COMPILED-ONLY for the same eager-drain reason as the resumed-yield - // test: the interpreter does not bind `const x = yield* …`. + // undefined, not null (#481). The interpreter binds `const x = yield* …` via its lazy coroutine. var source = """ async function* inner() { yield 2; yield 3; } async function* g() { const x = yield* inner(); console.log("x=" + x); yield 4; } @@ -1186,19 +1182,16 @@ async function main() { #region Re-entrant next() — "already running" guard (#542) [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] - public void AsyncGenerator_ReentrantNext_Compiled_RejectsInsteadOfStackOverflow(ExecutionMode mode) + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_ReentrantNext_RejectsInsteadOfStackOverflow(ExecutionMode mode) { - // A compiled async generator whose body advances itself previously recursed into MoveNextAsync - // until the stack overflowed (RangeError: Maximum call stack size exceeded). ECMA-262 §27.6.3 - // queues such a request, but under this synchronous drive a real queue would deadlock the only - // reachable re-entrancy case (the body awaiting its own request), so the guard rejects with a - // catchable TypeError instead of crashing (#542). Observed here via a `for await…of` consumer, - // whose next()-drive surfaces the rejection to the enclosing try/catch. - // - // COMPILED-ONLY: the interpreter eagerly drains the body, so its re-entrant next() returns a - // (non-conformant) done result rather than recursing — it never hit the stack overflow this - // guards against, and asserting that divergent eager-drain output here would not test #542. + // An async generator whose body advances itself (a re-entrant next()) is rejected with a + // catchable TypeError rather than crashing. Compiled mode previously recursed into MoveNextAsync + // until the stack overflowed; ECMA-262 §27.6.3 queues such a request, but under both modes' drive + // the queued request could only be served by the body that blocks on it, so the guard rejects + // (#542). The interpreter's lazy coroutine guards the body's synchronous segment the same way + // (an earlier eager-drain model returned a non-conformant done result instead). Observed via a + // `for await…of` consumer, whose next()-drive surfaces the rejection to the enclosing try/catch. var source = """ const h: any = {}; async function* g() { await h.it.next(); yield 1; } @@ -1245,12 +1238,11 @@ async function main() { [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_PendingAwait_DirectNext_ResolvesInOrder(ExecutionMode mode) { - // Driving a pending-await async generator by awaiting next() directly. #631 (compiled). Now also - // covered in the interpreter: a second `await it.next()` previously read `it` against a scope the - // generator's eager drain had corrupted ("Only instances and objects have properties"); the drain - // now restores the caller's environment across each suspension (#690), so sequential next() calls - // resolve in order. The eager-drain model still collects values eagerly, but the observable result - // matches the spec for this finite, no-sent-value case. + // Driving a pending-await async generator by awaiting next() directly. #631 (compiled). Also + // covered in the interpreter: a second `await it.next()` previously read `it` against a scope an + // eager-drain model had corrupted ("Only instances and objects have properties"). The interpreter's + // lazy async-generator coroutine runs the body on demand and preserves the caller's environment + // across each suspension (#690), so sequential next() calls resolve in order. var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1296,11 +1288,11 @@ async function main() { public void AsyncGenerator_ConcurrentNext_QueuesInOrder(ExecutionMode mode) { // Two next() calls issued before the first settles must be serviced FIFO (ECMA-262 §27.6.3 - // AsyncGeneratorQueue): compiled mode models it as a task chain (#542); the interpreter serializes - // every caller on one shared body drain, then hands out collected values in call order, so a - // second next() can no longer race ahead and read the not-yet-populated values list as completion - // (#690). Previously the interpreter threw "Only instances and objects have properties" here (the - // env-leak symptom) and could not service concurrent next() at all. + // AsyncGeneratorQueue): compiled mode models it as a task chain (#542); the interpreter's lazy + // async-generator coroutine enqueues the second request and the body services it after the first + // yield, so it can no longer race ahead and read a half-populated state as completion (#690). + // Previously the interpreter threw "Only instances and objects have properties" here (the env-leak + // symptom of an eager-drain model) and could not service concurrent next() at all. var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1322,10 +1314,10 @@ async function main() { public void AsyncGenerator_ForAwaitBody_WritesOuterBinding(ExecutionMode mode) { // #689: a `let`/`const` declared before a `for await…of` over a (genuinely-async) async generator - // must remain visible inside the loop body. The interpreter's eager drain repointed the shared - // environment at the generator's closure and held it across the body's awaits, so the loop body - // resolved `out` against the wrong scope and threw "Undefined variable 'out'". The drain now - // restores the caller's environment across each suspension, so the body reaches the enclosing + // must remain visible inside the loop body. An eager-drain model repointed the shared environment + // at the generator's closure and held it across the body's awaits, so the loop body resolved `out` + // against the wrong scope and threw "Undefined variable 'out'". The interpreter's lazy coroutine + // preserves the caller's environment across each suspension, so the body reaches the enclosing // scope in both modes. var source = """ function later(n: number): Promise { @@ -1347,11 +1339,11 @@ async function main() { [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_DirectNext_PreservesCallerScope(ExecutionMode mode) { - // #690 (env-leak symptom): after the first `await it.next()` drives the generator's eager drain, - // bindings declared before it (here `it` itself and the outer `tag`) must still resolve in the - // caller's scope. Previously a second `it.next()` read `it` against the leaked generator closure - // and threw "Only instances and objects have properties", and an outer local read back as - // undefined. The drain now restores the caller's environment across each await suspension. + // #690 (env-leak symptom): after the first `await it.next()` drives the generator, bindings + // declared before it (here `it` itself and the outer `tag`) must still resolve in the caller's + // scope. Previously a second `it.next()` read `it` against a leaked generator closure and threw + // "Only instances and objects have properties", and an outer local read back as undefined. The + // interpreter's lazy coroutine preserves the caller's environment across each await suspension. var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1375,11 +1367,10 @@ async function main() { public void AsyncGenerator_ForAwaitBody_OuterBinding_SurvivesNestedAwaitBody(ExecutionMode mode) { // #689 hardening: a generator body whose await is nested inside a delegated expression - // (`yield dbl(await later(n))`) suspends through the interpreter's general async-expression path, - // not the generator's own await chokepoint, so the generator alone can't keep its closure from - // leaking for that shape. The for-await driver re-asserts the loop's lexical scope after each - // next(), so the loop body still reaches the outer `out` binding regardless of the body shape. - // (Direct `await it.next()` of such a body is the narrower residual tracked in #752.) + // (`yield dbl(await later(n))`) suspends through the interpreter's general async-expression path. + // The interpreter's lazy coroutine evaluates the yielded expression with the ambient environment + // preserved across that await (like any async function), so the loop body still reaches the outer + // `out` binding regardless of the body shape. var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1398,14 +1389,41 @@ async function main() { } [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] + public void AsyncGenerator_DirectNext_OuterBinding_SurvivesNestedAwaitBody(ExecutionMode mode) + { + // #752: the direct `await it.next()` form of the nested-await body shape + // (`yield dbl(await later(n))`). An eager-drain model leaked the generator's closure into the + // shared environment for this shape, so the outer `tag` read back as undefined ("T 2" became + // "undefined 2"). The interpreter's lazy coroutine preserves the environment across the nested + // await like any async function, so the caller binding survives even when driven by direct next(). + var source = """ + function later(n: number): Promise { + return new Promise(res => setTimeout(() => res(n), 5)); + } + function dbl(n: number): number { return n * 2; } + async function* g() { yield dbl(await later(1)); yield dbl(await later(2)); } + async function main() { + let tag = "T"; + const it = g(); + const a = await it.next(); + console.log(tag + " " + a.value); + } + main(); + """; + + Assert.Equal("T 2\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_ForAwaitOf_OverPendingAsyncGenerator_SuspendsNotBlocks(ExecutionMode mode) { // A `for await…of` INSIDE an async generator, consuming a genuinely-async (setTimeout-backed) - // source, must suspend on the inner iterator's next() instead of blocking on a synchronous - // GetResult — the latter deadlocked/crashed this stream-transform shape in compiled mode (#697, - // the async-generator sibling of #631). Compiled-only: the interpreter has a separate "cannot - // iterate" gap for `for await` over an async generator (its eager-drain model), tracked apart. + // source, must suspend on the inner iterator's next() instead of blocking — the compiled sibling + // deadlocked/crashed this stream-transform shape (#697, the async-generator sibling of #631). The + // interpreter now drives it natively: its lazy async-generator body runs the for-await through the + // real async-iterator protocol, fixing the prior "Cannot iterate over non-iterable value" (#717). var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1422,13 +1440,13 @@ async function main() { } [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_ForAwaitOf_BreakAwaitsReturn(ExecutionMode mode) { // Breaking out of a `for await…of` inside an async generator must await the inner iterator's // return() (the suspension-based cleanup path) and stop consuming — here only the first - // transformed value is produced before the break (#697). Compiled-only (interp for-await-in- - // async-generator gap). + // transformed value is produced before the break (#697). The interpreter now drives this natively + // (#717), closing the inner iterator on the break via AsyncIteratorClose. var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); @@ -1448,13 +1466,13 @@ async function main() { } [Theory] - [MemberData(nameof(ExecutionModes.CompiledOnly), MemberType = typeof(ExecutionModes))] + [MemberData(nameof(ExecutionModes.All), MemberType = typeof(ExecutionModes))] public void AsyncGenerator_ForAwaitOf_InsideTryFinally_RunsFinally(ExecutionMode mode) { - // A `for await…of` inside a try in an async generator now suspends, so it must take the - // flag-based try path: its resume labels would be illegal BranchIntoTry targets on the real-IL - // try path (the async-gen analog of the #631 ContainsAwait pitfall). The finally still runs after - // the loop drains. Compiled-only (interp for-await-in-async-generator gap). + // A `for await…of` inside a try in an async generator suspends; the finally still runs after the + // loop drains. Compiled mode must take its flag-based try path (resume labels would be illegal + // BranchIntoTry targets on the real-IL try path — the async-gen analog of the #631 ContainsAwait + // pitfall). The interpreter now drives the same shape natively via its lazy coroutine (#717). var source = """ function later(n: number): Promise { return new Promise(res => setTimeout(() => res(n), 5)); diff --git a/SharpTS.Tests/SharedTests/GeneratorTests.cs b/SharpTS.Tests/SharedTests/GeneratorTests.cs index fd4e7ef8..bd0cec49 100644 --- a/SharpTS.Tests/SharedTests/GeneratorTests.cs +++ b/SharpTS.Tests/SharedTests/GeneratorTests.cs @@ -1785,6 +1785,101 @@ public void GeneratorExpression_BlockScopedCapture_CompiledRejectsClearly() #endregion + #region ASYNC generator function EXPRESSION closing over a BLOCK-scoped binding — issue #734 + + // The async analog of #678: an `async function*() {}` expression that closes over a block-scoped + // binding (loop variable, catch parameter, nested-block let/const) likewise cannot be lifted, so the + // GeneratorArrowLifter leaves it in place. EvaluateArrowFunction dispatches IsAsync && IsGenerator to + // a SharpTSAsyncArrowGeneratorFunction so the interpreter runs it natively (previously it was + // mishandled as a plain async function, leaving the capture out of scope — "Undefined variable"). + // Interpreted-only: the compiler has no generator-expression IL path and reports a clear + // "Yield not supported in this context" error (asserted by the *_CompiledRejectsClearly test). + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void AsyncGeneratorExpression_CapturesForOfLoopVariable(ExecutionMode mode) + { + // The exact repro from issue #734. + var source = """ + async function main() { + for (const n of [10]) { + const g = async function* () { yield n; }; + const out: number[] = []; + for await (const v of g()) out.push(v); + console.log(out.join(',')); + } + } + main(); + """; + + Assert.Equal("10\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void AsyncGeneratorExpression_CapturesNestedBlockLet(ExecutionMode mode) + { + // A let in a nested block captured by an in-place async generator expression, driven by direct + // next() (which the lazy coroutine resolves asynchronously). + var source = """ + async function main() { + { + let y = 5; + const g = async function* () { yield y; yield y + 1; }; + const it = g(); + console.log((await it.next()).value + "," + (await it.next()).value); + } + } + main(); + """; + + Assert.Equal("5,6\n", TestHarness.Run(source, mode)); + } + + [Theory] + [MemberData(nameof(ExecutionModes.InterpretedOnly), MemberType = typeof(ExecutionModes))] + public void AsyncGeneratorExpression_CapturesLoopVariablePerIteration(ExecutionMode mode) + { + // Each iteration's for-of binding is distinct, so an async generator created in one iteration + // captures that iteration's value — confirming the in-place generator closes over the + // per-iteration binding rather than a single shared slot. + var source = """ + async function main() { + let out = ""; + for (const n of [1, 2, 3]) { + const g = async function* () { yield n * 10; }; + for await (const v of g()) out += v + ","; + } + console.log(out); + } + main(); + """; + + Assert.Equal("10,20,30,\n", TestHarness.Run(source, mode)); + } + + [Fact] + public void AsyncGeneratorExpression_BlockScopedCapture_CompiledRejectsClearly() + { + // The compiler has no generator-expression IL path for an async generator that closes over a + // block-scoped binding; it must FAIL FAST with a clear message rather than crash (matching the + // sync #678 case). + var source = """ + async function main() { + for (const n of [1, 2]) { + const g = async function* () { yield n; }; + for await (const v of g()) console.log(v); + } + } + main(); + """; + + var ex = Assert.Throws(() => TestHarness.RunCompiled(source)); + Assert.Contains("Yield not supported", ex.Message); + } + + #endregion + #region Named generator function EXPRESSION self-reference — issue #679 // A NAMED generator function expression can call itself by its own name for recursion. The From 3564041618205d266f0874a9e33cdbd4d4edf9d6 Mon Sep 17 00:00:00 2001 From: Nick Nassiri Date: Tue, 16 Jun 2026 10:57:25 -0700 Subject: [PATCH 2/2] Fix broken build: EmitGeneratorFunctionDCInit param type (ParameterInfo[]) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit origin/main currently does not compile: the two instance/private-method call sites of EmitGeneratorFunctionDCInit (ILCompiler.AsyncGenerators.cs and ILCompiler.Generators.cs) pass methodBuilder.GetParameters() — a ParameterInfo[] — to a parameter declared as Type[]?. This is a latent mismatch from the interaction of the #720 (async/generator private methods, which deliberately box from the method's ACTUAL IL signature) and #724/#725 (function display-class seeding) changes after both landed. Accept ParameterInfo[]? and box value types via .ParameterType, leaving a private method's all-object slots unboxed. Behavior is unchanged; this only makes the tree compile. Unrelated to the async-generator coroutine rewrite in the preceding commit — committed separately so it is easy to drop if main is fixed independently. --- Compilation/ILCompiler.Generators.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Compilation/ILCompiler.Generators.cs b/Compilation/ILCompiler.Generators.cs index f3e94b71..0becce58 100644 --- a/Compilation/ILCompiler.Generators.cs +++ b/Compilation/ILCompiler.Generators.cs @@ -279,7 +279,7 @@ private void EmitGeneratorFunctionDCInit( Stmt.Function funcStmt, string qualifiedName, int paramOffset, - Type[]? paramTypes = null) + System.Reflection.ParameterInfo[]? paramTypes = null) { if (functionDCField == null || !_closures.FunctionDisplayClassCtors.TryGetValue(qualifiedName, out var dcCtor)) @@ -302,9 +302,10 @@ private void EmitGeneratorFunctionDCInit( il.Emit(OpCodes.Ldarg, i + paramOffset); // [sm, dc, arg] // The DC field is object-typed; box a value-type parameter. Free-function stubs pass // null here (their params are already object slots); instance-method stubs pass the - // resolved typed-parameter array so value types are boxed before the store (#724). - if (paramTypes != null && i < paramTypes.Length && paramTypes[i].IsValueType) - il.Emit(OpCodes.Box, paramTypes[i]); + // method's actual IL parameters (methodBuilder.GetParameters()) so value types are boxed + // before the store (#724) — and a private method's all-`object` slots are left unboxed. + if (paramTypes != null && i < paramTypes.Length && paramTypes[i].ParameterType.IsValueType) + il.Emit(OpCodes.Box, paramTypes[i].ParameterType); il.Emit(OpCodes.Stfld, dcField); // [sm] } }