diff --git a/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableExpression.cs b/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableExpression.cs new file mode 100644 index 00000000..6c2377cf --- /dev/null +++ b/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableExpression.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +#if NET + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Scripting.Utils; + +namespace Microsoft.Scripting.Ast { + /// + /// Wraps an async-generator body (one that contains both nodes and + /// nodes targeting ) into an expression that + /// evaluates to of . + /// + /// + /// Await points are rewritten to yield AwaitPoint(task) against the same label as the + /// language-level yields, so a single GeneratorRewriter-produced + /// carries both kinds of items. + /// then awaits + /// items internally and emits the rest to the + /// consumer. This marker is what lets await and yield coexist: a yielded Task is not an + /// AwaitPoint, so it is surfaced as a value rather than awaited. + /// + public sealed class AsyncEnumerableExpression : Expression { + private Expression? _reduced; + + internal AsyncEnumerableExpression(string? name, Expression body, LabelTarget yieldLabel, + Expression? cancellationToken = null, + Expression? cancellationException = null) { + Name = name; + Body = body; + YieldLabel = yieldLabel; + CancellationToken = cancellationToken ?? Utils.DefaultCancellationToken; + CancellationException = cancellationException ?? Utils.DefaultCancellationException; + } + + /// Optional diagnostic name (forwarded to the inner generator). + public string? Name { get; } + + /// The generator body. May contain and nodes. + public Expression Body { get; } + + /// + /// The label both the language-level yields and the rewritten awaits target, so they + /// land in one generator. Supplied by the host (e.g. IronPython's shared generator label). + /// + public LabelTarget YieldLabel { get; } + + /// Expression evaluating to the cancellation token (see ). Default default(CancellationToken). + public Expression CancellationToken { get; } + + /// Expression evaluating to a StrongBox<Exception?> exception override (or null). Default null. + public Expression CancellationException { get; } + + public override bool CanReduce => true; + + public override Type Type => typeof(IAsyncEnumerable); + + public override ExpressionType NodeType => ExpressionType.Extension; + + public override Expression Reduce() { + return _reduced ??= new AsyncEnumerableRewriter(this).Reduce(); + } + + protected override Expression VisitChildren(ExpressionVisitor visitor) { + Expression b = visitor.Visit(Body); + Expression ct = visitor.Visit(CancellationToken); + Expression ce = visitor.Visit(CancellationException); + if (b == Body && ct == CancellationToken && ce == CancellationException) return this; + return new AsyncEnumerableExpression(Name, b, YieldLabel, ct, ce); + } + } + + public partial class Utils { + /// + /// Wraps an async-generator body in an producing IAsyncEnumerable<object>. + /// + /// + /// It must be the same label the body's language-level yields target. + /// + public static AsyncEnumerableExpression AsyncEnumerable(string? name, Expression body, LabelTarget yieldLabel) { + ContractUtils.RequiresNotNull(body, nameof(body)); + ContractUtils.RequiresNotNull(yieldLabel, nameof(yieldLabel)); + return new AsyncEnumerableExpression(name, body, yieldLabel); + } + + /// + /// Wraps an async-generator body in an producing IAsyncEnumerable<object>, + /// with a caller-provided and, optionally, an exception-override box. + /// + /// + /// When cancellation fires and the box's Value is non-null, that exception is delivered to + /// the body instead of a fresh . This lets a host inject + /// an arbitrary exception (e.g. Python's coro.throw(exc)) by populating the box and then + /// cancelling the token. defaults to null + /// — the plain OCE-on-cancellation behavior. + /// + /// + /// It must be the same label the body's language-level yields target. + /// + public static AsyncEnumerableExpression AsyncEnumerable(string? name, Expression body, LabelTarget yieldLabel, + Expression cancellationToken, + Expression? cancellationException = null) { + ContractUtils.RequiresNotNull(body, nameof(body)); + ContractUtils.RequiresNotNull(yieldLabel, nameof(yieldLabel)); + ContractUtils.RequiresType(cancellationToken, typeof(CancellationToken), nameof(cancellationToken)); + if (cancellationException is not null) { + ContractUtils.RequiresType(cancellationException, typeof(StrongBox), nameof(cancellationException)); + } + return new AsyncEnumerableExpression(name, body, yieldLabel, cancellationToken, cancellationException); + } + } +} + +#endif diff --git a/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableRewriter.cs b/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableRewriter.cs new file mode 100644 index 00000000..14a78e50 --- /dev/null +++ b/src/core/Microsoft.Dynamic/Ast/AsyncEnumerableRewriter.cs @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +#if NET + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; + +namespace Microsoft.Scripting.Ast { + /// + /// Reduces an to an IAsyncEnumerable<object>-valued + /// expression tree that yields each 's operand (wrapped in an + /// ) alongside language-level yields, and hands + /// the resulting state machine to . + /// + internal sealed class AsyncEnumerableRewriter { + private static readonly MethodInfo s_driveMethod + = typeof(Runtime.AsyncHelpers).GetMethod(nameof(Runtime.AsyncHelpers.DriveAsyncEnumerable))!; + private static readonly ConstructorInfo s_awaitPointCtor + = typeof(Runtime.AwaitPoint).GetConstructor([typeof(Task)])!; + private static readonly FieldInfo s_valueSlotField + = typeof(StrongBox).GetField(nameof(StrongBox.Value))!; + private static readonly FieldInfo s_exceptionSlotField + = typeof(StrongBox).GetField(nameof(StrongBox.Value))!; + private static readonly ConstructorInfo s_valueSlotCtor + = typeof(StrongBox).GetConstructor(Type.EmptyTypes)!; + private static readonly ConstructorInfo s_exceptionSlotCtor + = typeof(StrongBox).GetConstructor(Type.EmptyTypes)!; + + private readonly AsyncEnumerableExpression _node; + + public AsyncEnumerableRewriter(AsyncEnumerableExpression node) { + _node = node; + } + + public Expression Reduce() { + // valueSlot / exceptionSlot carry the per-await result / fault back into the body at each + // await's resume point (same role as in AsyncRewriter). The generator's final value is + // irrelevant — generators don't return a value — so there is no capture step here. + ParameterExpression valueSlot = Expression.Variable(typeof(StrongBox), "$asyncValue"); + ParameterExpression exceptionSlot = Expression.Variable(typeof(StrongBox), "$awaitException"); + + var rewriter = new AwaitToAwaitPointRewriter(_node.YieldLabel, valueSlot, exceptionSlot); + Expression rewrittenBody = rewriter.Visit(_node.Body); + + // Coerce to void for Utils.Generator (the generator body's value is discarded). + Expression generatorBody = rewrittenBody.Type == typeof(void) + ? rewrittenBody + : Expression.Block(typeof(void), rewrittenBody); + + Expression generator = Utils.Generator( + _node.Name ?? "$asyncgen", + _node.YieldLabel, + generatorBody, + typeof(IEnumerator), + rewriteAssignments: false); + + // Argument order matches DriveAsyncEnumerable: ..., cancellationToken, cancellationException + // (same as DriveAsync — cancellationToken is the last required parameter). + Expression drive = Expression.Call( + s_driveMethod, + generator, + valueSlot, + exceptionSlot, + _node.CancellationToken, + _node.CancellationException); + + return Expression.Block( + typeof(IAsyncEnumerable), + [valueSlot, exceptionSlot], + Expression.Assign(valueSlot, Expression.New(s_valueSlotCtor)), + Expression.Assign(exceptionSlot, Expression.New(s_exceptionSlotCtor)), + drive); + } + + /// + /// Rewrites AwaitExpression(task) → { yield AwaitPoint(task); rethrow-if-pending; valueSlot.Value }, + /// targeting the shared yield label. Mirrors AsyncRewriter.AwaitToYieldRewriter but wraps the awaited + /// Task in an so the driver distinguishes it from a + /// value yielded by a language-level yield. + /// + private sealed class AwaitToAwaitPointRewriter : ExpressionVisitor { + private static readonly MethodInfo s_captureMethod + = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Capture))!; + private static readonly MethodInfo s_throwMethod + = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Throw), Type.EmptyTypes)!; + + private readonly LabelTarget _yieldLabel; + private readonly ParameterExpression _valueSlot; + private readonly ParameterExpression _exceptionSlot; + + public AwaitToAwaitPointRewriter(LabelTarget yieldLabel, ParameterExpression valueSlot, ParameterExpression exceptionSlot) { + _yieldLabel = yieldLabel; + _valueSlot = valueSlot; + _exceptionSlot = exceptionSlot; + } + + protected override Expression VisitExtension(Expression node) { + if (node is AwaitExpression aw) { + Expression operand = Visit(aw.Operand); + // Wrap the awaited Task in an AwaitPoint marker, then box to object for the yield. + Expression awaitPoint = Expression.New(s_awaitPointCtor, Expression.Convert(operand, typeof(Task))); + Expression yielded = Expression.Convert(awaitPoint, typeof(object)); + + Expression readException = Expression.Field(_exceptionSlot, s_exceptionSlotField); + Expression readSlot = Expression.Field(_valueSlot, s_valueSlotField); + + Expression rethrow = Expression.IfThen( + Expression.ReferenceNotEqual(readException, Expression.Constant(null, typeof(Exception))), + Expression.Call( + Expression.Call(s_captureMethod, readException), + s_throwMethod)); + + return Expression.Block( + typeof(object), + Utils.YieldReturn(_yieldLabel, yielded), + rethrow, + readSlot); + } + return base.VisitExtension(node); + } + } + } +} + +#endif diff --git a/src/core/Microsoft.Dynamic/Ast/AsyncExpression.cs b/src/core/Microsoft.Dynamic/Ast/AsyncExpression.cs new file mode 100644 index 00000000..49edab19 --- /dev/null +++ b/src/core/Microsoft.Dynamic/Ast/AsyncExpression.cs @@ -0,0 +1,132 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.Linq.Expressions; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Scripting.Runtime; +using Microsoft.Scripting.Utils; + +namespace Microsoft.Scripting.Ast { + /// + /// Wraps an async function body (possibly containing + /// nodes) into an expression that evaluates to of + /// . + /// + /// + /// Unlike , this is a + /// sub-expression: it does not introduce a new lambda scope, so the body has + /// direct access to parameters and locals of the enclosing scope. Callers + /// typically wrap the resulting Task in their own coroutine façade type. + ///
+ /// The body may evaluate to any type, including . A + /// body produces a Task result; + /// any other type is converted to (value types are boxed) + /// and that value becomes the Task's result. + ///
+ /// State-machine splitting at await sites is delegated to + /// via ; the await handling comes from + /// . + /// + public sealed class AsyncExpression : Expression { + private Expression? _reduced; + + internal AsyncExpression(string? name, Expression body, + Expression? cancellationToken = null, + Expression? cancellationException = null) { + Name = name; + Body = body; + CancellationToken = cancellationToken ?? Utils.DefaultCancellationToken; + CancellationException = cancellationException ?? Utils.DefaultCancellationException; + } + + /// + /// Optional diagnostic name (forwarded to the inner generator). + /// + public string? Name { get; } + + /// + /// The function body. May contain nodes. + /// + public Expression Body { get; } + + /// + /// Expression evaluating to a that samples + /// between iterations and links to each suspended task. Defaults to default(CancellationToken). + /// + public Expression CancellationToken { get; } + + /// + /// Expression evaluating to a StrongBox<Exception?> (null allowed) — see + /// 's cancellationException parameter. When the box is + /// non-null and its Value is non-null at cancellation time, that exception is surfaced to + /// the body instead of . Defaults to a null + /// constant (the plain-cancellation behavior). + /// + public Expression CancellationException { get; } + + public override bool CanReduce => true; + + public override Type Type => typeof(Task); + + public override ExpressionType NodeType => ExpressionType.Extension; + + public override Expression Reduce() { + return _reduced ??= new AsyncRewriter(this).Reduce(); + } + + protected override Expression VisitChildren(ExpressionVisitor visitor) { + Expression b = visitor.Visit(Body); + Expression ct = visitor.Visit(CancellationToken); + Expression ce = visitor.Visit(CancellationException); + if (b == Body && ct == CancellationToken && ce == CancellationException) return this; + return new AsyncExpression(Name, b, ct, ce); + } + } + + public partial class Utils { + /// + /// Wraps an async-function body in an . + /// + /// + /// The body may contain suspension points and may evaluate to any + /// type (including ): non-void values are converted to + /// — value types are boxed — and become the result of the resulting Task<object>; a + /// void body produces a result. Cancellation defaults to + /// default(CancellationToken). + /// + public static AsyncExpression Async(string? name, Expression body) { + ContractUtils.RequiresNotNull(body, nameof(body)); + return new AsyncExpression(name, body); + } + + /// + /// Wraps an async-function body in an with a caller-provided + /// and, optionally, an exception-override box. + /// + /// + /// When cancellation fires and the box's Value is non-null, that exception is delivered to + /// the body instead of a fresh . This lets a host inject + /// an arbitrary exception (e.g. Python's coro.throw(exc)) by populating the box and then + /// cancelling the token. defaults to null + /// — the plain OCE-on-cancellation behavior. + /// + public static AsyncExpression Async(string? name, Expression body, + Expression cancellationToken, + Expression? cancellationException = null) { + ContractUtils.RequiresNotNull(body, nameof(body)); + ContractUtils.RequiresNotNull(cancellationToken, nameof(cancellationToken)); + ContractUtils.RequiresType(cancellationToken, typeof(CancellationToken), nameof(cancellationToken)); + if (cancellationException is not null) { + ContractUtils.RequiresType(cancellationException, typeof(StrongBox), nameof(cancellationException)); + } + return new AsyncExpression(name, body, cancellationToken, cancellationException); + } + } +} diff --git a/src/core/Microsoft.Dynamic/Ast/AsyncRewriter.cs b/src/core/Microsoft.Dynamic/Ast/AsyncRewriter.cs new file mode 100644 index 00000000..ee52b13f --- /dev/null +++ b/src/core/Microsoft.Dynamic/Ast/AsyncRewriter.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading.Tasks; + +namespace Microsoft.Scripting.Ast { + /// + /// Reduces an to a Task<object>-valued + /// expression tree that yields each 's operand and + /// hands the resulting state machine to . + /// + internal sealed class AsyncRewriter { + private static readonly MethodInfo s_driveMethod + = typeof(Runtime.AsyncHelpers).GetMethod(nameof(Runtime.AsyncHelpers.DriveAsync))!; + private static readonly FieldInfo s_valueSlotField + = typeof(StrongBox).GetField(nameof(StrongBox.Value))!; + private static readonly FieldInfo s_exceptionSlotField + = typeof(StrongBox).GetField(nameof(StrongBox.Value))!; + private static readonly ConstructorInfo s_valueSlotCtor + = typeof(StrongBox).GetConstructor(Type.EmptyTypes)!; + private static readonly ConstructorInfo s_exceptionSlotCtor + = typeof(StrongBox).GetConstructor(Type.EmptyTypes)!; + + private readonly AsyncExpression _node; + + public AsyncRewriter(AsyncExpression node) { + _node = node; + } + + public Expression Reduce() { + // valueSlot is value cell shared with AsyncHelpers.DriveAsync. + // - At each await: the runner writes the awaited result here just before resuming the body, and the body reads it via the + // `readSlot` expression the rewriter inserts after each yield. + // - At the end of the body: the body's final return value is written here (see captureFinalValue below). After MoveNext() + // returns false, DriveAsync reads the same slot and returns it as the Task's result. + // + // The two uses do not overlap — by the time captureFinalValue runs, the last per-await read has already been consumed or + // discarded by the surrounding expression. + ParameterExpression valueSlot = Expression.Variable(typeof(StrongBox), "$asyncValue"); + + // exceptionSlot is where the runner stashes a faulted-await exception so the state machine can rethrow at the resumed position + // (lets a Python try/except around `await` observe e.g. StopAsyncIteration). + ParameterExpression exceptionSlot = Expression.Variable(typeof(StrongBox), "$awaitException"); + LabelTarget yieldLabel = Expression.Label(typeof(object), "$asyncYield"); + + // Rewrite AwaitExpression(e) -> { yield e; rethrow-if-pending; valueSlot.Value } + var rewriter = new AwaitToYieldRewriter(yieldLabel, valueSlot, exceptionSlot); + Expression rewrittenBody = rewriter.Visit(_node.Body); + + // After the body completes, the function's final value must live in valueSlot for DriveAsync to pick up. For a value-typed body this + // is a single assignment of the body expression into the slot. For a void body, the body has no value to assign, but we still + // must clear the slot — otherwise DriveAsync would return whatever the last await happened to stash there. (IronPython doesn't emit + // void async bodies today, but AsyncExpression is language-agnostic.) + Expression valueField = Expression.Field(valueSlot, s_valueSlotField); + Expression captureFinalValue; + if (_node.Body.Type == typeof(void)) { + captureFinalValue = Expression.Block( + typeof(void), + rewrittenBody, + Expression.Assign(valueField, Expression.Constant(null, typeof(object)))); + } else { + Expression asObject = rewrittenBody.Type == typeof(object) + ? rewrittenBody + : Expression.Convert(rewrittenBody, typeof(object)); + captureFinalValue = Expression.Assign(valueField, asObject); + } + Expression generatorBody = Expression.Block(typeof(void), captureFinalValue); + + Expression generator = Utils.Generator( + _node.Name ?? "$async", + yieldLabel, + generatorBody, + typeof(IEnumerator), + rewriteAssignments: false); + + Expression drive = Expression.Call( + s_driveMethod, + generator, + valueSlot, + exceptionSlot, + _node.CancellationToken, + _node.CancellationException); + + return Expression.Block( + typeof(Task), + [valueSlot, exceptionSlot], + Expression.Assign(valueSlot, Expression.New(s_valueSlotCtor)), + Expression.Assign(exceptionSlot, Expression.New(s_exceptionSlotCtor)), + drive); + } + + private sealed class AwaitToYieldRewriter : ExpressionVisitor { + private static readonly MethodInfo s_captureMethod + = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Capture))!; + private static readonly MethodInfo s_throwMethod + = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Throw), Type.EmptyTypes)!; + + private readonly LabelTarget _yieldLabel; + private readonly ParameterExpression _resultSlot; + private readonly ParameterExpression _exceptionSlot; + + public AwaitToYieldRewriter(LabelTarget yieldLabel, ParameterExpression resultSlot, ParameterExpression exceptionSlot) { + _yieldLabel = yieldLabel; + _resultSlot = resultSlot; + _exceptionSlot = exceptionSlot; + } + + protected override Expression VisitExtension(Expression node) { + if (node is AwaitExpression aw) { + Expression operand = Visit(aw.Operand); + Expression boxed = operand.Type == typeof(object) ? operand : Expression.Convert(operand, typeof(object)); + Expression readException = Expression.Field(_exceptionSlot, s_exceptionSlotField); + Expression readSlot = Expression.Field(_resultSlot, s_valueSlotField); + + // After the yield, if the runner stored an exception, rethrow it preserving the original stack trace + // (so the body's try/except observes the right exception object). + // Otherwise return the awaited result. + Expression rethrow = Expression.IfThen( + Expression.ReferenceNotEqual(readException, Expression.Constant(null, typeof(Exception))), + Expression.Call( + Expression.Call(s_captureMethod, readException), + s_throwMethod)); + + return Expression.Block( + typeof(object), + Utils.YieldReturn(_yieldLabel, boxed), + rethrow, + readSlot); + } + return base.VisitExtension(node); + } + } + } +} diff --git a/src/core/Microsoft.Dynamic/Ast/AwaitExpression.cs b/src/core/Microsoft.Dynamic/Ast/AwaitExpression.cs new file mode 100644 index 00000000..a40fd2cb --- /dev/null +++ b/src/core/Microsoft.Dynamic/Ast/AwaitExpression.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.Linq.Expressions; + +using Microsoft.Scripting.Utils; + +namespace Microsoft.Scripting.Ast { + /// + /// A suspension point inside an . The + /// is expected to evaluate to a + /// or . + /// + /// + /// The drives + /// the lambda by calling its GeneratorNext form; each await yields the + /// task to the runner, which performs a real runtime-async await and + /// resumes the body with the boxed result. + ///
+ /// Standalone reduction is not supported - + /// rewrites these nodes into yield+resume pairs before the body is lowered. + /// + public sealed class AwaitExpression : Expression { + internal AwaitExpression(Expression operand) { + Operand = operand; + } + + /// The awaitable being awaited. Must produce a Task or Task<T>. + public Expression Operand { get; } + + public override bool CanReduce => false; + + public override Type Type => typeof(object); + + public override ExpressionType NodeType => ExpressionType.Extension; + + protected override Expression VisitChildren(ExpressionVisitor visitor) { + Expression o = visitor.Visit(Operand); + if (o == Operand) return this; + return new AwaitExpression(o); + } + } + + public partial class Utils { + /// Wraps in an . + public static AwaitExpression Await(Expression awaitable) { + ContractUtils.RequiresNotNull(awaitable, nameof(awaitable)); + return new AwaitExpression(awaitable); + } + } +} diff --git a/src/core/Microsoft.Dynamic/Ast/Utils.cs b/src/core/Microsoft.Dynamic/Ast/Utils.cs index 76c3b04f..674b6e2f 100644 --- a/src/core/Microsoft.Dynamic/Ast/Utils.cs +++ b/src/core/Microsoft.Dynamic/Ast/Utils.cs @@ -2,11 +2,13 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using System.Linq.Expressions; using System; -using System.Reflection; using System.Dynamic; +using System.Linq.Expressions; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Threading; using Microsoft.Scripting.Utils; using AstUtils = Microsoft.Scripting.Ast.Utils; @@ -92,5 +94,11 @@ public static bool IsLValue(this ExpressionType type) { return false; } + + internal static Expression DefaultCancellationException + => field ??= Expression.Constant(null, typeof(StrongBox)); + + internal static Expression DefaultCancellationToken + => field ??= Expression.Default(typeof(CancellationToken)); } } diff --git a/src/core/Microsoft.Dynamic/Microsoft.Dynamic.csproj b/src/core/Microsoft.Dynamic/Microsoft.Dynamic.csproj index 7ad7b3e5..7acb50d9 100644 --- a/src/core/Microsoft.Dynamic/Microsoft.Dynamic.csproj +++ b/src/core/Microsoft.Dynamic/Microsoft.Dynamic.csproj @@ -14,6 +14,8 @@ M:System.Reflection.MethodInfo.CreateDelegate``1; M:System.String.IndexOf(System.Char,System.StringComparison); M:System.String.StartsWith(System.Char); + M:System.Threading.Tasks.Task.WaitAsync(System.Threading.CancellationToken); + M:System.Threading.Tasks.Task`1.WaitAsync(System.Threading.CancellationToken); diff --git a/src/core/Microsoft.Dynamic/Runtime/AsyncHelpers.cs b/src/core/Microsoft.Dynamic/Runtime/AsyncHelpers.cs new file mode 100644 index 00000000..b084828d --- /dev/null +++ b/src/core/Microsoft.Dynamic/Runtime/AsyncHelpers.cs @@ -0,0 +1,218 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Scripting.Runtime { + /// + /// Runtime-async orchestrator for . + /// + public static class AsyncHelpers { + /// + /// Drives a language function body, awaiting each yielded and feeding the result back through . Faulted awaits are routed through so the body's try/except can observe + /// them at the resumption point — this is how async for's StopAsyncIteration catch works. + /// + /// The body's state machine — an enumerator of yielded awaitables. + /// + /// While the body is running, this is where the driver writes each awaited result before resuming the body (read by the body at + /// the resume point of each await). After the body completes, this is where the body has stored its final return value, + /// which becomes the returned Task's result. The two uses do not overlap: the body has consumed (or discarded) any per-await + /// value before its final assignment runs. + /// + /// + /// Faulted/cancelled await is parked here so the body's rewriter can rethrow it at the resume point via . + /// + /// + /// Caller's cancellation token. Sampled at each loop iteration and linked to each + /// awaited task; see remarks for the cancellation model. + /// + /// + /// Optional override for the exception surfaced when + /// fires. If non-null and its Value is non-null at the moment cancellation is + /// observed, that exception is delivered into instead + /// of a fresh . It lets a host inject an arbitrary + /// exception (e.g. for Python's coro.throw(exc)) by setting the box's value and + /// then cancelling the token. Default null preserves the plain-cancellation behavior. + /// + /// + /// This method is itself an async Task whose IL is produced by Roslyn (on .NET 11+, under feature + /// runtime-async=on), so each await below becomes a real .NET 11+ runtime-async + /// opcode. The state machine of the language function body is delegated to a GeneratorRewriter-produced of yielded tasks. On older runtimes, the same method is compiled by Roslyn without runtime-async + /// support, so it compiles to a Roslyn-generated async state machine. + /// + /// Continuation scheduling is determined by the caller's / + /// — the awaits here do not call ConfigureAwait(false), so a host that installs a single-threaded context (e.g. an + /// asyncio-style event loop) pins every resumption to its loop thread. + /// + /// Cancellation is cooperative and follows asyncio's model: is sampled at each loop + /// iteration (so a stretch of synchronously-resolved awaits is still cancellable) and is linked to each suspended task so a + /// cancellation request unblocks an in-flight await even when the awaited task does not honor the token itself. In either case + /// the resulting is routed through so it surfaces at + /// the body's next resume point — exactly the place a body-level try/except around the await expects to observe it. + /// If the body lets the exception propagate, it bubbles out of and the returned Task transitions to because the OCE's token matches. + /// + /// This method is not obsolete but is not part of the public API surface; do not call it directly from source-level code. + /// It is made public only for internal use by the DLR. + /// + [Obsolete("do not call this method directly from source-level code", error: true)] + [EditorBrowsable(EditorBrowsableState.Never)] + public static async Task DriveAsync(IEnumerator states, + StrongBox valueSlot, + StrongBox exceptionSlot, + CancellationToken cancellationToken , + StrongBox? cancellationException = null) { + + while (states.MoveNext()) { + object? item = states.Current; + + // Surface cancellation at the just-yielded suspension point so the body's try/except around + // `await` can observe it (matches asyncio's "CancelledError raised at the await" model). + // Caught here as well as in the WaitAsync branch so that a stretch of synchronously-resolved + // yields is still cancellable. If the host pre-populated cancellationException, deliver that + // instead of a fresh OCE — lets coro.throw(arbitrary) inject any exception type. + if (cancellationToken.IsCancellationRequested) { + valueSlot.Value = null; + exceptionSlot.Value = cancellationException?.Value + ?? new OperationCanceledException(cancellationToken); + continue; + } + + if (item is Task task) { + try { + // Task.WaitAsync is BCL on net6+, polyfilled by Meziantou.Polyfill on net4x/netstandard2.0. + // No ConfigureAwait(false): honor caller's SyncContext / TaskScheduler. + await task.WaitAsync(cancellationToken); + valueSlot.Value = ExtractTaskResult(task); + exceptionSlot.Value = null; + } catch (Exception ex) { + // The body's rewriter inserts a check after each yield that rethrows this on the resumed thread, + // so a try/except inside `await` can catch it (including the cancellation OCE from WaitAsync). + valueSlot.Value = null; + exceptionSlot.Value = ex; + } + } else { + // Synchronously-produced value forwards straight through. + valueSlot.Value = item; + exceptionSlot.Value = null; + } + } + // Body has completed: its final assignment has just written into valueSlot + // (see AsyncExpression.BuildReduction), so the per-await role of the slot + // is over and the same slot now carries the function's return value. + return valueSlot.Value; + } + + +#if NET + /// + /// Drives a language async-generator body, producing an of object. + /// The async-enumerable version of . + /// + /// + /// The body's state machine (an from GeneratorRewriter) yields two + /// kinds of items, discriminated by type: + /// + /// — an internal suspension point. The driver awaits its Task and feeds + /// the result back through (faults/cancellation through + /// ), exactly like . Not emitted to the consumer. + /// + /// anything else — a value produced by a language-level yield; emitted to the consumer via + /// yield return. A yielded Task is NOT an , so it is surfaced as a value, + /// never awaited — this is the disambiguation that lets await and yield coexist. + /// + /// + /// This method is not obsolete but is not part of the public API surface; do not call it directly from source-level code. + /// It is made public only for internal use by the DLR. + /// + [Obsolete("do not call this method directly from source-level code", error: true)] + [EditorBrowsable(EditorBrowsableState.Never)] + public static async IAsyncEnumerable DriveAsyncEnumerable(IEnumerator states, + StrongBox valueSlot, + StrongBox exceptionSlot, + [EnumeratorCancellation] CancellationToken cancellationToken, + StrongBox? cancellationException = null) { + + while (states.MoveNext()) { + object? item = states.Current; + + if (cancellationToken.IsCancellationRequested) { + valueSlot.Value = null; + exceptionSlot.Value = cancellationException?.Value + ?? new OperationCanceledException(cancellationToken); + continue; + } + + if (item is AwaitPoint awaitPoint) { + // Suspension point — await internally, do not surface to the consumer. + try { + await awaitPoint.Task.WaitAsync(cancellationToken); + valueSlot.Value = ExtractTaskResult(awaitPoint.Task); + exceptionSlot.Value = null; + } catch (Exception ex) { + valueSlot.Value = null; + exceptionSlot.Value = ex; + } + } else { + // Value from a language-level `yield` — emit it. + yield return item; + + // No value flows back in for a plain `async for`; + // Clear the slots so the body's resume of the yield expression observes null. + valueSlot.Value = null; + exceptionSlot.Value = null; + } + } + } +#endif + + + private static object? ExtractTaskResult(Task task) { + // Fast-track for the most common case (e.g. IronPython): all awaitables are normalized to Task. + if (task is Task to) return to.Result; + + Type t = task.GetType(); + + // Non-generic Task subclass: no Result property exists. Covers Task.CompletedTask, Task.Delay's DelayPromise, + // Task.WhenAll's non-generic overload, etc. IsGenericType is a flag check on Type — far cheaper than GetProperty. + if (!t.IsGenericType) return null; + + // The runtime type may be a Task subclass (e.g. AsyncStateMachineBox, or RuntimeAsyncTask); + // so find Task.Result through inheritance hierarchy. + // This may be incorrect in the unlikely (and bad) case if the subclass shadows Result (e.g. new T2 Result {...} - not in BCL/CLR) + var prop = t.GetProperty("Result"); + + // Non-generic Task subclass that still wasn't caught above (defensive — shouldn't happen given IsGenericType). + if (prop is null) return null; + + // Roslyn-emitted or CLR async Task uses an internal VoidTaskResult type argument; surface that as null. + if (!prop.PropertyType.IsVisible) return null; + + // Task.Result, may be null, which is OK (and is null if task has not completed yet; not happening here) + return prop.GetValue(task); + } + } + + + /// + /// Marker wrapping a to flag an await-suspension point inside an async-generator body, + /// distinguishing it from a value produced by a language-level yield. Emitted by + /// 's await rewrite and consumed by + /// . + /// + internal sealed class AwaitPoint(Task task) { + public Task Task { get; } = task; + } +} diff --git a/src/core/Microsoft.Dynamic/Utils/ContractUtils.cs b/src/core/Microsoft.Dynamic/Utils/ContractUtils.cs index 3a409440..f15f2f82 100644 --- a/src/core/Microsoft.Dynamic/Utils/ContractUtils.cs +++ b/src/core/Microsoft.Dynamic/Utils/ContractUtils.cs @@ -182,6 +182,19 @@ public static void RequiresNotNullItems(IEnumerable collection, string col } } + + /// + /// Requires the expression to evaluate to the expected type. + /// + internal static void RequiresType(Expression expr, Type expected, string paramName) { + if (expr.Type != expected) { + throw new ArgumentException( + $"Expression must evaluate to {expected.Name}, got {expr.Type}.", + paramName); + } + } + + [Conditional("FALSE")] public static void Invariant(bool condition) { Debug.Assert(condition);