Support async expressions

src/core/Microsoft.Dynamic/Ast/AsyncEnumerableExpression.cs

@@ -0,0 +1,124 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+#if NET
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Scripting.Utils;
+
+namespace Microsoft.Scripting.Ast {
+    /// <summary>
+    ///   Wraps an async-generator body (one that contains both <see cref="AwaitExpression"/> nodes and
+    ///   <see cref="YieldExpression"/> nodes targeting <see cref="YieldLabel"/>) into an expression that
+    ///   evaluates to <see cref="IAsyncEnumerable{T}"/> of <see cref="object"/>.
+    /// </summary>
+    /// <remarks>
+    ///   Await points are rewritten to <c>yield AwaitPoint(task)</c> against the <em>same</em> label as the
+    ///   language-level <c>yield</c>s, so a single <c>GeneratorRewriter</c>-produced
+    ///   <see cref="IEnumerator{T}"/> carries both kinds of items.
+    ///   <see cref="Microsoft.Scripting.Runtime.AsyncHelpers.DriveAsyncEnumerable"/> then awaits
+    ///   <see cref="Microsoft.Scripting.Runtime.AwaitPoint"/> items internally and emits the rest to the
+    ///   consumer. This marker is what lets <c>await</c> and <c>yield</c> coexist: a yielded Task is not an
+    ///   AwaitPoint, so it is surfaced as a value rather than awaited.
+    /// </remarks>
+    public sealed class AsyncEnumerableExpression : Expression {
+        private Expression? _reduced;
+
+        internal AsyncEnumerableExpression(string? name, Expression body, LabelTarget yieldLabel,
+                                           Expression? cancellationToken = null,
+                                           Expression? cancellationException = null) {
+            Name = name;
+            Body = body;
+            YieldLabel = yieldLabel;
+            CancellationToken = cancellationToken ?? Utils.DefaultCancellationToken;
+            CancellationException = cancellationException ?? Utils.DefaultCancellationException;
+        }
+
+        /// <summary>Optional diagnostic name (forwarded to the inner generator).</summary>
+        public string? Name { get; }
+
+        /// <summary>The generator body. May contain <see cref="AwaitExpression"/> and <see cref="YieldExpression"/> nodes.</summary>
+        public Expression Body { get; }
+
+        /// <summary>
+        ///   The label both the language-level <c>yield</c>s and the rewritten <c>await</c>s target, so they
+        ///   land in one generator. Supplied by the host (e.g. IronPython's shared generator label).
+        /// </summary>
+        public LabelTarget YieldLabel { get; }
+
+        /// <summary>Expression evaluating to the cancellation token (see <see cref="AsyncExpression"/>). Default <c>default(CancellationToken)</c>.</summary>
+        public Expression CancellationToken { get; }
+
+        /// <summary>Expression evaluating to a <c>StrongBox&lt;Exception?&gt;</c> exception override (or null). Default null.</summary>
+        public Expression CancellationException { get; }
+
+        public override bool CanReduce => true;
+
+        public override Type Type => typeof(IAsyncEnumerable<object?>);
+
+        public override ExpressionType NodeType => ExpressionType.Extension;
+
+        public override Expression Reduce() {
+            return _reduced ??= new AsyncEnumerableRewriter(this).Reduce();
+        }
+
+        protected override Expression VisitChildren(ExpressionVisitor visitor) {
+            Expression b = visitor.Visit(Body);
+            Expression ct = visitor.Visit(CancellationToken);
+            Expression ce = visitor.Visit(CancellationException);
+            if (b == Body && ct == CancellationToken && ce == CancellationException) return this;
+            return new AsyncEnumerableExpression(Name, b, YieldLabel, ct, ce);
+        }
+    }
+
+    public partial class Utils {
+        /// <summary>
+        ///   Wraps an async-generator body in an <see cref="AsyncEnumerableExpression"/> producing <c>IAsyncEnumerable&lt;object&gt;</c>.
+        /// </summary>
+        /// <param name="yieldLabel">
+        ///   It must be the same label the body's language-level <c>yield</c>s target.
+        /// </param>
+        public static AsyncEnumerableExpression AsyncEnumerable(string? name, Expression body, LabelTarget yieldLabel) {
+            ContractUtils.RequiresNotNull(body, nameof(body));
+            ContractUtils.RequiresNotNull(yieldLabel, nameof(yieldLabel));
+            return new AsyncEnumerableExpression(name, body, yieldLabel);
+        }
+
+        /// <summary>
+        ///   Wraps an async-generator body in an <see cref="AsyncEnumerableExpression"/> producing <c>IAsyncEnumerable&lt;object&gt;</c>,
+        ///   with a caller-provided <see cref="System.Threading.CancellationToken"/> and, optionally, an exception-override box.
+        /// </summary>
+        /// <remarks>
+        ///   When cancellation fires and the box's <c>Value</c> is non-null, that exception is delivered to
+        ///   the body instead of a fresh <see cref="System.OperationCanceledException"/>. This lets a host inject
+        ///   an arbitrary exception (e.g. Python's <c>coro.throw(exc)</c>) by populating the box and then
+        ///   cancelling the token. <paramref name="cancellationException"/> defaults to <c>null</c>
+        ///   — the plain OCE-on-cancellation behavior.
+        /// </remarks>
+        /// <param name="yieldLabel">
+        ///   It must be the same label the body's language-level <c>yield</c>s target.
+        /// </param>
+        public static AsyncEnumerableExpression AsyncEnumerable(string? name, Expression body, LabelTarget yieldLabel,
+                                                                Expression cancellationToken,
+                                                                Expression? cancellationException = null) {
+            ContractUtils.RequiresNotNull(body, nameof(body));
+            ContractUtils.RequiresNotNull(yieldLabel, nameof(yieldLabel));
+            ContractUtils.RequiresType(cancellationToken, typeof(CancellationToken), nameof(cancellationToken));
+            if (cancellationException is not null) {
+                ContractUtils.RequiresType(cancellationException, typeof(StrongBox<Exception?>), nameof(cancellationException));
+            }
+            return new AsyncEnumerableExpression(name, body, yieldLabel, cancellationToken, cancellationException);
+        }
+    }
+}
+
+#endif

src/core/Microsoft.Dynamic/Ast/AsyncEnumerableRewriter.cs

@@ -0,0 +1,134 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+#if NET
+
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.ExceptionServices;
+using System.Threading.Tasks;
+
+namespace Microsoft.Scripting.Ast {
+    /// <summary>
+    ///   Reduces an <see cref="AsyncEnumerableExpression"/> to an <c>IAsyncEnumerable&lt;object&gt;</c>-valued
+    ///   expression tree that yields each <see cref="AwaitExpression"/>'s operand (wrapped in an
+    ///   <see cref="Runtime.AwaitPoint"/>) alongside language-level yields, and hands
+    ///   the resulting state machine to <see cref="Runtime.AsyncHelpers.DriveAsyncEnumerable"/>.
+    /// </summary>
+    internal sealed class AsyncEnumerableRewriter {
+        private static readonly MethodInfo s_driveMethod
+            = typeof(Runtime.AsyncHelpers).GetMethod(nameof(Runtime.AsyncHelpers.DriveAsyncEnumerable))!;
+        private static readonly ConstructorInfo s_awaitPointCtor
+            = typeof(Runtime.AwaitPoint).GetConstructor([typeof(Task)])!;
+        private static readonly FieldInfo s_valueSlotField
+            = typeof(StrongBox<object?>).GetField(nameof(StrongBox<object?>.Value))!;
+        private static readonly FieldInfo s_exceptionSlotField
+            = typeof(StrongBox<Exception?>).GetField(nameof(StrongBox<Exception?>.Value))!;
+        private static readonly ConstructorInfo s_valueSlotCtor
+            = typeof(StrongBox<object?>).GetConstructor(Type.EmptyTypes)!;
+        private static readonly ConstructorInfo s_exceptionSlotCtor
+            = typeof(StrongBox<Exception?>).GetConstructor(Type.EmptyTypes)!;
+
+        private readonly AsyncEnumerableExpression _node;
+
+        public AsyncEnumerableRewriter(AsyncEnumerableExpression node) {
+            _node = node;
+        }
+
+        public Expression Reduce() {
+            // valueSlot / exceptionSlot carry the per-await result / fault back into the body at each
+            // await's resume point (same role as in AsyncRewriter). The generator's final value is
+            // irrelevant — generators don't return a value — so there is no capture step here.
+            ParameterExpression valueSlot = Expression.Variable(typeof(StrongBox<object?>), "$asyncValue");
+            ParameterExpression exceptionSlot = Expression.Variable(typeof(StrongBox<Exception?>), "$awaitException");
+
+            var rewriter = new AwaitToAwaitPointRewriter(_node.YieldLabel, valueSlot, exceptionSlot);
+            Expression rewrittenBody = rewriter.Visit(_node.Body);
+
+            // Coerce to void for Utils.Generator (the generator body's value is discarded).
+            Expression generatorBody = rewrittenBody.Type == typeof(void)
+                ? rewrittenBody
+                : Expression.Block(typeof(void), rewrittenBody);
+
+            Expression generator = Utils.Generator(
+                _node.Name ?? "$asyncgen",
+                _node.YieldLabel,
+                generatorBody,
+                typeof(IEnumerator<object>),
+                rewriteAssignments: false);
+
+            // Argument order matches DriveAsyncEnumerable: ..., cancellationToken, cancellationException
+            // (same as DriveAsync — cancellationToken is the last required parameter).
+            Expression drive = Expression.Call(
+                s_driveMethod,
+                generator,
+                valueSlot,
+                exceptionSlot,
+                _node.CancellationToken,
+                _node.CancellationException);
+
+            return Expression.Block(
+                typeof(IAsyncEnumerable<object?>),
+                [valueSlot, exceptionSlot],
+                Expression.Assign(valueSlot, Expression.New(s_valueSlotCtor)),
+                Expression.Assign(exceptionSlot, Expression.New(s_exceptionSlotCtor)),
+                drive);
+        }
+
+        /// <summary>
+        ///   Rewrites <c>AwaitExpression(task)</c> → <c>{ yield AwaitPoint(task); rethrow-if-pending; valueSlot.Value }</c>,
+        ///   targeting the shared yield label. Mirrors <c>AsyncRewriter.AwaitToYieldRewriter</c> but wraps the awaited
+        ///   Task in an <see cref="Microsoft.Scripting.Runtime.AwaitPoint"/> so the driver distinguishes it from a
+        ///   value yielded by a language-level <c>yield</c>.
+        /// </summary>
+        private sealed class AwaitToAwaitPointRewriter : ExpressionVisitor {
+            private static readonly MethodInfo s_captureMethod
+                = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Capture))!;
+            private static readonly MethodInfo s_throwMethod
+                = typeof(ExceptionDispatchInfo).GetMethod(nameof(ExceptionDispatchInfo.Throw), Type.EmptyTypes)!;
+
+            private readonly LabelTarget _yieldLabel;
+            private readonly ParameterExpression _valueSlot;
+            private readonly ParameterExpression _exceptionSlot;
+
+            public AwaitToAwaitPointRewriter(LabelTarget yieldLabel, ParameterExpression valueSlot, ParameterExpression exceptionSlot) {
+                _yieldLabel = yieldLabel;
+                _valueSlot = valueSlot;
+                _exceptionSlot = exceptionSlot;
+            }
+
+            protected override Expression VisitExtension(Expression node) {
+                if (node is AwaitExpression aw) {
+                    Expression operand = Visit(aw.Operand);
+                    // Wrap the awaited Task in an AwaitPoint marker, then box to object for the yield.
+                    Expression awaitPoint = Expression.New(s_awaitPointCtor, Expression.Convert(operand, typeof(Task)));
+                    Expression yielded = Expression.Convert(awaitPoint, typeof(object));
+
+                    Expression readException = Expression.Field(_exceptionSlot, s_exceptionSlotField);
+                    Expression readSlot = Expression.Field(_valueSlot, s_valueSlotField);
+
+                    Expression rethrow = Expression.IfThen(
+                        Expression.ReferenceNotEqual(readException, Expression.Constant(null, typeof(Exception))),
+                        Expression.Call(
+                            Expression.Call(s_captureMethod, readException),
+                            s_throwMethod));
+
+                    return Expression.Block(
+                        typeof(object),
+                        Utils.YieldReturn(_yieldLabel, yielded),
+                        rethrow,
+                        readSlot);
+                }
+                return base.VisitExtension(node);
+            }
+        }
+    }
+}
+
+#endif

src/core/Microsoft.Dynamic/Ast/AsyncExpression.cs

@@ -0,0 +1,132 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+#nullable enable
+
+using System;
+using System.Linq.Expressions;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+
+using Microsoft.Scripting.Runtime;
+using Microsoft.Scripting.Utils;
+
+namespace Microsoft.Scripting.Ast {
+    /// <summary>
+    ///   Wraps an async function body (possibly containing <see cref="AwaitExpression"/>
+    ///   nodes) into an expression that evaluates to <see cref="Task{TResult}"/> of
+    ///   <see cref="object"/>.
+    /// </summary>
+    /// <remarks>
+    ///   Unlike <see cref="LambdaExpression"/>, this is a
+    ///   sub-expression: it does not introduce a new lambda scope, so the body has
+    ///   direct access to parameters and locals of the enclosing scope. Callers
+    ///   typically wrap the resulting Task in their own coroutine façade type.
+    ///   <br/>
+    ///   The body may evaluate to any type, including <see langword="void"/>. A
+    ///   <see langword="void"/> body produces a <see langword="null"/> Task result;
+    ///   any other type is converted to <see cref="object"/> (value types are boxed)
+    ///   and that value becomes the Task's result.
+    ///   <br/>
+    ///   State-machine splitting at await sites is delegated to <see cref="GeneratorExpression"/>
+    ///   via <see cref="AsyncRewriter"/>; the <c>await</c> handling comes from
+    ///   <see cref="AsyncHelpers.DriveAsync"/>.
+    /// </remarks>
+    public sealed class AsyncExpression : Expression {
+        private Expression? _reduced;
+
+        internal AsyncExpression(string? name, Expression body,
+                                  Expression? cancellationToken = null,
+                                  Expression? cancellationException = null) {
+            Name = name;
+            Body = body;
+            CancellationToken = cancellationToken ?? Utils.DefaultCancellationToken;
+            CancellationException = cancellationException ?? Utils.DefaultCancellationException;
+        }
+
+        /// <summary>
+        ///   Optional diagnostic name (forwarded to the inner generator).
+        /// </summary>
+        public string? Name { get; }
+
+        /// <summary>
+        ///   The function body. May contain <see cref="AwaitExpression"/> nodes.
+        /// </summary>
+        public Expression Body { get; }
+
+        /// <summary>
+        ///   Expression evaluating to a <see cref="System.Threading.CancellationToken"/> that <see cref="AsyncHelpers.DriveAsync"/> samples
+        ///   between iterations and links to each suspended task. Defaults to <c>default(CancellationToken)</c>.
+        /// </summary>
+        public Expression CancellationToken { get; }
+
+        /// <summary>
+        ///   Expression evaluating to a <c>StrongBox&lt;Exception?&gt;</c> (null allowed) — see
+        ///   <see cref="AsyncHelpers.DriveAsync"/>'s <c>cancellationException</c> parameter. When the box is
+        ///   non-null and its <c>Value</c> is non-null at cancellation time, that exception is surfaced to
+        ///   the body instead of <see cref="System.OperationCanceledException"/>. Defaults to a null
+        ///   constant (the plain-cancellation behavior).
+        /// </summary>
+        public Expression CancellationException { get; }
+
+        public override bool CanReduce => true;
+
+        public override Type Type => typeof(Task<object?>);
+
+        public override ExpressionType NodeType => ExpressionType.Extension;
+
+        public override Expression Reduce() {
+            return _reduced ??= new AsyncRewriter(this).Reduce();
+        }
+
+        protected override Expression VisitChildren(ExpressionVisitor visitor) {
+            Expression b = visitor.Visit(Body);
+            Expression ct = visitor.Visit(CancellationToken);
+            Expression ce = visitor.Visit(CancellationException);
+            if (b == Body && ct == CancellationToken && ce == CancellationException) return this;
+            return new AsyncExpression(Name, b, ct, ce);
+        }
+    }
+
+    public partial class Utils {
+        /// <summary>
+        ///   Wraps an async-function body in an <see cref="AsyncExpression"/>.
+        /// </summary>
+        /// <remarks>
+        ///   The body may contain <see cref="AwaitExpression"/> suspension points and may evaluate to any
+        ///   type (including <see langword="void"/>): non-void values are converted to <see cref="object"/>
+        ///   — value types are boxed — and become the result of the resulting <c>Task&lt;object&gt;</c>; a
+        ///   void body produces a <see langword="null"/> result. Cancellation defaults to
+        ///   <c>default(CancellationToken)</c>.
+        /// </remarks>
+        public static AsyncExpression Async(string? name, Expression body) {
+            ContractUtils.RequiresNotNull(body, nameof(body));
+            return new AsyncExpression(name, body);
+        }
+
+        /// <summary>
+        ///   Wraps an async-function body in an <see cref="AsyncExpression"/> with a caller-provided
+        ///   <see cref="System.Threading.CancellationToken"/> and, optionally, an exception-override box.
+        /// </summary>
+        /// <remarks>
+        ///   When cancellation fires and the box's <c>Value</c> is non-null, that exception is delivered to
+        ///   the body instead of a fresh <see cref="System.OperationCanceledException"/>. This lets a host inject
+        ///   an arbitrary exception (e.g. Python's <c>coro.throw(exc)</c>) by populating the box and then
+        ///   cancelling the token. <paramref name="cancellationException"/> defaults to <c>null</c>
+        ///   — the plain OCE-on-cancellation behavior.
+        /// </remarks>
+        public static AsyncExpression Async(string? name, Expression body,
+                                            Expression cancellationToken,
+                                            Expression? cancellationException = null) {
+            ContractUtils.RequiresNotNull(body, nameof(body));
+            ContractUtils.RequiresNotNull(cancellationToken, nameof(cancellationToken));
+            ContractUtils.RequiresType(cancellationToken, typeof(CancellationToken), nameof(cancellationToken));
+            if (cancellationException is not null) {
+                ContractUtils.RequiresType(cancellationException, typeof(StrongBox<Exception?>), nameof(cancellationException));
+            }
+            return new AsyncExpression(name, body, cancellationToken, cancellationException);
+        }
+    }
+}