Add opt-in Helix work item batching

Documentation/AzureDevOps/SendingJobsToHelix.md

@@ -127,6 +127,33 @@ XUnit v3 test projects are self-hosting executables and do not need an external
 
 For anything more complex than the above example, you'll want to create your own MSBuild proj file to specify the work items and correlation payloads you want to send up to Helix. Full documentation on how to do this can be found [in the SDK's readme](../../src/Microsoft.DotNet.Helix/Sdk/Readme.md).
 
+## Batching short work items
+
+If a job sends many short xUnit projects or `PayloadDirectory` work items, the Helix SDK can batch compatible work items to reduce per-item setup/download/upload overhead:
+
+```yaml
+- template: /eng/common/templates/steps/send-to-helix.yml
+  parameters:
+    EnableHelixWorkItemBatching: true
+    HelixBatchTargetDuration: 10           # minutes
+    HelixBatchTimeoutPadding: 2            # minutes
+    HelixBatchMaxItems: 10
+    HelixBatchMinItems: 2
+```
+
+For custom Helix project files, set the same properties directly:
+
+```xml
+<PropertyGroup>
+  <EnableHelixWorkItemBatching>true</EnableHelixWorkItemBatching>
+  <HelixBatchTargetDuration>10</HelixBatchTargetDuration> <!-- minutes -->
+  <HelixBatchTimeoutPadding>2</HelixBatchTimeoutPadding> <!-- minutes -->
+  <HelixBatchMaxItems>10</HelixBatchMaxItems>
+</PropertyGroup>
+```
+
+Batching is opt-in and conservative. It currently batches simple `PayloadDirectory` work items, preserves unbatchable items unchanged, and writes each member's logs/results to a separate upload subdirectory. Use `HelixBatchable=false` on tests that require isolation.
+
 ## Viewing test results
 
 All test results will be downloaded to the Azure DevOps build and viewable through the **Tests** tab.

eng/common/core-templates/steps/send-to-helix.yml

@@ -19,6 +19,11 @@ parameters:
   XUnitPublishTargetFramework: ''        # optional -- framework to use to publish your xUnit projects
   XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
   XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
+  EnableHelixWorkItemBatching: false     # optional -- batch compatible short Helix work items to reduce per-item overhead
+  HelixBatchTargetDuration: 10           # optional -- target aggregate duration per batch, in minutes
+  HelixBatchTimeoutPadding: 2            # optional -- additional time added to each batch's aggregate timeout, in minutes
+  HelixBatchMaxItems: 10                 # optional -- maximum original work items per batch
+  HelixBatchMinItems: 2                  # optional -- minimum original work items required to form a batch
   IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
   DotNetCliPackageType: ''               # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
   DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
@@ -52,6 +57,11 @@ steps:
       XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
       XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      EnableHelixWorkItemBatching: ${{ parameters.EnableHelixWorkItemBatching }}
+      HelixBatchTargetDuration: ${{ parameters.HelixBatchTargetDuration }}
+      HelixBatchTimeoutPadding: ${{ parameters.HelixBatchTimeoutPadding }}
+      HelixBatchMaxItems: ${{ parameters.HelixBatchMaxItems }}
+      HelixBatchMinItems: ${{ parameters.HelixBatchMinItems }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
       DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
@@ -82,6 +92,11 @@ steps:
       XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }}
       XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }}
       XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }}
+      EnableHelixWorkItemBatching: ${{ parameters.EnableHelixWorkItemBatching }}
+      HelixBatchTargetDuration: ${{ parameters.HelixBatchTargetDuration }}
+      HelixBatchTimeoutPadding: ${{ parameters.HelixBatchTimeoutPadding }}
+      HelixBatchMaxItems: ${{ parameters.HelixBatchMaxItems }}
+      HelixBatchMinItems: ${{ parameters.HelixBatchMinItems }}
       IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
       DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
       DotNetCliVersion: ${{ parameters.DotNetCliVersion }}

src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/BatchHelixWorkItemsTests.cs

@@ -0,0 +1,166 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using AwesomeAssertions;
+using Microsoft.Arcade.Test.Common;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Xunit;
+
+namespace Microsoft.DotNet.Helix.Sdk.Tests
+{
+    public class BatchHelixWorkItemsTests : IDisposable
+    {
+        private readonly string _root;
+
+        public BatchHelixWorkItemsTests()
+        {
+            _root = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(_root);
+        }
+
+        public void Dispose()
+        {
+            if (Directory.Exists(_root))
+            {
+                Directory.Delete(_root, recursive: true);
+            }
+        }
+
+        [Fact]
+        public void PayloadDirectoryWorkItemsAreBatched()
+        {
+            ITaskItem first = CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00");
+            ITaskItem second = CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00");
+
+            var task = CreateTask(first, second);
+
+            task.Execute().Should().BeTrue();
+
+            task.BatchedWorkItems.Should().ContainSingle();
+            ITaskItem batch = task.BatchedWorkItems.Single();
+            batch.ItemSpec.Should().StartWith("Batch_0001_First_Tests_dll");
+            batch.GetMetadata("Command").Should().Be("run-batch.cmd");
+            batch.GetMetadata("Timeout").Should().Be("00:04:00");
+            batch.GetMetadata("BatchedWorkItemNames").Should().Be("First.Tests.dll;Second.Tests.dll");
+
+            string batchPayload = batch.GetMetadata("PayloadDirectory");
+            File.Exists(Path.Combine(batchPayload, "run-batch.cmd")).Should().BeTrue();
+            File.Exists(Path.Combine(batchPayload, "batch-manifest.json")).Should().BeTrue();
+            Directory.GetDirectories(Path.Combine(batchPayload, "payloads")).Should().HaveCount(2);
+            File.ReadAllText(Path.Combine(batchPayload, "batch-manifest.json")).Should().Contain("First.Tests.dll");
+            File.ReadAllText(Path.Combine(batchPayload, "run-batch.cmd")).Should().Contain("console.log");
+        }
+
+        [Fact]
+        public void SingleEligibleWorkItemPassesThrough()
+        {
+            ITaskItem item = CreateWorkItem("Only.Tests.dll", "dotnet exec Only.Tests.dll", "00:01:00");
+
+            var task = CreateTask(item);
+
+            task.Execute().Should().BeTrue();
+
+            task.BatchedWorkItems.Should().ContainSingle().Which.Should().BeSameAs(item);
+        }
+
+        [Fact]
+        public void UnbatchableItemsArePreserved()
+        {
+            ITaskItem first = CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00");
+            ITaskItem skipped = CreateWorkItem("Skipped.Tests.dll", "dotnet exec Skipped.Tests.dll", "00:01:00");
+            skipped.SetMetadata("HelixBatchable", "false");
+            ITaskItem second = CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00");
+
+            var task = CreateTask(first, skipped, second);
+
+            task.Execute().Should().BeTrue();
+
+            task.BatchedWorkItems.Should().HaveCount(3);
+            task.BatchedWorkItems[0].Should().BeSameAs(first);
+            task.BatchedWorkItems[1].Should().BeSameAs(skipped);
+            task.BatchedWorkItems[2].Should().BeSameAs(second);
+        }
+
+        [Fact]
+        public void MaxItemsSplitsBatches()
+        {
+            var task = CreateTask(
+                CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"),
+                CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00"),
+                CreateWorkItem("Third.Tests.dll", "dotnet exec Third.Tests.dll", "00:01:00"),
+                CreateWorkItem("Fourth.Tests.dll", "dotnet exec Fourth.Tests.dll", "00:01:00"));
+            task.MaxItemsPerBatch = 2;
+
+            task.Execute().Should().BeTrue();
+
+            task.BatchedWorkItems.Should().HaveCount(2);
+            task.BatchedWorkItems.All(i => i.ItemSpec.StartsWith("Batch_")).Should().BeTrue();
+        }
+
+        [Fact]
+        public void PosixBatchUsesShellRunner()
+        {
+            var task = CreateTask(
+                CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"),
+                CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00"));
+            task.IsPosixShell = true;
+
+            task.Execute().Should().BeTrue();
+
+            ITaskItem batch = task.BatchedWorkItems.Single();
+            batch.GetMetadata("Command").Should().Be("./run-batch.sh");
+            File.ReadAllText(Path.Combine(batch.GetMetadata("PayloadDirectory"), "run-batch.sh")).Should().Contain("/bin/sh ./run-member.sh");
+        }
+
+        [Fact]
+        public void ManifestContainsMemberMetadata()
+        {
+            var task = CreateTask(
+                CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"),
+                CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:02:00"));
+
+            task.Execute().Should().BeTrue();
+
+            string manifestPath = Path.Combine(task.BatchedWorkItems.Single().GetMetadata("PayloadDirectory"), "batch-manifest.json");
+            using JsonDocument document = JsonDocument.Parse(File.ReadAllText(manifestPath));
+            JsonElement manifest = document.RootElement;
+
+            manifest.GetProperty("version").GetInt32().Should().Be(1);
+            manifest.GetProperty("workItems").GetArrayLength().Should().Be(2);
+            manifest.GetProperty("workItems")[1].GetProperty("timeout").GetString().Should().Be("00:02:00");
+        }
+
+        private BatchHelixWorkItems CreateTask(params ITaskItem[] workItems)
+        {
+            return new BatchHelixWorkItems
+            {
+                BuildEngine = new MockBuildEngine(),
+                WorkItems = workItems,
+                IntermediateOutputPath = Path.Combine(_root, "obj"),
+                IsPosixShell = false,
+                TargetDuration = 10,
+                TimeoutPadding = 2,
+                MaxItemsPerBatch = 10,
+                MinItemsPerBatch = 2
+            };
+        }
+
+        private ITaskItem CreateWorkItem(string name, string command, string timeout)
+        {
+            string payloadDirectory = Path.Combine(_root, "payloads", name);
+            Directory.CreateDirectory(payloadDirectory);
+            File.WriteAllText(Path.Combine(payloadDirectory, name), "payload");
+
+            var result = new TaskItem(name);
+            result.SetMetadata("PayloadDirectory", payloadDirectory);
+            result.SetMetadata("Command", command);
+            result.SetMetadata("Timeout", timeout);
+            return result;
+        }
+    }
+}