diff --git a/Documentation/AzureDevOps/SendingJobsToHelix.md b/Documentation/AzureDevOps/SendingJobsToHelix.md index d4fc054d308..b4fd3594b6c 100644 --- a/Documentation/AzureDevOps/SendingJobsToHelix.md +++ b/Documentation/AzureDevOps/SendingJobsToHelix.md @@ -127,6 +127,33 @@ XUnit v3 test projects are self-hosting executables and do not need an external For anything more complex than the above example, you'll want to create your own MSBuild proj file to specify the work items and correlation payloads you want to send up to Helix. Full documentation on how to do this can be found [in the SDK's readme](../../src/Microsoft.DotNet.Helix/Sdk/Readme.md). +## Batching short work items + +If a job sends many short xUnit projects or `PayloadDirectory` work items, the Helix SDK can batch compatible work items to reduce per-item setup/download/upload overhead: + +```yaml +- template: /eng/common/templates/steps/send-to-helix.yml + parameters: + EnableHelixWorkItemBatching: true + HelixBatchTargetDuration: 10 # minutes + HelixBatchTimeoutPadding: 2 # minutes + HelixBatchMaxItems: 10 + HelixBatchMinItems: 2 +``` + +For custom Helix project files, set the same properties directly: + +```xml + + true + 10 + 2 + 10 + +``` + +Batching is opt-in and conservative. It currently batches simple `PayloadDirectory` work items, preserves unbatchable items unchanged, and writes each member's logs/results to a separate upload subdirectory. Use `HelixBatchable=false` on tests that require isolation. + ## Viewing test results All test results will be downloaded to the Azure DevOps build and viewable through the **Tests** tab. diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml index 68fa739c4ab..168e6380344 100644 --- a/eng/common/core-templates/steps/send-to-helix.yml +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -19,6 +19,11 @@ parameters: XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects + EnableHelixWorkItemBatching: false # optional -- batch compatible short Helix work items to reduce per-item overhead + HelixBatchTargetDuration: 10 # optional -- target aggregate duration per batch, in minutes + HelixBatchTimeoutPadding: 2 # optional -- additional time added to each batch's aggregate timeout, in minutes + HelixBatchMaxItems: 10 # optional -- maximum original work items per batch + HelixBatchMinItems: 2 # optional -- minimum original work items required to form a batch IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json @@ -52,6 +57,11 @@ steps: XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + EnableHelixWorkItemBatching: ${{ parameters.EnableHelixWorkItemBatching }} + HelixBatchTargetDuration: ${{ parameters.HelixBatchTargetDuration }} + HelixBatchTimeoutPadding: ${{ parameters.HelixBatchTimeoutPadding }} + HelixBatchMaxItems: ${{ parameters.HelixBatchMaxItems }} + HelixBatchMinItems: ${{ parameters.HelixBatchMinItems }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }} @@ -82,6 +92,11 @@ steps: XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} + EnableHelixWorkItemBatching: ${{ parameters.EnableHelixWorkItemBatching }} + HelixBatchTargetDuration: ${{ parameters.HelixBatchTargetDuration }} + HelixBatchTimeoutPadding: ${{ parameters.HelixBatchTimeoutPadding }} + HelixBatchMaxItems: ${{ parameters.HelixBatchMaxItems }} + HelixBatchMinItems: ${{ parameters.HelixBatchMinItems }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/BatchHelixWorkItemsTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/BatchHelixWorkItemsTests.cs new file mode 100644 index 00000000000..a957331cfa4 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/BatchHelixWorkItemsTests.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Text.Json; +using AwesomeAssertions; +using Microsoft.Arcade.Test.Common; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xunit; + +namespace Microsoft.DotNet.Helix.Sdk.Tests +{ + public class BatchHelixWorkItemsTests : IDisposable + { + private readonly string _root; + + public BatchHelixWorkItemsTests() + { + _root = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + public void Dispose() + { + if (Directory.Exists(_root)) + { + Directory.Delete(_root, recursive: true); + } + } + + [Fact] + public void PayloadDirectoryWorkItemsAreBatched() + { + ITaskItem first = CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"); + ITaskItem second = CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00"); + + var task = CreateTask(first, second); + + task.Execute().Should().BeTrue(); + + task.BatchedWorkItems.Should().ContainSingle(); + ITaskItem batch = task.BatchedWorkItems.Single(); + batch.ItemSpec.Should().StartWith("Batch_0001_First_Tests_dll"); + batch.GetMetadata("Command").Should().Be("run-batch.cmd"); + batch.GetMetadata("Timeout").Should().Be("00:04:00"); + batch.GetMetadata("BatchedWorkItemNames").Should().Be("First.Tests.dll;Second.Tests.dll"); + + string batchPayload = batch.GetMetadata("PayloadDirectory"); + File.Exists(Path.Combine(batchPayload, "run-batch.cmd")).Should().BeTrue(); + File.Exists(Path.Combine(batchPayload, "batch-manifest.json")).Should().BeTrue(); + Directory.GetDirectories(Path.Combine(batchPayload, "payloads")).Should().HaveCount(2); + File.ReadAllText(Path.Combine(batchPayload, "batch-manifest.json")).Should().Contain("First.Tests.dll"); + File.ReadAllText(Path.Combine(batchPayload, "run-batch.cmd")).Should().Contain("console.log"); + } + + [Fact] + public void SingleEligibleWorkItemPassesThrough() + { + ITaskItem item = CreateWorkItem("Only.Tests.dll", "dotnet exec Only.Tests.dll", "00:01:00"); + + var task = CreateTask(item); + + task.Execute().Should().BeTrue(); + + task.BatchedWorkItems.Should().ContainSingle().Which.Should().BeSameAs(item); + } + + [Fact] + public void UnbatchableItemsArePreserved() + { + ITaskItem first = CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"); + ITaskItem skipped = CreateWorkItem("Skipped.Tests.dll", "dotnet exec Skipped.Tests.dll", "00:01:00"); + skipped.SetMetadata("HelixBatchable", "false"); + ITaskItem second = CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00"); + + var task = CreateTask(first, skipped, second); + + task.Execute().Should().BeTrue(); + + task.BatchedWorkItems.Should().HaveCount(3); + task.BatchedWorkItems[0].Should().BeSameAs(first); + task.BatchedWorkItems[1].Should().BeSameAs(skipped); + task.BatchedWorkItems[2].Should().BeSameAs(second); + } + + [Fact] + public void MaxItemsSplitsBatches() + { + var task = CreateTask( + CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"), + CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00"), + CreateWorkItem("Third.Tests.dll", "dotnet exec Third.Tests.dll", "00:01:00"), + CreateWorkItem("Fourth.Tests.dll", "dotnet exec Fourth.Tests.dll", "00:01:00")); + task.MaxItemsPerBatch = 2; + + task.Execute().Should().BeTrue(); + + task.BatchedWorkItems.Should().HaveCount(2); + task.BatchedWorkItems.All(i => i.ItemSpec.StartsWith("Batch_")).Should().BeTrue(); + } + + [Fact] + public void PosixBatchUsesShellRunner() + { + var task = CreateTask( + CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"), + CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:01:00")); + task.IsPosixShell = true; + + task.Execute().Should().BeTrue(); + + ITaskItem batch = task.BatchedWorkItems.Single(); + batch.GetMetadata("Command").Should().Be("./run-batch.sh"); + File.ReadAllText(Path.Combine(batch.GetMetadata("PayloadDirectory"), "run-batch.sh")).Should().Contain("/bin/sh ./run-member.sh"); + } + + [Fact] + public void ManifestContainsMemberMetadata() + { + var task = CreateTask( + CreateWorkItem("First.Tests.dll", "dotnet exec First.Tests.dll", "00:01:00"), + CreateWorkItem("Second.Tests.dll", "dotnet exec Second.Tests.dll", "00:02:00")); + + task.Execute().Should().BeTrue(); + + string manifestPath = Path.Combine(task.BatchedWorkItems.Single().GetMetadata("PayloadDirectory"), "batch-manifest.json"); + using JsonDocument document = JsonDocument.Parse(File.ReadAllText(manifestPath)); + JsonElement manifest = document.RootElement; + + manifest.GetProperty("version").GetInt32().Should().Be(1); + manifest.GetProperty("workItems").GetArrayLength().Should().Be(2); + manifest.GetProperty("workItems")[1].GetProperty("timeout").GetString().Should().Be("00:02:00"); + } + + private BatchHelixWorkItems CreateTask(params ITaskItem[] workItems) + { + return new BatchHelixWorkItems + { + BuildEngine = new MockBuildEngine(), + WorkItems = workItems, + IntermediateOutputPath = Path.Combine(_root, "obj"), + IsPosixShell = false, + TargetDuration = 10, + TimeoutPadding = 2, + MaxItemsPerBatch = 10, + MinItemsPerBatch = 2 + }; + } + + private ITaskItem CreateWorkItem(string name, string command, string timeout) + { + string payloadDirectory = Path.Combine(_root, "payloads", name); + Directory.CreateDirectory(payloadDirectory); + File.WriteAllText(Path.Combine(payloadDirectory, name), "payload"); + + var result = new TaskItem(name); + result.SetMetadata("PayloadDirectory", payloadDirectory); + result.SetMetadata("Command", command); + result.SetMetadata("Timeout", timeout); + return result; + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk/BatchHelixWorkItems.cs b/src/Microsoft.DotNet.Helix/Sdk/BatchHelixWorkItems.cs new file mode 100644 index 00000000000..12ea449ed07 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/BatchHelixWorkItems.cs @@ -0,0 +1,404 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Sdk +{ + public class BatchHelixWorkItems : BaseTask + { + private const int DefaultTargetDurationMinutes = 10; + private const int DefaultTimeoutPaddingMinutes = 2; + + public ITaskItem[] WorkItems { get; set; } = Array.Empty(); + + [Required] + public string IntermediateOutputPath { get; set; } + + [Required] + public bool IsPosixShell { get; set; } + + public int TargetDuration { get; set; } = DefaultTargetDurationMinutes; + + public int TimeoutPadding { get; set; } = DefaultTimeoutPaddingMinutes; + + public int MaxItemsPerBatch { get; set; } = 10; + + public int MinItemsPerBatch { get; set; } = 2; + + [Output] + public ITaskItem[] BatchedWorkItems { get; set; } + + public override bool Execute() + { + TimeSpan targetDuration = TimeSpan.FromMinutes(Math.Max(1, TargetDuration)); + TimeSpan timeoutPadding = TimeSpan.FromMinutes(Math.Max(0, TimeoutPadding)); + int maxItemsPerBatch = Math.Max(1, MaxItemsPerBatch); + int minItemsPerBatch = Math.Max(1, MinItemsPerBatch); + + string batchRoot = Path.GetFullPath(Path.Combine(IntermediateOutputPath, "helix-work-item-batches")); + if (Directory.Exists(batchRoot)) + { + Directory.Delete(batchRoot, recursive: true); + } + Directory.CreateDirectory(batchRoot); + + var results = new List(); + var currentBatch = new List(); + TimeSpan currentDuration = TimeSpan.Zero; + int batchNumber = 1; + + foreach (ITaskItem workItem in WorkItems ?? Array.Empty()) + { + if (!TryCreateBatchMember(workItem, out BatchMember member)) + { + FlushBatchIfNeeded(); + results.Add(workItem); + continue; + } + + bool wouldExceedTarget = currentBatch.Count > 0 && currentDuration + member.ExpectedDuration > targetDuration; + bool wouldExceedCount = currentBatch.Count >= maxItemsPerBatch; + if (wouldExceedTarget || wouldExceedCount) + { + FlushBatchIfNeeded(); + } + + currentBatch.Add(member); + currentDuration += member.ExpectedDuration; + } + + FlushBatchIfNeeded(); + BatchedWorkItems = results.ToArray(); + return !Log.HasLoggedErrors; + + void FlushBatchIfNeeded() + { + if (currentBatch.Count == 0) + { + return; + } + + if (currentBatch.Count < minItemsPerBatch) + { + results.AddRange(currentBatch.Select(m => m.WorkItem)); + } + else + { + results.Add(CreateBatchWorkItem(currentBatch, batchRoot, batchNumber++, timeoutPadding)); + } + + currentBatch = new List(); + currentDuration = TimeSpan.Zero; + } + } + + private bool TryCreateBatchMember(ITaskItem workItem, out BatchMember member) + { + member = null; + + string name = GetMetadataOrItemSpec(workItem, SendHelixJob.MetadataNames.Identity); + string command = workItem.GetMetadata(SendHelixJob.MetadataNames.Command); + string payloadDirectory = workItem.GetMetadata(SendHelixJob.MetadataNames.PayloadDirectory); + string payloadArchive = workItem.GetMetadata(SendHelixJob.MetadataNames.PayloadArchive); + string payloadUri = workItem.GetMetadata(SendHelixJob.MetadataNames.PayloadUri); + string preCommands = workItem.GetMetadata(SendHelixJob.MetadataNames.PreCommands); + string postCommands = workItem.GetMetadata(SendHelixJob.MetadataNames.PostCommands); + string batchable = workItem.GetMetadata("HelixBatchable"); + + if (string.Equals(batchable, "false", StringComparison.OrdinalIgnoreCase)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping Helix work item '{name}' because HelixBatchable=false."); + return false; + } + + if (string.IsNullOrWhiteSpace(command) || string.IsNullOrWhiteSpace(payloadDirectory)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping Helix work item '{name}' because it does not have a simple PayloadDirectory and Command."); + return false; + } + + if (!string.IsNullOrWhiteSpace(payloadArchive) || !string.IsNullOrWhiteSpace(payloadUri)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping Helix work item '{name}' because archive and URI payload batching is not supported."); + return false; + } + + if (!string.IsNullOrWhiteSpace(preCommands) || !string.IsNullOrWhiteSpace(postCommands)) + { + Log.LogMessage(MessageImportance.Low, $"Skipping Helix work item '{name}' because per-work-item pre/post commands are not supported by batching."); + return false; + } + + if (!Directory.Exists(payloadDirectory)) + { + Log.LogWarning($"Skipping Helix work item '{name}' because payload directory '{payloadDirectory}' does not exist."); + return false; + } + + TimeSpan expectedDuration = GetExpectedDuration(workItem); + TimeSpan timeout = GetTimeout(workItem, expectedDuration); + member = new BatchMember(workItem, name, command, payloadDirectory, expectedDuration, timeout); + return true; + } + + private ITaskItem CreateBatchWorkItem(IReadOnlyList members, string batchRoot, int batchNumber, TimeSpan timeoutPadding) + { + string batchName = $"Batch_{batchNumber:0000}_{SanitizeName(members[0].Name)}"; + string batchDirectory = Path.Combine(batchRoot, batchName); + string payloadsDirectory = Path.Combine(batchDirectory, "payloads"); + Directory.CreateDirectory(payloadsDirectory); + + var manifestEntries = new List(); + for (int i = 0; i < members.Count; i++) + { + BatchMember member = members[i]; + string memberDirectoryName = $"{i + 1:000}_{ShortHash(member.Name)}"; + string memberPayloadDirectory = Path.Combine(payloadsDirectory, memberDirectoryName); + CopyDirectory(member.PayloadDirectory, memberPayloadDirectory); + WriteMemberCommand(batchDirectory, memberDirectoryName, member.Command); + manifestEntries.Add(new ManifestEntry + { + Name = member.Name, + Command = member.Command, + PayloadDirectory = $"payloads/{memberDirectoryName}", + Timeout = member.Timeout.ToString(), + ExpectedDuration = member.ExpectedDuration.ToString() + }); + } + + File.WriteAllText( + Path.Combine(batchDirectory, "batch-manifest.json"), + JsonConvert.SerializeObject(new BatchManifest { Version = 1, WorkItems = manifestEntries }, Formatting.Indented), + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + + string runnerFileName = IsPosixShell ? "run-batch.sh" : "run-batch.cmd"; + File.WriteAllText( + Path.Combine(batchDirectory, runnerFileName), + IsPosixShell ? CreatePosixRunner(manifestEntries) : CreateWindowsRunner(manifestEntries), + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + + TimeSpan timeout = TimeSpan.FromTicks(members.Sum(m => m.Timeout.Ticks) + timeoutPadding.Ticks); + var batchWorkItem = new TaskItem(batchName, new Dictionary + { + [SendHelixJob.MetadataNames.PayloadDirectory] = batchDirectory, + [SendHelixJob.MetadataNames.Command] = IsPosixShell ? "./run-batch.sh" : "run-batch.cmd", + [SendHelixJob.MetadataNames.Timeout] = timeout.ToString(), + ["BatchedWorkItemNames"] = string.Join(";", members.Select(m => m.Name)), + ["HelixBatchManifest"] = "batch-manifest.json" + }); + + Log.LogMessage( + MessageImportance.High, + $"Batched {members.Count} Helix work items into '{batchName}': {string.Join(", ", members.Select(m => m.Name))}"); + return batchWorkItem; + } + + private static string CreatePosixRunner(IReadOnlyList entries) + { + var builder = new StringBuilder(); + builder.AppendLine("#!/bin/sh"); + builder.AppendLine("set +e"); + builder.AppendLine("batch_exit=0"); + builder.AppendLine("batch_root=$(pwd)"); + builder.AppendLine("upload_root=${HELIX_WORKITEM_UPLOAD_ROOT:-$batch_root/uploads}"); + builder.AppendLine("mkdir -p \"$upload_root\""); + for (int i = 0; i < entries.Count; i++) + { + string memberId = GetMemberId(i, entries[i].Name); + string payload = entries[i].PayloadDirectory; + builder.AppendLine($"echo \"##[group]Starting batched Helix work item {memberId}\""); + builder.AppendLine($"member_upload=\"$upload_root/{memberId}\""); + builder.AppendLine("mkdir -p \"$member_upload\""); + builder.AppendLine("("); + builder.AppendLine($" cd \"$batch_root/{payload}\""); + builder.AppendLine(" export HELIX_WORKITEM_PAYLOAD=$(pwd)"); + builder.AppendLine(" export HELIX_WORKITEM_ROOT=$(pwd)"); + builder.AppendLine(" export HELIX_WORKITEM_UPLOAD_ROOT=\"$member_upload\""); + builder.AppendLine($" /bin/sh ./run-member.sh"); + builder.AppendLine($") > \"$member_upload/console.log\" 2>&1"); + builder.AppendLine("member_exit=$?"); + builder.AppendLine($"find \"$batch_root/{payload}\" -maxdepth 5 \\( -iname '*.trx' -o -iname 'testResults.xml' -o -iname 'test-results.xml' -o -iname 'test_results.xml' -o -iname 'junit-results.xml' -o -iname 'junitresults.xml' \\) -exec cp {{}} \"$member_upload/\" \\; 2>/dev/null"); + builder.AppendLine("if [ $member_exit -ne 0 ]; then batch_exit=$member_exit; fi"); + builder.AppendLine("cat \"$member_upload/console.log\""); + builder.AppendLine($"echo \"##[endgroup]Finished batched Helix work item {memberId} with exit code $member_exit\""); + } + builder.AppendLine("exit $batch_exit"); + return builder.ToString(); + } + + private static string CreateWindowsRunner(IReadOnlyList entries) + { + var builder = new StringBuilder(); + builder.AppendLine("@echo off"); + builder.AppendLine("setlocal EnableExtensions"); + builder.AppendLine("set batch_exit=0"); + builder.AppendLine("set batch_root=%CD%"); + builder.AppendLine("set batch_upload_root=%HELIX_WORKITEM_UPLOAD_ROOT%"); + builder.AppendLine("if \"%batch_upload_root%\"==\"\" set batch_upload_root=%batch_root%\\uploads"); + builder.AppendLine("if not exist \"%batch_upload_root%\" mkdir \"%batch_upload_root%\""); + for (int i = 0; i < entries.Count; i++) + { + string memberId = GetMemberId(i, entries[i].Name); + string payload = entries[i].PayloadDirectory.Replace('/', '\\'); + builder.AppendLine($"echo ##[group]Starting batched Helix work item {memberId}"); + builder.AppendLine($"set member_upload=%batch_upload_root%\\{memberId}"); + builder.AppendLine("if not exist \"%member_upload%\" mkdir \"%member_upload%\""); + builder.AppendLine("pushd \"%batch_root%\\" + payload + "\""); + builder.AppendLine("set HELIX_WORKITEM_PAYLOAD=%CD%"); + builder.AppendLine("set HELIX_WORKITEM_ROOT=%CD%"); + builder.AppendLine("set HELIX_WORKITEM_UPLOAD_ROOT=%member_upload%"); + builder.AppendLine("call run-member.cmd > \"%member_upload%\\console.log\" 2>&1"); + builder.AppendLine("set member_exit=%ERRORLEVEL%"); + builder.AppendLine("popd"); + builder.AppendLine("powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command \"Get-ChildItem -Path '%batch_root%\\" + payload + "' -Recurse -File -Depth 5 -Include *.trx,testResults.xml,test-results.xml,test_results.xml,junit-results.xml,junitresults.xml -ErrorAction SilentlyContinue | Copy-Item -Destination '%member_upload%' -Force -ErrorAction SilentlyContinue\""); + builder.AppendLine("if not \"%member_exit%\"==\"0\" set batch_exit=%member_exit%"); + builder.AppendLine("type \"%member_upload%\\console.log\""); + builder.AppendLine($"echo ##[endgroup]Finished batched Helix work item {memberId} with exit code %member_exit%"); + } + builder.AppendLine("exit /b %batch_exit%"); + return builder.ToString(); + } + + private void WriteMemberCommand(string batchDirectory, string memberDirectoryName, string command) + { + string payloadDirectory = Path.Combine(batchDirectory, "payloads", memberDirectoryName); + string fileName = IsPosixShell ? "run-member.sh" : "run-member.cmd"; + string contents = IsPosixShell + ? "#!/bin/sh\n" + command + "\n" + : "@echo off\r\n" + command + "\r\nexit /b %ERRORLEVEL%\r\n"; + File.WriteAllText(Path.Combine(payloadDirectory, fileName), contents, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + } + + private TimeSpan GetExpectedDuration(ITaskItem workItem) + { + foreach (string metadataName in new[] { "ExpectedExecutionTime", "HelixExpectedDuration", "EstimatedDuration" }) + { + string value = workItem.GetMetadata(metadataName); + if (TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan duration) && duration > TimeSpan.Zero) + { + return duration; + } + } + + return GetTimeout(workItem, TimeSpan.FromMinutes(1)); + } + + private TimeSpan GetTimeout(ITaskItem workItem, TimeSpan fallback) + { + string timeout = workItem.GetMetadata(SendHelixJob.MetadataNames.Timeout); + if (TimeSpan.TryParse(timeout, CultureInfo.InvariantCulture, out TimeSpan parsed) && parsed > TimeSpan.Zero) + { + return parsed; + } + + return fallback; + } + + private static void CopyDirectory(string sourceDirectory, string destinationDirectory) + { + Directory.CreateDirectory(destinationDirectory); + foreach (string directory in Directory.EnumerateDirectories(sourceDirectory, "*", SearchOption.AllDirectories)) + { + Directory.CreateDirectory(Path.Combine(destinationDirectory, Path.GetRelativePath(sourceDirectory, directory))); + } + + foreach (string file in Directory.EnumerateFiles(sourceDirectory, "*", SearchOption.AllDirectories)) + { + string destination = Path.Combine(destinationDirectory, Path.GetRelativePath(sourceDirectory, file)); + Directory.CreateDirectory(Path.GetDirectoryName(destination)); + File.Copy(file, destination, overwrite: true); + } + } + + private static string GetMetadataOrItemSpec(ITaskItem item, string metadataName) + { + string metadata = item.GetMetadata(metadataName); + return string.IsNullOrWhiteSpace(metadata) ? item.ItemSpec : metadata; + } + + private static string GetMemberId(int index, string name) => $"{index + 1:000}_{ShortHash(name)}"; + + private static string SanitizeName(string name) + { + var builder = new StringBuilder(name.Length); + foreach (char c in name) + { + builder.Append(char.IsLetterOrDigit(c) ? c : '_'); + } + + string sanitized = builder.ToString().Trim('_'); + return sanitized.Length > 32 ? sanitized.Substring(0, 32) : sanitized; + } + + private static string ShortHash(string value) + { + unchecked + { + uint hash = 2166136261; + foreach (char c in value) + { + hash ^= c; + hash *= 16777619; + } + + return hash.ToString("x", CultureInfo.InvariantCulture); + } + } + + private sealed class BatchMember + { + public BatchMember(ITaskItem workItem, string name, string command, string payloadDirectory, TimeSpan expectedDuration, TimeSpan timeout) + { + WorkItem = workItem; + Name = name; + Command = command; + PayloadDirectory = payloadDirectory; + ExpectedDuration = expectedDuration; + Timeout = timeout; + } + + public ITaskItem WorkItem { get; } + public string Name { get; } + public string Command { get; } + public string PayloadDirectory { get; } + public TimeSpan ExpectedDuration { get; } + public TimeSpan Timeout { get; } + } + + private sealed class BatchManifest + { + [JsonProperty("version")] + public int Version { get; set; } + + [JsonProperty("workItems")] + public List WorkItems { get; set; } + } + + private sealed class ManifestEntry + { + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("command")] + public string Command { get; set; } + + [JsonProperty("payloadDirectory")] + public string PayloadDirectory { get; set; } + + [JsonProperty("timeout")] + public string Timeout { get; set; } + + [JsonProperty("expectedDuration")] + public string ExpectedDuration { get; set; } + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index 99377d0638b..c728eb619c7 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -232,6 +232,33 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r ``` +### Batching short work items + +Repos that send many short `HelixWorkItem`s can opt into submission-time batching. Batching combines compatible `PayloadDirectory` work items into a single Helix work item, runs each original command from its own staged payload directory, and namespaces each member's logs and test result files under `$HELIX_WORKITEM_UPLOAD_ROOT`. + +Batching is disabled by default. Enable it from the Helix project: + +```xml + + true + 10 + 2 + 10 + +``` + +Optional duration metadata improves grouping: + +```xml + + + 00:01:15 + + +``` + +Set `HelixBatchable=false` on a `HelixWorkItem` that needs machine-level isolation to opt it out of batching. (The metadata is read from the generated `HelixWorkItem` items, so when using `XUnitProject`/`XUnitProjects` the opt-out applies to the produced work items rather than being set directly on the project.) The initial batching implementation only batches simple `PayloadDirectory` items; archive/URI payloads and items with per-work-item pre/post commands are preserved unchanged. + ### All Possible Options ```xml diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props index c26e4f79ec3..83c22b41487 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props @@ -28,6 +28,12 @@ --> false + false + 10 + 2 + 10 + 2 + <_HelixMonoQueueTargets>$(_HelixMonoQueueTargets);$(MSBuildThisFileDirectory)helix-batching\HelixBatching.targets @@ -38,6 +44,7 @@ + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/helix-batching/HelixBatching.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/helix-batching/HelixBatching.targets new file mode 100644 index 00000000000..d94db0bef07 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/helix-batching/HelixBatching.targets @@ -0,0 +1,32 @@ + + + + + + + + + + + + + <_BatchedHelixWorkItem Remove="@(_BatchedHelixWorkItem)" /> + + + +