From a5f199246ba2dda865ba549f73af2e049cf8179b Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Mon, 4 May 2026 16:07:01 -0500 Subject: [PATCH 01/14] Add dotnet test integration tests for all Apple platforms Create DotNetTestTest that runs 'dotnet test' on projects created from the iostest, tvostest, maccatalysttest, and macostest templates. This verifies the MTP (Microsoft Testing Platform) pipeline works end-to-end, including the --server/--dotnet-test-pipe argument passthrough via RunArguments. Also add 'test' as a supported verb in DotNet.Execute. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/common/DotNet.cs | 1 + tests/dotnet/UnitTests/DotNetTestTest.cs | 39 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 tests/dotnet/UnitTests/DotNetTestTest.cs diff --git a/tests/common/DotNet.cs b/tests/common/DotNet.cs index 1f7a5c73e86..e362220cb9d 100644 --- a/tests/common/DotNet.cs +++ b/tests/common/DotNet.cs @@ -222,6 +222,7 @@ public static ExecutionResult Execute (string verb, string project, Dictionary (); args.Add (verb); args.Add (project); diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs new file mode 100644 index 00000000000..6892e704b81 --- /dev/null +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -0,0 +1,39 @@ +using Xamarin.Tests; + +#nullable enable + +namespace Xamarin.Tests { + [TestFixture] + public class DotNetTestTest : TestBaseClass { + [Test] + [TestCase (ApplePlatform.iOS, "iostest")] + [TestCase (ApplePlatform.TVOS, "tvostest")] + [TestCase (ApplePlatform.MacCatalyst, "maccatalysttest")] + [TestCase (ApplePlatform.MacOSX, "macostest")] + public void DotNetTest (ApplePlatform platform, string template) + { + Configuration.IgnoreIfIgnoredPlatform (platform); + + var tmpDir = Cache.CreateTemporaryDirectory (); + var outputDir = Path.Combine (tmpDir, template); + DotNet.AssertNew (outputDir, template); + var proj = Path.Combine (outputDir, $"{template}.csproj"); + + // Replace generated tests with a single passing test + var testFile = Path.Combine (outputDir, "Test1.cs"); + File.WriteAllText (testFile, $@"namespace {template}; + +[TestClass] +public sealed class Test1 {{ + [TestMethod] + public void TestMethod1 () + {{ + }} +}} +"); + + var properties = GetDefaultProperties (); + DotNet.Execute ("test", proj, properties); + } + } +} From 0d6a7902cd40e3b3a9c50abca102dec489310ce7 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 5 May 2026 08:37:11 -0500 Subject: [PATCH 02/14] Comment out macOS/MacCatalyst tests and fix test verb workaround macOS and Mac Catalyst don't use mlaunch (they use 'open' via Desktop.targets), so MTP support requires a different approach. Comment out those test cases for now. Also skip the /v:diag and /consoleloggerparameters workaround args when the verb is 'test' to prevent them leaking through to the MTP test runner via RunArguments. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/common/DotNet.cs | 14 +++++++++----- tests/dotnet/UnitTests/DotNetTestTest.cs | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/tests/common/DotNet.cs b/tests/common/DotNet.cs index e362220cb9d..5ee58a7720b 100644 --- a/tests/common/DotNet.cs +++ b/tests/common/DotNet.cs @@ -289,11 +289,15 @@ public static ExecutionResult Execute (string verb, string project, Dictionary Date: Tue, 12 May 2026 12:33:58 -0500 Subject: [PATCH 03/14] Bump Microsoft.Tools.Mlaunch to 1.1.115 Update mlaunch NuGet package to include MTP (Microsoft Testing Platform) support for iOS and tvOS simulator test execution. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- mk/xamarin.mk | 2 +- tests/dotnet/UnitTests/DotNetTestTest.cs | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/mk/xamarin.mk b/mk/xamarin.mk index 537af408111..1dc19353839 100644 --- a/mk/xamarin.mk +++ b/mk/xamarin.mk @@ -1,5 +1,5 @@ # Available versions can be seen here: # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.Tools.Mlaunch/versions -MLAUNCH_NUGET_VERSION=1.1.113 +MLAUNCH_NUGET_VERSION=1.1.115 # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/AppleDocReader/versions ADR_NUGET_VERSION=1.0.0 diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 18ab86eca29..a1d48809ace 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -35,14 +35,6 @@ public void TestMethod1 () "); var properties = GetDefaultProperties (); - - // Mobile platforms use mlaunch to run the app in the simulator. - // Override MlaunchPath if set via the environment variable. - var mlaunchPath = Environment.GetEnvironmentVariable ("MLAUNCH_PATH"); - if (!string.IsNullOrEmpty (mlaunchPath)) { - properties ["MlaunchPath"] = mlaunchPath; - } - DotNet.Execute ("test", proj, properties); } } From e086c8ef839bce43aa8333940744275b8483a2fd Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Wed, 13 May 2026 15:40:02 -0500 Subject: [PATCH 04/14] Split dotnet test into build + ComputeRunArguments + test --no-build The dotnet test MTP flow calls ComputeRunArguments via the MSBuild API with loggers: null, making errors invisible. Split into three steps: 1. dotnet build - ensures app bundle is created 2. dotnet build /t:ComputeRunArguments - verifies targets work with full diagnostic output 3. dotnet test --no-build - runs the actual test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index a1d48809ace..1ad3ad2c6c2 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -35,7 +35,16 @@ public void TestMethod1 () "); var properties = GetDefaultProperties (); - DotNet.Execute ("test", proj, properties); + + // Build first to ensure the app bundle is created with the correct properties. + DotNet.Execute ("build", proj, properties); + + // Verify ComputeRunArguments targets work with diagnostic output. + // This helps diagnose failures that dotnet test hides (it uses loggers: null internally). + DotNet.Execute ("build", proj, properties, target: "ComputeRunArguments"); + + // Now run dotnet test with --no-build since we already built above. + DotNet.Execute ("test", proj, properties, extraArguments: new [] { "--no-build" }); } } } From 062cf2745927e12984a4d5679ca7cb8f9d958e20 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 14 May 2026 08:53:33 -0500 Subject: [PATCH 05/14] Remove ComputeRunArguments diagnostic step The standalone ComputeRunArguments target fails on CI because GetAvailableDevices filters out all simulators. Since dotnet test handles ComputeRunArguments internally, just use build + test --no-build. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 1ad3ad2c6c2..74bd9cb3d23 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -36,14 +36,9 @@ public void TestMethod1 () var properties = GetDefaultProperties (); - // Build first to ensure the app bundle is created with the correct properties. + // Build first to ensure the app bundle is created correctly. + // dotnet test won't deploy properly on its own, so we need to build separately. DotNet.Execute ("build", proj, properties); - - // Verify ComputeRunArguments targets work with diagnostic output. - // This helps diagnose failures that dotnet test hides (it uses loggers: null internally). - DotNet.Execute ("build", proj, properties, target: "ComputeRunArguments"); - - // Now run dotnet test with --no-build since we already built above. DotNet.Execute ("test", proj, properties, extraArguments: new [] { "--no-build" }); } } From 6f672aab2055004ee548add22d178135a6ba8e2f Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 14 May 2026 11:28:44 -0500 Subject: [PATCH 06/14] Set UseFloatingTargetPlatformVersion in csproj for dotnet test dotnet test MTP internally calls ComputeRunArguments via MSBuild API without forwarding /p: properties from the command line. This caused ComputeRunArguments to fail because the TargetPlatformVersion did not match the installed SDK workload. Setting the property in the project file ensures it is available during all MSBuild evaluations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 74bd9cb3d23..81c4d1202aa 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -21,6 +21,12 @@ public void DotNetTest (ApplePlatform platform, string template) DotNet.AssertNew (outputDir, template); var proj = Path.Combine (outputDir, $"{template}.csproj"); + // dotnet test internally calls ComputeRunArguments via MSBuild API without + // forwarding /p: properties, so we must set them in the project file directly. + var csproj = File.ReadAllText (proj); + csproj = csproj.Replace ("", " true\n "); + File.WriteAllText (proj, csproj); + // Replace generated tests with a single passing test var testFile = Path.Combine (outputDir, "Test1.cs"); File.WriteAllText (testFile, $@"namespace {template}; From 612099da079006c069c8db83f4b0d0686299eb01 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 14 May 2026 14:28:34 -0500 Subject: [PATCH 07/14] Use Execution.RunAsync directly for dotnet test step dotnet test MTP flow doesn't forward /p: properties or produce binlogs for its internal ComputeRunArguments call. Using Execution.RunAsync directly avoids these issues and includes the full output in the assertion message for better CI diagnostics. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 81c4d1202aa..d6253a0a605 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -1,4 +1,5 @@ using Xamarin.Tests; +using Xamarin.Utils; #nullable enable @@ -45,7 +46,17 @@ public void TestMethod1 () // Build first to ensure the app bundle is created correctly. // dotnet test won't deploy properly on its own, so we need to build separately. DotNet.Execute ("build", proj, properties); - DotNet.Execute ("test", proj, properties, extraArguments: new [] { "--no-build" }); + + // Run 'dotnet test --no-build' directly using Execution.RunAsync. + // We can't use DotNet.Execute because dotnet test's MTP flow + // doesn't forward /p: properties to its internal ComputeRunArguments call, + // and --no-build means no binlog is produced (causing FileNotFound errors). + var env = new Dictionary (); + env ["MSBuildSDKsPath"] = null; + env ["MSBUILD_EXE_PATH"] = null; + var testArgs = new List { "test", proj, "--no-build" }; + var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; + Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test --no-build' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); } } } From b4c73706fbe10b082c03e1542be2f5222497a8ec Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 14 May 2026 14:53:58 -0500 Subject: [PATCH 08/14] Drop --no-build and use dotnet test directly The dotnet test MTP flow uses MSBuild API internally for ComputeRunArguments but doesn't load the workload SDK resolver. With --no-build, this means the iOS SDK can't be resolved at all. Without --no-build, dotnet test does a full build+run cycle where the workload resolver is properly initialized. Also use Execution.RunAsync directly instead of DotNet.Execute to avoid /p: and /bl: args that dotnet test doesn't forward. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index d6253a0a605..596ebe6feef 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -41,22 +41,15 @@ public void TestMethod1 () }} "); - var properties = GetDefaultProperties (); - - // Build first to ensure the app bundle is created correctly. - // dotnet test won't deploy properly on its own, so we need to build separately. - DotNet.Execute ("build", proj, properties); - - // Run 'dotnet test --no-build' directly using Execution.RunAsync. - // We can't use DotNet.Execute because dotnet test's MTP flow - // doesn't forward /p: properties to its internal ComputeRunArguments call, - // and --no-build means no binlog is produced (causing FileNotFound errors). + // Run 'dotnet test' directly using Execution.RunAsync. + // dotnet test's MTP flow doesn't forward /p: properties to its internal + // ComputeRunArguments MSBuild API call, so properties must be in the csproj. var env = new Dictionary (); env ["MSBuildSDKsPath"] = null; env ["MSBUILD_EXE_PATH"] = null; - var testArgs = new List { "test", proj, "--no-build" }; + var testArgs = new List { "test", proj }; var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; - Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test --no-build' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); + Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); } } } From 69e617ed284eec1cd2ff9fe0c03d67b17ea8d48d Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 14 May 2026 14:55:24 -0500 Subject: [PATCH 09/14] Remove unnecessary env var overrides from dotnet test call Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 596ebe6feef..7c371149aed 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -44,11 +44,8 @@ public void TestMethod1 () // Run 'dotnet test' directly using Execution.RunAsync. // dotnet test's MTP flow doesn't forward /p: properties to its internal // ComputeRunArguments MSBuild API call, so properties must be in the csproj. - var env = new Dictionary (); - env ["MSBuildSDKsPath"] = null; - env ["MSBUILD_EXE_PATH"] = null; var testArgs = new List { "test", proj }; - var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; + var testResult = Execution.RunAsync (DotNet.Executable, testArgs, environment: null, log: Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); } } From 1f8941c5e42f1ed1659c8b029dc96231921ca262 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 15 May 2026 14:16:59 -0500 Subject: [PATCH 10/14] Clear MSBuild env vars for dotnet test call The NUnit runner process sets MSBuildSDKsPath and MSBUILD_EXE_PATH which interfere with the child dotnet process SDK resolution. Every DotNet.Execute call clears these; we must do the same. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 7c371149aed..596ebe6feef 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -44,8 +44,11 @@ public void TestMethod1 () // Run 'dotnet test' directly using Execution.RunAsync. // dotnet test's MTP flow doesn't forward /p: properties to its internal // ComputeRunArguments MSBuild API call, so properties must be in the csproj. + var env = new Dictionary (); + env ["MSBuildSDKsPath"] = null; + env ["MSBUILD_EXE_PATH"] = null; var testArgs = new List { "test", proj }; - var testResult = Execution.RunAsync (DotNet.Executable, testArgs, environment: null, log: Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; + var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); } } From 167accd45103e827cd2106dafee6e76bb5aad2f0 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Mon, 18 May 2026 10:31:59 -0500 Subject: [PATCH 11/14] Add /bl: to dotnet test for CI diagnostics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 596ebe6feef..62965125981 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -47,9 +47,10 @@ public void TestMethod1 () var env = new Dictionary (); env ["MSBuildSDKsPath"] = null; env ["MSBUILD_EXE_PATH"] = null; - var testArgs = new List { "test", proj }; + var binlog = Path.Combine (outputDir, "log-test.binlog"); + var testArgs = new List { "test", proj, $"/bl:{binlog}" }; var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; - Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nOutput:\n{testResult.Output.MergedOutput}"); + Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nBinlog: {binlog}\nOutput:\n{testResult.Output.MergedOutput}"); } } } From 7fc6857985d4123d05763310bc3f88168be1e7fb Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Wed, 27 May 2026 16:28:52 -0500 Subject: [PATCH 12/14] Boot simulator before running dotnet test The ComputeRunArguments target requires a booted simulator to select a device. Query available devices via ComputeAvailableDevices target and boot one before running dotnet test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 46 ++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 62965125981..18bb3cf2498 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -1,3 +1,5 @@ +using System.Text.Json; + using Xamarin.Tests; using Xamarin.Utils; @@ -41,6 +43,11 @@ public void TestMethod1 () }} "); + // Boot a simulator so that ComputeRunArguments can find a device + var runtimeIdentifier = GetDefaultRuntimeIdentifier (platform); + var deviceUdid = GetDeviceUdid (outputDir, runtimeIdentifier); + BootSimulator (deviceUdid); + // Run 'dotnet test' directly using Execution.RunAsync. // dotnet test's MTP flow doesn't forward /p: properties to its internal // ComputeRunArguments MSBuild API call, so properties must be in the csproj. @@ -52,5 +59,44 @@ public void TestMethod1 () var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nBinlog: {binlog}\nOutput:\n{testResult.Output.MergedOutput}"); } + + static string GetDeviceUdid (string projectDirectory, string runtimeIdentifier) + { + var tmpdir = Cache.CreateTemporaryDirectory (); + var outputFile = Path.Combine (tmpdir, "AvailableDevices.json"); + var args = new List { + "build", + "-t:ComputeAvailableDevices", + "-getItem:Devices", + $"-getResultOutputFile:{outputFile}", + $"-p:RuntimeIdentifier={runtimeIdentifier}", + }; + + var env = new Dictionary (); + env ["MSBuildSDKsPath"] = null; + env ["MSBUILD_EXE_PATH"] = null; + var rv = Execution.RunAsync (DotNet.Executable, args, env, Console.Out, workingDirectory: projectDirectory, timeout: TimeSpan.FromMinutes (2)).Result; + Assert.AreEqual (0, rv.ExitCode, $"Failed to compute available devices. Output:\n{rv.Output.MergedOutput}"); + + var output = File.ReadAllText (outputFile); + var doc = JsonDocument.Parse (output); + var devices = doc.RootElement.GetProperty ("Items").GetProperty ("Devices").EnumerateArray ().Select (e => { + var identity = e.GetProperty ("Identity").GetString ()!; + var osVersion = Version.Parse (e.GetProperty ("OSVersion").GetString ()!); + var deviceTypeIdentifier = e.GetProperty ("DeviceTypeIdentifier").GetString ()!; + return (Identity: identity, OsVersion: osVersion, DeviceTypeIdentifier: deviceTypeIdentifier); + }).OrderByDescending (d => d.OsVersion).ThenBy (d => d.DeviceTypeIdentifier).ThenBy (d => d.Identity).ToList (); + + Assert.That (devices, Is.Not.Empty, $"No devices found. Output:\n{output}"); + return devices.First ().Identity; + } + + static void BootSimulator (string udid) + { + var rv = Execution.RunAsync ("xcrun", new List { "simctl", "boot", udid }, timeout: TimeSpan.FromMinutes (1)).Result; + // Exit code 149 means "already booted", which is fine + if (rv.ExitCode != 0 && rv.ExitCode != 149) + Assert.Fail ($"Failed to boot simulator {udid}. Exit code: {rv.ExitCode}\nOutput:\n{rv.Output.MergedOutput}"); + } } } From 7fb2b4f27d32fa0d4c43e984839da5007a5c7c72 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 28 May 2026 09:06:23 -0500 Subject: [PATCH 13/14] Fix NUnit v4 Assert.That syntax in DotNetTestTest The net11.0 branch upgraded to NUnit v4 which uses Assert.That() instead of Assert.AreEqual(). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index 18bb3cf2498..bdfdb647df8 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -57,7 +57,7 @@ public void TestMethod1 () var binlog = Path.Combine (outputDir, "log-test.binlog"); var testArgs = new List { "test", proj, $"/bl:{binlog}" }; var testResult = Execution.RunAsync (DotNet.Executable, testArgs, env, Console.Out, workingDirectory: outputDir, timeout: TimeSpan.FromMinutes (10)).Result; - Assert.AreEqual (0, testResult.ExitCode, $"'dotnet test' failed with exit code {testResult.ExitCode}.\nBinlog: {binlog}\nOutput:\n{testResult.Output.MergedOutput}"); + Assert.That (testResult.ExitCode, Is.EqualTo (0), $"'dotnet test' failed with exit code {testResult.ExitCode}.\nBinlog: {binlog}\nOutput:\n{testResult.Output.MergedOutput}"); } static string GetDeviceUdid (string projectDirectory, string runtimeIdentifier) @@ -76,7 +76,7 @@ static string GetDeviceUdid (string projectDirectory, string runtimeIdentifier) env ["MSBuildSDKsPath"] = null; env ["MSBUILD_EXE_PATH"] = null; var rv = Execution.RunAsync (DotNet.Executable, args, env, Console.Out, workingDirectory: projectDirectory, timeout: TimeSpan.FromMinutes (2)).Result; - Assert.AreEqual (0, rv.ExitCode, $"Failed to compute available devices. Output:\n{rv.Output.MergedOutput}"); + Assert.That (rv.ExitCode, Is.EqualTo (0), $"Failed to compute available devices. Output:\n{rv.Output.MergedOutput}"); var output = File.ReadAllText (outputFile); var doc = JsonDocument.Parse (output); From b42f06ca91f4e3ff72dcf6fd8c50bf441df63db7 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Thu, 28 May 2026 14:28:41 -0500 Subject: [PATCH 14/14] Use xcrun simctl to find simulators instead of MSBuild target The ComputeAvailableDevices MSBuild target returned empty results on CI, likely due to mlaunch not being available yet at that point. Use xcrun simctl list directly which is always available on the CI agents. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/dotnet/UnitTests/DotNetTestTest.cs | 51 +++++++++++------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/tests/dotnet/UnitTests/DotNetTestTest.cs b/tests/dotnet/UnitTests/DotNetTestTest.cs index bdfdb647df8..e5c549f5407 100644 --- a/tests/dotnet/UnitTests/DotNetTestTest.cs +++ b/tests/dotnet/UnitTests/DotNetTestTest.cs @@ -44,8 +44,7 @@ public void TestMethod1 () "); // Boot a simulator so that ComputeRunArguments can find a device - var runtimeIdentifier = GetDefaultRuntimeIdentifier (platform); - var deviceUdid = GetDeviceUdid (outputDir, runtimeIdentifier); + var deviceUdid = GetDeviceUdid (platform); BootSimulator (deviceUdid); // Run 'dotnet test' directly using Execution.RunAsync. @@ -60,35 +59,33 @@ public void TestMethod1 () Assert.That (testResult.ExitCode, Is.EqualTo (0), $"'dotnet test' failed with exit code {testResult.ExitCode}.\nBinlog: {binlog}\nOutput:\n{testResult.Output.MergedOutput}"); } - static string GetDeviceUdid (string projectDirectory, string runtimeIdentifier) + static string GetDeviceUdid (ApplePlatform platform) { - var tmpdir = Cache.CreateTemporaryDirectory (); - var outputFile = Path.Combine (tmpdir, "AvailableDevices.json"); - var args = new List { - "build", - "-t:ComputeAvailableDevices", - "-getItem:Devices", - $"-getResultOutputFile:{outputFile}", - $"-p:RuntimeIdentifier={runtimeIdentifier}", - }; + // Use xcrun simctl directly to find available simulator devices. + var rv = Execution.RunAsync ("xcrun", new List { "simctl", "list", "devices", "available", "--json" }, timeout: TimeSpan.FromMinutes (1)).Result; + Assert.That (rv.ExitCode, Is.EqualTo (0), $"Failed to list simulators. Output:\n{rv.Output.MergedOutput}"); - var env = new Dictionary (); - env ["MSBuildSDKsPath"] = null; - env ["MSBUILD_EXE_PATH"] = null; - var rv = Execution.RunAsync (DotNet.Executable, args, env, Console.Out, workingDirectory: projectDirectory, timeout: TimeSpan.FromMinutes (2)).Result; - Assert.That (rv.ExitCode, Is.EqualTo (0), $"Failed to compute available devices. Output:\n{rv.Output.MergedOutput}"); + var runtimePrefix = platform switch { + ApplePlatform.iOS => "com.apple.CoreSimulator.SimRuntime.iOS-", + ApplePlatform.TVOS => "com.apple.CoreSimulator.SimRuntime.tvOS-", + _ => throw new ArgumentException ($"Unsupported platform: {platform}"), + }; - var output = File.ReadAllText (outputFile); - var doc = JsonDocument.Parse (output); - var devices = doc.RootElement.GetProperty ("Items").GetProperty ("Devices").EnumerateArray ().Select (e => { - var identity = e.GetProperty ("Identity").GetString ()!; - var osVersion = Version.Parse (e.GetProperty ("OSVersion").GetString ()!); - var deviceTypeIdentifier = e.GetProperty ("DeviceTypeIdentifier").GetString ()!; - return (Identity: identity, OsVersion: osVersion, DeviceTypeIdentifier: deviceTypeIdentifier); - }).OrderByDescending (d => d.OsVersion).ThenBy (d => d.DeviceTypeIdentifier).ThenBy (d => d.Identity).ToList (); + var doc = JsonDocument.Parse (rv.Output.MergedOutput); + var devicesObj = doc.RootElement.GetProperty ("devices"); + var allDevices = new List<(string Udid, string Runtime)> (); + foreach (var runtimeProp in devicesObj.EnumerateObject ()) { + if (!runtimeProp.Name.StartsWith (runtimePrefix, StringComparison.Ordinal)) + continue; + foreach (var device in runtimeProp.Value.EnumerateArray ()) { + var udid = device.GetProperty ("udid").GetString ()!; + allDevices.Add ((udid, runtimeProp.Name)); + } + } - Assert.That (devices, Is.Not.Empty, $"No devices found. Output:\n{output}"); - return devices.First ().Identity; + Assert.That (allDevices, Is.Not.Empty, $"No {platform} simulators found. Output:\n{rv.Output.MergedOutput}"); + // Pick the last runtime (highest version) and first device + return allDevices.OrderByDescending (d => d.Runtime).First ().Udid; } static void BootSimulator (string udid)