diff --git a/Cesium.IntegrationTests/IntegrationTestContext.cs b/Cesium.IntegrationTests/IntegrationTestContext.cs index 64dfddb5..fa4411cb 100644 --- a/Cesium.IntegrationTests/IntegrationTestContext.cs +++ b/Cesium.IntegrationTests/IntegrationTestContext.cs @@ -24,6 +24,9 @@ public class IntegrationTestContext : IAsyncDisposable private Exception? _initializationException; public AbsolutePath? VisualStudioPath { get; private set; } + + /// Directory for timing results output. + public AbsolutePath TimingOutputDirectory { get; private set; } public async Task WrapTestBody(Func testBody) { @@ -66,12 +69,26 @@ private async Task InitializeOnce(ITestOutputHelper output) VisualStudioPath = await WindowsEnvUtil.FindVcCompilerInstallationFolder(output); } + // Initialize timing output directory + TimingOutputDirectory = SolutionMetadata.SourceRoot / "artifacts" / "timing"; + TestTimingCollector.Initialize(TimingOutputDirectory); + await BuildRuntime(output); await BuildCompiler(output); } public async ValueTask DisposeAsync() { + // Save timing results to JSON file + try + { + TestTimingCollector.SaveToJson(); + } + catch + { + // Ignore errors when saving timing results + } + await DotNetCliHelper.ShutdownBuildServer(); } diff --git a/Cesium.IntegrationTests/IntegrationTestRunner.cs b/Cesium.IntegrationTests/IntegrationTestRunner.cs index 0ec7333a..09c447a4 100644 --- a/Cesium.IntegrationTests/IntegrationTestRunner.cs +++ b/Cesium.IntegrationTests/IntegrationTestRunner.cs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +using System.Diagnostics; using System.Runtime.InteropServices; using Cesium.Solution.Metadata; using Cesium.TestFramework; @@ -16,6 +17,10 @@ public class IntegrationTestRunner : IClassFixture, IAsy private readonly ITestOutputHelper _output; private readonly IntegrationTestContext _context; + + /// Records timing for a test execution. + private record TestExecutionResult(string Output, TimeSpan CompileTime, TimeSpan ExecutionTime); + public IntegrationTestRunner(IntegrationTestContext context, ITestOutputHelper output) { _context = context; @@ -155,6 +160,11 @@ public Task MultiFileApplicationCompiles() => public Task CompilerAcceptsAnObjFileAsInput() => _context.WrapTestBody(async () => { + var totalStopwatch = Stopwatch.StartNew(); + TestTimingResult? timingResult = null; + bool success = false; + string? errorMessage = null; + var outRoot = Temporary.CreateTempFolder(); try { @@ -180,16 +190,66 @@ public Task CompilerAcceptsAnObjFileAsInput() => [functionObject, programObject], null); - Assert.Equal(nativeResult.ReplaceLineEndings("\n"), cesiumResult.ReplaceLineEndings("\n")); + Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n")); + + totalStopwatch.Stop(); + success = true; + + // Record timing result + timingResult = new TestTimingResult + { + TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput", + TargetFramework = TargetFramework.Net.ToString(), + TargetArch = TargetArch.Dynamic.ToString(), + SourceFiles = ["multi-file/function.c", "multi-file/program.c"], + OperatingSystem = Environment.OSVersion.Platform.ToString(), + NativeCompileTime = nativeResult.CompileTime, + CesiumCompileTime = cesiumResult.CompileTime, + NativeExecutionTime = nativeResult.ExecutionTime, + CesiumExecutionTime = cesiumResult.ExecutionTime, + TotalTime = totalStopwatch.Elapsed, + Success = success + }; + } + catch (Exception ex) + { + totalStopwatch.Stop(); + success = false; + errorMessage = ex.Message; + + // Record failed test timing result + timingResult = new TestTimingResult + { + TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput", + TargetFramework = TargetFramework.Net.ToString(), + TargetArch = TargetArch.Dynamic.ToString(), + SourceFiles = ["multi-file/function.c", "multi-file/program.c"], + OperatingSystem = Environment.OSVersion.Platform.ToString(), + TotalTime = totalStopwatch.Elapsed, + Success = success, + ErrorMessage = errorMessage + }; + throw; } finally { + // Always record the timing result + if (timingResult != null) + { + TestTimingCollector.RecordResult(timingResult); + } + Directory.Delete(outRoot.Value, recursive: true); } }); private async Task DoTest(TargetFramework targetFramework, TargetArch arch, params LocalPath[] relativeSourcePaths) { + var totalStopwatch = Stopwatch.StartNew(); + TestTimingResult? timingResult = null; + bool success = false; + string? errorMessage = null; + var outRoot = Temporary.CreateTempFolder(); try { @@ -214,30 +274,85 @@ private async Task DoTest(TargetFramework targetFramework, TargetArch arch, para .Select(x => SolutionMetadata.SourceRoot / "Cesium.IntegrationTests" / x) .ToList(); - await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent); - await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent); + var nativeResult = await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent); + var cesiumResult = await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent); + + Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n")); + + totalStopwatch.Stop(); + success = true; + + // Record timing result + timingResult = new TestTimingResult + { + TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}", + TargetFramework = targetFramework.ToString(), + TargetArch = arch.ToString(), + SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(), + OperatingSystem = Environment.OSVersion.Platform.ToString(), + NativeCompileTime = nativeResult.CompileTime, + CesiumCompileTime = cesiumResult.CompileTime, + NativeExecutionTime = nativeResult.ExecutionTime, + CesiumExecutionTime = cesiumResult.ExecutionTime, + TotalTime = totalStopwatch.Elapsed, + Success = success + }; + } + catch (Exception ex) + { + totalStopwatch.Stop(); + success = false; + errorMessage = ex.Message; + + // Record failed test timing result + timingResult = new TestTimingResult + { + TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}", + TargetFramework = targetFramework.ToString(), + TargetArch = arch.ToString(), + SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(), + OperatingSystem = Environment.OSVersion.Platform.ToString(), + TotalTime = totalStopwatch.Elapsed, + Success = success, + ErrorMessage = errorMessage + }; + throw; } finally { + // Always record the timing result + if (timingResult != null) + { + TestTimingCollector.RecordResult(timingResult); + } + Directory.Delete(outRoot.Value, recursive: true); } } - private async Task CompileAndRunWithNative( + private async Task CompileAndRunWithNative( AbsolutePath binDir, AbsolutePath objDir, AbsolutePath outRoot, IList sources, string? inputContent) { + var compileStopwatch = Stopwatch.StartNew(); var nativeExecutable = await BuildExecutableWithNativeCompiler(binDir, objDir, sources); + compileStopwatch.Stop(); + var compileTime = compileStopwatch.Elapsed; + + var executionStopwatch = Stopwatch.StartNew(); var nativeResult = await ExecUtil.Run(_output, nativeExecutable, outRoot, [], inputContent); + executionStopwatch.Stop(); + var executionTime = executionStopwatch.Elapsed; + Assert.Equal(42, nativeResult.ExitCode); Assert.Empty(nativeResult.StandardError); - return nativeResult.StandardOutput; + return new TestExecutionResult(nativeResult.StandardOutput, compileTime, executionTime); } - private async Task CompileAndRunWithCesium( + private async Task CompileAndRunWithCesium( AbsolutePath binDir, AbsolutePath objDir, AbsolutePath outRoot, @@ -246,21 +361,29 @@ private async Task CompileAndRunWithCesium( IList inputFiles, string? inputContent) { + var compileStopwatch = Stopwatch.StartNew(); var managedExecutable = await BuildExecutableWithCesium( binDir, objDir, inputFiles, targetFramework, arch); + compileStopwatch.Stop(); + var compileTime = compileStopwatch.Elapsed; + + var executionStopwatch = Stopwatch.StartNew(); var managedResult = await (targetFramework switch { TargetFramework.Net => DotNetCliHelper.RunDotNetDll(_output, outRoot, managedExecutable, inputContent), TargetFramework.NetFramework => ExecUtil.Run(_output, managedExecutable, outRoot, [], inputContent), _ => throw new ArgumentOutOfRangeException(nameof(targetFramework), targetFramework, null) }); + executionStopwatch.Stop(); + var executionTime = executionStopwatch.Elapsed; + Assert.Equal(42, managedResult.ExitCode); Assert.Empty(managedResult.StandardError); - return managedResult.StandardOutput; + return new TestExecutionResult(managedResult.StandardOutput, compileTime, executionTime); } private static readonly object _tempDirCreator = new(); diff --git a/Cesium.TestFramework/TimingHelper.cs b/Cesium.TestFramework/TimingHelper.cs new file mode 100644 index 00000000..d6a33297 --- /dev/null +++ b/Cesium.TestFramework/TimingHelper.cs @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2025 Cesium contributors +// +// SPDX-License-Identifier: MIT + +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Text.Json; +using TruePath; + +namespace Cesium.TestFramework; + +/// +/// Represents timing information for a single test execution. +/// +public record TestTimingResult +{ + public string TestName { get; init; } = string.Empty; + public string TargetFramework { get; init; } = string.Empty; + public string TargetArch { get; init; } = string.Empty; + public string[] SourceFiles { get; init; } = []; + public string OperatingSystem { get; init; } = string.Empty; + + /// Time to compile with native compiler (MSVC on Windows, GCC on Linux/Mac). + public TimeSpan? NativeCompileTime { get; init; } + + /// Time to compile with Cesium compiler. + public TimeSpan? CesiumCompileTime { get; init; } + + /// Time to run the native executable. + public TimeSpan? NativeExecutionTime { get; init; } + + /// Time to run the Cesium-compiled executable. + public TimeSpan? CesiumExecutionTime { get; init; } + + /// Total test time from start to finish. + public TimeSpan TotalTime { get; init; } + + /// Timestamp when the test was executed. + public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow; + + /// Whether the test passed or failed. + public bool Success { get; init; } + + /// Error message if the test failed. + public string? ErrorMessage { get; init; } +} + +/// +/// Collects and aggregates timing results for integration tests. +/// +public static class TestTimingCollector +{ + private static readonly ConcurrentBag _results = new(); + private static readonly object _fileLock = new(); + private static AbsolutePath? _outputDirectory; + + /// + /// Initializes the collector with the output directory for timing results. + /// + public static void Initialize(AbsolutePath outputDirectory) + { + _outputDirectory = outputDirectory; + Directory.CreateDirectory(outputDirectory.Value); + } + + /// + /// Records a timing result. + /// + public static void RecordResult(TestTimingResult result) + { + _results.Add(result); + } + + /// + /// Gets all recorded results. + /// + public static IReadOnlyList GetResults() => _results.ToList(); + + /// + /// Clears all recorded results. + /// + public static void Clear() => _results.Clear(); + + /// + /// Saves all timing results to a JSON file. + /// + public static void SaveToJson() + { + if (_outputDirectory == null) + { + throw new InvalidOperationException("TestTimingCollector has not been initialized with an output directory."); + } + + var results = _results.ToList(); + var timestamp = DateTimeOffset.UtcNow.ToString("yyyyMMdd_HHmmss"); + var fileName = $"integration_test_timing_{timestamp}.json"; + var filePath = _outputDirectory / fileName; + + var jsonOptions = new JsonSerializerOptions + { + WriteIndented = true + }; + + var summary = new + { + GeneratedAt = DateTimeOffset.UtcNow, + OperatingSystem = Environment.OSVersion.Platform.ToString(), + MachineName = Environment.MachineName, + TotalTests = results.Count, + PassedTests = results.Count(r => r.Success), + FailedTests = results.Count(r => !r.Success), + TotalTime = results.Aggregate(TimeSpan.Zero, (sum, r) => sum + r.TotalTime), + AverageNativeCompileTime = results.Where(r => r.NativeCompileTime.HasValue) + .Select(r => r.NativeCompileTime!.Value) + .DefaultIfEmpty(TimeSpan.Zero) + .Average(t => t.TotalMilliseconds), + AverageCesiumCompileTime = results.Where(r => r.CesiumCompileTime.HasValue) + .Select(r => r.CesiumCompileTime!.Value) + .DefaultIfEmpty(TimeSpan.Zero) + .Average(t => t.TotalMilliseconds), + AverageNativeExecutionTime = results.Where(r => r.NativeExecutionTime.HasValue) + .Select(r => r.NativeExecutionTime!.Value) + .DefaultIfEmpty(TimeSpan.Zero) + .Average(t => t.TotalMilliseconds), + AverageCesiumExecutionTime = results.Where(r => r.CesiumExecutionTime.HasValue) + .Select(r => r.CesiumExecutionTime!.Value) + .DefaultIfEmpty(TimeSpan.Zero) + .Average(t => t.TotalMilliseconds), + Results = results + }; + + lock (_fileLock) + { + var json = JsonSerializer.Serialize(summary, jsonOptions); + File.WriteAllText(filePath.Value, json); + } + } +} + +/// +/// Helper class for measuring execution time of operations. +/// +public class TestTimer +{ + private readonly Stopwatch _stopwatch = new(); + private readonly string _operationName; + + public TestTimer(string operationName) + { + _operationName = operationName; + } + + public void Start() + { + _stopwatch.Restart(); + } + + public TimeSpan Stop() + { + _stopwatch.Stop(); + return _stopwatch.Elapsed; + } + + public TimeSpan Elapsed => _stopwatch.Elapsed; + public string OperationName => _operationName; +} \ No newline at end of file