diff --git a/Cesium.IntegrationTests/IntegrationTestContext.cs b/Cesium.IntegrationTests/IntegrationTestContext.cs
index 64dfddb5..fa4411cb 100644
--- a/Cesium.IntegrationTests/IntegrationTestContext.cs
+++ b/Cesium.IntegrationTests/IntegrationTestContext.cs
@@ -24,6 +24,9 @@ public class IntegrationTestContext : IAsyncDisposable
private Exception? _initializationException;
public AbsolutePath? VisualStudioPath { get; private set; }
+
+ /// Directory for timing results output.
+ public AbsolutePath TimingOutputDirectory { get; private set; }
public async Task WrapTestBody(Func testBody)
{
@@ -66,12 +69,26 @@ private async Task InitializeOnce(ITestOutputHelper output)
VisualStudioPath = await WindowsEnvUtil.FindVcCompilerInstallationFolder(output);
}
+ // Initialize timing output directory
+ TimingOutputDirectory = SolutionMetadata.SourceRoot / "artifacts" / "timing";
+ TestTimingCollector.Initialize(TimingOutputDirectory);
+
await BuildRuntime(output);
await BuildCompiler(output);
}
public async ValueTask DisposeAsync()
{
+ // Save timing results to JSON file
+ try
+ {
+ TestTimingCollector.SaveToJson();
+ }
+ catch
+ {
+ // Ignore errors when saving timing results
+ }
+
await DotNetCliHelper.ShutdownBuildServer();
}
diff --git a/Cesium.IntegrationTests/IntegrationTestRunner.cs b/Cesium.IntegrationTests/IntegrationTestRunner.cs
index 0ec7333a..09c447a4 100644
--- a/Cesium.IntegrationTests/IntegrationTestRunner.cs
+++ b/Cesium.IntegrationTests/IntegrationTestRunner.cs
@@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: MIT
+using System.Diagnostics;
using System.Runtime.InteropServices;
using Cesium.Solution.Metadata;
using Cesium.TestFramework;
@@ -16,6 +17,10 @@ public class IntegrationTestRunner : IClassFixture, IAsy
private readonly ITestOutputHelper _output;
private readonly IntegrationTestContext _context;
+
+ /// Records timing for a test execution.
+ private record TestExecutionResult(string Output, TimeSpan CompileTime, TimeSpan ExecutionTime);
+
public IntegrationTestRunner(IntegrationTestContext context, ITestOutputHelper output)
{
_context = context;
@@ -155,6 +160,11 @@ public Task MultiFileApplicationCompiles() =>
public Task CompilerAcceptsAnObjFileAsInput() =>
_context.WrapTestBody(async () =>
{
+ var totalStopwatch = Stopwatch.StartNew();
+ TestTimingResult? timingResult = null;
+ bool success = false;
+ string? errorMessage = null;
+
var outRoot = Temporary.CreateTempFolder();
try
{
@@ -180,16 +190,66 @@ public Task CompilerAcceptsAnObjFileAsInput() =>
[functionObject, programObject],
null);
- Assert.Equal(nativeResult.ReplaceLineEndings("\n"), cesiumResult.ReplaceLineEndings("\n"));
+ Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n"));
+
+ totalStopwatch.Stop();
+ success = true;
+
+ // Record timing result
+ timingResult = new TestTimingResult
+ {
+ TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput",
+ TargetFramework = TargetFramework.Net.ToString(),
+ TargetArch = TargetArch.Dynamic.ToString(),
+ SourceFiles = ["multi-file/function.c", "multi-file/program.c"],
+ OperatingSystem = Environment.OSVersion.Platform.ToString(),
+ NativeCompileTime = nativeResult.CompileTime,
+ CesiumCompileTime = cesiumResult.CompileTime,
+ NativeExecutionTime = nativeResult.ExecutionTime,
+ CesiumExecutionTime = cesiumResult.ExecutionTime,
+ TotalTime = totalStopwatch.Elapsed,
+ Success = success
+ };
+ }
+ catch (Exception ex)
+ {
+ totalStopwatch.Stop();
+ success = false;
+ errorMessage = ex.Message;
+
+ // Record failed test timing result
+ timingResult = new TestTimingResult
+ {
+ TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput",
+ TargetFramework = TargetFramework.Net.ToString(),
+ TargetArch = TargetArch.Dynamic.ToString(),
+ SourceFiles = ["multi-file/function.c", "multi-file/program.c"],
+ OperatingSystem = Environment.OSVersion.Platform.ToString(),
+ TotalTime = totalStopwatch.Elapsed,
+ Success = success,
+ ErrorMessage = errorMessage
+ };
+ throw;
}
finally
{
+ // Always record the timing result
+ if (timingResult != null)
+ {
+ TestTimingCollector.RecordResult(timingResult);
+ }
+
Directory.Delete(outRoot.Value, recursive: true);
}
});
private async Task DoTest(TargetFramework targetFramework, TargetArch arch, params LocalPath[] relativeSourcePaths)
{
+ var totalStopwatch = Stopwatch.StartNew();
+ TestTimingResult? timingResult = null;
+ bool success = false;
+ string? errorMessage = null;
+
var outRoot = Temporary.CreateTempFolder();
try
{
@@ -214,30 +274,85 @@ private async Task DoTest(TargetFramework targetFramework, TargetArch arch, para
.Select(x => SolutionMetadata.SourceRoot / "Cesium.IntegrationTests" / x)
.ToList();
- await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent);
- await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent);
+ var nativeResult = await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent);
+ var cesiumResult = await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent);
+
+ Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n"));
+
+ totalStopwatch.Stop();
+ success = true;
+
+ // Record timing result
+ timingResult = new TestTimingResult
+ {
+ TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}",
+ TargetFramework = targetFramework.ToString(),
+ TargetArch = arch.ToString(),
+ SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(),
+ OperatingSystem = Environment.OSVersion.Platform.ToString(),
+ NativeCompileTime = nativeResult.CompileTime,
+ CesiumCompileTime = cesiumResult.CompileTime,
+ NativeExecutionTime = nativeResult.ExecutionTime,
+ CesiumExecutionTime = cesiumResult.ExecutionTime,
+ TotalTime = totalStopwatch.Elapsed,
+ Success = success
+ };
+ }
+ catch (Exception ex)
+ {
+ totalStopwatch.Stop();
+ success = false;
+ errorMessage = ex.Message;
+
+ // Record failed test timing result
+ timingResult = new TestTimingResult
+ {
+ TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}",
+ TargetFramework = targetFramework.ToString(),
+ TargetArch = arch.ToString(),
+ SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(),
+ OperatingSystem = Environment.OSVersion.Platform.ToString(),
+ TotalTime = totalStopwatch.Elapsed,
+ Success = success,
+ ErrorMessage = errorMessage
+ };
+ throw;
}
finally
{
+ // Always record the timing result
+ if (timingResult != null)
+ {
+ TestTimingCollector.RecordResult(timingResult);
+ }
+
Directory.Delete(outRoot.Value, recursive: true);
}
}
- private async Task CompileAndRunWithNative(
+ private async Task CompileAndRunWithNative(
AbsolutePath binDir,
AbsolutePath objDir,
AbsolutePath outRoot,
IList sources,
string? inputContent)
{
+ var compileStopwatch = Stopwatch.StartNew();
var nativeExecutable = await BuildExecutableWithNativeCompiler(binDir, objDir, sources);
+ compileStopwatch.Stop();
+ var compileTime = compileStopwatch.Elapsed;
+
+ var executionStopwatch = Stopwatch.StartNew();
var nativeResult = await ExecUtil.Run(_output, nativeExecutable, outRoot, [], inputContent);
+ executionStopwatch.Stop();
+ var executionTime = executionStopwatch.Elapsed;
+
Assert.Equal(42, nativeResult.ExitCode);
Assert.Empty(nativeResult.StandardError);
- return nativeResult.StandardOutput;
+ return new TestExecutionResult(nativeResult.StandardOutput, compileTime, executionTime);
}
- private async Task CompileAndRunWithCesium(
+ private async Task CompileAndRunWithCesium(
AbsolutePath binDir,
AbsolutePath objDir,
AbsolutePath outRoot,
@@ -246,21 +361,29 @@ private async Task CompileAndRunWithCesium(
IList inputFiles,
string? inputContent)
{
+ var compileStopwatch = Stopwatch.StartNew();
var managedExecutable = await BuildExecutableWithCesium(
binDir,
objDir,
inputFiles,
targetFramework,
arch);
+ compileStopwatch.Stop();
+ var compileTime = compileStopwatch.Elapsed;
+
+ var executionStopwatch = Stopwatch.StartNew();
var managedResult = await (targetFramework switch
{
TargetFramework.Net => DotNetCliHelper.RunDotNetDll(_output, outRoot, managedExecutable, inputContent),
TargetFramework.NetFramework => ExecUtil.Run(_output, managedExecutable, outRoot, [], inputContent),
_ => throw new ArgumentOutOfRangeException(nameof(targetFramework), targetFramework, null)
});
+ executionStopwatch.Stop();
+ var executionTime = executionStopwatch.Elapsed;
+
Assert.Equal(42, managedResult.ExitCode);
Assert.Empty(managedResult.StandardError);
- return managedResult.StandardOutput;
+ return new TestExecutionResult(managedResult.StandardOutput, compileTime, executionTime);
}
private static readonly object _tempDirCreator = new();
diff --git a/Cesium.TestFramework/TimingHelper.cs b/Cesium.TestFramework/TimingHelper.cs
new file mode 100644
index 00000000..d6a33297
--- /dev/null
+++ b/Cesium.TestFramework/TimingHelper.cs
@@ -0,0 +1,166 @@
+// SPDX-FileCopyrightText: 2025 Cesium contributors
+//
+// SPDX-License-Identifier: MIT
+
+using System.Collections.Concurrent;
+using System.Diagnostics;
+using System.Text.Json;
+using TruePath;
+
+namespace Cesium.TestFramework;
+
+///
+/// Represents timing information for a single test execution.
+///
+public record TestTimingResult
+{
+ public string TestName { get; init; } = string.Empty;
+ public string TargetFramework { get; init; } = string.Empty;
+ public string TargetArch { get; init; } = string.Empty;
+ public string[] SourceFiles { get; init; } = [];
+ public string OperatingSystem { get; init; } = string.Empty;
+
+ /// Time to compile with native compiler (MSVC on Windows, GCC on Linux/Mac).
+ public TimeSpan? NativeCompileTime { get; init; }
+
+ /// Time to compile with Cesium compiler.
+ public TimeSpan? CesiumCompileTime { get; init; }
+
+ /// Time to run the native executable.
+ public TimeSpan? NativeExecutionTime { get; init; }
+
+ /// Time to run the Cesium-compiled executable.
+ public TimeSpan? CesiumExecutionTime { get; init; }
+
+ /// Total test time from start to finish.
+ public TimeSpan TotalTime { get; init; }
+
+ /// Timestamp when the test was executed.
+ public DateTimeOffset Timestamp { get; init; } = DateTimeOffset.UtcNow;
+
+ /// Whether the test passed or failed.
+ public bool Success { get; init; }
+
+ /// Error message if the test failed.
+ public string? ErrorMessage { get; init; }
+}
+
+///
+/// Collects and aggregates timing results for integration tests.
+///
+public static class TestTimingCollector
+{
+ private static readonly ConcurrentBag _results = new();
+ private static readonly object _fileLock = new();
+ private static AbsolutePath? _outputDirectory;
+
+ ///
+ /// Initializes the collector with the output directory for timing results.
+ ///
+ public static void Initialize(AbsolutePath outputDirectory)
+ {
+ _outputDirectory = outputDirectory;
+ Directory.CreateDirectory(outputDirectory.Value);
+ }
+
+ ///
+ /// Records a timing result.
+ ///
+ public static void RecordResult(TestTimingResult result)
+ {
+ _results.Add(result);
+ }
+
+ ///
+ /// Gets all recorded results.
+ ///
+ public static IReadOnlyList GetResults() => _results.ToList();
+
+ ///
+ /// Clears all recorded results.
+ ///
+ public static void Clear() => _results.Clear();
+
+ ///
+ /// Saves all timing results to a JSON file.
+ ///
+ public static void SaveToJson()
+ {
+ if (_outputDirectory == null)
+ {
+ throw new InvalidOperationException("TestTimingCollector has not been initialized with an output directory.");
+ }
+
+ var results = _results.ToList();
+ var timestamp = DateTimeOffset.UtcNow.ToString("yyyyMMdd_HHmmss");
+ var fileName = $"integration_test_timing_{timestamp}.json";
+ var filePath = _outputDirectory / fileName;
+
+ var jsonOptions = new JsonSerializerOptions
+ {
+ WriteIndented = true
+ };
+
+ var summary = new
+ {
+ GeneratedAt = DateTimeOffset.UtcNow,
+ OperatingSystem = Environment.OSVersion.Platform.ToString(),
+ MachineName = Environment.MachineName,
+ TotalTests = results.Count,
+ PassedTests = results.Count(r => r.Success),
+ FailedTests = results.Count(r => !r.Success),
+ TotalTime = results.Aggregate(TimeSpan.Zero, (sum, r) => sum + r.TotalTime),
+ AverageNativeCompileTime = results.Where(r => r.NativeCompileTime.HasValue)
+ .Select(r => r.NativeCompileTime!.Value)
+ .DefaultIfEmpty(TimeSpan.Zero)
+ .Average(t => t.TotalMilliseconds),
+ AverageCesiumCompileTime = results.Where(r => r.CesiumCompileTime.HasValue)
+ .Select(r => r.CesiumCompileTime!.Value)
+ .DefaultIfEmpty(TimeSpan.Zero)
+ .Average(t => t.TotalMilliseconds),
+ AverageNativeExecutionTime = results.Where(r => r.NativeExecutionTime.HasValue)
+ .Select(r => r.NativeExecutionTime!.Value)
+ .DefaultIfEmpty(TimeSpan.Zero)
+ .Average(t => t.TotalMilliseconds),
+ AverageCesiumExecutionTime = results.Where(r => r.CesiumExecutionTime.HasValue)
+ .Select(r => r.CesiumExecutionTime!.Value)
+ .DefaultIfEmpty(TimeSpan.Zero)
+ .Average(t => t.TotalMilliseconds),
+ Results = results
+ };
+
+ lock (_fileLock)
+ {
+ var json = JsonSerializer.Serialize(summary, jsonOptions);
+ File.WriteAllText(filePath.Value, json);
+ }
+ }
+}
+
+///
+/// Helper class for measuring execution time of operations.
+///
+public class TestTimer
+{
+ private readonly Stopwatch _stopwatch = new();
+ private readonly string _operationName;
+
+ public TestTimer(string operationName)
+ {
+ _operationName = operationName;
+ }
+
+ public void Start()
+ {
+ _stopwatch.Restart();
+ }
+
+ public TimeSpan Stop()
+ {
+ _stopwatch.Stop();
+ return _stopwatch.Elapsed;
+ }
+
+ public TimeSpan Elapsed => _stopwatch.Elapsed;
+ public string OperationName => _operationName;
+}
\ No newline at end of file