Skip to content

feat(test): Add timing measurement for integration tests (#985) #987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
xingzihai wants to merge 1 commit into from
Closed

feat(test): Add timing measurement for integration tests (#985) #987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions Cesium.IntegrationTests/IntegrationTestContext.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ public class IntegrationTestContext : IAsyncDisposable
private Exception? _initializationException;

public AbsolutePath? VisualStudioPath { get; private set; }

/// <summary>Directory for timing results output.</summary>
public AbsolutePath TimingOutputDirectory { get; private set; }

public async Task WrapTestBody(Func<Task> testBody)
{
Expand Down Expand Up @@ -66,12 +69,26 @@ private async Task InitializeOnce(ITestOutputHelper output)
VisualStudioPath = await WindowsEnvUtil.FindVcCompilerInstallationFolder(output);
}

// Initialize timing output directory
TimingOutputDirectory = SolutionMetadata.SourceRoot / "artifacts" / "timing";
TestTimingCollector.Initialize(TimingOutputDirectory);

await BuildRuntime(output);
await BuildCompiler(output);
}

public async ValueTask DisposeAsync()
{
// Save timing results to JSON file
try
{
TestTimingCollector.SaveToJson();
}
catch
{
// Ignore errors when saving timing results
}

await DotNetCliHelper.ShutdownBuildServer();
}

Expand Down
137 changes: 130 additions & 7 deletions Cesium.IntegrationTests/IntegrationTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
//
// SPDX-License-Identifier: MIT

using System.Diagnostics;
using System.Runtime.InteropServices;
using Cesium.Solution.Metadata;
using Cesium.TestFramework;
Expand All @@ -16,6 +17,10 @@ public class IntegrationTestRunner : IClassFixture<IntegrationTestContext>, IAsy

private readonly ITestOutputHelper _output;
private readonly IntegrationTestContext _context;

/// <summary>Records timing for a test execution.</summary>
private record TestExecutionResult(string Output, TimeSpan CompileTime, TimeSpan ExecutionTime);

public IntegrationTestRunner(IntegrationTestContext context, ITestOutputHelper output)
{
_context = context;
Expand Down Expand Up @@ -155,6 +160,11 @@ public Task MultiFileApplicationCompiles() =>
public Task CompilerAcceptsAnObjFileAsInput() =>
_context.WrapTestBody(async () =>
{
var totalStopwatch = Stopwatch.StartNew();
TestTimingResult? timingResult = null;
bool success = false;
string? errorMessage = null;

var outRoot = Temporary.CreateTempFolder();
try
{
Expand All @@ -180,16 +190,66 @@ public Task CompilerAcceptsAnObjFileAsInput() =>
[functionObject, programObject],
null);

Assert.Equal(nativeResult.ReplaceLineEndings("\n"), cesiumResult.ReplaceLineEndings("\n"));
Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n"));

totalStopwatch.Stop();
success = true;

// Record timing result
timingResult = new TestTimingResult
{
TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput",
TargetFramework = TargetFramework.Net.ToString(),
TargetArch = TargetArch.Dynamic.ToString(),
SourceFiles = ["multi-file/function.c", "multi-file/program.c"],
OperatingSystem = Environment.OSVersion.Platform.ToString(),
NativeCompileTime = nativeResult.CompileTime,
CesiumCompileTime = cesiumResult.CompileTime,
NativeExecutionTime = nativeResult.ExecutionTime,
CesiumExecutionTime = cesiumResult.ExecutionTime,
TotalTime = totalStopwatch.Elapsed,
Success = success
};
}
catch (Exception ex)
{
totalStopwatch.Stop();
success = false;
errorMessage = ex.Message;

// Record failed test timing result
timingResult = new TestTimingResult
{
TestName = "IntegrationTest.CompilerAcceptsAnObjFileAsInput",
TargetFramework = TargetFramework.Net.ToString(),
TargetArch = TargetArch.Dynamic.ToString(),
SourceFiles = ["multi-file/function.c", "multi-file/program.c"],
OperatingSystem = Environment.OSVersion.Platform.ToString(),
TotalTime = totalStopwatch.Elapsed,
Success = success,
ErrorMessage = errorMessage
};
throw;
}
finally
{
// Always record the timing result
if (timingResult != null)
{
TestTimingCollector.RecordResult(timingResult);
}

Directory.Delete(outRoot.Value, recursive: true);
}
});

private async Task DoTest(TargetFramework targetFramework, TargetArch arch, params LocalPath[] relativeSourcePaths)
{
var totalStopwatch = Stopwatch.StartNew();
TestTimingResult? timingResult = null;
bool success = false;
string? errorMessage = null;

var outRoot = Temporary.CreateTempFolder();
try
{
Expand All @@ -214,30 +274,85 @@ private async Task DoTest(TargetFramework targetFramework, TargetArch arch, para
.Select(x => SolutionMetadata.SourceRoot / "Cesium.IntegrationTests" / x)
.ToList();

await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent);
await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent);
var nativeResult = await CompileAndRunWithNative(binDir, objDir, outRoot, sourceFiles, inputContent);
var cesiumResult = await CompileAndRunWithCesium(binDir, objDir, outRoot, targetFramework, arch, sourceFiles, inputContent);

Assert.Equal(nativeResult.Output.ReplaceLineEndings("\n"), cesiumResult.Output.ReplaceLineEndings("\n"));

totalStopwatch.Stop();
success = true;

// Record timing result
timingResult = new TestTimingResult
{
TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}",
TargetFramework = targetFramework.ToString(),
TargetArch = arch.ToString(),
SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(),
OperatingSystem = Environment.OSVersion.Platform.ToString(),
NativeCompileTime = nativeResult.CompileTime,
CesiumCompileTime = cesiumResult.CompileTime,
NativeExecutionTime = nativeResult.ExecutionTime,
CesiumExecutionTime = cesiumResult.ExecutionTime,
TotalTime = totalStopwatch.Elapsed,
Success = success
};
}
catch (Exception ex)
{
totalStopwatch.Stop();
success = false;
errorMessage = ex.Message;

// Record failed test timing result
timingResult = new TestTimingResult
{
TestName = $"IntegrationTest.{targetFramework}.{arch}.{string.Join("_", relativeSourcePaths.Select(p => p.Value.Replace("/", "_").Replace(".c", "")))}",
TargetFramework = targetFramework.ToString(),
TargetArch = arch.ToString(),
SourceFiles = relativeSourcePaths.Select(p => p.Value).ToArray(),
OperatingSystem = Environment.OSVersion.Platform.ToString(),
TotalTime = totalStopwatch.Elapsed,
Success = success,
ErrorMessage = errorMessage
};
throw;
}
finally
{
// Always record the timing result
if (timingResult != null)
{
TestTimingCollector.RecordResult(timingResult);
}

Directory.Delete(outRoot.Value, recursive: true);
}
}

private async Task<string> CompileAndRunWithNative(
private async Task<TestExecutionResult> CompileAndRunWithNative(
AbsolutePath binDir,
AbsolutePath objDir,
AbsolutePath outRoot,
IList<AbsolutePath> sources,
string? inputContent)
{
var compileStopwatch = Stopwatch.StartNew();
var nativeExecutable = await BuildExecutableWithNativeCompiler(binDir, objDir, sources);
compileStopwatch.Stop();
var compileTime = compileStopwatch.Elapsed;

var executionStopwatch = Stopwatch.StartNew();
var nativeResult = await ExecUtil.Run(_output, nativeExecutable, outRoot, [], inputContent);
executionStopwatch.Stop();
var executionTime = executionStopwatch.Elapsed;

Assert.Equal(42, nativeResult.ExitCode);
Assert.Empty(nativeResult.StandardError);
return nativeResult.StandardOutput;
return new TestExecutionResult(nativeResult.StandardOutput, compileTime, executionTime);
}

private async Task<string> CompileAndRunWithCesium(
private async Task<TestExecutionResult> CompileAndRunWithCesium(
AbsolutePath binDir,
AbsolutePath objDir,
AbsolutePath outRoot,
Expand All @@ -246,21 +361,29 @@ private async Task<string> CompileAndRunWithCesium(
IList<AbsolutePath> inputFiles,
string? inputContent)
{
var compileStopwatch = Stopwatch.StartNew();
var managedExecutable = await BuildExecutableWithCesium(
binDir,
objDir,
inputFiles,
targetFramework,
arch);
compileStopwatch.Stop();
var compileTime = compileStopwatch.Elapsed;

var executionStopwatch = Stopwatch.StartNew();
var managedResult = await (targetFramework switch
{
TargetFramework.Net => DotNetCliHelper.RunDotNetDll(_output, outRoot, managedExecutable, inputContent),
TargetFramework.NetFramework => ExecUtil.Run(_output, managedExecutable, outRoot, [], inputContent),
_ => throw new ArgumentOutOfRangeException(nameof(targetFramework), targetFramework, null)
});
executionStopwatch.Stop();
var executionTime = executionStopwatch.Elapsed;

Assert.Equal(42, managedResult.ExitCode);
Assert.Empty(managedResult.StandardError);
return managedResult.StandardOutput;
return new TestExecutionResult(managedResult.StandardOutput, compileTime, executionTime);
}

private static readonly object _tempDirCreator = new();
Expand Down
Loading
Loading