From 550760dd9730a757b99ddc90cab961ecb5e9a863 Mon Sep 17 00:00:00 2001 From: Slideep Date: Tue, 19 May 2026 10:00:37 +0300 Subject: [PATCH 1/2] Add scenario reports and artifact policies --- Luotsi.Cli.Tests/ScenarioExecutorTests.cs | 563 ++++++++++++++++++ Luotsi.Cli.Tests/TestSupport.cs | 18 +- .../AppHostedCommandCompositionBuilder.cs | 5 +- Luotsi.Cli/Cli/Help.cs | 6 +- .../Cli/Routing/ScenarioCommandDispatcher.cs | 42 +- .../Cli/Routing/ScenarioQueryFactory.cs | 20 +- Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs | 10 +- .../Scenarios/ScenarioBatchExecutorFactory.cs | 7 +- Luotsi.Cli/Scenarios/ScenarioCatalog.cs | 136 ++++- Luotsi.Cli/Scenarios/ScenarioEvents.cs | 50 +- Luotsi.Cli/Scenarios/ScenarioExecutor.cs | 73 ++- .../Scenarios/ScenarioExecutorFactory.cs | 7 +- Luotsi.Cli/Scenarios/ScenarioReports.cs | 539 +++++++++++++++++ Luotsi.Cli/Scenarios/ScenarioValidator.cs | 4 +- 14 files changed, 1422 insertions(+), 58 deletions(-) create mode 100644 Luotsi.Cli/Scenarios/ScenarioReports.cs diff --git a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs index 40639e4..d84eeab 100644 --- a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs +++ b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using System.Xml.Linq; using Luotsi.Cli.Artifacts; using Luotsi.Cli.Cli; using Luotsi.Cli.Errors; @@ -184,6 +185,46 @@ public async Task RunAsync_Path_DryRun_Does_Not_Write_Jsonl_Events() Assert.False(fileSystem.FileExists("/tmp/events.jsonl")); } + [Fact] + public async Task ScenarioList_RecursiveGlob_Finds_Nested_Scenarios() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/a.json", """ + { + "name": "root", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + fileSystem.AddFile("/tmp/scenarios/nested/b.json", """ + { + "name": "nested", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["scenario-list", "--path", "/tmp/scenarios/**/*.json"]); + using var envelope = console.ParseSingleOutputAsJson(); + + Assert.Equal(0, exitCode); + var scenarios = envelope.RootElement.GetProperty("data").GetProperty("scenarios"); + Assert.Equal(["nested", "root"], scenarios.EnumerateArray().Select(static scenario => scenario.GetProperty("name").GetString()!).Order(StringComparer.Ordinal).ToArray()); + } + [Fact] public async Task RunAsync_File_DryRun_Returns_Usage_Error() @@ -476,6 +517,416 @@ public async Task RunAsync_Path_Writes_Jsonl_Run_And_Per_Scenario_Events() Assert.Equal("scenario_run_ended", events[^1].GetProperty("event").GetString()); Assert.Equal("passed", events[^1].GetProperty("status").GetString()); Assert.Equal(2, events[^1].GetProperty("passed_count").GetInt32()); + Assert.All(events.Where(static evt => evt.TryGetProperty("scenario", out _)), evt => Assert.Contains("::", evt.GetProperty("scenario_id").GetString(), StringComparison.Ordinal)); + } + + [Fact] + public async Task RunAsync_Path_PlanningFailure_Writes_Terminal_Event_And_Report() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/broken.json", "{"); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync([ + "run", + "--path", "/tmp/scenarios", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(2, exitCode); + Assert.Equal(["scenario_run_started", "scenario_run_ended"], events.Select(static evt => evt.GetProperty("event").GetString()!).ToArray()); + Assert.Equal("failed", events[^1].GetProperty("status").GetString()); + Assert.Equal("usage_error", events[^1].GetProperty("error").GetProperty("category").GetString()); + Assert.Equal("failed", report.RootElement.GetProperty("status").GetString()); + Assert.Equal("usage_error", report.RootElement.GetProperty("error").GetProperty("category").GetString()); + Assert.Equal(1, report.RootElement.GetProperty("failed_count").GetInt32()); + Assert.Equal("scenario discovery", report.RootElement.GetProperty("scenarios")[0].GetProperty("scenario").GetString()); + } + + [Fact] + public async Task RunAsync_Path_PostPlanUsageFailure_Writes_Batch_Metadata_Report() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/broken.json", """ + { + "name": "broken", + "steps": [ + { "action": "notYetImplemented" } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync([ + "run", + "--path", "/tmp/scenarios", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(2, exitCode); + Assert.Equal(1, events[^1].GetProperty("total_count").GetInt32()); + Assert.Equal(1, events[^1].GetProperty("selected_count").GetInt32()); + Assert.Equal(1, report.RootElement.GetProperty("total_count").GetInt32()); + Assert.Equal(1, report.RootElement.GetProperty("selected_count").GetInt32()); + Assert.Equal(1, report.RootElement.GetProperty("failed_count").GetInt32()); + Assert.Equal("scenario run", report.RootElement.GetProperty("scenarios")[0].GetProperty("scenario").GetString()); + } + + [Fact] + public async Task RunAsync_File_Writes_Json_Report() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "name": "pause", "action": "sleep", "milliseconds": 1 } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(0, exitCode); + Assert.Equal("luotsi-scenario-run-report.v1", report.RootElement.GetProperty("schema").GetString()); + Assert.Equal("passed", report.RootElement.GetProperty("status").GetString()); + Assert.Equal(1, report.RootElement.GetProperty("passed_count").GetInt32()); + Assert.Equal("single", report.RootElement.GetProperty("scenarios")[0].GetProperty("scenario").GetString()); + Assert.Equal("/tmp/scenario.json::single", report.RootElement.GetProperty("scenarios")[0].GetProperty("scenario_id").GetString()); + Assert.Equal("/tmp/scenario.json", report.RootElement.GetProperty("scenarios")[0].GetProperty("file").GetString()); + Assert.Equal("sleep", report.RootElement.GetProperty("scenarios")[0].GetProperty("steps")[0].GetProperty("action").GetString()); + } + + [Fact] + public async Task RunAsync_Path_Writes_JUnit_Report_For_Mixed_Result() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/fails.json", """ + { + "name": "fails", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + fileSystem.AddFile("/tmp/scenarios/passes.json", """ + { + "name": "passes", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + var host = new FakeDeviceHost + { + WaitVisibleException = new InvalidOperationException("not visible") + }; + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--path", "/tmp/scenarios", "--report-junit", "/tmp/junit.xml"]); + var report = XDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/junit.xml")); + + Assert.Equal(1, exitCode); + Assert.Equal("testsuite", report.Root!.Name.LocalName); + Assert.Equal("2", report.Root.Attribute("tests")!.Value); + Assert.Equal("1", report.Root.Attribute("failures")!.Value); + var failed = report.Root.Elements("testcase").Single(test => test.Attribute("name")!.Value == "fails"); + Assert.NotNull(failed.Element("failure")); + Assert.Equal("/tmp/scenarios/fails.json", failed.Attribute("classname")!.Value); + Assert.Equal("/tmp/scenarios/fails.json::fails", failed.Attribute("id")!.Value); + } + + [Fact] + public async Task RunAsync_File_FailureArtifactCaptureFailure_Still_Emits_Step_Failure() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = new FakeDeviceHost + { + WaitVisibleException = new InvalidOperationException("not visible"), + FailureArtifactException = new IOException("artifact disk full") + }; + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync([ + "run", + "--file", "/tmp/scenario.json", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(1, exitCode); + var failedStep = Assert.Single(events, static evt => evt.GetProperty("event").GetString() == "scenario_step_failed"); + Assert.Equal("not visible", failedStep.GetProperty("error").GetProperty("message").GetString()); + Assert.DoesNotContain("artifact disk full", report.RootElement.GetProperty("scenarios")[0].GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_File_Failure_Report_Honors_AttachArtifacts_Never() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = CreateFailingHostWithArtifacts(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json", "--attach-artifacts", "never"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(1, exitCode); + Assert.Equal("failed", report.RootElement.GetProperty("status").GetString()); + Assert.Equal(0, report.RootElement.GetProperty("scenarios")[0].GetProperty("artifacts").GetArrayLength()); + } + + [Fact] + public async Task RunAsync_File_Failure_Report_Attaches_Failure_Artifacts_By_Default() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = CreateFailingHostWithArtifacts(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + var artifacts = report.RootElement.GetProperty("scenarios")[0].GetProperty("artifacts"); + + Assert.Equal(1, exitCode); + Assert.Equal(2, artifacts.GetArrayLength()); + Assert.Equal("screenshot", artifacts[0].GetProperty("kind").GetString()); + Assert.Equal("failure.png", artifacts[0].GetProperty("file_name").GetString()); + Assert.Equal("metadata", artifacts[1].GetProperty("kind").GetString()); + } + + [Fact] + public async Task RunAsync_File_JUnit_Report_Attaches_Failure_Artifacts_In_SystemOut() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = CreateFailingHostWithArtifacts(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-junit", "/tmp/junit.xml"]); + var report = XDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/junit.xml")); + var systemOut = Assert.Single(report.Root!.Elements("testcase")).Element("system-out")!.Value; + + Assert.Equal(1, exitCode); + Assert.Contains("screenshot: failure.png", systemOut, StringComparison.Ordinal); + Assert.Contains("metadata: failure.json", systemOut, StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_File_CaptureOnNever_Skips_Failure_Artifact_Generation() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = CreateFailingHostWithArtifacts(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json", "--capture-on", "never"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(1, exitCode); + Assert.Empty(host.FailureArtifactRequests); + Assert.Equal(0, report.RootElement.GetProperty("scenarios")[0].GetProperty("artifacts").GetArrayLength()); + } + + [Fact] + public async Task RunAsync_File_Report_Attaches_Step_Artifacts_When_Always() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "captureArtifacts", "label": "checkpoint" } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json", "--attach-artifacts", "always"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + var artifacts = report.RootElement.GetProperty("scenarios")[0].GetProperty("artifacts"); + + Assert.Equal(0, exitCode); + Assert.Equal(4, artifacts.GetArrayLength()); + Assert.Contains(artifacts.EnumerateArray(), artifact => artifact.GetProperty("kind").GetString() == "logcat"); + } + + [Fact] + public async Task RunAsync_File_Failure_Report_Attaches_Prior_Step_Artifacts_When_Always() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "single", + "steps": [ + { "action": "captureArtifacts", "label": "checkpoint" }, + { "action": "waitVisible", "text": "Target" } + ] + } + """); + var host = CreateFailingHostWithArtifacts(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--report-json", "/tmp/report.json", "--attach-artifacts", "always"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + var artifacts = report.RootElement.GetProperty("scenarios")[0].GetProperty("artifacts"); + + Assert.Equal(1, exitCode); + Assert.Contains(artifacts.EnumerateArray(), artifact => artifact.GetProperty("file_name").GetString() == "checkpoint.png"); + Assert.Contains(artifacts.EnumerateArray(), artifact => artifact.GetProperty("file_name").GetString() == "failure.png"); } [Fact] @@ -574,6 +1025,75 @@ public async Task RunAsync_Path_Executes_Deterministic_Shard_Order() Assert.Equal("c", data.GetProperty("scenarios")[1].GetProperty("scenario").GetString()); } + [Fact] + public async Task RunAsync_Path_DryRun_Can_Use_Hash_Shard_Strategy() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + foreach (var name in new[] { "a", "b", "c" }) + { + fileSystem.AddFile($"/tmp/scenarios/{name}.json", $$""" + { + "name": "{{name}}", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + } + + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--path", "/tmp/scenarios", "--dry-run", "--shard-count", "2", "--shard-index", "0", "--shard-strategy", "hash"]); + using var envelope = console.ParseSingleOutputAsJson(); + var data = envelope.RootElement.GetProperty("data"); + + Assert.Equal(0, exitCode); + Assert.Equal("hash", data.GetProperty("shard_strategy").GetString()); + Assert.Equal(3, data.GetProperty("matched_count").GetInt32()); + Assert.Equal(3, data.GetProperty("selected_count").GetInt32() + data.GetProperty("sharded_out_count").GetInt32()); + } + + [Fact] + public async Task RunAsync_Path_InvalidShardStrategy_Returns_Usage_Error() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/a.json", """ + { + "name": "a", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--path", "/tmp/scenarios", "--dry-run", "--shard-strategy", "round-robin"]); + using var envelope = console.ParseSingleOutputAsJson(); + + Assert.Equal(2, exitCode); + Assert.Contains("--shard-strategy", envelope.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + } + [Theory] [InlineData("0", "0", "--shard-count must be greater than zero.")] @@ -673,6 +1193,27 @@ public async Task RunScenarioAsync_Unknown_Action_Throws_UsageException() Assert.Contains("Unknown scenario action 'launchApp'", error.Message, StringComparison.Ordinal); } + [Fact] + public async Task RunScenarioAsync_Missing_Action_Throws_UsageException() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var runner = new DeviceRunner(new FakeAdbClient(), ArtifactSession.Create(CliOptions.Parse(["run"]), fileSystem, timeProvider), timeProvider, new FakeDelay(timeProvider), fileSystem); + var scenarios = new ScenarioExecutor(runner, fileSystem, timeProvider, new FakeDelay(timeProvider)); + var scenarioPath = "/tmp/missing-action.json"; + fileSystem.AddFile(scenarioPath, """ + { + "name": "broken", + "steps": [ + { "name": "missing action" } + ] + } + """); + + var error = await Assert.ThrowsAsync(() => scenarios.RunAsync(scenarioPath)); + + Assert.Contains("must define a non-empty action", error.Message, StringComparison.Ordinal); + } [Fact] public async Task RunScenarioAsync_Corrupted_NonEmpty_Json_Throws_UsageException() @@ -983,4 +1524,26 @@ private static JsonElement[] ReadJsonlEvents(FakeFileSystem fileSystem, string p .Select(static line => JsonDocument.Parse(line).RootElement.Clone()) .ToArray(); + private static FakeDeviceHost CreateFailingHostWithArtifacts() => + new() + { + WaitVisibleException = new InvalidOperationException("not visible"), + FailureArtifacts = new FailureArtifactBundle( + ResultSchemas.FailureBundle, + DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind), + "scenario", + "single", + "/tmp/scenario.json", + 1, + "waitVisible", + "waitVisible", + typeof(InvalidOperationException).FullName!, + "not visible", + [new FailureArtifact("screenshot", "failure.png")], + []) + { + MetadataFile = "failure.json" + } + }; + } diff --git a/Luotsi.Cli.Tests/TestSupport.cs b/Luotsi.Cli.Tests/TestSupport.cs index da9780e..c3adc46 100644 --- a/Luotsi.Cli.Tests/TestSupport.cs +++ b/Luotsi.Cli.Tests/TestSupport.cs @@ -508,6 +508,12 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic public Exception? WaitVisibleException { get; set; } + public FailureArtifactBundle? FailureArtifacts { get; set; } + + public Exception? FailureArtifactException { get; set; } + + public List FailureArtifactRequests { get; } = []; + public Task GetDevicesAsync() { if (GetDevicesException is not null) @@ -842,8 +848,16 @@ public Task WriteDeviceFingerprintAsync() return Task.FromResult(new DeviceFingerprint(ResultSchemas.DeviceFingerprint, DateTimeOffset.UtcNow, "SER", "Model", "16", "36", "fingerprint", "arm64-v8a", "focus")); } - public Task CaptureFailureArtifactsAsync(FailureCaptureRequest request, Exception exception) => - Task.FromResult(new FailureArtifactBundle(ResultSchemas.FailureBundle, DateTimeOffset.UtcNow, request.Scope, request.Name, request.File, request.StepIndex, request.StepName, request.Action, exception.GetType().FullName ?? exception.GetType().Name, exception.Message, [], [])); + public Task CaptureFailureArtifactsAsync(FailureCaptureRequest request, Exception exception) + { + FailureArtifactRequests.Add(request); + if (FailureArtifactException is not null) + { + throw FailureArtifactException; + } + + return Task.FromResult(FailureArtifacts ?? new FailureArtifactBundle(ResultSchemas.FailureBundle, DateTimeOffset.UtcNow, request.Scope, request.Name, request.File, request.StepIndex, request.StepName, request.Action, exception.GetType().FullName ?? exception.GetType().Name, exception.Message, [], [])); + } public Task LogcatAsync(int tail) => Task.FromResult(new LogcatResult([])); diff --git a/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs b/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs index 48c75d7..f18a733 100644 --- a/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs +++ b/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs @@ -18,10 +18,11 @@ public static AppHostedCommandComposition Build(AppHostedCommandCompositionBuild var scenarioExecutorFactory = new ScenarioExecutorFactory(dependencies.FileSystem, dependencies.TimeProvider, dependencies.Delay, scenarioTemplateResolver); var scenarioBatchExecutorFactory = new ScenarioBatchExecutorFactory(scenarioExecutorFactory); var scenarioRunEventCoordinatorFactory = new ScenarioRunEventCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider); + var scenarioRunReportCoordinatorFactory = new ScenarioRunReportCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider); var envelopeWriter = new AppCommandEnvelopeWriter(dependencies.Console, dependencies.TimeProvider); var commandDispatcher = new AppCommandDispatcher( new AdbSubcommandDispatcher(), - new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioExecutorFactory, scenarioBatchExecutorFactory, scenarioRunEventCoordinatorFactory), + new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioExecutorFactory, scenarioBatchExecutorFactory, scenarioRunEventCoordinatorFactory, scenarioRunReportCoordinatorFactory), dependencies.ProfileCoordinator); return new( @@ -44,4 +45,4 @@ internal sealed record AppHostedCommandCompositionBuilderDependencies( internal sealed record AppHostedCommandComposition( AppCommandEnvelopeWriter EnvelopeWriter, - AppCommandHost CommandHost); \ No newline at end of file + AppCommandHost CommandHost); diff --git a/Luotsi.Cli/Cli/Help.cs b/Luotsi.Cli/Cli/Help.cs index c4f8989..b749a06 100644 --- a/Luotsi.Cli/Cli/Help.cs +++ b/Luotsi.Cli/Cli/Help.cs @@ -58,7 +58,7 @@ view setup --device [--profile ] [--preset safe|balanced|high list-installed-packages [--third-party] grant-permission --package --permission revoke-permission --package --permission - scenario-list --path [--include-tag ] [--exclude-tag ] [--name ] [--action ] + scenario-list --path [--include-tag ] [--exclude-tag ] [--name ] [--action ] telemetry-tail [--tail 200] telemetry-watch [--timeout-sec 15] wait-step --step [--timeout-sec 15] @@ -71,8 +71,8 @@ view setup --device [--profile ] [--preset safe|balanced|high logcat [--tail 200] wait-log --contains [--timeout-sec 15] record --output [--time-limit-sec 30] - run --file [--events-jsonl ] - run --path [--dry-run] [--events-jsonl ] [--include-tag ] [--exclude-tag ] [--name ] [--action ] [--shard-count --shard-index ] + run --file [--events-jsonl ] [--report-json ] [--report-junit ] [--capture-on failure|never] [--attach-artifacts never|on-failure|always] + run --path [--dry-run] [--events-jsonl ] [--report-json ] [--report-junit ] [--capture-on failure|never] [--attach-artifacts never|on-failure|always] [--include-tag ] [--exclude-tag ] [--name ] [--action ] [--shard-count --shard-index ] [--shard-strategy index|hash] Common options: --device diff --git a/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs b/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs index 3487568..21968d7 100644 --- a/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs +++ b/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs @@ -8,12 +8,14 @@ internal sealed class ScenarioCommandDispatcher( ScenarioRunPlanner runPlanner, ScenarioExecutorFactory scenarioExecutorFactory, ScenarioBatchExecutorFactory scenarioBatchExecutorFactory, - ScenarioRunEventCoordinatorFactory scenarioRunEventCoordinatorFactory) + ScenarioRunEventCoordinatorFactory scenarioRunEventCoordinatorFactory, + ScenarioRunReportCoordinatorFactory scenarioRunReportCoordinatorFactory) { private readonly ScenarioRunPlanner _runPlanner = runPlanner ?? throw new ArgumentNullException(nameof(runPlanner)); private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory)); private readonly ScenarioBatchExecutorFactory _scenarioBatchExecutorFactory = scenarioBatchExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioBatchExecutorFactory)); private readonly ScenarioRunEventCoordinatorFactory _scenarioRunEventCoordinatorFactory = scenarioRunEventCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunEventCoordinatorFactory)); + private readonly ScenarioRunReportCoordinatorFactory _scenarioRunReportCoordinatorFactory = scenarioRunReportCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunReportCoordinatorFactory)); public async Task ListAsync(CliOptions options) { @@ -29,6 +31,8 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(runner); + var failureArtifactCapturePolicy = ParseFailureArtifactCapturePolicy(options.Get("capture-on")); + if (!ScenarioQueryFactory.UsesCatalogExecution(options)) { if (options.HasFlag("dry-run")) @@ -38,16 +42,18 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) var file = options.Require("file"); await using var singleRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl")); - return await singleRunEvents.RunFileAsync( + var singleRunReports = _scenarioRunReportCoordinatorFactory.Create(options); + return await singleRunReports.RunFileAsync( file, - sink => _scenarioExecutorFactory.Create(runner, sink).RunAsync(file)).ConfigureAwait(false); + () => singleRunEvents.RunFileAsync( + file, + sink => _scenarioExecutorFactory.Create(runner, sink, failureArtifactCapturePolicy).RunAsync(file))).ConfigureAwait(false); } var query = ScenarioQueryFactory.CreateCatalogRunQuery(options); - var plan = await _runPlanner.CreateAsync(query).ConfigureAwait(false); - if (query.DryRun) { + var plan = await _runPlanner.CreateAsync(query).ConfigureAwait(false); return new ScenarioRunPlanResult( query.Path, true, @@ -57,12 +63,32 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) plan.ShardedOutCount, query.ShardCount, query.ShardIndex, + query.ShardStrategy, plan.SelectedScenarios); } await using var batchRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl")); - return await batchRunEvents.RunBatchAsync( - plan, - sink => _scenarioBatchExecutorFactory.Create(runner, sink).RunAsync(plan)).ConfigureAwait(false); + var batchRunReports = _scenarioRunReportCoordinatorFactory.Create(options); + return await batchRunEvents.RunPathAsync( + query, + _ => batchRunReports.PlanPathAsync(query, () => _runPlanner.CreateAsync(query)), + (preparedPlan, sink) => batchRunReports.RunBatchAsync( + preparedPlan, + () => _scenarioBatchExecutorFactory.Create(runner, sink, failureArtifactCapturePolicy).RunAsync(preparedPlan))).ConfigureAwait(false); + } + + private static ScenarioFailureArtifactCapturePolicy ParseFailureArtifactCapturePolicy(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ScenarioFailureArtifactCapturePolicy.Failure; + } + + return value.Trim().ToLowerInvariant() switch + { + "failure" or "on-failure" or "onfailure" => ScenarioFailureArtifactCapturePolicy.Failure, + "never" => ScenarioFailureArtifactCapturePolicy.Never, + _ => throw new UsageException("--capture-on must be one of: failure, never.") + }; } } diff --git a/Luotsi.Cli/Cli/Routing/ScenarioQueryFactory.cs b/Luotsi.Cli/Cli/Routing/ScenarioQueryFactory.cs index dfdd59b..8b71777 100644 --- a/Luotsi.Cli/Cli/Routing/ScenarioQueryFactory.cs +++ b/Luotsi.Cli/Cli/Routing/ScenarioQueryFactory.cs @@ -42,11 +42,27 @@ private static ScenarioQuery CreateQuery(CliOptions options, bool requirePath) options.Get("action"), shardCount, shardIndex, - options.HasFlag("dry-run")); + options.HasFlag("dry-run"), + NormalizeShardStrategy(options.Get("shard-strategy"))); } private static string[] SplitOption(string? value) => string.IsNullOrWhiteSpace(value) ? [] : value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries); -} \ No newline at end of file + + private static string NormalizeShardStrategy(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ScenarioShardStrategies.Index; + } + + return value.Trim().ToLowerInvariant() switch + { + ScenarioShardStrategies.Index => ScenarioShardStrategies.Index, + ScenarioShardStrategies.Hash => ScenarioShardStrategies.Hash, + _ => throw new UsageException("--shard-strategy must be one of: index, hash.") + }; + } +} diff --git a/Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs b/Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs index a2e9627..0fb540f 100644 --- a/Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs +++ b/Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs @@ -18,7 +18,7 @@ public async Task RunAsync(ScenarioRunPlan plan) { try { - results.Add(ScenarioBatchItemResult.FromSuccess(await _scenarios.RunAsync(scenario.File).ConfigureAwait(false))); + results.Add(ScenarioBatchItemResult.FromSuccess(await _scenarios.RunAsync(scenario.File).ConfigureAwait(false), scenario)); passedCount++; } catch (Exception ex) when (ex is not UsageException) @@ -39,7 +39,8 @@ public async Task RunAsync(ScenarioRunPlan plan) plan.ShardedOutCount, plan.Query.ShardCount, plan.Query.ShardIndex, - results); + results, + plan.Query.ShardStrategy); } private static ScenarioBatchItemResult CreateFailureResult(ScenarioCatalogEntry scenario, Exception exception) @@ -49,6 +50,7 @@ private static ScenarioBatchItemResult CreateFailureResult(ScenarioCatalogEntry scenario.Name, scenario.File, failure?.DataPayload as ScenarioRunFailureData, - ScenarioErrorInfo.From(exception)); + ScenarioErrorInfo.From(exception), + scenario.Id); } -} \ No newline at end of file +} diff --git a/Luotsi.Cli/Scenarios/ScenarioBatchExecutorFactory.cs b/Luotsi.Cli/Scenarios/ScenarioBatchExecutorFactory.cs index f55171c..64c6e42 100644 --- a/Luotsi.Cli/Scenarios/ScenarioBatchExecutorFactory.cs +++ b/Luotsi.Cli/Scenarios/ScenarioBatchExecutorFactory.cs @@ -6,9 +6,12 @@ internal sealed class ScenarioBatchExecutorFactory(ScenarioExecutorFactory scena { private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory)); - public ScenarioBatchExecutor Create(IDeviceHost runner, IScenarioEventSink? eventSink = null) + public ScenarioBatchExecutor Create( + IDeviceHost runner, + IScenarioEventSink? eventSink = null, + ScenarioFailureArtifactCapturePolicy failureArtifactCapturePolicy = ScenarioFailureArtifactCapturePolicy.Failure) { ArgumentNullException.ThrowIfNull(runner); - return new ScenarioBatchExecutor(_scenarioExecutorFactory.Create(runner, eventSink)); + return new ScenarioBatchExecutor(_scenarioExecutorFactory.Create(runner, eventSink, failureArtifactCapturePolicy)); } } diff --git a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs index 8c5602c..ae8ae18 100644 --- a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs +++ b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs @@ -29,6 +29,7 @@ public sealed record ScenarioRunPlanResult( int ShardedOutCount, int? ShardCount, int? ShardIndex, + string ShardStrategy, IReadOnlyList Scenarios); public sealed record ScenarioRunTiming( @@ -63,7 +64,9 @@ public sealed record ScenarioRunResult( string Scenario, string Status, ScenarioRunTiming Timing, - IReadOnlyList Steps); + IReadOnlyList Steps, + string? ScenarioId = null, + string? File = null); public sealed record ScenarioRunFailureData( string Scenario, @@ -72,7 +75,8 @@ public sealed record ScenarioRunFailureData( ScenarioRunTiming Timing, ScenarioFailedStepResult FailedStep, IReadOnlyList Steps, - FailureArtifactBundle FailureArtifacts); + FailureArtifactBundle FailureArtifacts, + string? ScenarioId = null); public sealed record ScenarioBatchItemResult( string Scenario, @@ -81,18 +85,31 @@ public sealed record ScenarioBatchItemResult( IReadOnlyList? Steps = null, string? File = null, ScenarioRunFailureData? Data = null, - ErrorInfo? Error = null) + ErrorInfo? Error = null, + string? ScenarioId = null) { - public static ScenarioBatchItemResult FromSuccess(ScenarioRunResult result) + public static ScenarioBatchItemResult FromSuccess(ScenarioRunResult result, ScenarioCatalogEntry? catalogEntry = null) { ArgumentNullException.ThrowIfNull(result); - return new ScenarioBatchItemResult(result.Scenario, result.Status, result.Timing, result.Steps); + return new ScenarioBatchItemResult( + result.Scenario, + result.Status, + result.Timing, + result.Steps, + result.File ?? catalogEntry?.File, + ScenarioId: result.ScenarioId ?? catalogEntry?.Id); } - public static ScenarioBatchItemResult FromFailure(string scenario, string file, ScenarioRunFailureData? data, ErrorInfo error) + public static ScenarioBatchItemResult FromFailure(string scenario, string file, ScenarioRunFailureData? data, ErrorInfo error, string? scenarioId = null) { ArgumentNullException.ThrowIfNull(error); - return new ScenarioBatchItemResult(scenario, "failed", File: file, Data: data, Error: error); + return new ScenarioBatchItemResult( + scenario, + "failed", + File: file, + Data: data, + Error: error, + ScenarioId: scenarioId ?? data?.ScenarioId ?? ScenarioIdentity.Create(file, scenario)); } } @@ -107,7 +124,8 @@ public sealed record ScenarioRunBatchResult( int ShardedOutCount, int? ShardCount, int? ShardIndex, - IReadOnlyList Scenarios); + IReadOnlyList Scenarios, + string ShardStrategy = ScenarioShardStrategies.Index); public sealed record ScenarioQuery( string Path, @@ -117,7 +135,25 @@ public sealed record ScenarioQuery( string? Action, int? ShardCount, int? ShardIndex, - bool DryRun); + bool DryRun, + string ShardStrategy = ScenarioShardStrategies.Index); + +public static class ScenarioShardStrategies +{ + public const string Index = "index"; + public const string Hash = "hash"; +} + +internal enum ScenarioFailureArtifactCapturePolicy +{ + Failure, + Never +} + +public static class ScenarioIdentity +{ + public static string Create(string file, string scenarioName) => $"{file}::{scenarioName}"; +} internal sealed class ScenarioCatalog( IFileSystem fileSystem, @@ -198,11 +234,18 @@ public static IReadOnlyList SelectShard(IReadOnlyList new {entry, index}) - .Where(item => item.index % shardCount == shardIndex) - .Select(static item => item.entry) - .ToArray(); + return query.ShardStrategy.ToLowerInvariant() switch + { + ScenarioShardStrategies.Index => entries + .Select((entry, index) => new {entry, index}) + .Where(item => item.index % shardCount == shardIndex) + .Select(static item => item.entry) + .ToArray(), + ScenarioShardStrategies.Hash => entries + .Where(entry => GetStableShardIndex(entry.Id, shardCount) == shardIndex) + .ToArray(), + _ => throw new UsageException("--shard-strategy must be one of: index, hash.") + }; } private string[] ResolveScenarioFiles(string path) @@ -229,6 +272,18 @@ private string[] ResolveScenarioFiles(string path) throw new UsageException($"Scenario path '{path}' does not exist."); } + if (TrySplitRecursiveGlob(path, out var recursiveRoot, out var recursivePattern)) + { + if (!_fileSystem.DirectoryExists(recursiveRoot)) + { + throw new UsageException($"Scenario path '{path}' does not exist."); + } + + return _fileSystem.GetFiles(recursiveRoot, recursivePattern, SearchOption.AllDirectories) + .OrderBy(static file => file, StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + var directory = Path.GetDirectoryName(path); var searchRoot = string.IsNullOrWhiteSpace(directory) ? "." : directory; @@ -245,6 +300,40 @@ private string[] ResolveScenarioFiles(string path) .ToArray(); } + private static bool TrySplitRecursiveGlob(string path, out string root, out string pattern) + { + var normalized = path.Replace('\\', '/'); + if (normalized.StartsWith("**/", StringComparison.Ordinal)) + { + root = "."; + pattern = Path.GetFileName(normalized[3..]); + if (string.IsNullOrWhiteSpace(pattern)) + { + pattern = "*.json"; + } + + return true; + } + + var markerIndex = normalized.IndexOf("/**/", StringComparison.Ordinal); + if (markerIndex < 0) + { + root = string.Empty; + pattern = string.Empty; + return false; + } + + root = markerIndex == 0 ? "." : path[..markerIndex]; + var remainder = normalized[(markerIndex + 4)..]; + pattern = Path.GetFileName(remainder); + if (string.IsNullOrWhiteSpace(pattern)) + { + pattern = "*.json"; + } + + return true; + } + private async Task LoadAsync(string file) { if (!_fileSystem.FileExists(file)) @@ -279,11 +368,28 @@ private static ScenarioCatalogEntry ToEntry(string file, ScenarioFile scenario) .ToArray(); return new ScenarioCatalogEntry( - $"{file}::{scenario.Name}", + ScenarioIdentity.Create(file, scenario.Name), scenario.Name, file, tags, scenario.Steps.Count, actions); } + + private static int GetStableShardIndex(string value, int shardCount) + { + unchecked + { + const uint offsetBasis = 2166136261; + const uint prime = 16777619; + var hash = offsetBasis; + foreach (var character in value) + { + hash ^= char.ToUpperInvariant(character); + hash *= prime; + } + + return (int)(hash % (uint)shardCount); + } + } } diff --git a/Luotsi.Cli/Scenarios/ScenarioEvents.cs b/Luotsi.Cli/Scenarios/ScenarioEvents.cs index 0e75e08..97b3594 100644 --- a/Luotsi.Cli/Scenarios/ScenarioEvents.cs +++ b/Luotsi.Cli/Scenarios/ScenarioEvents.cs @@ -118,7 +118,8 @@ public async Task RunBatchAsync(ScenarioRunPlan plan, Fu SelectedCount: plan.SelectedCount, ShardedOutCount: plan.ShardedOutCount, ShardCount: plan.Query.ShardCount, - ShardIndex: plan.Query.ShardIndex), + ShardIndex: plan.Query.ShardIndex, + ShardStrategy: plan.Query.ShardStrategy), runAsync, result => new ScenarioEvent( "scenario_run_ended", @@ -132,7 +133,8 @@ public async Task RunBatchAsync(ScenarioRunPlan plan, Fu FailedCount: result.FailedCount, ShardedOutCount: result.ShardedOutCount, ShardCount: result.ShardCount, - ShardIndex: result.ShardIndex), + ShardIndex: result.ShardIndex, + ShardStrategy: result.ShardStrategy), ex => CreateFailedRunEndedEvent( plan.Query.Path, ex, @@ -141,7 +143,43 @@ public async Task RunBatchAsync(ScenarioRunPlan plan, Fu selectedCount: plan.SelectedCount, shardedOutCount: plan.ShardedOutCount, shardCount: plan.Query.ShardCount, - shardIndex: plan.Query.ShardIndex)).ConfigureAwait(false); + shardIndex: plan.Query.ShardIndex, + shardStrategy: plan.Query.ShardStrategy)).ConfigureAwait(false); + } + + public async Task RunPathAsync( + ScenarioQuery query, + Func> planAsync, + Func> runAsync) + { + ArgumentNullException.ThrowIfNull(query); + ArgumentNullException.ThrowIfNull(planAsync); + ArgumentNullException.ThrowIfNull(runAsync); + + ScenarioRunPlan plan; + try + { + plan = await planAsync(_eventSink).ConfigureAwait(false); + } + catch (Exception ex) + { + await _eventSink.EmitAsync(new ScenarioEvent( + "scenario_run_started", + _timeProvider.GetUtcNow(), + Path: query.Path, + ShardCount: query.ShardCount, + ShardIndex: query.ShardIndex, + ShardStrategy: query.ShardStrategy)).ConfigureAwait(false); + await _eventSink.EmitAsync(CreateFailedRunEndedEvent( + query.Path, + ex, + shardCount: query.ShardCount, + shardIndex: query.ShardIndex, + shardStrategy: query.ShardStrategy)).ConfigureAwait(false); + throw; + } + + return await RunBatchAsync(plan, sink => runAsync(plan, sink)).ConfigureAwait(false); } public ValueTask DisposeAsync() => _eventSink.DisposeAsync(); @@ -176,7 +214,8 @@ private ScenarioEvent CreateFailedRunEndedEvent( int? failedCount = null, int? shardedOutCount = null, int? shardCount = null, - int? shardIndex = null) => + int? shardIndex = null, + string? shardStrategy = null) => new( "scenario_run_ended", _timeProvider.GetUtcNow(), @@ -190,6 +229,7 @@ private ScenarioEvent CreateFailedRunEndedEvent( ShardedOutCount: shardedOutCount, ShardCount: shardCount, ShardIndex: shardIndex, + ShardStrategy: shardStrategy, Error: ScenarioErrorInfo.From(exception)); } @@ -216,6 +256,7 @@ internal sealed record ScenarioEvent( [property: JsonPropertyName("status")] string? Status = null, [property: JsonPropertyName("path")] string? Path = null, [property: JsonPropertyName("file")] string? File = null, + [property: JsonPropertyName("scenario_id")] string? ScenarioId = null, [property: JsonPropertyName("scenario")] string? Scenario = null, [property: JsonPropertyName("step_index")] int? StepIndex = null, [property: JsonPropertyName("step")] string? Step = null, @@ -229,4 +270,5 @@ internal sealed record ScenarioEvent( [property: JsonPropertyName("sharded_out_count")] int? ShardedOutCount = null, [property: JsonPropertyName("shard_count")] int? ShardCount = null, [property: JsonPropertyName("shard_index")] int? ShardIndex = null, + [property: JsonPropertyName("shard_strategy")] string? ShardStrategy = null, [property: JsonPropertyName("error")] object? Error = null); diff --git a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs index 803bec2..a6579b1 100644 --- a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs +++ b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs @@ -91,6 +91,7 @@ public sealed class ScenarioExecutor private readonly ScenarioActionDispatcher _actionDispatcher; private readonly ScenarioCatalog _scenarioCatalog; private readonly IScenarioEventSink _eventSink; + private readonly ScenarioFailureArtifactCapturePolicy _failureArtifactCapturePolicy; public ScenarioExecutor(IScenarioActionHost actionHost, IFileSystem fileSystem, TimeProvider timeProvider, IDelay delay) : this( @@ -116,7 +117,14 @@ public ScenarioExecutor(IScenarioActionHost actionHost, IFileSystem fileSystem, { } - internal ScenarioExecutor(IScenarioActionHost actionHost, IFileSystem fileSystem, TimeProvider timeProvider, IDelay delay, IScenarioTemplateResolver templateResolver, IScenarioEventSink? eventSink = null) + internal ScenarioExecutor( + IScenarioActionHost actionHost, + IFileSystem fileSystem, + TimeProvider timeProvider, + IDelay delay, + IScenarioTemplateResolver templateResolver, + IScenarioEventSink? eventSink = null, + ScenarioFailureArtifactCapturePolicy failureArtifactCapturePolicy = ScenarioFailureArtifactCapturePolicy.Failure) { _actionHost = actionHost ?? throw new ArgumentNullException(nameof(actionHost)); _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); @@ -127,6 +135,7 @@ internal ScenarioExecutor(IScenarioActionHost actionHost, IFileSystem fileSystem fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)), templateResolver ?? throw new ArgumentNullException(nameof(templateResolver))); _eventSink = eventSink ?? NullScenarioEventSink.Instance; + _failureArtifactCapturePolicy = failureArtifactCapturePolicy; } /// @@ -138,21 +147,24 @@ public async Task RunAsync(string file) { var scenarioStarted = _timeProvider.GetUtcNow(); var scenario = await LoadValidatedScenarioAsync(file).ConfigureAwait(false); - await EmitAsync(new ScenarioEvent("scenario_started", scenarioStarted, File: file, Scenario: scenario.Name)).ConfigureAwait(false); + var scenarioId = ScenarioIdentity.Create(file, scenario.Name); + await EmitAsync(new ScenarioEvent("scenario_started", scenarioStarted, File: file, ScenarioId: scenarioId, Scenario: scenario.Name)).ConfigureAwait(false); var status = "failed"; try { await _actionHost.WriteDeviceFingerprintAsync().ConfigureAwait(false); var prologueMs = (_timeProvider.GetUtcNow() - scenarioStarted).TotalMilliseconds; - var execution = await ExecuteStepsAsync(scenario, file, scenarioStarted, prologueMs).ConfigureAwait(false); + var execution = await ExecuteStepsAsync(scenario, file, scenarioId, scenarioStarted, prologueMs).ConfigureAwait(false); status = "passed"; return new ScenarioRunResult( scenario.Name, status, CreateScenarioRunTiming((_timeProvider.GetUtcNow() - scenarioStarted).TotalMilliseconds, prologueMs, execution.ExecutedStepMs), - execution.Steps); + execution.Steps, + scenarioId, + file); } finally { @@ -162,6 +174,7 @@ await EmitAsync(new ScenarioEvent( endedAt, status, File: file, + ScenarioId: scenarioId, Scenario: scenario.Name, DurationMs: (endedAt - scenarioStarted).TotalMilliseconds)).ConfigureAwait(false); } @@ -173,7 +186,7 @@ private Task LoadValidatedScenarioAsync(string file) => private async Task ExecuteStepAsync(ScenarioStep step, DateTimeOffset? previousStepStartedAt) => await _actionDispatcher.ExecuteAsync(step, previousStepStartedAt).ConfigureAwait(false); - private async Task ExecuteStepsAsync(ScenarioFile scenario, string file, DateTimeOffset scenarioStarted, double prologueMs) + private async Task ExecuteStepsAsync(ScenarioFile scenario, string file, string scenarioId, DateTimeOffset scenarioStarted, double prologueMs) { var steps = new List(scenario.Steps.Count); var executedStepMs = 0d; @@ -184,7 +197,7 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s var step = scenario.Steps[index]; using var delayScope = DelayMetrics.BeginScope(); var started = _timeProvider.GetUtcNow(); - await EmitStepAsync("scenario_step_started", file, scenario.Name, index, step, started).ConfigureAwait(false); + await EmitStepAsync("scenario_step_started", file, scenarioId, scenario.Name, index, step, started).ConfigureAwait(false); try { @@ -192,7 +205,7 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; executedStepMs += durationMs; previousStepStartedAt = started; - await EmitStepAsync("scenario_step_passed", file, scenario.Name, index, step, _timeProvider.GetUtcNow(), "passed", durationMs).ConfigureAwait(false); + await EmitStepAsync("scenario_step_passed", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "passed", durationMs).ConfigureAwait(false); steps.Add(new ScenarioStepResult( step.Name ?? step.Action, @@ -207,7 +220,7 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; executedStepMs += durationMs; previousStepStartedAt = started; - await EmitStepAsync("scenario_step_continued_on_error", file, scenario.Name, index, step, _timeProvider.GetUtcNow(), "continued_on_error", durationMs, error).ConfigureAwait(false); + await EmitStepAsync("scenario_step_continued_on_error", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "continued_on_error", durationMs, error).ConfigureAwait(false); steps.Add(new ScenarioStepResult( step.Name ?? step.Action, step.Action, @@ -224,11 +237,11 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s { var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; executedStepMs += durationMs; - var failureArtifacts = await _actionHost.CaptureFailureArtifactsAsync( + var error = ScenarioErrorInfo.From(ex); + await EmitStepAsync("scenario_step_failed", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "failed", durationMs, error).ConfigureAwait(false); + var failureArtifacts = await CaptureFailureArtifactsBestEffortAsync( new FailureCaptureRequest("scenario", scenario.Name, file, index + 1, step.Name ?? step.Action, step.Action), ex).ConfigureAwait(false); - var error = ScenarioErrorInfo.From(ex); - await EmitStepAsync("scenario_step_failed", file, scenario.Name, index, step, _timeProvider.GetUtcNow(), "failed", durationMs, error).ConfigureAwait(false); throw new ScenarioStepFailureException( $"Scenario '{scenario.Name}' failed at step {index + 1} ({step.Name ?? step.Action}).", error.Category, @@ -244,7 +257,8 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s durationMs, CreateTimingData(step, durationMs, delayScope.TotalMilliseconds)), steps, - failureArtifacts), + failureArtifacts, + scenarioId), ex); } } @@ -276,6 +290,7 @@ private static ScenarioRunTiming CreateScenarioRunTiming(double totalMs, double private Task EmitStepAsync( string eventName, string file, + string scenarioId, string scenario, int index, ScenarioStep step, @@ -288,6 +303,7 @@ private Task EmitStepAsync( timestamp, status, File: file, + ScenarioId: scenarioId, Scenario: scenario, StepIndex: index + 1, Step: step.Name ?? step.Action, @@ -297,6 +313,39 @@ private Task EmitStepAsync( private Task EmitAsync(ScenarioEvent scenarioEvent) => _eventSink.EmitAsync(scenarioEvent); + private async Task CaptureFailureArtifactsBestEffortAsync(FailureCaptureRequest request, Exception failure) + { + if (_failureArtifactCapturePolicy == ScenarioFailureArtifactCapturePolicy.Never) + { + return CreateEmptyFailureArtifactBundle(request, failure); + } + + try + { + return await _actionHost.CaptureFailureArtifactsAsync(request, failure).ConfigureAwait(false); + } + catch (Exception captureException) + { + var bundle = CreateEmptyFailureArtifactBundle(request, failure); + return bundle with { CaptureErrors = [new FailureCaptureError("failure_artifacts", captureException.Message)] }; + } + } + + private FailureArtifactBundle CreateEmptyFailureArtifactBundle(FailureCaptureRequest request, Exception failure) => + new( + ResultSchemas.FailureBundle, + _timeProvider.GetUtcNow(), + request.Scope, + request.Name, + request.File, + request.StepIndex, + request.StepName, + request.Action, + failure.GetType().FullName ?? failure.GetType().Name, + failure.Message, + [], + []); + private sealed record ScenarioExecution(double ExecutedStepMs, IReadOnlyList Steps); } diff --git a/Luotsi.Cli/Scenarios/ScenarioExecutorFactory.cs b/Luotsi.Cli/Scenarios/ScenarioExecutorFactory.cs index 090a07c..27450d3 100644 --- a/Luotsi.Cli/Scenarios/ScenarioExecutorFactory.cs +++ b/Luotsi.Cli/Scenarios/ScenarioExecutorFactory.cs @@ -13,9 +13,12 @@ internal sealed class ScenarioExecutorFactory( private readonly IDelay _delay = delay ?? throw new ArgumentNullException(nameof(delay)); private readonly IScenarioTemplateResolver _templateResolver = templateResolver ?? throw new ArgumentNullException(nameof(templateResolver)); - public ScenarioExecutor Create(IScenarioActionHost actionHost, IScenarioEventSink? eventSink = null) + public ScenarioExecutor Create( + IScenarioActionHost actionHost, + IScenarioEventSink? eventSink = null, + ScenarioFailureArtifactCapturePolicy failureArtifactCapturePolicy = ScenarioFailureArtifactCapturePolicy.Failure) { ArgumentNullException.ThrowIfNull(actionHost); - return new ScenarioExecutor(actionHost, _fileSystem, _timeProvider, _delay, _templateResolver, eventSink); + return new ScenarioExecutor(actionHost, _fileSystem, _timeProvider, _delay, _templateResolver, eventSink, failureArtifactCapturePolicy); } } diff --git a/Luotsi.Cli/Scenarios/ScenarioReports.cs b/Luotsi.Cli/Scenarios/ScenarioReports.cs new file mode 100644 index 0000000..c684c44 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioReports.cs @@ -0,0 +1,539 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Xml.Linq; +using Luotsi.Cli.Cli; +using Luotsi.Cli.Errors; +using Luotsi.Cli.Infrastructure.Contracts; +using Luotsi.Cli.Models; + +namespace Luotsi.Cli.Scenarios; + +internal enum ScenarioArtifactAttachmentPolicy +{ + Never, + OnFailure, + Always +} + +internal sealed class ScenarioRunReportCoordinatorFactory(IFileSystem fileSystem, TimeProvider timeProvider) +{ + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly TimeProvider _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + + public ScenarioRunReportCoordinator Create(CliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + var writers = new List(); + if (!string.IsNullOrWhiteSpace(options.Get("report-json"))) + { + writers.Add(new JsonScenarioRunReportWriter(_fileSystem, options.Require("report-json"))); + } + + if (!string.IsNullOrWhiteSpace(options.Get("report-junit"))) + { + writers.Add(new JUnitScenarioRunReportWriter(_fileSystem, options.Require("report-junit"))); + } + + return new ScenarioRunReportCoordinator( + _timeProvider, + new CompositeScenarioRunReportWriter(writers), + ParseAttachmentPolicy(options.Get("attach-artifacts"))); + } + + private static ScenarioArtifactAttachmentPolicy ParseAttachmentPolicy(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ScenarioArtifactAttachmentPolicy.OnFailure; + } + + return value.Trim().ToLowerInvariant() switch + { + "never" => ScenarioArtifactAttachmentPolicy.Never, + "on-failure" or "onfailure" => ScenarioArtifactAttachmentPolicy.OnFailure, + "always" => ScenarioArtifactAttachmentPolicy.Always, + _ => throw new UsageException("--attach-artifacts must be one of: never, on-failure, always.") + }; + } +} + +internal sealed class ScenarioRunReportCoordinator( + TimeProvider timeProvider, + IScenarioRunReportWriter writer, + ScenarioArtifactAttachmentPolicy attachmentPolicy) +{ + private readonly TimeProvider _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); + private readonly IScenarioRunReportWriter _writer = writer ?? throw new ArgumentNullException(nameof(writer)); + + public async Task RunFileAsync(string file, Func> runAsync) + { + ArgumentException.ThrowIfNullOrWhiteSpace(file); + ArgumentNullException.ThrowIfNull(runAsync); + + var startedAt = _timeProvider.GetUtcNow(); + try + { + var result = await runAsync().ConfigureAwait(false); + await WriteAsync(ScenarioRunReport.FromSingle(file, result, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + await WriteAsync(ScenarioRunReport.FromSingleFailure(file, ex, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + throw; + } + } + + public async Task RunBatchAsync(ScenarioRunPlan plan, Func> runAsync) + { + ArgumentNullException.ThrowIfNull(plan); + ArgumentNullException.ThrowIfNull(runAsync); + + var startedAt = _timeProvider.GetUtcNow(); + try + { + var result = await runAsync().ConfigureAwait(false); + await WriteAsync(ScenarioRunReport.FromBatch(result, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + return result; + } + catch (Exception ex) + { + await WriteAsync(ScenarioRunReport.FromBatchFailure(plan, ex, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + throw; + } + } + + public async Task PlanPathAsync(ScenarioQuery query, Func> planAsync) + { + ArgumentNullException.ThrowIfNull(query); + ArgumentNullException.ThrowIfNull(planAsync); + + var startedAt = _timeProvider.GetUtcNow(); + try + { + return await planAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + await WriteAsync(ScenarioRunReport.FromQueryFailure(query, ex, startedAt, _timeProvider.GetUtcNow())).ConfigureAwait(false); + throw; + } + } + + private Task WriteAsync(ScenarioRunReport report) => _writer.WriteAsync(report); +} + +internal interface IScenarioRunReportWriter +{ + Task WriteAsync(ScenarioRunReport report); +} + +internal sealed class CompositeScenarioRunReportWriter(IReadOnlyList writers) : IScenarioRunReportWriter +{ + private readonly IReadOnlyList _writers = writers ?? throw new ArgumentNullException(nameof(writers)); + + public async Task WriteAsync(ScenarioRunReport report) + { + foreach (var writer in _writers) + { + await writer.WriteAsync(report).ConfigureAwait(false); + } + } +} + +internal sealed class JsonScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true + }; + + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; + + public async Task WriteAsync(ScenarioRunReport report) + { + ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); + await _fileSystem.WriteAllTextAsync(_path, JsonSerializer.Serialize(report, JsonOptions), Encoding.UTF8).ConfigureAwait(false); + } +} + +internal sealed class JUnitScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter +{ + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; + + public async Task WriteAsync(ScenarioRunReport report) + { + ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); + var xml = ToXml(report); + await _fileSystem.WriteAllTextAsync(_path, xml.ToString(SaveOptions.DisableFormatting), Encoding.UTF8).ConfigureAwait(false); + } + + private static XDocument ToXml(ScenarioRunReport report) + { + var testCases = report.Scenarios.Select(ToTestCase).ToArray(); + return new XDocument( + new XElement( + "testsuite", + new XAttribute("name", report.Path), + new XAttribute("tests", report.Scenarios.Count), + new XAttribute("failures", report.FailedCount), + new XAttribute("skipped", 0), + new XAttribute("time", Seconds(report.DurationMs)), + new XAttribute("timestamp", report.StartedAt.ToString("O", CultureInfo.InvariantCulture)), + testCases)); + } + + private static XElement ToTestCase(ScenarioReportScenario scenario) + { + var element = new XElement( + "testcase", + new XAttribute("classname", string.IsNullOrWhiteSpace(scenario.File) ? "luotsi.scenario" : scenario.File), + new XAttribute("name", scenario.Scenario), + new XAttribute("id", scenario.ScenarioId ?? scenario.Scenario), + new XAttribute("time", Seconds(scenario.DurationMs ?? 0))); + + if (scenario.Status != "passed") + { + element.Add(new XElement( + "failure", + new XAttribute("type", scenario.Error?.Category ?? "scenario_error"), + new XAttribute("message", scenario.Error?.Message ?? $"Scenario '{scenario.Scenario}' failed."), + scenario.FailedStep is null + ? scenario.Error?.Message + : $"Step {scenario.FailedStep.Index} ({scenario.FailedStep.Name}) failed during {scenario.FailedStep.Action}. {scenario.Error?.Message}".Trim())); + } + + if (scenario.Artifacts.Count > 0) + { + element.Add(new XElement( + "system-out", + string.Join( + Environment.NewLine, + scenario.Artifacts.Select(static artifact => + $"{artifact.Kind}: {artifact.FileName}" + + (artifact.StepIndex is null ? string.Empty : $" (step {artifact.StepIndex}: {artifact.StepName})"))))); + } + + return element; + } + + private static string Seconds(double milliseconds) => + Math.Max(0, milliseconds / 1000d).ToString("0.###", CultureInfo.InvariantCulture); +} + +internal sealed record ScenarioRunReport( + string Schema, + string Path, + string Status, + DateTimeOffset StartedAt, + DateTimeOffset EndedAt, + double DurationMs, + int TotalCount, + int MatchedCount, + int SelectedCount, + int PassedCount, + int FailedCount, + int ShardedOutCount, + int? ShardCount, + int? ShardIndex, + string? ShardStrategy, + IReadOnlyList Scenarios, + ErrorInfo? Error = null) +{ + private const string ReportSchema = "luotsi-scenario-run-report.v1"; + + public static ScenarioRunReport FromSingle( + string file, + ScenarioRunResult result, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + ReportSchema, + file, + result.Status, + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 1, + 1, + 1, + result.Status == "passed" ? 1 : 0, + result.Status == "passed" ? 0 : 1, + 0, + null, + null, + null, + [ScenarioReportScenario.FromSuccess(result, file, attachmentPolicy)]); + + public static ScenarioRunReport FromSingleFailure( + string file, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; + var scenario = failureData is null + ? ScenarioReportScenario.FromException(file, exception) + : ScenarioReportScenario.FromFailure(failureData, ScenarioErrorInfo.From(exception), attachmentPolicy); + return new ScenarioRunReport( + ReportSchema, + file, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 1, + 1, + 1, + 0, + 1, + 0, + null, + null, + null, + [scenario], + ScenarioErrorInfo.From(exception)); + } + + public static ScenarioRunReport FromBatch( + ScenarioRunBatchResult result, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + ReportSchema, + result.Path, + result.Status, + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + result.TotalCount, + result.MatchedCount, + result.SelectedCount, + result.PassedCount, + result.FailedCount, + result.ShardedOutCount, + result.ShardCount, + result.ShardIndex, + result.ShardStrategy, + result.Scenarios.Select(scenario => ScenarioReportScenario.FromBatchItem(scenario, attachmentPolicy)).ToArray()); + + public static ScenarioRunReport FromBatchFailure( + ScenarioRunPlan plan, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; + ScenarioReportScenario[] scenarios = failureData is null + ? [ScenarioReportScenario.FromException(plan.Query.Path, exception, "scenario run", $"{plan.Query.Path}::run")] + : [ScenarioReportScenario.FromFailure(failureData, ScenarioErrorInfo.From(exception), attachmentPolicy)]; + return new ScenarioRunReport( + ReportSchema, + plan.Query.Path, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + plan.TotalCount, + plan.MatchedCount, + plan.SelectedCount, + 0, + 1, + plan.ShardedOutCount, + plan.Query.ShardCount, + plan.Query.ShardIndex, + plan.Query.ShardStrategy, + scenarios, + ScenarioErrorInfo.From(exception)); + } + + public static ScenarioRunReport FromQueryFailure( + ScenarioQuery query, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt) => + new( + ReportSchema, + query.Path, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 0, + 0, + 0, + 0, + 1, + 0, + query.ShardCount, + query.ShardIndex, + query.ShardStrategy, + [ScenarioReportScenario.FromException(query.Path, exception, "scenario discovery", $"{query.Path}::discovery")], + ScenarioErrorInfo.From(exception)); + + private static double CalculateDurationMs(DateTimeOffset startedAt, DateTimeOffset endedAt) => + Math.Max(0, (endedAt - startedAt).TotalMilliseconds); +} + +internal sealed record ScenarioReportScenario( + string Scenario, + string? ScenarioId, + string Status, + string? File, + double? DurationMs, + ScenarioRunTiming? Timing, + IReadOnlyList Steps, + ScenarioFailedStepResult? FailedStep, + IReadOnlyList Artifacts, + ErrorInfo? Error) +{ + public static ScenarioReportScenario FromSuccess(ScenarioRunResult result, string? file, ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + result.Scenario, + result.ScenarioId ?? (file is null ? null : ScenarioIdentity.Create(file, result.Scenario)), + result.Status, + result.File ?? file, + result.Timing.TotalMs, + result.Timing, + result.Steps, + null, + GetStepArtifacts(result.Steps, attachmentPolicy), + null); + + public static ScenarioReportScenario FromFailure(ScenarioRunFailureData data, ErrorInfo error, ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + data.Scenario, + data.ScenarioId ?? ScenarioIdentity.Create(data.File, data.Scenario), + data.Status, + data.File, + data.Timing.TotalMs, + data.Timing, + data.Steps, + data.FailedStep, + GetFailureAndStepArtifacts(data.Steps, data.FailureArtifacts, attachmentPolicy), + error); + + public static ScenarioReportScenario FromBatchItem(ScenarioBatchItemResult item, ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + if (item.Data is not null) + { + return FromFailure(item.Data, item.Error ?? new ErrorInfo("Exception", "Scenario failed.", "scenario_error"), attachmentPolicy); + } + + return new ScenarioReportScenario( + item.Scenario, + item.ScenarioId ?? (item.File is null ? null : ScenarioIdentity.Create(item.File, item.Scenario)), + item.Status, + item.File, + item.Timing?.TotalMs, + item.Timing, + item.Steps ?? [], + null, + item.Steps is null ? [] : GetStepArtifacts(item.Steps, attachmentPolicy), + item.Error); + } + + public static ScenarioReportScenario FromException(string file, Exception exception, string? scenario = null, string? scenarioId = null) => + new( + scenario ?? Path.GetFileNameWithoutExtension(file), + scenarioId ?? file, + "failed", + file, + null, + null, + [], + null, + [], + ScenarioErrorInfo.From(exception)); + + private static IReadOnlyList GetFailureArtifacts(FailureArtifactBundle bundle, ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) + { + return []; + } + + var artifacts = bundle.Artifacts + .Select(artifact => new ScenarioReportArtifact(artifact.Kind, artifact.FileName, bundle.StepIndex, bundle.StepName)) + .ToList(); + if (!string.IsNullOrWhiteSpace(bundle.MetadataFile)) + { + artifacts.Add(new ScenarioReportArtifact("metadata", bundle.MetadataFile, bundle.StepIndex, bundle.StepName)); + } + + return artifacts; + } + + private static IReadOnlyList GetFailureAndStepArtifacts( + IReadOnlyList steps, + FailureArtifactBundle bundle, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) + { + return []; + } + + var artifacts = new List(); + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Always) + { + artifacts.AddRange(GetStepArtifacts(steps, attachmentPolicy)); + } + + artifacts.AddRange(GetFailureArtifacts(bundle, attachmentPolicy)); + return artifacts; + } + + private static IReadOnlyList GetStepArtifacts(IReadOnlyList steps, ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + if (attachmentPolicy != ScenarioArtifactAttachmentPolicy.Always) + { + return []; + } + + return steps + .SelectMany((step, index) => FromStepResult(step, index + 1)) + .ToArray(); + } + + private static IEnumerable FromStepResult(ScenarioStepResult step, int index) + { + if (step.Result is TakeScreenshotResult screenshot) + { + yield return new ScenarioReportArtifact("screenshot", screenshot.File, index, step.Step); + } + + if (step.Result is CaptureArtifactsResult artifacts) + { + yield return new ScenarioReportArtifact("screenshot", artifacts.Screenshot, index, step.Step); + yield return new ScenarioReportArtifact("logcat", artifacts.Logcat, index, step.Step); + yield return new ScenarioReportArtifact("screen_state", artifacts.ScreenState, index, step.Step); + yield return new ScenarioReportArtifact("hierarchy", artifacts.Hierarchy, index, step.Step); + } + } +} + +internal sealed record ScenarioReportArtifact(string Kind, string FileName, int? StepIndex, string? StepName); + +internal static class ScenarioReportFileSystem +{ + public static void CreateReportDirectory(IFileSystem fileSystem, string path) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + fileSystem.CreateDirectory(directory); + } + } +} diff --git a/Luotsi.Cli/Scenarios/ScenarioValidator.cs b/Luotsi.Cli/Scenarios/ScenarioValidator.cs index 36caa58..a34d9d7 100644 --- a/Luotsi.Cli/Scenarios/ScenarioValidator.cs +++ b/Luotsi.Cli/Scenarios/ScenarioValidator.cs @@ -29,13 +29,13 @@ public static ScenarioFile ValidateScenario(ScenarioFile scenario, string file, private static void ValidateStep(ScenarioFile scenario, ScenarioStep step, int index, IReadOnlySet supportedScenarioActions) { var stepLabel = $"Scenario '{scenario.Name}' step {index}"; - var action = step.Action.Trim(); - if (string.IsNullOrWhiteSpace(step.Action)) { throw new UsageException($"{stepLabel} must define a non-empty action."); } + var action = step.Action.Trim(); + if (!supportedScenarioActions.Contains(action) || (string.Equals(action, "doubleTap", StringComparison.OrdinalIgnoreCase) && step.HeaderLogo is not true)) { From 80c22aa8f25581ece3856ea4bfe86ecdccaa91c1 Mon Sep 17 00:00:00 2001 From: Slideep Date: Tue, 19 May 2026 10:47:44 +0300 Subject: [PATCH 2/2] Refactor scenario run reporting boundaries --- Luotsi.Cli.Tests/ScenarioExecutorTests.cs | 55 +++ .../ScenarioRunConfigurationTests.cs | 39 ++ .../AppHostedCommandCompositionBuilder.cs | 8 +- .../Cli/Routing/ScenarioCommandDispatcher.cs | 44 +- Luotsi.Cli/Scenarios/ScenarioCatalog.cs | 37 +- Luotsi.Cli/Scenarios/ScenarioEvents.cs | 18 - Luotsi.Cli/Scenarios/ScenarioExecutor.cs | 11 +- .../ScenarioReportArtifactProjection.cs | 83 ++++ Luotsi.Cli/Scenarios/ScenarioReports.cs | 467 +----------------- .../Scenarios/ScenarioRunConfiguration.cs | 65 +++ .../Scenarios/ScenarioRunOrchestrator.cs | 74 +++ .../Scenarios/ScenarioRunReportFactory.cs | 226 +++++++++ .../Scenarios/ScenarioRunReportModels.cs | 36 ++ .../Scenarios/ScenarioRunReportWriters.cs | 122 +++++ Luotsi.Cli/Scenarios/ScenarioRunSupport.cs | 36 ++ 15 files changed, 786 insertions(+), 535 deletions(-) create mode 100644 Luotsi.Cli.Tests/ScenarioRunConfigurationTests.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioReportArtifactProjection.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunConfiguration.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunOrchestrator.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunReportFactory.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunReportModels.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunReportWriters.cs create mode 100644 Luotsi.Cli/Scenarios/ScenarioRunSupport.cs diff --git a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs index d84eeab..1032b29 100644 --- a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs +++ b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs @@ -225,6 +225,37 @@ public async Task ScenarioList_RecursiveGlob_Finds_Nested_Scenarios() Assert.Equal(["nested", "root"], scenarios.EnumerateArray().Select(static scenario => scenario.GetProperty("name").GetString()!).Order(StringComparer.Ordinal).ToArray()); } + [Fact] + public async Task ScenarioList_RecursiveGlob_With_Directory_Remainder_Returns_Usage_Error() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenarios/nested/b.json", """ + { + "name": "nested", + "steps": [ + { "action": "sleep", "milliseconds": 1 } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["scenario-list", "--path", "/tmp/scenarios/**/nested/*.json"]); + using var envelope = console.ParseSingleOutputAsJson(); + + Assert.Equal(2, exitCode); + Assert.Contains("only supports recursive globs", envelope.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + } + [Fact] public async Task RunAsync_File_DryRun_Returns_Usage_Error() @@ -633,6 +664,30 @@ public async Task RunAsync_File_Writes_Json_Report() Assert.Equal("sleep", report.RootElement.GetProperty("scenarios")[0].GetProperty("steps")[0].GetProperty("action").GetString()); } + [Fact] + public async Task RunAsync_File_Report_Uses_Deterministic_ScenarioId_For_Parse_Failure() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/broken.json", "{"); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/broken.json", "--report-json", "/tmp/report.json"]); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(2, exitCode); + Assert.Equal("/tmp/broken.json::broken", report.RootElement.GetProperty("scenarios")[0].GetProperty("scenario_id").GetString()); + } + [Fact] public async Task RunAsync_Path_Writes_JUnit_Report_For_Mixed_Result() { diff --git a/Luotsi.Cli.Tests/ScenarioRunConfigurationTests.cs b/Luotsi.Cli.Tests/ScenarioRunConfigurationTests.cs new file mode 100644 index 0000000..912b83c --- /dev/null +++ b/Luotsi.Cli.Tests/ScenarioRunConfigurationTests.cs @@ -0,0 +1,39 @@ +using Luotsi.Cli.Cli; +using Luotsi.Cli.Errors; +using Luotsi.Cli.Scenarios; +using Xunit; + +namespace Luotsi.Cli.Tests; + +public sealed class ScenarioRunConfigurationTests +{ + [Fact] + public void Create_Parses_Run_Output_Policies() + { + var options = CliOptions.Parse([ + "run", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json", + "--report-junit", "/tmp/junit.xml", + "--capture-on", "never", + "--attach-artifacts", "always"]); + + var configuration = ScenarioRunConfiguration.Create(options); + + Assert.Equal("/tmp/events.jsonl", configuration.EventsJsonlPath); + Assert.Equal("/tmp/report.json", configuration.JsonReportPath); + Assert.Equal("/tmp/junit.xml", configuration.JUnitReportPath); + Assert.Equal(ScenarioFailureArtifactCapturePolicy.Never, configuration.FailureArtifactCapturePolicy); + Assert.Equal(ScenarioArtifactAttachmentPolicy.Always, configuration.ArtifactAttachmentPolicy); + } + + [Fact] + public void Create_Invalid_AttachArtifacts_Throws_UsageException() + { + var options = CliOptions.Parse(["run", "--attach-artifacts", "sometimes"]); + + var error = Assert.Throws(() => ScenarioRunConfiguration.Create(options)); + + Assert.Contains("--attach-artifacts", error.Message, StringComparison.Ordinal); + } +} \ No newline at end of file diff --git a/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs b/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs index f18a733..b4756b8 100644 --- a/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs +++ b/Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs @@ -19,10 +19,16 @@ public static AppHostedCommandComposition Build(AppHostedCommandCompositionBuild var scenarioBatchExecutorFactory = new ScenarioBatchExecutorFactory(scenarioExecutorFactory); var scenarioRunEventCoordinatorFactory = new ScenarioRunEventCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider); var scenarioRunReportCoordinatorFactory = new ScenarioRunReportCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider); + var scenarioRunOrchestrator = new ScenarioRunOrchestrator( + scenarioRunPlanner, + scenarioExecutorFactory, + scenarioBatchExecutorFactory, + scenarioRunEventCoordinatorFactory, + scenarioRunReportCoordinatorFactory); var envelopeWriter = new AppCommandEnvelopeWriter(dependencies.Console, dependencies.TimeProvider); var commandDispatcher = new AppCommandDispatcher( new AdbSubcommandDispatcher(), - new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioExecutorFactory, scenarioBatchExecutorFactory, scenarioRunEventCoordinatorFactory, scenarioRunReportCoordinatorFactory), + new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioRunOrchestrator), dependencies.ProfileCoordinator); return new( diff --git a/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs b/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs index 21968d7..ce62157 100644 --- a/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs +++ b/Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs @@ -6,16 +6,10 @@ namespace Luotsi.Cli.Cli.Routing; internal sealed class ScenarioCommandDispatcher( ScenarioRunPlanner runPlanner, - ScenarioExecutorFactory scenarioExecutorFactory, - ScenarioBatchExecutorFactory scenarioBatchExecutorFactory, - ScenarioRunEventCoordinatorFactory scenarioRunEventCoordinatorFactory, - ScenarioRunReportCoordinatorFactory scenarioRunReportCoordinatorFactory) + ScenarioRunOrchestrator scenarioRunOrchestrator) { private readonly ScenarioRunPlanner _runPlanner = runPlanner ?? throw new ArgumentNullException(nameof(runPlanner)); - private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory)); - private readonly ScenarioBatchExecutorFactory _scenarioBatchExecutorFactory = scenarioBatchExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioBatchExecutorFactory)); - private readonly ScenarioRunEventCoordinatorFactory _scenarioRunEventCoordinatorFactory = scenarioRunEventCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunEventCoordinatorFactory)); - private readonly ScenarioRunReportCoordinatorFactory _scenarioRunReportCoordinatorFactory = scenarioRunReportCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunReportCoordinatorFactory)); + private readonly ScenarioRunOrchestrator _scenarioRunOrchestrator = scenarioRunOrchestrator ?? throw new ArgumentNullException(nameof(scenarioRunOrchestrator)); public async Task ListAsync(CliOptions options) { @@ -31,7 +25,7 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(runner); - var failureArtifactCapturePolicy = ParseFailureArtifactCapturePolicy(options.Get("capture-on")); + var configuration = ScenarioRunConfiguration.Create(options); if (!ScenarioQueryFactory.UsesCatalogExecution(options)) { @@ -41,13 +35,7 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) } var file = options.Require("file"); - await using var singleRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl")); - var singleRunReports = _scenarioRunReportCoordinatorFactory.Create(options); - return await singleRunReports.RunFileAsync( - file, - () => singleRunEvents.RunFileAsync( - file, - sink => _scenarioExecutorFactory.Create(runner, sink, failureArtifactCapturePolicy).RunAsync(file))).ConfigureAwait(false); + return await _scenarioRunOrchestrator.RunFileAsync(file, runner, configuration).ConfigureAwait(false); } var query = ScenarioQueryFactory.CreateCatalogRunQuery(options); @@ -67,28 +55,6 @@ public async Task RunAsync(CliOptions options, IDeviceHost runner) plan.SelectedScenarios); } - await using var batchRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl")); - var batchRunReports = _scenarioRunReportCoordinatorFactory.Create(options); - return await batchRunEvents.RunPathAsync( - query, - _ => batchRunReports.PlanPathAsync(query, () => _runPlanner.CreateAsync(query)), - (preparedPlan, sink) => batchRunReports.RunBatchAsync( - preparedPlan, - () => _scenarioBatchExecutorFactory.Create(runner, sink, failureArtifactCapturePolicy).RunAsync(preparedPlan))).ConfigureAwait(false); - } - - private static ScenarioFailureArtifactCapturePolicy ParseFailureArtifactCapturePolicy(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return ScenarioFailureArtifactCapturePolicy.Failure; - } - - return value.Trim().ToLowerInvariant() switch - { - "failure" or "on-failure" or "onfailure" => ScenarioFailureArtifactCapturePolicy.Failure, - "never" => ScenarioFailureArtifactCapturePolicy.Never, - _ => throw new UsageException("--capture-on must be one of: failure, never.") - }; + return await _scenarioRunOrchestrator.RunPathAsync(query, runner, configuration).ConfigureAwait(false); } } diff --git a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs index ae8ae18..099549f 100644 --- a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs +++ b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs @@ -144,17 +144,6 @@ public static class ScenarioShardStrategies public const string Hash = "hash"; } -internal enum ScenarioFailureArtifactCapturePolicy -{ - Failure, - Never -} - -public static class ScenarioIdentity -{ - public static string Create(string file, string scenarioName) => $"{file}::{scenarioName}"; -} - internal sealed class ScenarioCatalog( IFileSystem fileSystem, IScenarioTemplateResolver templateResolver) @@ -306,12 +295,7 @@ private static bool TrySplitRecursiveGlob(string path, out string root, out stri if (normalized.StartsWith("**/", StringComparison.Ordinal)) { root = "."; - pattern = Path.GetFileName(normalized[3..]); - if (string.IsNullOrWhiteSpace(pattern)) - { - pattern = "*.json"; - } - + pattern = NormalizeRecursivePattern(path, normalized[3..]); return true; } @@ -325,13 +309,24 @@ private static bool TrySplitRecursiveGlob(string path, out string root, out stri root = markerIndex == 0 ? "." : path[..markerIndex]; var remainder = normalized[(markerIndex + 4)..]; - pattern = Path.GetFileName(remainder); - if (string.IsNullOrWhiteSpace(pattern)) + pattern = NormalizeRecursivePattern(path, remainder); + return true; + } + + private static string NormalizeRecursivePattern(string originalPath, string remainder) + { + if (string.IsNullOrWhiteSpace(remainder)) { - pattern = "*.json"; + return "*.json"; } - return true; + if (remainder.Contains('/', StringComparison.Ordinal)) + { + throw new UsageException( + $"Scenario path '{originalPath}' only supports recursive globs with a file pattern after '**/' (for example '**/*.json')."); + } + + return remainder; } private async Task LoadAsync(string file) diff --git a/Luotsi.Cli/Scenarios/ScenarioEvents.cs b/Luotsi.Cli/Scenarios/ScenarioEvents.cs index 97b3594..4824e22 100644 --- a/Luotsi.Cli/Scenarios/ScenarioEvents.cs +++ b/Luotsi.Cli/Scenarios/ScenarioEvents.cs @@ -2,7 +2,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using Luotsi.Cli.Infrastructure.Contracts; -using Luotsi.Cli.Models; namespace Luotsi.Cli.Scenarios; @@ -233,23 +232,6 @@ private ScenarioEvent CreateFailedRunEndedEvent( Error: ScenarioErrorInfo.From(exception)); } -internal static class ScenarioErrorInfo -{ - public static ErrorInfo From(Exception exception) - { - ArgumentNullException.ThrowIfNull(exception); - return ErrorInfo.From(exception, GetCategory(exception)); - } - - public static string GetCategory(Exception exception) - { - ArgumentNullException.ThrowIfNull(exception); - return exception is ICommandFailureDetails failure - ? failure.CategoryOverride - : ErrorInfo.Classify(exception.Message); - } -} - internal sealed record ScenarioEvent( [property: JsonPropertyName("event")] string Event, [property: JsonPropertyName("timestamp")] DateTimeOffset Timestamp, diff --git a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs index a6579b1..26a769f 100644 --- a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs +++ b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs @@ -324,13 +324,22 @@ private async Task CaptureFailureArtifactsBestEffortAsync { return await _actionHost.CaptureFailureArtifactsAsync(request, failure).ConfigureAwait(false); } - catch (Exception captureException) + catch (Exception captureException) when (!IsFatalException(captureException)) { var bundle = CreateEmptyFailureArtifactBundle(request, failure); return bundle with { CaptureErrors = [new FailureCaptureError("failure_artifacts", captureException.Message)] }; } } + private static bool IsFatalException(Exception exception) => + exception is OutOfMemoryException + or StackOverflowException + or AccessViolationException + or AppDomainUnloadedException + or BadImageFormatException + or CannotUnloadAppDomainException + or InvalidProgramException; + private FailureArtifactBundle CreateEmptyFailureArtifactBundle(FailureCaptureRequest request, Exception failure) => new( ResultSchemas.FailureBundle, diff --git a/Luotsi.Cli/Scenarios/ScenarioReportArtifactProjection.cs b/Luotsi.Cli/Scenarios/ScenarioReportArtifactProjection.cs new file mode 100644 index 0000000..ff85ef0 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioReportArtifactProjection.cs @@ -0,0 +1,83 @@ +using Luotsi.Cli.Models; + +namespace Luotsi.Cli.Scenarios; + +internal static class ScenarioReportArtifactProjection +{ + public static IReadOnlyList FromSteps( + IReadOnlyList steps, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + ArgumentNullException.ThrowIfNull(steps); + + if (attachmentPolicy != ScenarioArtifactAttachmentPolicy.Always) + { + return []; + } + + return steps + .SelectMany((step, index) => FromStepResult(step, index + 1)) + .ToArray(); + } + + public static IReadOnlyList FromFailure( + FailureArtifactBundle bundle, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + ArgumentNullException.ThrowIfNull(bundle); + + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) + { + return []; + } + + var artifacts = bundle.Artifacts + .Select(artifact => new ScenarioReportArtifact(artifact.Kind, artifact.FileName, bundle.StepIndex, bundle.StepName)) + .ToList(); + if (!string.IsNullOrWhiteSpace(bundle.MetadataFile)) + { + artifacts.Add(new ScenarioReportArtifact("metadata", bundle.MetadataFile, bundle.StepIndex, bundle.StepName)); + } + + return artifacts; + } + + public static IReadOnlyList FromFailureAndSteps( + IReadOnlyList steps, + FailureArtifactBundle bundle, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + ArgumentNullException.ThrowIfNull(steps); + ArgumentNullException.ThrowIfNull(bundle); + + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) + { + return []; + } + + var artifacts = new List(); + if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Always) + { + artifacts.AddRange(FromSteps(steps, attachmentPolicy)); + } + + artifacts.AddRange(FromFailure(bundle, attachmentPolicy)); + return artifacts; + } + + private static IEnumerable FromStepResult(ScenarioStepResult step, int index) + { + if (step.Result is TakeScreenshotResult screenshot) + { + yield return new ScenarioReportArtifact("screenshot", screenshot.File, index, step.Step); + } + + if (step.Result is CaptureArtifactsResult artifacts) + { + yield return new ScenarioReportArtifact("screenshot", artifacts.Screenshot, index, step.Step); + yield return new ScenarioReportArtifact("logcat", artifacts.Logcat, index, step.Step); + yield return new ScenarioReportArtifact("screen_state", artifacts.ScreenState, index, step.Step); + yield return new ScenarioReportArtifact("hierarchy", artifacts.Hierarchy, index, step.Step); + } + } +} \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioReports.cs b/Luotsi.Cli/Scenarios/ScenarioReports.cs index c684c44..281937f 100644 --- a/Luotsi.Cli/Scenarios/ScenarioReports.cs +++ b/Luotsi.Cli/Scenarios/ScenarioReports.cs @@ -1,62 +1,31 @@ -using System.Globalization; -using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Xml.Linq; -using Luotsi.Cli.Cli; -using Luotsi.Cli.Errors; using Luotsi.Cli.Infrastructure.Contracts; -using Luotsi.Cli.Models; namespace Luotsi.Cli.Scenarios; -internal enum ScenarioArtifactAttachmentPolicy -{ - Never, - OnFailure, - Always -} - internal sealed class ScenarioRunReportCoordinatorFactory(IFileSystem fileSystem, TimeProvider timeProvider) { private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); private readonly TimeProvider _timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider)); - public ScenarioRunReportCoordinator Create(CliOptions options) + public ScenarioRunReportCoordinator Create(ScenarioRunConfiguration configuration) { - ArgumentNullException.ThrowIfNull(options); + ArgumentNullException.ThrowIfNull(configuration); var writers = new List(); - if (!string.IsNullOrWhiteSpace(options.Get("report-json"))) + if (!string.IsNullOrWhiteSpace(configuration.JsonReportPath)) { - writers.Add(new JsonScenarioRunReportWriter(_fileSystem, options.Require("report-json"))); + writers.Add(new JsonScenarioRunReportWriter(_fileSystem, configuration.JsonReportPath)); } - if (!string.IsNullOrWhiteSpace(options.Get("report-junit"))) + if (!string.IsNullOrWhiteSpace(configuration.JUnitReportPath)) { - writers.Add(new JUnitScenarioRunReportWriter(_fileSystem, options.Require("report-junit"))); + writers.Add(new JUnitScenarioRunReportWriter(_fileSystem, configuration.JUnitReportPath)); } return new ScenarioRunReportCoordinator( _timeProvider, new CompositeScenarioRunReportWriter(writers), - ParseAttachmentPolicy(options.Get("attach-artifacts"))); - } - - private static ScenarioArtifactAttachmentPolicy ParseAttachmentPolicy(string? value) - { - if (string.IsNullOrWhiteSpace(value)) - { - return ScenarioArtifactAttachmentPolicy.OnFailure; - } - - return value.Trim().ToLowerInvariant() switch - { - "never" => ScenarioArtifactAttachmentPolicy.Never, - "on-failure" or "onfailure" => ScenarioArtifactAttachmentPolicy.OnFailure, - "always" => ScenarioArtifactAttachmentPolicy.Always, - _ => throw new UsageException("--attach-artifacts must be one of: never, on-failure, always.") - }; + configuration.ArtifactAttachmentPolicy); } } @@ -77,12 +46,12 @@ public async Task RunFileAsync(string file, Func RunBatchAsync(ScenarioRunPlan plan, Fu try { var result = await runAsync().ConfigureAwait(false); - await WriteAsync(ScenarioRunReport.FromBatch(result, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + await WriteAsync(ScenarioRunReportFactory.FromBatch(result, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); return result; } catch (Exception ex) { - await WriteAsync(ScenarioRunReport.FromBatchFailure(plan, ex, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); + await WriteAsync(ScenarioRunReportFactory.FromBatchFailure(plan, ex, startedAt, _timeProvider.GetUtcNow(), attachmentPolicy)).ConfigureAwait(false); throw; } } @@ -118,422 +87,10 @@ public async Task PlanPathAsync(ScenarioQuery query, Func _writer.WriteAsync(report); } - -internal interface IScenarioRunReportWriter -{ - Task WriteAsync(ScenarioRunReport report); -} - -internal sealed class CompositeScenarioRunReportWriter(IReadOnlyList writers) : IScenarioRunReportWriter -{ - private readonly IReadOnlyList _writers = writers ?? throw new ArgumentNullException(nameof(writers)); - - public async Task WriteAsync(ScenarioRunReport report) - { - foreach (var writer in _writers) - { - await writer.WriteAsync(report).ConfigureAwait(false); - } - } -} - -internal sealed class JsonScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter -{ - private static readonly JsonSerializerOptions JsonOptions = new() - { - PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - WriteIndented = true - }; - - private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; - - public async Task WriteAsync(ScenarioRunReport report) - { - ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); - await _fileSystem.WriteAllTextAsync(_path, JsonSerializer.Serialize(report, JsonOptions), Encoding.UTF8).ConfigureAwait(false); - } -} - -internal sealed class JUnitScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter -{ - private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); - private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; - - public async Task WriteAsync(ScenarioRunReport report) - { - ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); - var xml = ToXml(report); - await _fileSystem.WriteAllTextAsync(_path, xml.ToString(SaveOptions.DisableFormatting), Encoding.UTF8).ConfigureAwait(false); - } - - private static XDocument ToXml(ScenarioRunReport report) - { - var testCases = report.Scenarios.Select(ToTestCase).ToArray(); - return new XDocument( - new XElement( - "testsuite", - new XAttribute("name", report.Path), - new XAttribute("tests", report.Scenarios.Count), - new XAttribute("failures", report.FailedCount), - new XAttribute("skipped", 0), - new XAttribute("time", Seconds(report.DurationMs)), - new XAttribute("timestamp", report.StartedAt.ToString("O", CultureInfo.InvariantCulture)), - testCases)); - } - - private static XElement ToTestCase(ScenarioReportScenario scenario) - { - var element = new XElement( - "testcase", - new XAttribute("classname", string.IsNullOrWhiteSpace(scenario.File) ? "luotsi.scenario" : scenario.File), - new XAttribute("name", scenario.Scenario), - new XAttribute("id", scenario.ScenarioId ?? scenario.Scenario), - new XAttribute("time", Seconds(scenario.DurationMs ?? 0))); - - if (scenario.Status != "passed") - { - element.Add(new XElement( - "failure", - new XAttribute("type", scenario.Error?.Category ?? "scenario_error"), - new XAttribute("message", scenario.Error?.Message ?? $"Scenario '{scenario.Scenario}' failed."), - scenario.FailedStep is null - ? scenario.Error?.Message - : $"Step {scenario.FailedStep.Index} ({scenario.FailedStep.Name}) failed during {scenario.FailedStep.Action}. {scenario.Error?.Message}".Trim())); - } - - if (scenario.Artifacts.Count > 0) - { - element.Add(new XElement( - "system-out", - string.Join( - Environment.NewLine, - scenario.Artifacts.Select(static artifact => - $"{artifact.Kind}: {artifact.FileName}" + - (artifact.StepIndex is null ? string.Empty : $" (step {artifact.StepIndex}: {artifact.StepName})"))))); - } - - return element; - } - - private static string Seconds(double milliseconds) => - Math.Max(0, milliseconds / 1000d).ToString("0.###", CultureInfo.InvariantCulture); -} - -internal sealed record ScenarioRunReport( - string Schema, - string Path, - string Status, - DateTimeOffset StartedAt, - DateTimeOffset EndedAt, - double DurationMs, - int TotalCount, - int MatchedCount, - int SelectedCount, - int PassedCount, - int FailedCount, - int ShardedOutCount, - int? ShardCount, - int? ShardIndex, - string? ShardStrategy, - IReadOnlyList Scenarios, - ErrorInfo? Error = null) -{ - private const string ReportSchema = "luotsi-scenario-run-report.v1"; - - public static ScenarioRunReport FromSingle( - string file, - ScenarioRunResult result, - DateTimeOffset startedAt, - DateTimeOffset endedAt, - ScenarioArtifactAttachmentPolicy attachmentPolicy) => - new( - ReportSchema, - file, - result.Status, - startedAt, - endedAt, - CalculateDurationMs(startedAt, endedAt), - 1, - 1, - 1, - result.Status == "passed" ? 1 : 0, - result.Status == "passed" ? 0 : 1, - 0, - null, - null, - null, - [ScenarioReportScenario.FromSuccess(result, file, attachmentPolicy)]); - - public static ScenarioRunReport FromSingleFailure( - string file, - Exception exception, - DateTimeOffset startedAt, - DateTimeOffset endedAt, - ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; - var scenario = failureData is null - ? ScenarioReportScenario.FromException(file, exception) - : ScenarioReportScenario.FromFailure(failureData, ScenarioErrorInfo.From(exception), attachmentPolicy); - return new ScenarioRunReport( - ReportSchema, - file, - "failed", - startedAt, - endedAt, - CalculateDurationMs(startedAt, endedAt), - 1, - 1, - 1, - 0, - 1, - 0, - null, - null, - null, - [scenario], - ScenarioErrorInfo.From(exception)); - } - - public static ScenarioRunReport FromBatch( - ScenarioRunBatchResult result, - DateTimeOffset startedAt, - DateTimeOffset endedAt, - ScenarioArtifactAttachmentPolicy attachmentPolicy) => - new( - ReportSchema, - result.Path, - result.Status, - startedAt, - endedAt, - CalculateDurationMs(startedAt, endedAt), - result.TotalCount, - result.MatchedCount, - result.SelectedCount, - result.PassedCount, - result.FailedCount, - result.ShardedOutCount, - result.ShardCount, - result.ShardIndex, - result.ShardStrategy, - result.Scenarios.Select(scenario => ScenarioReportScenario.FromBatchItem(scenario, attachmentPolicy)).ToArray()); - - public static ScenarioRunReport FromBatchFailure( - ScenarioRunPlan plan, - Exception exception, - DateTimeOffset startedAt, - DateTimeOffset endedAt, - ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; - ScenarioReportScenario[] scenarios = failureData is null - ? [ScenarioReportScenario.FromException(plan.Query.Path, exception, "scenario run", $"{plan.Query.Path}::run")] - : [ScenarioReportScenario.FromFailure(failureData, ScenarioErrorInfo.From(exception), attachmentPolicy)]; - return new ScenarioRunReport( - ReportSchema, - plan.Query.Path, - "failed", - startedAt, - endedAt, - CalculateDurationMs(startedAt, endedAt), - plan.TotalCount, - plan.MatchedCount, - plan.SelectedCount, - 0, - 1, - plan.ShardedOutCount, - plan.Query.ShardCount, - plan.Query.ShardIndex, - plan.Query.ShardStrategy, - scenarios, - ScenarioErrorInfo.From(exception)); - } - - public static ScenarioRunReport FromQueryFailure( - ScenarioQuery query, - Exception exception, - DateTimeOffset startedAt, - DateTimeOffset endedAt) => - new( - ReportSchema, - query.Path, - "failed", - startedAt, - endedAt, - CalculateDurationMs(startedAt, endedAt), - 0, - 0, - 0, - 0, - 1, - 0, - query.ShardCount, - query.ShardIndex, - query.ShardStrategy, - [ScenarioReportScenario.FromException(query.Path, exception, "scenario discovery", $"{query.Path}::discovery")], - ScenarioErrorInfo.From(exception)); - - private static double CalculateDurationMs(DateTimeOffset startedAt, DateTimeOffset endedAt) => - Math.Max(0, (endedAt - startedAt).TotalMilliseconds); -} - -internal sealed record ScenarioReportScenario( - string Scenario, - string? ScenarioId, - string Status, - string? File, - double? DurationMs, - ScenarioRunTiming? Timing, - IReadOnlyList Steps, - ScenarioFailedStepResult? FailedStep, - IReadOnlyList Artifacts, - ErrorInfo? Error) -{ - public static ScenarioReportScenario FromSuccess(ScenarioRunResult result, string? file, ScenarioArtifactAttachmentPolicy attachmentPolicy) => - new( - result.Scenario, - result.ScenarioId ?? (file is null ? null : ScenarioIdentity.Create(file, result.Scenario)), - result.Status, - result.File ?? file, - result.Timing.TotalMs, - result.Timing, - result.Steps, - null, - GetStepArtifacts(result.Steps, attachmentPolicy), - null); - - public static ScenarioReportScenario FromFailure(ScenarioRunFailureData data, ErrorInfo error, ScenarioArtifactAttachmentPolicy attachmentPolicy) => - new( - data.Scenario, - data.ScenarioId ?? ScenarioIdentity.Create(data.File, data.Scenario), - data.Status, - data.File, - data.Timing.TotalMs, - data.Timing, - data.Steps, - data.FailedStep, - GetFailureAndStepArtifacts(data.Steps, data.FailureArtifacts, attachmentPolicy), - error); - - public static ScenarioReportScenario FromBatchItem(ScenarioBatchItemResult item, ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - if (item.Data is not null) - { - return FromFailure(item.Data, item.Error ?? new ErrorInfo("Exception", "Scenario failed.", "scenario_error"), attachmentPolicy); - } - - return new ScenarioReportScenario( - item.Scenario, - item.ScenarioId ?? (item.File is null ? null : ScenarioIdentity.Create(item.File, item.Scenario)), - item.Status, - item.File, - item.Timing?.TotalMs, - item.Timing, - item.Steps ?? [], - null, - item.Steps is null ? [] : GetStepArtifacts(item.Steps, attachmentPolicy), - item.Error); - } - - public static ScenarioReportScenario FromException(string file, Exception exception, string? scenario = null, string? scenarioId = null) => - new( - scenario ?? Path.GetFileNameWithoutExtension(file), - scenarioId ?? file, - "failed", - file, - null, - null, - [], - null, - [], - ScenarioErrorInfo.From(exception)); - - private static IReadOnlyList GetFailureArtifacts(FailureArtifactBundle bundle, ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) - { - return []; - } - - var artifacts = bundle.Artifacts - .Select(artifact => new ScenarioReportArtifact(artifact.Kind, artifact.FileName, bundle.StepIndex, bundle.StepName)) - .ToList(); - if (!string.IsNullOrWhiteSpace(bundle.MetadataFile)) - { - artifacts.Add(new ScenarioReportArtifact("metadata", bundle.MetadataFile, bundle.StepIndex, bundle.StepName)); - } - - return artifacts; - } - - private static IReadOnlyList GetFailureAndStepArtifacts( - IReadOnlyList steps, - FailureArtifactBundle bundle, - ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Never) - { - return []; - } - - var artifacts = new List(); - if (attachmentPolicy == ScenarioArtifactAttachmentPolicy.Always) - { - artifacts.AddRange(GetStepArtifacts(steps, attachmentPolicy)); - } - - artifacts.AddRange(GetFailureArtifacts(bundle, attachmentPolicy)); - return artifacts; - } - - private static IReadOnlyList GetStepArtifacts(IReadOnlyList steps, ScenarioArtifactAttachmentPolicy attachmentPolicy) - { - if (attachmentPolicy != ScenarioArtifactAttachmentPolicy.Always) - { - return []; - } - - return steps - .SelectMany((step, index) => FromStepResult(step, index + 1)) - .ToArray(); - } - - private static IEnumerable FromStepResult(ScenarioStepResult step, int index) - { - if (step.Result is TakeScreenshotResult screenshot) - { - yield return new ScenarioReportArtifact("screenshot", screenshot.File, index, step.Step); - } - - if (step.Result is CaptureArtifactsResult artifacts) - { - yield return new ScenarioReportArtifact("screenshot", artifacts.Screenshot, index, step.Step); - yield return new ScenarioReportArtifact("logcat", artifacts.Logcat, index, step.Step); - yield return new ScenarioReportArtifact("screen_state", artifacts.ScreenState, index, step.Step); - yield return new ScenarioReportArtifact("hierarchy", artifacts.Hierarchy, index, step.Step); - } - } -} - -internal sealed record ScenarioReportArtifact(string Kind, string FileName, int? StepIndex, string? StepName); - -internal static class ScenarioReportFileSystem -{ - public static void CreateReportDirectory(IFileSystem fileSystem, string path) - { - var directory = Path.GetDirectoryName(path); - if (!string.IsNullOrWhiteSpace(directory)) - { - fileSystem.CreateDirectory(directory); - } - } -} diff --git a/Luotsi.Cli/Scenarios/ScenarioRunConfiguration.cs b/Luotsi.Cli/Scenarios/ScenarioRunConfiguration.cs new file mode 100644 index 0000000..a90781f --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunConfiguration.cs @@ -0,0 +1,65 @@ +using Luotsi.Cli.Cli; +using Luotsi.Cli.Errors; + +namespace Luotsi.Cli.Scenarios; + +internal enum ScenarioArtifactAttachmentPolicy +{ + Never, + OnFailure, + Always +} + +internal sealed record ScenarioRunConfiguration( + string? EventsJsonlPath, + string? JsonReportPath, + string? JUnitReportPath, + ScenarioFailureArtifactCapturePolicy FailureArtifactCapturePolicy, + ScenarioArtifactAttachmentPolicy ArtifactAttachmentPolicy) +{ + public static ScenarioRunConfiguration Create(CliOptions options) + { + ArgumentNullException.ThrowIfNull(options); + + return new ScenarioRunConfiguration( + NormalizePath(options.Get("events-jsonl")), + NormalizePath(options.Get("report-json")), + NormalizePath(options.Get("report-junit")), + ParseFailureArtifactCapturePolicy(options.Get("capture-on")), + ParseArtifactAttachmentPolicy(options.Get("attach-artifacts"))); + } + + private static string? NormalizePath(string? value) => + string.IsNullOrWhiteSpace(value) ? null : value; + + private static ScenarioFailureArtifactCapturePolicy ParseFailureArtifactCapturePolicy(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ScenarioFailureArtifactCapturePolicy.Failure; + } + + return value.Trim().ToLowerInvariant() switch + { + "failure" or "on-failure" or "onfailure" => ScenarioFailureArtifactCapturePolicy.Failure, + "never" => ScenarioFailureArtifactCapturePolicy.Never, + _ => throw new UsageException("--capture-on must be one of: failure, never.") + }; + } + + private static ScenarioArtifactAttachmentPolicy ParseArtifactAttachmentPolicy(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return ScenarioArtifactAttachmentPolicy.OnFailure; + } + + return value.Trim().ToLowerInvariant() switch + { + "never" => ScenarioArtifactAttachmentPolicy.Never, + "on-failure" or "onfailure" => ScenarioArtifactAttachmentPolicy.OnFailure, + "always" => ScenarioArtifactAttachmentPolicy.Always, + _ => throw new UsageException("--attach-artifacts must be one of: never, on-failure, always.") + }; + } +} \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioRunOrchestrator.cs b/Luotsi.Cli/Scenarios/ScenarioRunOrchestrator.cs new file mode 100644 index 0000000..c491f77 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunOrchestrator.cs @@ -0,0 +1,74 @@ +using Luotsi.Cli.Infrastructure.Contracts; + +namespace Luotsi.Cli.Scenarios; + +internal sealed class ScenarioRunOrchestrator( + ScenarioRunPlanner runPlanner, + ScenarioExecutorFactory scenarioExecutorFactory, + ScenarioBatchExecutorFactory scenarioBatchExecutorFactory, + ScenarioRunEventCoordinatorFactory scenarioRunEventCoordinatorFactory, + ScenarioRunReportCoordinatorFactory scenarioRunReportCoordinatorFactory) +{ + private readonly ScenarioRunPlanner _runPlanner = runPlanner ?? throw new ArgumentNullException(nameof(runPlanner)); + private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory)); + private readonly ScenarioBatchExecutorFactory _scenarioBatchExecutorFactory = scenarioBatchExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioBatchExecutorFactory)); + private readonly ScenarioRunEventCoordinatorFactory _scenarioRunEventCoordinatorFactory = scenarioRunEventCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunEventCoordinatorFactory)); + private readonly ScenarioRunReportCoordinatorFactory _scenarioRunReportCoordinatorFactory = scenarioRunReportCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunReportCoordinatorFactory)); + + public async Task RunFileAsync(string file, IDeviceHost runner, ScenarioRunConfiguration configuration) + { + ArgumentException.ThrowIfNullOrWhiteSpace(file); + ArgumentNullException.ThrowIfNull(runner); + ArgumentNullException.ThrowIfNull(configuration); + + await using var runEvents = _scenarioRunEventCoordinatorFactory.Create(configuration.EventsJsonlPath); + var runReports = _scenarioRunReportCoordinatorFactory.Create(configuration); + return await RunFileCoreAsync( + file, + runner, + configuration, + runEvents, + runReports).ConfigureAwait(false); + } + + public async Task RunPathAsync(ScenarioQuery query, IDeviceHost runner, ScenarioRunConfiguration configuration) + { + ArgumentNullException.ThrowIfNull(query); + ArgumentNullException.ThrowIfNull(runner); + ArgumentNullException.ThrowIfNull(configuration); + + await using var runEvents = _scenarioRunEventCoordinatorFactory.Create(configuration.EventsJsonlPath); + var runReports = _scenarioRunReportCoordinatorFactory.Create(configuration); + return await RunPathCoreAsync( + query, + runner, + configuration, + runEvents, + runReports).ConfigureAwait(false); + } + + private Task RunFileCoreAsync( + string file, + IDeviceHost runner, + ScenarioRunConfiguration configuration, + ScenarioRunEventCoordinator runEvents, + ScenarioRunReportCoordinator runReports) => + runReports.RunFileAsync( + file, + () => runEvents.RunFileAsync( + file, + sink => _scenarioExecutorFactory.Create(runner, sink, configuration.FailureArtifactCapturePolicy).RunAsync(file))); + + private Task RunPathCoreAsync( + ScenarioQuery query, + IDeviceHost runner, + ScenarioRunConfiguration configuration, + ScenarioRunEventCoordinator runEvents, + ScenarioRunReportCoordinator runReports) => + runEvents.RunPathAsync( + query, + _ => runReports.PlanPathAsync(query, () => _runPlanner.CreateAsync(query)), + (preparedPlan, sink) => runReports.RunBatchAsync( + preparedPlan, + () => _scenarioBatchExecutorFactory.Create(runner, sink, configuration.FailureArtifactCapturePolicy).RunAsync(preparedPlan))); +} \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioRunReportFactory.cs b/Luotsi.Cli/Scenarios/ScenarioRunReportFactory.cs new file mode 100644 index 0000000..c33a3a2 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunReportFactory.cs @@ -0,0 +1,226 @@ +using Luotsi.Cli.Models; + +namespace Luotsi.Cli.Scenarios; + +internal static class ScenarioRunReportFactory +{ + private const string ReportSchema = "luotsi-scenario-run-report.v1"; + + public static ScenarioRunReport FromSingle( + string file, + ScenarioRunResult result, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + ReportSchema, + file, + result.Status, + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 1, + 1, + 1, + result.Status == "passed" ? 1 : 0, + result.Status == "passed" ? 0 : 1, + 0, + null, + null, + null, + [CreateScenarioFromSuccess(result, file, attachmentPolicy)]); + + public static ScenarioRunReport FromSingleFailure( + string file, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + var error = ScenarioErrorInfo.From(exception); + var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; + var scenario = failureData is null + ? CreateScenarioFromException(file, exception) + : CreateScenarioFromFailure(failureData, error, attachmentPolicy); + return new ScenarioRunReport( + ReportSchema, + file, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 1, + 1, + 1, + 0, + 1, + 0, + null, + null, + null, + [scenario], + error); + } + + public static ScenarioRunReport FromBatch( + ScenarioRunBatchResult result, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + ReportSchema, + result.Path, + result.Status, + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + result.TotalCount, + result.MatchedCount, + result.SelectedCount, + result.PassedCount, + result.FailedCount, + result.ShardedOutCount, + result.ShardCount, + result.ShardIndex, + result.ShardStrategy, + result.Scenarios.Select(scenario => CreateScenarioFromBatchItem(scenario, attachmentPolicy)).ToArray()); + + public static ScenarioRunReport FromBatchFailure( + ScenarioRunPlan plan, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + var error = ScenarioErrorInfo.From(exception); + var failureData = (exception as ICommandFailureDetails)?.DataPayload as ScenarioRunFailureData; + ScenarioReportScenario[] scenarios = failureData is null + ? [CreateScenarioFromException(plan.Query.Path, exception, "scenario run", $"{plan.Query.Path}::run")] + : [CreateScenarioFromFailure(failureData, error, attachmentPolicy)]; + return new ScenarioRunReport( + ReportSchema, + plan.Query.Path, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + plan.TotalCount, + plan.MatchedCount, + plan.SelectedCount, + 0, + 1, + plan.ShardedOutCount, + plan.Query.ShardCount, + plan.Query.ShardIndex, + plan.Query.ShardStrategy, + scenarios, + error); + } + + public static ScenarioRunReport FromQueryFailure( + ScenarioQuery query, + Exception exception, + DateTimeOffset startedAt, + DateTimeOffset endedAt) + { + var error = ScenarioErrorInfo.From(exception); + return new ScenarioRunReport( + ReportSchema, + query.Path, + "failed", + startedAt, + endedAt, + CalculateDurationMs(startedAt, endedAt), + 0, + 0, + 0, + 0, + 1, + 0, + query.ShardCount, + query.ShardIndex, + query.ShardStrategy, + [CreateScenarioFromException(query.Path, exception, "scenario discovery", $"{query.Path}::discovery")], + error); + } + + private static ScenarioReportScenario CreateScenarioFromSuccess( + ScenarioRunResult result, + string? file, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + result.Scenario, + result.ScenarioId ?? (file is null ? null : ScenarioIdentity.Create(file, result.Scenario)), + result.Status, + result.File ?? file, + result.Timing.TotalMs, + result.Timing, + result.Steps, + null, + ScenarioReportArtifactProjection.FromSteps(result.Steps, attachmentPolicy), + null); + + private static ScenarioReportScenario CreateScenarioFromFailure( + ScenarioRunFailureData data, + ErrorInfo error, + ScenarioArtifactAttachmentPolicy attachmentPolicy) => + new( + data.Scenario, + data.ScenarioId ?? ScenarioIdentity.Create(data.File, data.Scenario), + data.Status, + data.File, + data.Timing.TotalMs, + data.Timing, + data.Steps, + data.FailedStep, + ScenarioReportArtifactProjection.FromFailureAndSteps(data.Steps, data.FailureArtifacts, attachmentPolicy), + error); + + private static ScenarioReportScenario CreateScenarioFromBatchItem( + ScenarioBatchItemResult item, + ScenarioArtifactAttachmentPolicy attachmentPolicy) + { + if (item.Data is not null) + { + return CreateScenarioFromFailure( + item.Data, + item.Error ?? new ErrorInfo("Exception", "Scenario failed.", "scenario_error"), + attachmentPolicy); + } + + return new ScenarioReportScenario( + item.Scenario, + item.ScenarioId ?? (item.File is null ? null : ScenarioIdentity.Create(item.File, item.Scenario)), + item.Status, + item.File, + item.Timing?.TotalMs, + item.Timing, + item.Steps ?? [], + null, + item.Steps is null ? [] : ScenarioReportArtifactProjection.FromSteps(item.Steps, attachmentPolicy), + item.Error); + } + + private static ScenarioReportScenario CreateScenarioFromException( + string file, + Exception exception, + string? scenario = null, + string? scenarioId = null) + { + var scenarioName = scenario ?? Path.GetFileNameWithoutExtension(file); + return new ScenarioReportScenario( + scenarioName, + scenarioId ?? ScenarioIdentity.Create(file, scenarioName), + "failed", + file, + null, + null, + [], + null, + [], + ScenarioErrorInfo.From(exception)); + } + + private static double CalculateDurationMs(DateTimeOffset startedAt, DateTimeOffset endedAt) => + Math.Max(0, (endedAt - startedAt).TotalMilliseconds); +} \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioRunReportModels.cs b/Luotsi.Cli/Scenarios/ScenarioRunReportModels.cs new file mode 100644 index 0000000..474a896 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunReportModels.cs @@ -0,0 +1,36 @@ +using Luotsi.Cli.Models; + +namespace Luotsi.Cli.Scenarios; + +internal sealed record ScenarioRunReport( + string Schema, + string Path, + string Status, + DateTimeOffset StartedAt, + DateTimeOffset EndedAt, + double DurationMs, + int TotalCount, + int MatchedCount, + int SelectedCount, + int PassedCount, + int FailedCount, + int ShardedOutCount, + int? ShardCount, + int? ShardIndex, + string? ShardStrategy, + IReadOnlyList Scenarios, + ErrorInfo? Error = null); + +internal sealed record ScenarioReportScenario( + string Scenario, + string? ScenarioId, + string Status, + string? File, + double? DurationMs, + ScenarioRunTiming? Timing, + IReadOnlyList Steps, + ScenarioFailedStepResult? FailedStep, + IReadOnlyList Artifacts, + ErrorInfo? Error); + +internal sealed record ScenarioReportArtifact(string Kind, string FileName, int? StepIndex, string? StepName); \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioRunReportWriters.cs b/Luotsi.Cli/Scenarios/ScenarioRunReportWriters.cs new file mode 100644 index 0000000..aaec9be --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunReportWriters.cs @@ -0,0 +1,122 @@ +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Xml.Linq; +using Luotsi.Cli.Infrastructure.Contracts; + +namespace Luotsi.Cli.Scenarios; + +internal interface IScenarioRunReportWriter +{ + Task WriteAsync(ScenarioRunReport report); +} + +internal sealed class CompositeScenarioRunReportWriter(IReadOnlyList writers) : IScenarioRunReportWriter +{ + private readonly IReadOnlyList _writers = writers ?? throw new ArgumentNullException(nameof(writers)); + + public async Task WriteAsync(ScenarioRunReport report) + { + foreach (var writer in _writers) + { + await writer.WriteAsync(report).ConfigureAwait(false); + } + } +} + +internal sealed class JsonScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = true + }; + + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; + + public async Task WriteAsync(ScenarioRunReport report) + { + ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); + await _fileSystem.WriteAllTextAsync(_path, JsonSerializer.Serialize(report, JsonOptions), Encoding.UTF8).ConfigureAwait(false); + } +} + +internal sealed class JUnitScenarioRunReportWriter(IFileSystem fileSystem, string path) : IScenarioRunReportWriter +{ + private readonly IFileSystem _fileSystem = fileSystem ?? throw new ArgumentNullException(nameof(fileSystem)); + private readonly string _path = string.IsNullOrWhiteSpace(path) ? throw new ArgumentException("Report path must be non-empty.", nameof(path)) : path; + + public async Task WriteAsync(ScenarioRunReport report) + { + ScenarioReportFileSystem.CreateReportDirectory(_fileSystem, _path); + var xml = ToXml(report); + await _fileSystem.WriteAllTextAsync(_path, xml.ToString(SaveOptions.DisableFormatting), Encoding.UTF8).ConfigureAwait(false); + } + + private static XDocument ToXml(ScenarioRunReport report) + { + var testCases = report.Scenarios.Select(ToTestCase).ToArray(); + return new XDocument( + new XElement( + "testsuite", + new XAttribute("name", report.Path), + new XAttribute("tests", report.Scenarios.Count), + new XAttribute("failures", report.FailedCount), + new XAttribute("skipped", 0), + new XAttribute("time", Seconds(report.DurationMs)), + new XAttribute("timestamp", report.StartedAt.ToString("O", CultureInfo.InvariantCulture)), + testCases)); + } + + private static XElement ToTestCase(ScenarioReportScenario scenario) + { + var element = new XElement( + "testcase", + new XAttribute("classname", string.IsNullOrWhiteSpace(scenario.File) ? "luotsi.scenario" : scenario.File), + new XAttribute("name", scenario.Scenario), + new XAttribute("id", scenario.ScenarioId ?? scenario.Scenario), + new XAttribute("time", Seconds(scenario.DurationMs ?? 0))); + + if (scenario.Status != "passed") + { + element.Add(new XElement( + "failure", + new XAttribute("type", scenario.Error?.Category ?? "scenario_error"), + new XAttribute("message", scenario.Error?.Message ?? $"Scenario '{scenario.Scenario}' failed."), + scenario.FailedStep is null + ? scenario.Error?.Message + : $"Step {scenario.FailedStep.Index} ({scenario.FailedStep.Name}) failed during {scenario.FailedStep.Action}. {scenario.Error?.Message}".Trim())); + } + + if (scenario.Artifacts.Count > 0) + { + element.Add(new XElement( + "system-out", + string.Join( + Environment.NewLine, + scenario.Artifacts.Select(static artifact => + $"{artifact.Kind}: {artifact.FileName}" + + (artifact.StepIndex is null ? string.Empty : $" (step {artifact.StepIndex}: {artifact.StepName})"))))); + } + + return element; + } + + private static string Seconds(double milliseconds) => + Math.Max(0, milliseconds / 1000d).ToString("0.###", CultureInfo.InvariantCulture); +} + +internal static class ScenarioReportFileSystem +{ + public static void CreateReportDirectory(IFileSystem fileSystem, string path) + { + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrWhiteSpace(directory)) + { + fileSystem.CreateDirectory(directory); + } + } +} \ No newline at end of file diff --git a/Luotsi.Cli/Scenarios/ScenarioRunSupport.cs b/Luotsi.Cli/Scenarios/ScenarioRunSupport.cs new file mode 100644 index 0000000..22feac1 --- /dev/null +++ b/Luotsi.Cli/Scenarios/ScenarioRunSupport.cs @@ -0,0 +1,36 @@ +using Luotsi.Cli.Models; + +namespace Luotsi.Cli.Scenarios; + +internal enum ScenarioFailureArtifactCapturePolicy +{ + Failure, + Never +} + +public static class ScenarioIdentity +{ + public static string Create(string file, string scenarioName) + { + ArgumentException.ThrowIfNullOrWhiteSpace(file); + ArgumentException.ThrowIfNullOrWhiteSpace(scenarioName); + return $"{file}::{scenarioName}"; + } +} + +internal static class ScenarioErrorInfo +{ + public static ErrorInfo From(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + return ErrorInfo.From(exception, GetCategory(exception)); + } + + public static string GetCategory(Exception exception) + { + ArgumentNullException.ThrowIfNull(exception); + return exception is ICommandFailureDetails failure + ? failure.CategoryOverride + : ErrorInfo.Classify(exception.Message); + } +} \ No newline at end of file