diff --git a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs index 1032b29..6cc62af 100644 --- a/Luotsi.Cli.Tests/ScenarioExecutorTests.cs +++ b/Luotsi.Cli.Tests/ScenarioExecutorTests.cs @@ -431,6 +431,120 @@ public async Task RunAsync_File_Writes_Jsonl_Events_With_Terminal_Result() Assert.Equal(0, events[^1].GetProperty("failed_count").GetInt32()); } + [Fact] + public async Task RunAsync_File_Runs_Setup_Steps_Teardown_In_Order() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "lifecycle", + "setup": [ + { "name": "launch", "action": "startApp", "package": "dev.luotsi.app", "activity": ".MainActivity", "wait": true } + ], + "steps": [ + { "name": "press back", "action": "keyevent", "code": "KEYCODE_BACK" } + ], + "teardown": [ + { "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" } + ] + } + """); + var host = new FakeDeviceHost(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--events-jsonl", "/tmp/events.jsonl"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + var passedSteps = events + .Where(static evt => evt.GetProperty("event").GetString() == "scenario_step_passed") + .ToArray(); + + Assert.Equal(0, exitCode); + Assert.Equal([("setup", 1), ("main", 2), ("teardown", 3)], passedSteps + .Select(static evt => (evt.GetProperty("phase").GetString()!, evt.GetProperty("step_index").GetInt32())) + .ToArray()); + Assert.Equal("dev.luotsi.app", host.StartAppRequests.Single().Package); + Assert.Equal("KEYCODE_BACK", host.KeyEventRequests.Single()); + Assert.Equal("dev.luotsi.app", host.ForceStopRequests.Single()); + } + + [Fact] + public async Task RunAsync_File_First_Setup_Step_Cannot_Observe_Previous_Step() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "invalid lifecycle", + "setup": [ + { "name": "event", "action": "assertEvent", "event": "ready", "observeFromPreviousStep": true } + ], + "steps": [ + { "name": "pause", "action": "sleep", "milliseconds": 1 } + ] + } + """); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json"]); + using var envelope = console.ParseSingleOutputAsJson(); + + Assert.Equal(2, exitCode); + Assert.Contains("no previous lifecycle step", envelope.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_File_First_Main_Step_Can_Observe_Setup_Step() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new SteppingTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind), TimeSpan.FromMilliseconds(10)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "observe setup", + "setup": [ + { "name": "pause", "action": "sleep", "milliseconds": 25 } + ], + "steps": [ + { "name": "event", "action": "assertEvent", "event": "ready", "observeFromPreviousStep": true } + ] + } + """); + var host = new FakeDeviceHost(); + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new SteppingDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json"]); + + Assert.Equal(0, exitCode); + Assert.NotNull(host.AssertEventRequests.Single().Since); + } + [Fact] public async Task RunAsync_File_Writes_Consistent_Scenario_Ended_Timestamp_And_Duration() { @@ -506,6 +620,110 @@ public async Task RunAsync_File_Failure_Writes_Jsonl_Terminal_Result() Assert.Equal(1, events[^1].GetProperty("failed_count").GetInt32()); } + [Fact] + public async Task RunAsync_File_Runs_Teardown_After_Main_Failure_And_Reports_Phases() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "failing lifecycle", + "setup": [ + { "name": "launch", "action": "startApp", "package": "dev.luotsi.app" } + ], + "steps": [ + { "name": "target", "action": "waitVisible", "text": "Target" } + ], + "teardown": [ + { "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" } + ] + } + """); + var host = new FakeDeviceHost + { + WaitVisibleException = new InvalidOperationException("not visible") + }; + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync([ + "run", + "--file", "/tmp/scenario.json", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + var scenario = report.RootElement.GetProperty("scenarios")[0]; + + Assert.Equal(1, exitCode); + Assert.Equal("dev.luotsi.app", host.ForceStopRequests.Single()); + Assert.Contains(events, static evt => + evt.GetProperty("event").GetString() == "scenario_step_passed" && + evt.GetProperty("phase").GetString() == "teardown"); + Assert.Equal("main", scenario.GetProperty("failed_step").GetProperty("phase").GetString()); + Assert.Contains(scenario.GetProperty("steps").EnumerateArray(), static step => + step.GetProperty("phase").GetString() == "teardown" && + step.GetProperty("action").GetString() == "forceStop"); + } + + [Fact] + public async Task RunAsync_File_Teardown_Failure_Does_Not_Mask_Main_Failure() + { + var fileSystem = new FakeFileSystem(); + var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind)); + var console = new FakeConsole(); + fileSystem.AddFile("/tmp/scenario.json", """ + { + "name": "double failure lifecycle", + "steps": [ + { "name": "target", "action": "waitVisible", "text": "Target" } + ], + "teardown": [ + { "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" } + ] + } + """); + var host = new FakeDeviceHost + { + WaitVisibleException = new InvalidOperationException("not visible"), + ForceStopException = new InvalidOperationException("cleanup failed") + }; + var app = new App(new AppDependencies + { + TimeProvider = timeProvider, + FileSystem = fileSystem, + ProcessRunner = new DefaultProcessRunner(), + Delay = new FakeDelay(timeProvider), + DeviceHostFactory = new FakeDeviceHostFactory(host), + Console = console + }); + + var exitCode = await app.RunAsync([ + "run", + "--file", "/tmp/scenario.json", + "--events-jsonl", "/tmp/events.jsonl", + "--report-json", "/tmp/report.json"]); + var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl"); + using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json")); + + Assert.Equal(1, exitCode); + Assert.Contains("main step", report.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.DoesNotContain("cleanup failed", report.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal); + Assert.Equal("main", report.RootElement.GetProperty("scenarios")[0].GetProperty("failed_step").GetProperty("phase").GetString()); + Assert.Contains(events, static evt => + evt.GetProperty("event").GetString() == "scenario_step_failed" && + evt.GetProperty("phase").GetString() == "teardown" && + evt.GetProperty("error").GetProperty("message").GetString() == "cleanup failed"); + } + [Fact] public async Task RunAsync_Path_Writes_Jsonl_Run_And_Per_Scenario_Events() { diff --git a/Luotsi.Cli.Tests/TestSupport.cs b/Luotsi.Cli.Tests/TestSupport.cs index c3adc46..ae8af7a 100644 --- a/Luotsi.Cli.Tests/TestSupport.cs +++ b/Luotsi.Cli.Tests/TestSupport.cs @@ -496,6 +496,8 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic public List<(string Package, string Permission)> RevokePermissionRequests { get; } = []; + public List<(string Name, DateTimeOffset? Since)> AssertEventRequests { get; } = []; + public List ConnectedDevices { get; } = []; public PreflightResult PreflightTemplate { get; set; } = new("Model", "16", "36", "focus", null, null, "fingerprint", "arm64-v8a", "SER"); @@ -508,6 +510,8 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic public Exception? WaitVisibleException { get; set; } + public Exception? ForceStopException { get; set; } + public FailureArtifactBundle? FailureArtifacts { get; set; } public Exception? FailureArtifactException { get; set; } @@ -609,8 +613,11 @@ public Task TapPointAsync(string? label, int? x, int? y, double? public Task ResetLogAsync() => Task.FromResult(new ResetLogResult(true)); - public Task AssertEventAsync(string name, IReadOnlyList contains, string? detailsPattern, int timeoutSec, DateTimeOffset? since = null) => - Task.FromResult(new AssertEventResult(name, contains, detailsPattern, string.Empty)); + public Task AssertEventAsync(string name, IReadOnlyList contains, string? detailsPattern, int timeoutSec, DateTimeOffset? since = null) + { + AssertEventRequests.Add((name, since)); + return Task.FromResult(new AssertEventResult(name, contains, detailsPattern, string.Empty)); + } public Task TakeScreenshotAsync(string label) { @@ -791,6 +798,11 @@ public Task StartUriAsync(string uri, string? packageName, strin public Task ForceStopAsync(string packageName) { ForceStopRequests.Add(packageName); + if (ForceStopException is not null) + { + throw ForceStopException; + } + return Task.FromResult(new AppPackageCommandResult(packageName)); } diff --git a/Luotsi.Cli/Models/Contracts.cs b/Luotsi.Cli/Models/Contracts.cs index a18971c..ea0de43 100644 --- a/Luotsi.Cli/Models/Contracts.cs +++ b/Luotsi.Cli/Models/Contracts.cs @@ -257,11 +257,15 @@ public sealed record Bounds(int Left, int Top, int Right, int Bottom); /// Scenario steps. /// Optional scenario variables. /// Optional runner tags used for filtering and reporting. +/// Optional setup steps that run before scenario steps. +/// Optional teardown steps that run after setup and scenario steps. public sealed record ScenarioFile( string Name, IReadOnlyList Steps, IReadOnlyDictionary? Variables = null, - IReadOnlyList? Tags = null); + IReadOnlyList? Tags = null, + IReadOnlyList? Setup = null, + IReadOnlyList? Teardown = null); /// /// Scenario playbook step. diff --git a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs index 099549f..55dc305 100644 --- a/Luotsi.Cli/Scenarios/ScenarioCatalog.cs +++ b/Luotsi.Cli/Scenarios/ScenarioCatalog.cs @@ -51,14 +51,16 @@ public sealed record ScenarioStepResult( ScenarioStepTiming Timing, object? Result = null, string? Status = null, - ErrorInfo? Error = null); + ErrorInfo? Error = null, + string Phase = ScenarioStepPhases.Main); public sealed record ScenarioFailedStepResult( int Index, string Name, string Action, double DurationMs, - ScenarioStepTiming Timing); + ScenarioStepTiming Timing, + string Phase = ScenarioStepPhases.Main); public sealed record ScenarioRunResult( string Scenario, @@ -144,6 +146,13 @@ public static class ScenarioShardStrategies public const string Hash = "hash"; } +public static class ScenarioStepPhases +{ + public const string Setup = "setup"; + public const string Main = "main"; + public const string Teardown = "teardown"; +} + internal sealed class ScenarioCatalog( IFileSystem fileSystem, IScenarioTemplateResolver templateResolver) @@ -355,7 +364,7 @@ private static ScenarioCatalogEntry ToEntry(string file, ScenarioFile scenario) .Distinct(StringComparer.OrdinalIgnoreCase) .OrderBy(static tag => tag, StringComparer.OrdinalIgnoreCase) .ToArray() ?? []; - var actions = scenario.Steps + var actions = EnumerateLifecycleSteps(scenario) .Select(static step => step.Action) .Where(static action => !string.IsNullOrWhiteSpace(action)) .Distinct(StringComparer.OrdinalIgnoreCase) @@ -367,10 +376,30 @@ private static ScenarioCatalogEntry ToEntry(string file, ScenarioFile scenario) scenario.Name, file, tags, - scenario.Steps.Count, + EnumerateLifecycleSteps(scenario).Count(), actions); } + internal static IEnumerable EnumerateLifecycleSteps(ScenarioFile scenario) + { + ArgumentNullException.ThrowIfNull(scenario); + + foreach (var step in scenario.Setup ?? []) + { + yield return step; + } + + foreach (var step in scenario.Steps) + { + yield return step; + } + + foreach (var step in scenario.Teardown ?? []) + { + yield return step; + } + } + private static int GetStableShardIndex(string value, int shardCount) { unchecked diff --git a/Luotsi.Cli/Scenarios/ScenarioEvents.cs b/Luotsi.Cli/Scenarios/ScenarioEvents.cs index 4824e22..fb6199f 100644 --- a/Luotsi.Cli/Scenarios/ScenarioEvents.cs +++ b/Luotsi.Cli/Scenarios/ScenarioEvents.cs @@ -240,6 +240,7 @@ internal sealed record ScenarioEvent( [property: JsonPropertyName("file")] string? File = null, [property: JsonPropertyName("scenario_id")] string? ScenarioId = null, [property: JsonPropertyName("scenario")] string? Scenario = null, + [property: JsonPropertyName("phase")] string? Phase = null, [property: JsonPropertyName("step_index")] int? StepIndex = null, [property: JsonPropertyName("step")] string? Step = null, [property: JsonPropertyName("action")] string? Action = null, diff --git a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs index 26a769f..2514d16 100644 --- a/Luotsi.Cli/Scenarios/ScenarioExecutor.cs +++ b/Luotsi.Cli/Scenarios/ScenarioExecutor.cs @@ -155,7 +155,7 @@ public async Task RunAsync(string file) { await _actionHost.WriteDeviceFingerprintAsync().ConfigureAwait(false); var prologueMs = (_timeProvider.GetUtcNow() - scenarioStarted).TotalMilliseconds; - var execution = await ExecuteStepsAsync(scenario, file, scenarioId, scenarioStarted, prologueMs).ConfigureAwait(false); + var execution = await ExecuteLifecycleAsync(scenario, file, scenarioId, scenarioStarted, prologueMs).ConfigureAwait(false); status = "passed"; return new ScenarioRunResult( @@ -186,48 +186,80 @@ private Task LoadValidatedScenarioAsync(string file) => private async Task ExecuteStepAsync(ScenarioStep step, DateTimeOffset? previousStepStartedAt) => await _actionDispatcher.ExecuteAsync(step, previousStepStartedAt).ConfigureAwait(false); - private async Task ExecuteStepsAsync(ScenarioFile scenario, string file, string scenarioId, DateTimeOffset scenarioStarted, double prologueMs) + private async Task ExecuteLifecycleAsync(ScenarioFile scenario, string file, string scenarioId, DateTimeOffset scenarioStarted, double prologueMs) { - var steps = new List(scenario.Steps.Count); - var executedStepMs = 0d; - DateTimeOffset? previousStepStartedAt = null; + var context = new ScenarioExecutionContext(scenario, file, scenarioId, scenarioStarted, prologueMs); + Exception? firstFailure = null; - for (var index = 0; index < scenario.Steps.Count; index++) + try + { + await ExecuteStepsAsync(context, scenario.Setup ?? [], ScenarioStepPhases.Setup).ConfigureAwait(false); + await ExecuteStepsAsync(context, scenario.Steps, ScenarioStepPhases.Main).ConfigureAwait(false); + } + catch (Exception ex) + { + firstFailure = ex; + } + + try + { + await ExecuteStepsAsync(context, scenario.Teardown ?? [], ScenarioStepPhases.Teardown).ConfigureAwait(false); + } + catch (Exception ex) + { + firstFailure ??= ex; + } + + if (firstFailure is not null) + { + throw firstFailure; + } + + return new ScenarioExecution(context.ExecutedStepMs, context.Steps); + } + + private async Task ExecuteStepsAsync(ScenarioExecutionContext context, IReadOnlyList scenarioSteps, string phase) + { + for (var index = 0; index < scenarioSteps.Count; index++) { - var step = scenario.Steps[index]; + var step = scenarioSteps[index]; using var delayScope = DelayMetrics.BeginScope(); var started = _timeProvider.GetUtcNow(); - await EmitStepAsync("scenario_step_started", file, scenarioId, scenario.Name, index, step, started).ConfigureAwait(false); + await EmitStepAsync("scenario_step_started", context.File, context.ScenarioId, context.Scenario.Name, phase, context.NextStepIndex, step, started).ConfigureAwait(false); try { - var result = await ExecuteStepAsync(step, previousStepStartedAt).ConfigureAwait(false); + var result = await ExecuteStepAsync(step, context.PreviousStepStartedAt).ConfigureAwait(false); var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; - executedStepMs += durationMs; - previousStepStartedAt = started; - await EmitStepAsync("scenario_step_passed", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "passed", durationMs).ConfigureAwait(false); + context.ExecutedStepMs += durationMs; + context.PreviousStepStartedAt = started; + await EmitStepAsync("scenario_step_passed", context.File, context.ScenarioId, context.Scenario.Name, phase, context.NextStepIndex, step, _timeProvider.GetUtcNow(), "passed", durationMs).ConfigureAwait(false); - steps.Add(new ScenarioStepResult( + context.Steps.Add(new ScenarioStepResult( step.Name ?? step.Action, step.Action, durationMs, CreateTimingData(step, durationMs, delayScope.TotalMilliseconds), - Result: result)); + Result: result, + Phase: phase)); + context.NextStepIndex++; } catch (Exception ex) when (step.ContinueOnError is true && ex is not UsageException) { var error = ScenarioErrorInfo.From(ex); var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; - executedStepMs += durationMs; - previousStepStartedAt = started; - await EmitStepAsync("scenario_step_continued_on_error", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "continued_on_error", durationMs, error).ConfigureAwait(false); - steps.Add(new ScenarioStepResult( + context.ExecutedStepMs += durationMs; + context.PreviousStepStartedAt = started; + await EmitStepAsync("scenario_step_continued_on_error", context.File, context.ScenarioId, context.Scenario.Name, phase, context.NextStepIndex, step, _timeProvider.GetUtcNow(), "continued_on_error", durationMs, error).ConfigureAwait(false); + context.Steps.Add(new ScenarioStepResult( step.Name ?? step.Action, step.Action, durationMs, CreateTimingData(step, durationMs, delayScope.TotalMilliseconds), Status: "continued_on_error", - Error: error)); + Error: error, + Phase: phase)); + context.NextStepIndex++; } catch (UsageException) { @@ -236,34 +268,33 @@ private async Task ExecuteStepsAsync(ScenarioFile scenario, s catch (Exception ex) { var durationMs = (_timeProvider.GetUtcNow() - started).TotalMilliseconds; - executedStepMs += durationMs; + context.ExecutedStepMs += durationMs; var error = ScenarioErrorInfo.From(ex); - await EmitStepAsync("scenario_step_failed", file, scenarioId, scenario.Name, index, step, _timeProvider.GetUtcNow(), "failed", durationMs, error).ConfigureAwait(false); + await EmitStepAsync("scenario_step_failed", context.File, context.ScenarioId, context.Scenario.Name, phase, context.NextStepIndex, step, _timeProvider.GetUtcNow(), "failed", durationMs, error).ConfigureAwait(false); var failureArtifacts = await CaptureFailureArtifactsBestEffortAsync( - new FailureCaptureRequest("scenario", scenario.Name, file, index + 1, step.Name ?? step.Action, step.Action), + new FailureCaptureRequest("scenario", context.Scenario.Name, context.File, context.NextStepIndex, step.Name ?? step.Action, step.Action), ex).ConfigureAwait(false); throw new ScenarioStepFailureException( - $"Scenario '{scenario.Name}' failed at step {index + 1} ({step.Name ?? step.Action}).", + $"Scenario '{context.Scenario.Name}' failed during {phase} step {context.NextStepIndex} ({step.Name ?? step.Action}).", error.Category, new ScenarioRunFailureData( - scenario.Name, - file, + context.Scenario.Name, + context.File, "failed", - CreateScenarioRunTiming((_timeProvider.GetUtcNow() - scenarioStarted).TotalMilliseconds, prologueMs, executedStepMs), + CreateScenarioRunTiming((_timeProvider.GetUtcNow() - context.ScenarioStarted).TotalMilliseconds, context.PrologueMs, context.ExecutedStepMs), new ScenarioFailedStepResult( - index + 1, + context.NextStepIndex, step.Name ?? step.Action, step.Action, durationMs, - CreateTimingData(step, durationMs, delayScope.TotalMilliseconds)), - steps, + CreateTimingData(step, durationMs, delayScope.TotalMilliseconds), + phase), + context.Steps, failureArtifacts, - scenarioId), + context.ScenarioId), ex); } } - - return new ScenarioExecution(executedStepMs, steps); } private static ScenarioStepTiming CreateTimingData(ScenarioStep step, double durationMs, int harnessDelayMs) @@ -292,6 +323,7 @@ private Task EmitStepAsync( string file, string scenarioId, string scenario, + string phase, int index, ScenarioStep step, DateTimeOffset timestamp, @@ -305,7 +337,8 @@ private Task EmitStepAsync( File: file, ScenarioId: scenarioId, Scenario: scenario, - StepIndex: index + 1, + Phase: phase, + StepIndex: index, Step: step.Name ?? step.Action, Action: step.Action, DurationMs: durationMs, @@ -356,6 +389,32 @@ private FailureArtifactBundle CreateEmptyFailureArtifactBundle(FailureCaptureReq []); private sealed record ScenarioExecution(double ExecutedStepMs, IReadOnlyList Steps); + + private sealed class ScenarioExecutionContext( + ScenarioFile scenario, + string file, + string scenarioId, + DateTimeOffset scenarioStarted, + double prologueMs) + { + public ScenarioFile Scenario { get; } = scenario; + + public string File { get; } = file; + + public string ScenarioId { get; } = scenarioId; + + public DateTimeOffset ScenarioStarted { get; } = scenarioStarted; + + public double PrologueMs { get; } = prologueMs; + + public List Steps { get; } = []; + + public double ExecutedStepMs { get; set; } + + public DateTimeOffset? PreviousStepStartedAt { get; set; } + + public int NextStepIndex { get; set; } = 1; + } } public interface ICommandFailureDetails diff --git a/Luotsi.Cli/Scenarios/ScenarioValidator.cs b/Luotsi.Cli/Scenarios/ScenarioValidator.cs index a34d9d7..9f18683 100644 --- a/Luotsi.Cli/Scenarios/ScenarioValidator.cs +++ b/Luotsi.Cli/Scenarios/ScenarioValidator.cs @@ -18,17 +18,39 @@ public static ScenarioFile ValidateScenario(ScenarioFile scenario, string file, throw new UsageException($"Scenario file '{file}' must define at least one step."); } - for (var index = 0; index < scenario.Steps.Count; index++) - { - ValidateStep(scenario, scenario.Steps[index], index + 1, supportedScenarioActions); - } + var hasPreviousLifecycleStep = false; + ValidateSteps(scenario, scenario.Setup ?? [], ScenarioStepPhases.Setup, supportedScenarioActions, ref hasPreviousLifecycleStep); + ValidateSteps(scenario, scenario.Steps, ScenarioStepPhases.Main, supportedScenarioActions, ref hasPreviousLifecycleStep); + ValidateSteps(scenario, scenario.Teardown ?? [], ScenarioStepPhases.Teardown, supportedScenarioActions, ref hasPreviousLifecycleStep); return scenario; } - private static void ValidateStep(ScenarioFile scenario, ScenarioStep step, int index, IReadOnlySet supportedScenarioActions) + private static void ValidateSteps( + ScenarioFile scenario, + IReadOnlyList steps, + string phase, + IReadOnlySet supportedScenarioActions, + ref bool hasPreviousLifecycleStep) + { + for (var index = 0; index < steps.Count; index++) + { + ValidateStep(scenario, steps[index], phase, index + 1, hasPreviousLifecycleStep, supportedScenarioActions); + hasPreviousLifecycleStep = true; + } + } + + private static void ValidateStep( + ScenarioFile scenario, + ScenarioStep step, + string phase, + int index, + bool hasPreviousLifecycleStep, + IReadOnlySet supportedScenarioActions) { - var stepLabel = $"Scenario '{scenario.Name}' step {index}"; + var stepLabel = string.Equals(phase, ScenarioStepPhases.Main, StringComparison.Ordinal) + ? $"Scenario '{scenario.Name}' step {index}" + : $"Scenario '{scenario.Name}' {phase} step {index}"; if (string.IsNullOrWhiteSpace(step.Action)) { throw new UsageException($"{stepLabel} must define a non-empty action."); @@ -200,9 +222,9 @@ private static void ValidateStep(ScenarioFile scenario, ScenarioStep step, int i if (string.Equals(action, "assertEvent", StringComparison.OrdinalIgnoreCase) && step.ObserveFromPreviousStep is true && - index == 1) + !hasPreviousLifecycleStep) { - throw new UsageException($"{stepLabel} assertEvent cannot observe from the previous step when it is the first step."); + throw new UsageException($"{stepLabel} assertEvent cannot observe from the previous step when no previous lifecycle step has run."); } }