Skip to content

Add scenario setup and teardown lifecycle #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slideep merged 1 commit into from
May 19, 2026
Merged

Add scenario setup and teardown lifecycle #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 218 additions & 0 deletions Luotsi.Cli.Tests/ScenarioExecutorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,120 @@ public async Task RunAsync_File_Writes_Jsonl_Events_With_Terminal_Result()
Assert.Equal(0, events[^1].GetProperty("failed_count").GetInt32());
}

[Fact]
public async Task RunAsync_File_Runs_Setup_Steps_Teardown_In_Order()
{
var fileSystem = new FakeFileSystem();
var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind));
var console = new FakeConsole();
fileSystem.AddFile("/tmp/scenario.json", """
{
"name": "lifecycle",
"setup": [
{ "name": "launch", "action": "startApp", "package": "dev.luotsi.app", "activity": ".MainActivity", "wait": true }
],
"steps": [
{ "name": "press back", "action": "keyevent", "code": "KEYCODE_BACK" }
],
"teardown": [
{ "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" }
]
}
""");
var host = new FakeDeviceHost();
var app = new App(new AppDependencies
{
TimeProvider = timeProvider,
FileSystem = fileSystem,
ProcessRunner = new DefaultProcessRunner(),
Delay = new FakeDelay(timeProvider),
DeviceHostFactory = new FakeDeviceHostFactory(host),
Console = console
});

var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json", "--events-jsonl", "/tmp/events.jsonl"]);
var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl");
var passedSteps = events
.Where(static evt => evt.GetProperty("event").GetString() == "scenario_step_passed")
.ToArray();

Assert.Equal(0, exitCode);
Assert.Equal([("setup", 1), ("main", 2), ("teardown", 3)], passedSteps
.Select(static evt => (evt.GetProperty("phase").GetString()!, evt.GetProperty("step_index").GetInt32()))
.ToArray());
Assert.Equal("dev.luotsi.app", host.StartAppRequests.Single().Package);
Assert.Equal("KEYCODE_BACK", host.KeyEventRequests.Single());
Assert.Equal("dev.luotsi.app", host.ForceStopRequests.Single());
}

[Fact]
public async Task RunAsync_File_First_Setup_Step_Cannot_Observe_Previous_Step()
{
var fileSystem = new FakeFileSystem();
var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind));
var console = new FakeConsole();
fileSystem.AddFile("/tmp/scenario.json", """
{
"name": "invalid lifecycle",
"setup": [
{ "name": "event", "action": "assertEvent", "event": "ready", "observeFromPreviousStep": true }
],
"steps": [
{ "name": "pause", "action": "sleep", "milliseconds": 1 }
]
}
""");
var app = new App(new AppDependencies
{
TimeProvider = timeProvider,
FileSystem = fileSystem,
ProcessRunner = new DefaultProcessRunner(),
Delay = new FakeDelay(timeProvider),
DeviceHostFactory = new FakeDeviceHostFactory(new FakeDeviceHost()),
Console = console
});

var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json"]);
using var envelope = console.ParseSingleOutputAsJson();

Assert.Equal(2, exitCode);
Assert.Contains("no previous lifecycle step", envelope.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal);
}

[Fact]
public async Task RunAsync_File_First_Main_Step_Can_Observe_Setup_Step()
{
var fileSystem = new FakeFileSystem();
var timeProvider = new SteppingTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind), TimeSpan.FromMilliseconds(10));
var console = new FakeConsole();
fileSystem.AddFile("/tmp/scenario.json", """
{
"name": "observe setup",
"setup": [
{ "name": "pause", "action": "sleep", "milliseconds": 25 }
],
"steps": [
{ "name": "event", "action": "assertEvent", "event": "ready", "observeFromPreviousStep": true }
]
}
""");
var host = new FakeDeviceHost();
var app = new App(new AppDependencies
{
TimeProvider = timeProvider,
FileSystem = fileSystem,
ProcessRunner = new DefaultProcessRunner(),
Delay = new SteppingDelay(timeProvider),
DeviceHostFactory = new FakeDeviceHostFactory(host),
Console = console
});

var exitCode = await app.RunAsync(["run", "--file", "/tmp/scenario.json"]);

Assert.Equal(0, exitCode);
Assert.NotNull(host.AssertEventRequests.Single().Since);
}

[Fact]
public async Task RunAsync_File_Writes_Consistent_Scenario_Ended_Timestamp_And_Duration()
{
Expand Down Expand Up @@ -506,6 +620,110 @@ public async Task RunAsync_File_Failure_Writes_Jsonl_Terminal_Result()
Assert.Equal(1, events[^1].GetProperty("failed_count").GetInt32());
}

[Fact]
public async Task RunAsync_File_Runs_Teardown_After_Main_Failure_And_Reports_Phases()
{
var fileSystem = new FakeFileSystem();
var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind));
var console = new FakeConsole();
fileSystem.AddFile("/tmp/scenario.json", """
{
"name": "failing lifecycle",
"setup": [
{ "name": "launch", "action": "startApp", "package": "dev.luotsi.app" }
],
"steps": [
{ "name": "target", "action": "waitVisible", "text": "Target" }
],
"teardown": [
{ "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" }
]
}
""");
var host = new FakeDeviceHost
{
WaitVisibleException = new InvalidOperationException("not visible")
};
var app = new App(new AppDependencies
{
TimeProvider = timeProvider,
FileSystem = fileSystem,
ProcessRunner = new DefaultProcessRunner(),
Delay = new FakeDelay(timeProvider),
DeviceHostFactory = new FakeDeviceHostFactory(host),
Console = console
});

var exitCode = await app.RunAsync([
"run",
"--file", "/tmp/scenario.json",
"--events-jsonl", "/tmp/events.jsonl",
"--report-json", "/tmp/report.json"]);
var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl");
using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json"));
var scenario = report.RootElement.GetProperty("scenarios")[0];

Assert.Equal(1, exitCode);
Assert.Equal("dev.luotsi.app", host.ForceStopRequests.Single());
Assert.Contains(events, static evt =>
evt.GetProperty("event").GetString() == "scenario_step_passed" &&
evt.GetProperty("phase").GetString() == "teardown");
Assert.Equal("main", scenario.GetProperty("failed_step").GetProperty("phase").GetString());
Assert.Contains(scenario.GetProperty("steps").EnumerateArray(), static step =>
step.GetProperty("phase").GetString() == "teardown" &&
step.GetProperty("action").GetString() == "forceStop");
}

[Fact]
public async Task RunAsync_File_Teardown_Failure_Does_Not_Mask_Main_Failure()
{
var fileSystem = new FakeFileSystem();
var timeProvider = new ManualTimeProvider(DateTimeOffset.Parse("2026-05-15T12:00:00Z", null, System.Globalization.DateTimeStyles.RoundtripKind));
var console = new FakeConsole();
fileSystem.AddFile("/tmp/scenario.json", """
{
"name": "double failure lifecycle",
"steps": [
{ "name": "target", "action": "waitVisible", "text": "Target" }
],
"teardown": [
{ "name": "stop", "action": "forceStop", "package": "dev.luotsi.app" }
]
}
""");
var host = new FakeDeviceHost
{
WaitVisibleException = new InvalidOperationException("not visible"),
ForceStopException = new InvalidOperationException("cleanup failed")
};
var app = new App(new AppDependencies
{
TimeProvider = timeProvider,
FileSystem = fileSystem,
ProcessRunner = new DefaultProcessRunner(),
Delay = new FakeDelay(timeProvider),
DeviceHostFactory = new FakeDeviceHostFactory(host),
Console = console
});

var exitCode = await app.RunAsync([
"run",
"--file", "/tmp/scenario.json",
"--events-jsonl", "/tmp/events.jsonl",
"--report-json", "/tmp/report.json"]);
var events = ReadJsonlEvents(fileSystem, "/tmp/events.jsonl");
using var report = JsonDocument.Parse(await fileSystem.ReadAllTextAsync("/tmp/report.json"));

Assert.Equal(1, exitCode);
Assert.Contains("main step", report.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal);
Assert.DoesNotContain("cleanup failed", report.RootElement.GetProperty("error").GetProperty("message").GetString(), StringComparison.Ordinal);
Assert.Equal("main", report.RootElement.GetProperty("scenarios")[0].GetProperty("failed_step").GetProperty("phase").GetString());
Assert.Contains(events, static evt =>
evt.GetProperty("event").GetString() == "scenario_step_failed" &&
evt.GetProperty("phase").GetString() == "teardown" &&
evt.GetProperty("error").GetProperty("message").GetString() == "cleanup failed");
}

[Fact]
public async Task RunAsync_Path_Writes_Jsonl_Run_And_Per_Scenario_Events()
{
Expand Down
16 changes: 14 additions & 2 deletions Luotsi.Cli.Tests/TestSupport.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -496,6 +496,8 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic

public List<(string Package, string Permission)> RevokePermissionRequests { get; } = [];

public List<(string Name, DateTimeOffset? Since)> AssertEventRequests { get; } = [];

public List<DeviceInfo> ConnectedDevices { get; } = [];

public PreflightResult PreflightTemplate { get; set; } = new("Model", "16", "36", "focus", null, null, "fingerprint", "arm64-v8a", "SER");
Expand All @@ -508,6 +510,8 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic

public Exception? WaitVisibleException { get; set; }

public Exception? ForceStopException { get; set; }

public FailureArtifactBundle? FailureArtifacts { get; set; }

public Exception? FailureArtifactException { get; set; }
Expand Down Expand Up @@ -609,8 +613,11 @@ public Task<TapPointResult> TapPointAsync(string? label, int? x, int? y, double?

public Task<ResetLogResult> ResetLogAsync() => Task.FromResult(new ResetLogResult(true));

public Task<AssertEventResult> AssertEventAsync(string name, IReadOnlyList<string> contains, string? detailsPattern, int timeoutSec, DateTimeOffset? since = null) =>
Task.FromResult(new AssertEventResult(name, contains, detailsPattern, string.Empty));
public Task<AssertEventResult> AssertEventAsync(string name, IReadOnlyList<string> contains, string? detailsPattern, int timeoutSec, DateTimeOffset? since = null)
{
AssertEventRequests.Add((name, since));
return Task.FromResult(new AssertEventResult(name, contains, detailsPattern, string.Empty));
}

public Task<TakeScreenshotResult> TakeScreenshotAsync(string label)
{
Expand Down Expand Up @@ -791,6 +798,11 @@ public Task<StartUriResult> StartUriAsync(string uri, string? packageName, strin
public Task<AppPackageCommandResult> ForceStopAsync(string packageName)
{
ForceStopRequests.Add(packageName);
if (ForceStopException is not null)
{
throw ForceStopException;
}

return Task.FromResult(new AppPackageCommandResult(packageName));
}

Expand Down
6 changes: 5 additions & 1 deletion Luotsi.Cli/Models/Contracts.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,11 +257,15 @@ public sealed record Bounds(int Left, int Top, int Right, int Bottom);
/// <param name="Steps">Scenario steps.</param>
/// <param name="Variables">Optional scenario variables.</param>
/// <param name="Tags">Optional runner tags used for filtering and reporting.</param>
/// <param name="Setup">Optional setup steps that run before scenario steps.</param>
/// <param name="Teardown">Optional teardown steps that run after setup and scenario steps.</param>
public sealed record ScenarioFile(
string Name,
IReadOnlyList<ScenarioStep> Steps,
IReadOnlyDictionary<string, string>? Variables = null,
IReadOnlyList<string>? Tags = null);
IReadOnlyList<string>? Tags = null,
IReadOnlyList<ScenarioStep>? Setup = null,
IReadOnlyList<ScenarioStep>? Teardown = null);

/// <summary>
/// Scenario playbook step.
Expand Down
37 changes: 33 additions & 4 deletions Luotsi.Cli/Scenarios/ScenarioCatalog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,16 @@ public sealed record ScenarioStepResult(
ScenarioStepTiming Timing,
object? Result = null,
string? Status = null,
ErrorInfo? Error = null);
ErrorInfo? Error = null,
string Phase = ScenarioStepPhases.Main);

public sealed record ScenarioFailedStepResult(
int Index,
string Name,
string Action,
double DurationMs,
ScenarioStepTiming Timing);
ScenarioStepTiming Timing,
string Phase = ScenarioStepPhases.Main);

public sealed record ScenarioRunResult(
string Scenario,
Expand Down Expand Up @@ -144,6 +146,13 @@ public static class ScenarioShardStrategies
public const string Hash = "hash";
}

public static class ScenarioStepPhases
{
public const string Setup = "setup";
public const string Main = "main";
public const string Teardown = "teardown";
}

internal sealed class ScenarioCatalog(
IFileSystem fileSystem,
IScenarioTemplateResolver templateResolver)
Expand Down Expand Up @@ -355,7 +364,7 @@ private static ScenarioCatalogEntry ToEntry(string file, ScenarioFile scenario)
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static tag => tag, StringComparer.OrdinalIgnoreCase)
.ToArray() ?? [];
var actions = scenario.Steps
var actions = EnumerateLifecycleSteps(scenario)
.Select(static step => step.Action)
.Where(static action => !string.IsNullOrWhiteSpace(action))
.Distinct(StringComparer.OrdinalIgnoreCase)
Expand All @@ -367,10 +376,30 @@ private static ScenarioCatalogEntry ToEntry(string file, ScenarioFile scenario)
scenario.Name,
file,
tags,
scenario.Steps.Count,
EnumerateLifecycleSteps(scenario).Count(),
actions);
Comment thread
slideep marked this conversation as resolved.
}

internal static IEnumerable<ScenarioStep> EnumerateLifecycleSteps(ScenarioFile scenario)
{
ArgumentNullException.ThrowIfNull(scenario);

foreach (var step in scenario.Setup ?? [])
{
yield return step;
}

foreach (var step in scenario.Steps)
{
yield return step;
}

foreach (var step in scenario.Teardown ?? [])
{
yield return step;
}
}

private static int GetStableShardIndex(string value, int shardCount)
{
unchecked
Expand Down
1 change: 1 addition & 0 deletions Luotsi.Cli/Scenarios/ScenarioEvents.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,7 @@ internal sealed record ScenarioEvent(
[property: JsonPropertyName("file")] string? File = null,
[property: JsonPropertyName("scenario_id")] string? ScenarioId = null,
[property: JsonPropertyName("scenario")] string? Scenario = null,
[property: JsonPropertyName("phase")] string? Phase = null,
[property: JsonPropertyName("step_index")] int? StepIndex = null,
[property: JsonPropertyName("step")] string? Step = null,
[property: JsonPropertyName("action")] string? Action = null,
Expand Down
Loading
Loading