Add scenario reports and artifact policies

Luotsi.Cli.Tests/ScenarioRunConfigurationTests.cs

@@ -0,0 +1,39 @@
+using Luotsi.Cli.Cli;
+using Luotsi.Cli.Errors;
+using Luotsi.Cli.Scenarios;
+using Xunit;
+
+namespace Luotsi.Cli.Tests;
+
+public sealed class ScenarioRunConfigurationTests
+{
+    [Fact]
+    public void Create_Parses_Run_Output_Policies()
+    {
+        var options = CliOptions.Parse([
+            "run",
+            "--events-jsonl", "/tmp/events.jsonl",
+            "--report-json", "/tmp/report.json",
+            "--report-junit", "/tmp/junit.xml",
+            "--capture-on", "never",
+            "--attach-artifacts", "always"]);
+
+        var configuration = ScenarioRunConfiguration.Create(options);
+
+        Assert.Equal("/tmp/events.jsonl", configuration.EventsJsonlPath);
+        Assert.Equal("/tmp/report.json", configuration.JsonReportPath);
+        Assert.Equal("/tmp/junit.xml", configuration.JUnitReportPath);
+        Assert.Equal(ScenarioFailureArtifactCapturePolicy.Never, configuration.FailureArtifactCapturePolicy);
+        Assert.Equal(ScenarioArtifactAttachmentPolicy.Always, configuration.ArtifactAttachmentPolicy);
+    }
+
+    [Fact]
+    public void Create_Invalid_AttachArtifacts_Throws_UsageException()
+    {
+        var options = CliOptions.Parse(["run", "--attach-artifacts", "sometimes"]);
+
+        var error = Assert.Throws<UsageException>(() => ScenarioRunConfiguration.Create(options));
+
+        Assert.Contains("--attach-artifacts", error.Message, StringComparison.Ordinal);
+    }
+}

Luotsi.Cli.Tests/TestSupport.cs

@@ -508,6 +508,12 @@ internal sealed class FakeDeviceHost(params ScreenState[] screenStates) : IDevic
 
     public Exception? WaitVisibleException { get; set; }
 
+    public FailureArtifactBundle? FailureArtifacts { get; set; }
+
+    public Exception? FailureArtifactException { get; set; }
+
+    public List<FailureCaptureRequest> FailureArtifactRequests { get; } = [];
+
     public Task<DeviceListResult> GetDevicesAsync()
     {
         if (GetDevicesException is not null)
@@ -842,8 +848,16 @@ public Task<DeviceFingerprint> WriteDeviceFingerprintAsync()
         return Task.FromResult(new DeviceFingerprint(ResultSchemas.DeviceFingerprint, DateTimeOffset.UtcNow, "SER", "Model", "16", "36", "fingerprint", "arm64-v8a", "focus"));
     }
 
-    public Task<FailureArtifactBundle> CaptureFailureArtifactsAsync(FailureCaptureRequest request, Exception exception) =>
-        Task.FromResult(new FailureArtifactBundle(ResultSchemas.FailureBundle, DateTimeOffset.UtcNow, request.Scope, request.Name, request.File, request.StepIndex, request.StepName, request.Action, exception.GetType().FullName ?? exception.GetType().Name, exception.Message, [], []));
+    public Task<FailureArtifactBundle> CaptureFailureArtifactsAsync(FailureCaptureRequest request, Exception exception)
+    {
+        FailureArtifactRequests.Add(request);
+        if (FailureArtifactException is not null)
+        {
+            throw FailureArtifactException;
+        }
+
+        return Task.FromResult(FailureArtifacts ?? new FailureArtifactBundle(ResultSchemas.FailureBundle, DateTimeOffset.UtcNow, request.Scope, request.Name, request.File, request.StepIndex, request.StepName, request.Action, exception.GetType().FullName ?? exception.GetType().Name, exception.Message, [], []));
+    }
 
     public Task<LogcatResult> LogcatAsync(int tail) => Task.FromResult(new LogcatResult([]));
 

Luotsi.Cli/Cli/Composition/AppHostedCommandCompositionBuilder.cs

@@ -18,10 +18,17 @@ public static AppHostedCommandComposition Build(AppHostedCommandCompositionBuild
         var scenarioExecutorFactory = new ScenarioExecutorFactory(dependencies.FileSystem, dependencies.TimeProvider, dependencies.Delay, scenarioTemplateResolver);
         var scenarioBatchExecutorFactory = new ScenarioBatchExecutorFactory(scenarioExecutorFactory);
         var scenarioRunEventCoordinatorFactory = new ScenarioRunEventCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider);
+        var scenarioRunReportCoordinatorFactory = new ScenarioRunReportCoordinatorFactory(dependencies.FileSystem, dependencies.TimeProvider);
+        var scenarioRunOrchestrator = new ScenarioRunOrchestrator(
+            scenarioRunPlanner,
+            scenarioExecutorFactory,
+            scenarioBatchExecutorFactory,
+            scenarioRunEventCoordinatorFactory,
+            scenarioRunReportCoordinatorFactory);
         var envelopeWriter = new AppCommandEnvelopeWriter(dependencies.Console, dependencies.TimeProvider);
         var commandDispatcher = new AppCommandDispatcher(
             new AdbSubcommandDispatcher(),
-            new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioExecutorFactory, scenarioBatchExecutorFactory, scenarioRunEventCoordinatorFactory),
+            new ScenarioCommandDispatcher(scenarioRunPlanner, scenarioRunOrchestrator),
             dependencies.ProfileCoordinator);
 
         return new(
@@ -44,4 +51,4 @@ internal sealed record AppHostedCommandCompositionBuilderDependencies(
 
 internal sealed record AppHostedCommandComposition(
     AppCommandEnvelopeWriter EnvelopeWriter,
-    AppCommandHost CommandHost);
+    AppCommandHost CommandHost);

Luotsi.Cli/Cli/Help.cs

@@ -58,7 +58,7 @@ view setup --device <adb serial> [--profile <name>] [--preset safe|balanced|high
   list-installed-packages [--third-party]
   grant-permission --package <app.id> --permission <permission>
   revoke-permission --package <app.id> --permission <permission>
-  scenario-list --path <scenario-file-or-directory> [--include-tag <tag>] [--exclude-tag <tag>] [--name <text>] [--action <action>]
+  scenario-list --path <scenario-file-or-directory-or-glob> [--include-tag <tag>] [--exclude-tag <tag>] [--name <text>] [--action <action>]
   telemetry-tail [--tail 200]
   telemetry-watch [--timeout-sec 15]
   wait-step --step <STEP_NAME> [--timeout-sec 15]
@@ -71,8 +71,8 @@ view setup --device <adb serial> [--profile <name>] [--preset safe|balanced|high
   logcat [--tail 200]
   wait-log --contains <text> [--timeout-sec 15]
   record --output <file.mp4> [--time-limit-sec 30]
-  run --file <scenario.json> [--events-jsonl <file>]
-  run --path <scenario-file-or-directory> [--dry-run] [--events-jsonl <file>] [--include-tag <tag>] [--exclude-tag <tag>] [--name <text>] [--action <action>] [--shard-count <n> --shard-index <zero-based>]
+  run --file <scenario.json> [--events-jsonl <file>] [--report-json <file>] [--report-junit <file>] [--capture-on failure|never] [--attach-artifacts never|on-failure|always]
+  run --path <scenario-file-or-directory-or-glob> [--dry-run] [--events-jsonl <file>] [--report-json <file>] [--report-junit <file>] [--capture-on failure|never] [--attach-artifacts never|on-failure|always] [--include-tag <tag>] [--exclude-tag <tag>] [--name <text>] [--action <action>] [--shard-count <n> --shard-index <zero-based>] [--shard-strategy index|hash]
 
 Common options:
   --device <adb serial>

Luotsi.Cli/Cli/Routing/ScenarioCommandDispatcher.cs

@@ -6,14 +6,10 @@ namespace Luotsi.Cli.Cli.Routing;
 
 internal sealed class ScenarioCommandDispatcher(
     ScenarioRunPlanner runPlanner,
-    ScenarioExecutorFactory scenarioExecutorFactory,
-    ScenarioBatchExecutorFactory scenarioBatchExecutorFactory,
-    ScenarioRunEventCoordinatorFactory scenarioRunEventCoordinatorFactory)
+    ScenarioRunOrchestrator scenarioRunOrchestrator)
 {
     private readonly ScenarioRunPlanner _runPlanner = runPlanner ?? throw new ArgumentNullException(nameof(runPlanner));
-    private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory));
-    private readonly ScenarioBatchExecutorFactory _scenarioBatchExecutorFactory = scenarioBatchExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioBatchExecutorFactory));
-    private readonly ScenarioRunEventCoordinatorFactory _scenarioRunEventCoordinatorFactory = scenarioRunEventCoordinatorFactory ?? throw new ArgumentNullException(nameof(scenarioRunEventCoordinatorFactory));
+    private readonly ScenarioRunOrchestrator _scenarioRunOrchestrator = scenarioRunOrchestrator ?? throw new ArgumentNullException(nameof(scenarioRunOrchestrator));
 
     public async Task<ScenarioListResult> ListAsync(CliOptions options)
     {
@@ -29,6 +25,8 @@ public async Task<object> RunAsync(CliOptions options, IDeviceHost runner)
         ArgumentNullException.ThrowIfNull(options);
         ArgumentNullException.ThrowIfNull(runner);
 
+        var configuration = ScenarioRunConfiguration.Create(options);
+
         if (!ScenarioQueryFactory.UsesCatalogExecution(options))
         {
             if (options.HasFlag("dry-run"))
@@ -37,17 +35,13 @@ public async Task<object> RunAsync(CliOptions options, IDeviceHost runner)
             }
 
             var file = options.Require("file");
-            await using var singleRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl"));
-            return await singleRunEvents.RunFileAsync(
-                file,
-                sink => _scenarioExecutorFactory.Create(runner, sink).RunAsync(file)).ConfigureAwait(false);
+            return await _scenarioRunOrchestrator.RunFileAsync(file, runner, configuration).ConfigureAwait(false);
         }
 
         var query = ScenarioQueryFactory.CreateCatalogRunQuery(options);
-        var plan = await _runPlanner.CreateAsync(query).ConfigureAwait(false);
-
         if (query.DryRun)
         {
+            var plan = await _runPlanner.CreateAsync(query).ConfigureAwait(false);
             return new ScenarioRunPlanResult(
                 query.Path,
                 true,
@@ -57,12 +51,10 @@ public async Task<object> RunAsync(CliOptions options, IDeviceHost runner)
                 plan.ShardedOutCount,
                 query.ShardCount,
                 query.ShardIndex,
+                query.ShardStrategy,
                 plan.SelectedScenarios);
         }
 
-        await using var batchRunEvents = _scenarioRunEventCoordinatorFactory.Create(options.Get("events-jsonl"));
-        return await batchRunEvents.RunBatchAsync(
-            plan,
-            sink => _scenarioBatchExecutorFactory.Create(runner, sink).RunAsync(plan)).ConfigureAwait(false);
+        return await _scenarioRunOrchestrator.RunPathAsync(query, runner, configuration).ConfigureAwait(false);
     }
 }

Luotsi.Cli/Cli/Routing/ScenarioQueryFactory.cs

@@ -42,11 +42,27 @@ private static ScenarioQuery CreateQuery(CliOptions options, bool requirePath)
             options.Get("action"),
             shardCount,
             shardIndex,
-            options.HasFlag("dry-run"));
+            options.HasFlag("dry-run"),
+            NormalizeShardStrategy(options.Get("shard-strategy")));
     }
 
     private static string[] SplitOption(string? value) =>
         string.IsNullOrWhiteSpace(value)
             ? []
             : value.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
-}
+
+    private static string NormalizeShardStrategy(string? value)
+    {
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            return ScenarioShardStrategies.Index;
+        }
+
+        return value.Trim().ToLowerInvariant() switch
+        {
+            ScenarioShardStrategies.Index => ScenarioShardStrategies.Index,
+            ScenarioShardStrategies.Hash => ScenarioShardStrategies.Hash,
+            _ => throw new UsageException("--shard-strategy must be one of: index, hash.")
+        };
+    }
+}

Luotsi.Cli/Scenarios/ScenarioBatchExecutor.cs

@@ -18,7 +18,7 @@ public async Task<ScenarioRunBatchResult> RunAsync(ScenarioRunPlan plan)
         {
             try
             {
-                results.Add(ScenarioBatchItemResult.FromSuccess(await _scenarios.RunAsync(scenario.File).ConfigureAwait(false)));
+                results.Add(ScenarioBatchItemResult.FromSuccess(await _scenarios.RunAsync(scenario.File).ConfigureAwait(false), scenario));
                 passedCount++;
             }
             catch (Exception ex) when (ex is not UsageException)
@@ -39,7 +39,8 @@ public async Task<ScenarioRunBatchResult> RunAsync(ScenarioRunPlan plan)
             plan.ShardedOutCount,
             plan.Query.ShardCount,
             plan.Query.ShardIndex,
-            results);
+            results,
+            plan.Query.ShardStrategy);
     }
 
     private static ScenarioBatchItemResult CreateFailureResult(ScenarioCatalogEntry scenario, Exception exception)
@@ -49,6 +50,7 @@ private static ScenarioBatchItemResult CreateFailureResult(ScenarioCatalogEntry
             scenario.Name,
             scenario.File,
             failure?.DataPayload as ScenarioRunFailureData,
-            ScenarioErrorInfo.From(exception));
+            ScenarioErrorInfo.From(exception),
+            scenario.Id);
     }
-}
+}

Luotsi.Cli/Scenarios/ScenarioBatchExecutorFactory.cs

@@ -6,9 +6,12 @@ internal sealed class ScenarioBatchExecutorFactory(ScenarioExecutorFactory scena
 {
     private readonly ScenarioExecutorFactory _scenarioExecutorFactory = scenarioExecutorFactory ?? throw new ArgumentNullException(nameof(scenarioExecutorFactory));
 
-    public ScenarioBatchExecutor Create(IDeviceHost runner, IScenarioEventSink? eventSink = null)
+    public ScenarioBatchExecutor Create(
+        IDeviceHost runner,
+        IScenarioEventSink? eventSink = null,
+        ScenarioFailureArtifactCapturePolicy failureArtifactCapturePolicy = ScenarioFailureArtifactCapturePolicy.Failure)
     {
         ArgumentNullException.ThrowIfNull(runner);
-        return new ScenarioBatchExecutor(_scenarioExecutorFactory.Create(runner, eventSink));
+        return new ScenarioBatchExecutor(_scenarioExecutorFactory.Create(runner, eventSink, failureArtifactCapturePolicy));
     }
 }