Batch planning

Ares.Core.Grpc/Services/AutomationService.cs

@@ -399,7 +399,18 @@ public override Task<Empty> SetReplicateRate(ReplicateRate request, ServerCallCo
 
   public override Task<ReplicateRate> GetReplicateRate(Empty request, ServerCallContext? context)
   {
-    return Task.FromResult(new ReplicateRate { ReplicateRate_ = _executionManager.ReplanRate });
+    return Task.FromResult(new ReplicateRate { ReplicateRate_ = _executionManager.ReplicateRate });
+  }
+
+  public override Task<Empty> SetPlanningBatchSize(PlanningBatchSize request, ServerCallContext? context)
+  {
+    _executionManager.UpdateBatchPlanningSize(request.BatchSize);
+    return Task.FromResult(new Empty());
+  }
+
+  public override Task<PlanningBatchSize> GetPlanningBatchSize(Empty request, ServerCallContext? context)
+  {
+    return Task.FromResult(new PlanningBatchSize { BatchSize = _executionManager.PlanningBatchSize });
   }
 
   public override Task<Empty> SetAnalysisResultStopCondition(AnalysisResultCondition request, ServerCallContext? context)

Ares.Core.Tests/Execution/CampaignExecutorTests.cs

@@ -59,6 +59,8 @@ public void SetUp()
         It.IsAny<IEnumerable<Parameter>>(),
         It.IsAny<IEnumerable<Ares.Datamodel.Analyzing.Analysis>>(),
         It.IsAny<IEnumerable<ExperimentOverview>>(),
+        It.IsAny<int>(),
+        It.IsAny<List<PlanStatusCode>>(),
         It.IsAny<CancellationToken>()))
       .ReturnsAsync(true);
 
@@ -218,6 +220,8 @@ public async Task Replan_Composes_And_Executes_Experiment_Again()
       It.IsAny<IEnumerable<Parameter>>(),
       It.IsAny<IEnumerable<Ares.Datamodel.Analyzing.Analysis>>(),
       It.IsAny<IEnumerable<ExperimentOverview>>(),
+      It.IsAny<int>(),
+      It.IsAny<List<PlanStatusCode>>(),
       It.IsAny<CancellationToken>()), Times.Once);
   }
 

Ares.Core/Execution/Enums/ExperimentLoopOutcome.cs

@@ -0,0 +1,8 @@
+namespace Ares.Core.Execution.Enums;
+
+public enum ExperimentLoopOutcome
+{
+  Succeeded,
+  Failed,
+  Canceled
+}

Ares.Core/Execution/Enums/ExperimentPhase.cs

@@ -0,0 +1,15 @@
+namespace Ares.Core.Execution.Enums;
+
+public enum ExperimentPhase
+{
+  Initialize,
+  Plan,
+  Compose,
+  Execute,
+  Analyze,
+  Retry,
+  Replan,
+  Complete,
+  Failed,
+  Canceled
+}

Ares.Core/Execution/ExecutionManager.cs

@@ -55,7 +55,7 @@ public async Task<bool> CanRun()
     var startConditions = await Task.WhenAll(startConditionTasks);
     return startConditions.All(condition => condition?.Success ?? true);
   }
-  public int ReplanRate { get; private set; } = 1;
+
   public async Task Start(string executionNotes, List<AresCampaignTag> campaignTags)
   {
     var err = await CheckCampaignStartPrerequisites();
@@ -73,7 +73,8 @@ public async Task Start(string executionNotes, List<AresCampaignTag> campaignTag
       executor.UpdateCampaignTags(campaignTags);
 
     executor.StopConditions.AddRange(CampaignStopConditions);
-    executor.ReplanRate = ReplanRate;
+    executor.ReplicateRate = ReplicateRate;
+    executor.BatchPlanningSize = PlanningBatchSize;
     _executionControlTokenSource = new ExecutionControlTokenSource();
     CampaignExecutionSummary campaignExecutionSummary;
     ExecutionStartTime = DateTime.UtcNow;
@@ -111,7 +112,7 @@ public void Resume()
   public async Task<string> CheckCampaignStartPrerequisites()
   {
     if(_activeCampaignTemplateStore.CampaignTemplate is null)
-      return "CampaignTemplate was not assigned to the active template store.";
+      return "Campaign Template was not assigned to the active template store.";
 
     if(!CampaignStopConditions.Any())
       return "The Campaign has no stop conditions, please set a stop condition before starting campaign.";
@@ -142,7 +143,12 @@ public bool EnsureParameterAssignment()
 
   public void UpdateReplicateRate(int newRate)
   {
-    ReplanRate = newRate;
+    ReplicateRate = newRate;
+  }
+
+  public void UpdateBatchPlanningSize(int newBatchSize)
+  {
+    PlanningBatchSize = newBatchSize;
   }
 
   public void SubmitUserDecision(ErrorHandling decision)
@@ -171,8 +177,11 @@ private async Task StoreCompletedCampaign(CampaignExecutionSummary result)
     {
       throw;
     }
-
   }
 
   public DateTime? ExecutionStartTime { get; set; }
+
+  public int ReplicateRate { get; private set; } = 1;
+
+  public int PlanningBatchSize { get; private set; } = 1;
 }

Ares.Core/Execution/Executors/CampaignExecutor.cs

@@ -3,6 +3,7 @@
 using Ares.Core.Device.State.Logging;
 using Ares.Core.Exceptions;
 using Ares.Core.Execution.ControlTokens;
+using Ares.Core.Execution.Enums;
 using Ares.Core.Execution.Executors.Composers;
 using Ares.Core.Execution.Extensions;
 using Ares.Core.Execution.Safety;
@@ -13,6 +14,7 @@
 using Ares.Core.Settings;
 using Ares.Datamodel;
 using Ares.Datamodel.Analyzing;
+using Ares.Datamodel.Planning;
 using Ares.Datamodel.Templates;
 using Google.Protobuf.WellKnownTypes;
 using Microsoft.Extensions.Logging;
@@ -22,28 +24,7 @@
 namespace Ares.Core.Execution.Executors;
 
 public class CampaignExecutor : ICampaignExecutor
-{
-  private enum ExperimentPhase
-  {
-    Initialize,
-    Plan,
-    Compose,
-    Execute,
-    Analyze,
-    Retry,
-    Replan,
-    Complete,
-    Failed,
-    Canceled
-  }
-
-  private enum ExperimentLoopOutcome
-  {
-    Succeeded,
-    Failed,
-    Canceled
-  }
-
+{ 
   private readonly IExecutionReporter _executionReporter;
   private readonly ISubject<CampaignExecutionStatus> _executionStatusSubject;
   private readonly ICommandComposer<ExperimentTemplate, ExperimentExecutor> _experimentComposer;
@@ -66,6 +47,7 @@ private enum ExperimentLoopOutcome
   private ExperimentTemplate? _currentExperimentTemplate = null;
   private int _experimentCount = 0;
   private TaskCompletionSource<ErrorHandling>? _userDecisionSource;
+  private List<PlanStatusCode> _latestPlanStatusCodes = new List<PlanStatusCode>();
 
   internal CampaignExecutor(ICommandComposer<ExperimentTemplate, ExperimentExecutor> experimentComposer,
     IPlanningHelper planningHelper,
@@ -226,8 +208,13 @@ private async Task NotifyCampaignStart()
     return (true, startupSummary);
   }
 
-  private async Task<ExperimentLoopOutcome> ExecuteExperimentLoop(string campaignPath, List<Analysis> analyses, List<ExperimentExecutionSummary> experimentSummaries, ExecutionControlToken token, ExperimentExecutionSummary startupSummary)
+  private async Task<ExperimentLoopOutcome> ExecuteExperimentLoop(string campaignPath, 
+    List<Analysis> analyses, 
+    List<ExperimentExecutionSummary> experimentSummaries, 
+    ExecutionControlToken token, 
+    ExperimentExecutionSummary startupSummary)
   {
+    _latestPlanStatusCodes = new List<PlanStatusCode>();
     var currentPhase = ExperimentPhase.Initialize;
     var currentExperimentPath = "";
     var failedExperimentRetryCount = 0;
@@ -317,7 +304,7 @@ private async Task<ExperimentPhase> PrepareExperiment(List<Analysis> analyses, L
     if(!_currentExperimentTemplate.IsResolved())
     {
       _logger.LogTrace("Experiment has not been resolved, ARES will now begin the planning process.");
-      if(analyses.Count % ReplanRate == 0)
+      if(analyses.Count % ReplicateRate == 0)
       {
         if(!await PlanExperiment(analyses, _currentExperimentTemplate, experimentSummaries, token))
         {
@@ -397,9 +384,17 @@ private async Task<ExperimentPhase> ExecuteCurrentExperiment(string currentExper
       .SelectMany(s => s.CommandSummaries)
       .FirstOrDefault(c => !c.Result.Success);
 
-    return failedCommandSummary is null
-      ? ExperimentPhase.Analyze
-      : await HandleError(failedCommandSummary, token);
+    if(failedCommandSummary is null)
+    {
+      _latestPlanStatusCodes.Add(PlanStatusCode.PlanAccepted);
+      return ExperimentPhase.Analyze;
+    }
+
+    else
+    {
+      UpdatePlanStatus(failedCommandSummary.StatusCode);
+      return await HandleError(failedCommandSummary, token);
+    }
   }
 
   private async Task<ExperimentPhase> AnalyzeCurrentExperiment(ExperimentExecutionSummary startupSummary, List<Analysis> analyses, List<ExperimentExecutionSummary> experimentSummaries, ExecutionControlToken token)
@@ -479,6 +474,7 @@ private async Task<ExperimentPhase> HandleError(CommandExecutionSummary cmdSumma
 
       var decisionSource = new TaskCompletionSource<ErrorHandling>(TaskCreationOptions.RunContinuationsAsynchronously);
       _userDecisionSource = decisionSource;
+
       try
       {
         errorHandling = await decisionSource.Task.WaitAsync(token.CancellationToken);
@@ -516,14 +512,25 @@ private async Task<ExperimentPhase> HandleError(CommandExecutionSummary cmdSumma
     }
   }
 
+
+  private void UpdatePlanStatus(CommandStatusCode failCode)
+  {
+    if(failCode == CommandStatusCode.OutOfRange || failCode == CommandStatusCode.ParametersUnachievable)
+      _latestPlanStatusCodes.Add(PlanStatusCode.PlanUnachievable);
+
+    else
+      _latestPlanStatusCodes.Add(PlanStatusCode.PlanFailed);
+  }
+
   private async Task<bool> PlanExperiment(List<Analysis> analyses, 
     ExperimentTemplate currentExperimentTemplate, 
     List<ExperimentExecutionSummary> experimentSummaries, 
     ExecutionControlToken token)
   {
     Status.PlannerState = PlannerState.PlanningInProgress;
     ReportCampaignStatus();
-    _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplanRate);
+    _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplicateRate);
+
 
     var metadata = new RequestMetadata
     {
@@ -540,6 +547,8 @@ private async Task<bool> PlanExperiment(List<Analysis> analyses,
       currentExperimentTemplate.GetAllPlannedParameters(),
       analyses,
       experimentSummaries.Select(es => es.ExperimentOverview),
+      BatchPlanningSize,
+      _latestPlanStatusCodes,
       token.CancellationToken);
 
     if(!resolveSuccess)
@@ -579,7 +588,7 @@ private async Task<bool> PlanExperiment(List<Analysis> analyses,
     // The following are top level checks for analysis failure in case the
     // failure is not properly handled on the Analysis itself
     // which also has support for "success" and "error" message
-    if (analysis is null || analysis.Result == float.NaN)
+    if(analysis is null || analysis.Result == float.NaN)
     {
       Status.AnalysisState = AnalysisState.AnalysisError;
       await _notifier.Notify("Analysis Failure", $"Analysis was reported as successful, but no actual analysis was provided. {analysis?.ErrorString ?? "No error string provided"}", NotificationSeverityEnum.Error);
@@ -725,9 +734,9 @@ private static bool IsAwaitingResponse(ExperimentExecutionStatus status)
     .Any(step => step.CommandExecutionStatuses
     .Any(cmd => cmd.State == ExecutionState.AwaitingUser));
 
-  private async Task<ExperimentExecutorResult> GenerateExperimentExecutor(ExperimentTemplate template, 
-    IEnumerable<Analysis> analyses, 
-    IEnumerable<ExperimentOverview> previousExperiments, 
+  private async Task<ExperimentExecutorResult> GenerateExperimentExecutor(ExperimentTemplate template,
+    IEnumerable<Analysis> analyses,
+    IEnumerable<ExperimentOverview> previousExperiments,
     CancellationToken cancellationToken)
   {
     var result = new ExperimentExecutorResult();
@@ -737,11 +746,11 @@ private async Task<ExperimentExecutorResult> GenerateExperimentExecutor(Experime
     if(!experimentTemplate.IsResolved())
     {
       _logger.LogTrace("Experiment was not resolved");
-      if(analyses.Count() % ReplanRate == 0)
+      if(analyses.Count() % ReplicateRate == 0)
       {
         Status.PlannerState = PlannerState.PlanningInProgress;
         ReportCampaignStatus();
-        _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplanRate);
+        _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplicateRate);
 
         var metadata = new RequestMetadata 
         { 
@@ -757,7 +766,9 @@ private async Task<ExperimentExecutorResult> GenerateExperimentExecutor(Experime
           metadata, 
           experimentTemplate.GetAllPlannedParameters(), 
           analyses, 
-          previousExperiments, 
+          previousExperiments,
+          BatchPlanningSize,
+          _latestPlanStatusCodes,
           cancellationToken);
 
         if(!resolveSuccess)
@@ -840,7 +851,8 @@ private async Task PostExperimentExecution(ExperimentExecutionSummary summary)
 
   public CampaignTemplate Template { get; }
   public IList<IStopCondition> StopConditions { get; } = [];
-  public double ReplanRate { get; set; } = 1;
+  public int ReplicateRate { get; set; } = 1;
+  public int BatchPlanningSize { get; set; } = 1;
   public string? ExecutionNotes { get; set; }
   public List<AresCampaignTag> CampaignTags { get; set; } = [];
   public IObservable<CampaignExecutionStatus> ExperimentStatusObservable { get; }

Ares.Core/Execution/Executors/CommandExecutor.cs

@@ -109,7 +109,6 @@ public async Task<CommandExecutionSummary> Execute(ExecutionControlToken token,
 
     Status.Result = result.Result;
     _stateSubject.OnNext(Status);
-    _stateSubject.OnCompleted();
 
     return ExecutorSummaryHelpers.CreateCommandExecutionSummary(Template, result, timeStarted, DateTime.UtcNow);
   }

Ares.Core/Execution/Executors/ExecutorSummaryHelpers.cs

@@ -58,7 +58,8 @@ public static CommandExecutionSummary CreateCommandExecutionSummary(CommandTempl
       CommandDescription = template.Metadata.Description,
       CommandName = template.Metadata.Name,
       StatusCode = deviceResult?.StatusCode ?? CommandStatusCode.StatusUnspecified
-    };
+     };
+
 
     if(template.HasOutputVarName)
       commandExecutionSummary.VarName = template.OutputVarName;

Ares.Core/Execution/Executors/ExperimentExecutor.cs

@@ -20,6 +20,7 @@ public ExperimentExecutor(ExperimentTemplate template, IExecutor<StepExecutionSu
 
     Status.StepExecutionStatuses.AddRange(experimentStepExecutors.Select(executor => executor.Status));
 
+
     var experimentStepExecutionObservation = experimentStepExecutors.Select(executor =>
     {
       return executor.ExperimentStatusObservable.Select(_ =>

Ares.Core/Execution/Executors/ICampaignExecutor.cs

@@ -6,9 +6,9 @@ namespace Ares.Core.Execution.Executors;
 public interface ICampaignExecutor : IExecutor<CampaignExecutionSummary, CampaignExecutionStatus>
 {
   IList<IStopCondition> StopConditions { get; }
-  double ReplanRate { get; set; }
+  int ReplicateRate { get; set; }
+  int BatchPlanningSize { get; set; }
   void UpdateExecutionNotes(string executionNotes);
-
   void UpdateCampaignTags(List<AresCampaignTag> campaignTags);
 
   void SubmitUserDecision(ErrorHandling decision);

Ares.Core/Execution/IExecutionManager.cs

@@ -11,9 +11,14 @@ public interface IExecutionManager
   public IList<IStopCondition> CampaignStopConditions { get; }
 
   /// <summary>
-  /// A double value that determines how often a campaign will re-plan it's experiment, defaults to one
+  /// A int value that determines how often a campaign will re-plan it's experiment, defaults to one
   /// </summary>
-  public int ReplanRate { get; }
+  public int ReplicateRate { get; }
+
+  /// <summary>
+  /// An int value that determines how many experiments are planned for per planning request
+  /// </summary>
+  public int PlanningBatchSize { get; }
 
   /// <summary>
   /// Indicates whether the currently loaded campaign has all the prerequisites in order to start and run
@@ -55,6 +60,11 @@ public interface IExecutionManager
   void UpdateReplicateRate(int newRate);
 
   /// <summary>
+  /// Updates the batch planning size
+  /// </summary>
+  /// <param name="batchSize"></param>
+  void UpdateBatchPlanningSize(int batchSize);
+
   /// Submits a user decision for how to handle an error that has occurred during execution
   /// </summary>
   /// <param name="decision"></param>

Ares.Core/Planning/IPlanningHelper.cs

@@ -23,6 +23,8 @@ Task<bool> TryResolveParameters(IEnumerable<PlannerAllocation> plannerAllocation
     IEnumerable<Parameter> parameters,
     IEnumerable<Analysis> seedAnalyses,
     IEnumerable<ExperimentOverview> seedExperiments,
+    int batchSize,
+    List<PlanStatusCode> codes,
     CancellationToken cancellationToken);
 
   /// <summary>