diff --git a/Ares.Core.Grpc/Services/AutomationService.cs b/Ares.Core.Grpc/Services/AutomationService.cs index 626b3629..ee7b4665 100644 --- a/Ares.Core.Grpc/Services/AutomationService.cs +++ b/Ares.Core.Grpc/Services/AutomationService.cs @@ -399,7 +399,18 @@ public override Task SetReplicateRate(ReplicateRate request, ServerCallCo public override Task GetReplicateRate(Empty request, ServerCallContext? context) { - return Task.FromResult(new ReplicateRate { ReplicateRate_ = _executionManager.ReplanRate }); + return Task.FromResult(new ReplicateRate { ReplicateRate_ = _executionManager.ReplicateRate }); + } + + public override Task SetPlanningBatchSize(PlanningBatchSize request, ServerCallContext? context) + { + _executionManager.UpdateBatchPlanningSize(request.BatchSize); + return Task.FromResult(new Empty()); + } + + public override Task GetPlanningBatchSize(Empty request, ServerCallContext? context) + { + return Task.FromResult(new PlanningBatchSize { BatchSize = _executionManager.PlanningBatchSize }); } public override Task SetAnalysisResultStopCondition(AnalysisResultCondition request, ServerCallContext? context) diff --git a/Ares.Core.Tests/Execution/CampaignExecutorTests.cs b/Ares.Core.Tests/Execution/CampaignExecutorTests.cs index f0f74be2..a184c911 100644 --- a/Ares.Core.Tests/Execution/CampaignExecutorTests.cs +++ b/Ares.Core.Tests/Execution/CampaignExecutorTests.cs @@ -59,6 +59,8 @@ public void SetUp() It.IsAny>(), It.IsAny>(), It.IsAny>(), + It.IsAny(), + It.IsAny>(), It.IsAny())) .ReturnsAsync(true); @@ -218,6 +220,8 @@ public async Task Replan_Composes_And_Executes_Experiment_Again() It.IsAny>(), It.IsAny>(), It.IsAny>(), + It.IsAny(), + It.IsAny>(), It.IsAny()), Times.Once); } diff --git a/Ares.Core/Execution/Enums/ExperimentLoopOutcome.cs b/Ares.Core/Execution/Enums/ExperimentLoopOutcome.cs new file mode 100644 index 00000000..33a07c72 --- /dev/null +++ b/Ares.Core/Execution/Enums/ExperimentLoopOutcome.cs @@ -0,0 +1,8 @@ +namespace Ares.Core.Execution.Enums; + +public enum ExperimentLoopOutcome +{ + Succeeded, + Failed, + Canceled +} \ No newline at end of file diff --git a/Ares.Core/Execution/Enums/ExperimentPhase.cs b/Ares.Core/Execution/Enums/ExperimentPhase.cs new file mode 100644 index 00000000..0a2cff41 --- /dev/null +++ b/Ares.Core/Execution/Enums/ExperimentPhase.cs @@ -0,0 +1,15 @@ +namespace Ares.Core.Execution.Enums; + +public enum ExperimentPhase +{ + Initialize, + Plan, + Compose, + Execute, + Analyze, + Retry, + Replan, + Complete, + Failed, + Canceled +} diff --git a/Ares.Core/Execution/ExecutionManager.cs b/Ares.Core/Execution/ExecutionManager.cs index e61a2282..08163c4e 100644 --- a/Ares.Core/Execution/ExecutionManager.cs +++ b/Ares.Core/Execution/ExecutionManager.cs @@ -55,7 +55,7 @@ public async Task CanRun() var startConditions = await Task.WhenAll(startConditionTasks); return startConditions.All(condition => condition?.Success ?? true); } - public int ReplanRate { get; private set; } = 1; + public async Task Start(string executionNotes, List campaignTags) { var err = await CheckCampaignStartPrerequisites(); @@ -73,7 +73,8 @@ public async Task Start(string executionNotes, List campaignTag executor.UpdateCampaignTags(campaignTags); executor.StopConditions.AddRange(CampaignStopConditions); - executor.ReplanRate = ReplanRate; + executor.ReplicateRate = ReplicateRate; + executor.BatchPlanningSize = PlanningBatchSize; _executionControlTokenSource = new ExecutionControlTokenSource(); CampaignExecutionSummary campaignExecutionSummary; ExecutionStartTime = DateTime.UtcNow; @@ -111,7 +112,7 @@ public void Resume() public async Task CheckCampaignStartPrerequisites() { if(_activeCampaignTemplateStore.CampaignTemplate is null) - return "CampaignTemplate was not assigned to the active template store."; + return "Campaign Template was not assigned to the active template store."; if(!CampaignStopConditions.Any()) return "The Campaign has no stop conditions, please set a stop condition before starting campaign."; @@ -142,7 +143,12 @@ public bool EnsureParameterAssignment() public void UpdateReplicateRate(int newRate) { - ReplanRate = newRate; + ReplicateRate = newRate; + } + + public void UpdateBatchPlanningSize(int newBatchSize) + { + PlanningBatchSize = newBatchSize; } public void SubmitUserDecision(ErrorHandling decision) @@ -171,8 +177,11 @@ private async Task StoreCompletedCampaign(CampaignExecutionSummary result) { throw; } - } public DateTime? ExecutionStartTime { get; set; } + + public int ReplicateRate { get; private set; } = 1; + + public int PlanningBatchSize { get; private set; } = 1; } diff --git a/Ares.Core/Execution/Executors/CampaignExecutor.cs b/Ares.Core/Execution/Executors/CampaignExecutor.cs index 14f176ed..eac1ee82 100644 --- a/Ares.Core/Execution/Executors/CampaignExecutor.cs +++ b/Ares.Core/Execution/Executors/CampaignExecutor.cs @@ -3,6 +3,7 @@ using Ares.Core.Device.State.Logging; using Ares.Core.Exceptions; using Ares.Core.Execution.ControlTokens; +using Ares.Core.Execution.Enums; using Ares.Core.Execution.Executors.Composers; using Ares.Core.Execution.Extensions; using Ares.Core.Execution.Safety; @@ -13,6 +14,7 @@ using Ares.Core.Settings; using Ares.Datamodel; using Ares.Datamodel.Analyzing; +using Ares.Datamodel.Planning; using Ares.Datamodel.Templates; using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; @@ -22,28 +24,7 @@ namespace Ares.Core.Execution.Executors; public class CampaignExecutor : ICampaignExecutor -{ - private enum ExperimentPhase - { - Initialize, - Plan, - Compose, - Execute, - Analyze, - Retry, - Replan, - Complete, - Failed, - Canceled - } - - private enum ExperimentLoopOutcome - { - Succeeded, - Failed, - Canceled - } - +{ private readonly IExecutionReporter _executionReporter; private readonly ISubject _executionStatusSubject; private readonly ICommandComposer _experimentComposer; @@ -66,6 +47,7 @@ private enum ExperimentLoopOutcome private ExperimentTemplate? _currentExperimentTemplate = null; private int _experimentCount = 0; private TaskCompletionSource? _userDecisionSource; + private List _latestPlanStatusCodes = new List(); internal CampaignExecutor(ICommandComposer experimentComposer, IPlanningHelper planningHelper, @@ -226,8 +208,13 @@ private async Task NotifyCampaignStart() return (true, startupSummary); } - private async Task ExecuteExperimentLoop(string campaignPath, List analyses, List experimentSummaries, ExecutionControlToken token, ExperimentExecutionSummary startupSummary) + private async Task ExecuteExperimentLoop(string campaignPath, + List analyses, + List experimentSummaries, + ExecutionControlToken token, + ExperimentExecutionSummary startupSummary) { + _latestPlanStatusCodes = new List(); var currentPhase = ExperimentPhase.Initialize; var currentExperimentPath = ""; var failedExperimentRetryCount = 0; @@ -317,7 +304,7 @@ private async Task PrepareExperiment(List analyses, L if(!_currentExperimentTemplate.IsResolved()) { _logger.LogTrace("Experiment has not been resolved, ARES will now begin the planning process."); - if(analyses.Count % ReplanRate == 0) + if(analyses.Count % ReplicateRate == 0) { if(!await PlanExperiment(analyses, _currentExperimentTemplate, experimentSummaries, token)) { @@ -397,9 +384,17 @@ private async Task ExecuteCurrentExperiment(string currentExper .SelectMany(s => s.CommandSummaries) .FirstOrDefault(c => !c.Result.Success); - return failedCommandSummary is null - ? ExperimentPhase.Analyze - : await HandleError(failedCommandSummary, token); + if(failedCommandSummary is null) + { + _latestPlanStatusCodes.Add(PlanStatusCode.PlanAccepted); + return ExperimentPhase.Analyze; + } + + else + { + UpdatePlanStatus(failedCommandSummary.StatusCode); + return await HandleError(failedCommandSummary, token); + } } private async Task AnalyzeCurrentExperiment(ExperimentExecutionSummary startupSummary, List analyses, List experimentSummaries, ExecutionControlToken token) @@ -479,6 +474,7 @@ private async Task HandleError(CommandExecutionSummary cmdSumma var decisionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); _userDecisionSource = decisionSource; + try { errorHandling = await decisionSource.Task.WaitAsync(token.CancellationToken); @@ -516,6 +512,16 @@ private async Task HandleError(CommandExecutionSummary cmdSumma } } + + private void UpdatePlanStatus(CommandStatusCode failCode) + { + if(failCode == CommandStatusCode.OutOfRange || failCode == CommandStatusCode.ParametersUnachievable) + _latestPlanStatusCodes.Add(PlanStatusCode.PlanUnachievable); + + else + _latestPlanStatusCodes.Add(PlanStatusCode.PlanFailed); + } + private async Task PlanExperiment(List analyses, ExperimentTemplate currentExperimentTemplate, List experimentSummaries, @@ -523,7 +529,8 @@ private async Task PlanExperiment(List analyses, { Status.PlannerState = PlannerState.PlanningInProgress; ReportCampaignStatus(); - _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplanRate); + _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplicateRate); + var metadata = new RequestMetadata { @@ -540,6 +547,8 @@ private async Task PlanExperiment(List analyses, currentExperimentTemplate.GetAllPlannedParameters(), analyses, experimentSummaries.Select(es => es.ExperimentOverview), + BatchPlanningSize, + _latestPlanStatusCodes, token.CancellationToken); if(!resolveSuccess) @@ -579,7 +588,7 @@ private async Task PlanExperiment(List analyses, // The following are top level checks for analysis failure in case the // failure is not properly handled on the Analysis itself // which also has support for "success" and "error" message - if (analysis is null || analysis.Result == float.NaN) + if(analysis is null || analysis.Result == float.NaN) { Status.AnalysisState = AnalysisState.AnalysisError; await _notifier.Notify("Analysis Failure", $"Analysis was reported as successful, but no actual analysis was provided. {analysis?.ErrorString ?? "No error string provided"}", NotificationSeverityEnum.Error); @@ -725,9 +734,9 @@ private static bool IsAwaitingResponse(ExperimentExecutionStatus status) .Any(step => step.CommandExecutionStatuses .Any(cmd => cmd.State == ExecutionState.AwaitingUser)); - private async Task GenerateExperimentExecutor(ExperimentTemplate template, - IEnumerable analyses, - IEnumerable previousExperiments, + private async Task GenerateExperimentExecutor(ExperimentTemplate template, + IEnumerable analyses, + IEnumerable previousExperiments, CancellationToken cancellationToken) { var result = new ExperimentExecutorResult(); @@ -737,11 +746,11 @@ private async Task GenerateExperimentExecutor(Experime if(!experimentTemplate.IsResolved()) { _logger.LogTrace("Experiment was not resolved"); - if(analyses.Count() % ReplanRate == 0) + if(analyses.Count() % ReplicateRate == 0) { Status.PlannerState = PlannerState.PlanningInProgress; ReportCampaignStatus(); - _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplanRate); + _logger.LogTrace("Analyses count is {count} and replan rate {rate}", analyses.Count(), ReplicateRate); var metadata = new RequestMetadata { @@ -757,7 +766,9 @@ private async Task GenerateExperimentExecutor(Experime metadata, experimentTemplate.GetAllPlannedParameters(), analyses, - previousExperiments, + previousExperiments, + BatchPlanningSize, + _latestPlanStatusCodes, cancellationToken); if(!resolveSuccess) @@ -840,7 +851,8 @@ private async Task PostExperimentExecution(ExperimentExecutionSummary summary) public CampaignTemplate Template { get; } public IList StopConditions { get; } = []; - public double ReplanRate { get; set; } = 1; + public int ReplicateRate { get; set; } = 1; + public int BatchPlanningSize { get; set; } = 1; public string? ExecutionNotes { get; set; } public List CampaignTags { get; set; } = []; public IObservable ExperimentStatusObservable { get; } diff --git a/Ares.Core/Execution/Executors/CommandExecutor.cs b/Ares.Core/Execution/Executors/CommandExecutor.cs index 63a152bd..4d9600d9 100644 --- a/Ares.Core/Execution/Executors/CommandExecutor.cs +++ b/Ares.Core/Execution/Executors/CommandExecutor.cs @@ -109,7 +109,6 @@ public async Task Execute(ExecutionControlToken token, Status.Result = result.Result; _stateSubject.OnNext(Status); - _stateSubject.OnCompleted(); return ExecutorSummaryHelpers.CreateCommandExecutionSummary(Template, result, timeStarted, DateTime.UtcNow); } diff --git a/Ares.Core/Execution/Executors/ExecutorSummaryHelpers.cs b/Ares.Core/Execution/Executors/ExecutorSummaryHelpers.cs index f2a1f198..7492dcc3 100644 --- a/Ares.Core/Execution/Executors/ExecutorSummaryHelpers.cs +++ b/Ares.Core/Execution/Executors/ExecutorSummaryHelpers.cs @@ -58,7 +58,8 @@ public static CommandExecutionSummary CreateCommandExecutionSummary(CommandTempl CommandDescription = template.Metadata.Description, CommandName = template.Metadata.Name, StatusCode = deviceResult?.StatusCode ?? CommandStatusCode.StatusUnspecified - }; + }; + if(template.HasOutputVarName) commandExecutionSummary.VarName = template.OutputVarName; diff --git a/Ares.Core/Execution/Executors/ExperimentExecutor.cs b/Ares.Core/Execution/Executors/ExperimentExecutor.cs index 08d0dba8..f375ea59 100644 --- a/Ares.Core/Execution/Executors/ExperimentExecutor.cs +++ b/Ares.Core/Execution/Executors/ExperimentExecutor.cs @@ -20,6 +20,7 @@ public ExperimentExecutor(ExperimentTemplate template, IExecutor executor.Status)); + var experimentStepExecutionObservation = experimentStepExecutors.Select(executor => { return executor.ExperimentStatusObservable.Select(_ => diff --git a/Ares.Core/Execution/Executors/ICampaignExecutor.cs b/Ares.Core/Execution/Executors/ICampaignExecutor.cs index 6e56cc87..cda8bf7e 100644 --- a/Ares.Core/Execution/Executors/ICampaignExecutor.cs +++ b/Ares.Core/Execution/Executors/ICampaignExecutor.cs @@ -6,9 +6,9 @@ namespace Ares.Core.Execution.Executors; public interface ICampaignExecutor : IExecutor { IList StopConditions { get; } - double ReplanRate { get; set; } + int ReplicateRate { get; set; } + int BatchPlanningSize { get; set; } void UpdateExecutionNotes(string executionNotes); - void UpdateCampaignTags(List campaignTags); void SubmitUserDecision(ErrorHandling decision); diff --git a/Ares.Core/Execution/IExecutionManager.cs b/Ares.Core/Execution/IExecutionManager.cs index 90a141dc..98f29343 100644 --- a/Ares.Core/Execution/IExecutionManager.cs +++ b/Ares.Core/Execution/IExecutionManager.cs @@ -11,9 +11,14 @@ public interface IExecutionManager public IList CampaignStopConditions { get; } /// - /// A double value that determines how often a campaign will re-plan it's experiment, defaults to one + /// A int value that determines how often a campaign will re-plan it's experiment, defaults to one /// - public int ReplanRate { get; } + public int ReplicateRate { get; } + + /// + /// An int value that determines how many experiments are planned for per planning request + /// + public int PlanningBatchSize { get; } /// /// Indicates whether the currently loaded campaign has all the prerequisites in order to start and run @@ -55,6 +60,11 @@ public interface IExecutionManager void UpdateReplicateRate(int newRate); /// + /// Updates the batch planning size + /// + /// + void UpdateBatchPlanningSize(int batchSize); + /// Submits a user decision for how to handle an error that has occurred during execution /// /// diff --git a/Ares.Core/Planning/IPlanningHelper.cs b/Ares.Core/Planning/IPlanningHelper.cs index 7ac1c5e9..99ecec7d 100644 --- a/Ares.Core/Planning/IPlanningHelper.cs +++ b/Ares.Core/Planning/IPlanningHelper.cs @@ -23,6 +23,8 @@ Task TryResolveParameters(IEnumerable plannerAllocation IEnumerable parameters, IEnumerable seedAnalyses, IEnumerable seedExperiments, + int batchSize, + List codes, CancellationToken cancellationToken); /// diff --git a/Ares.Core/Planning/PlanningHelper.cs b/Ares.Core/Planning/PlanningHelper.cs index 1f154544..748bff05 100644 --- a/Ares.Core/Planning/PlanningHelper.cs +++ b/Ares.Core/Planning/PlanningHelper.cs @@ -8,6 +8,7 @@ using Google.Protobuf.WellKnownTypes; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; +using System.Collections.Concurrent; namespace Ares.Core.Planning; @@ -18,9 +19,9 @@ public class PlanningHelper : IPlanningHelper private readonly INotifier _notifier; private readonly IDbContextFactory _dbContextFactory; - public PlanningHelper(IPlannerServiceRepo plannerManager, - ILogger logger, - INotifier notifier, + public PlanningHelper(IPlannerServiceRepo plannerManager, + ILogger logger, + INotifier notifier, IDbContextFactory dbContextFactory) { _plannerManager = plannerManager; @@ -34,103 +35,240 @@ public async Task TryResolveParameters(IEnumerable plan IEnumerable parameters, IEnumerable seedAnalyses, IEnumerable seedExperiments, + int batchSize, + List statusCodes, CancellationToken cancellationToken) { var parameterArray = parameters.ToArray(); + var plannerToMetadataMaps = MapParameterMetadataToPlanners(plannerAllocations); + + if(plannerToMetadataMaps is null) + return false; + + var planGroup = plannerToMetadataMaps.GroupBy(pair => pair.Planner); + var seedAnalysesArr = seedAnalyses.ToArray(); + + var planningTasks = planGroup.Select(async grouping => + { + var planner = grouping.Key; + + var planQueue = PlannerQueueDictionary.GetOrAdd(planner.UniqueId, _ => new ConcurrentQueue()); + + if(planQueue.TryDequeue(out var plan)) + { + ResolveParametersFromPlan(plan, parameterArray); + return true; + } + + return await RequestNewPlans(planner, grouping, seedExperiments, seedAnalysesArr, statusCodes, metadata, planQueue, parameterArray, batchSize, cancellationToken); + }); + + var results = await Task.WhenAll(planningTasks); + return results.All(success => success); + } + + /// + /// Requests a fresh set of plans from a planner service, resolves using the first one received and queues any additional. + /// + /// The planner service to request plans from + /// The grouping of planner and parameter metadata + /// Seed experiments for the planning request + /// Seed analyses for the planning request + /// The status code of the plan + /// Request metadata + /// + /// + /// + /// + private async Task RequestNewPlans(IPlannerService planner, + IGrouping grouping, + IEnumerable seedExperiments, + Analysis[] seedAnalysesArr, + List statusCodes, + RequestMetadata metadata, + ConcurrentQueue planQueue, + Parameter[] parameterArray, + int batchSize, + CancellationToken cancellationToken) + { + var planTransaction = new PlannerTransaction() + { + UniqueId = Guid.NewGuid().ToString(), + PlannerName = planner.Name, + PlannerType = planner.Type, + PlannerVersion = planner.Version, + PlannerId = planner.UniqueId + }; + + try + { + var plannableParameters = grouping.Select(pair => pair.Item2).ToArray(); + planTransaction.TimeRequestSent = DateTime.UtcNow.ToTimestamp(); + + //Create the plan request. Store it in the transaction. + var planRequest = CreatePlanningRequest(plannableParameters, seedExperiments, seedAnalysesArr, statusCodes, batchSize, metadata); + planTransaction.PlanningRequest = planRequest; + + var planResponse = await planner.Plan(planRequest, cancellationToken); + planTransaction.TimeResponseReceived = DateTime.UtcNow.ToTimestamp(); + planTransaction.PlanningResponse = planResponse; + + if(planResponse.Plans.Any()) + { + var currentPlan = planResponse.Plans.First(); + + foreach(var plan in planResponse.Plans.Skip(1)) + planQueue.Enqueue(plan); + + ResolveParametersFromPlan(currentPlan, parameterArray); + } + + else + { + var legacyResponseProcessed = await HandleLegacyPlanResponse(planResponse, parameterArray); + + if(!legacyResponseProcessed) + return false; + } + } + + catch(Exception e) + { + _logger.LogError("Failed to plan. {}", e.Message); + await _notifier.Notify("Planner Error!", e.Message, NotificationSeverityEnum.Error); + return false; + } + + await LogPlannerTransaction(planTransaction); + return true; + } + + /// + /// Creates a planning request based on the provided data. + /// + /// The parameters to be planned for + /// Seed data to pull experiment history from + /// Analysis results to be included in the planning request + /// The previous plans status code + /// Request metadata + /// An ARES + private PlanningRequest CreatePlanningRequest(ParameterMetadata[] plannableParameters, + IEnumerable seedExperiments, + Analysis[] seedAnalysesArr, + List statusCodes, + int batchSize, + RequestMetadata metadata) + { + //Create the plan request. Store it in the transaction. + var planRequest = new PlanningRequest(); + planRequest.PlanningParameters.AddRange(plannableParameters.Select(parameter => ConvertToPlanningParameter(parameter, seedExperiments))); + planRequest.AnalysisResults.AddRange(seedAnalysesArr.Select(a => (double)a.Result)); + planRequest.PreviousPlanStatusCodes.AddRange(statusCodes); + planRequest.Metadata = metadata; + planRequest.BatchSize = batchSize; + + return planRequest; + } + + /// + /// Uses the provided planner allocations to map parameter metadata to their respective planners. + /// + /// + /// A mapping of planners to the provided parameter metadata. + private List<(IPlannerService Planner, ParameterMetadata Metadata)>? MapParameterMetadataToPlanners(IEnumerable plannerAllocations) + { var plannerToMetadataMaps = new List<(IPlannerService Planner, ParameterMetadata Metadata)>(); foreach(var plannerAllocation in plannerAllocations) { var planner = _plannerManager.GetPlannerById(plannerAllocation.Planner.UniqueId); + if(planner is null) - return false; + return null; plannerToMetadataMaps.Add((planner, plannerAllocation.Parameter)); } - var planGroup = plannerToMetadataMaps.GroupBy(pair => pair.Planner); - var seedAnalysesArr = seedAnalyses.ToArray(); - foreach(var grouping in planGroup) + return plannerToMetadataMaps; + } + + /// + /// Handles legacy plan responses + /// + /// + /// + /// A bool indicating whether or not resolving variables was a success + private async Task HandleLegacyPlanResponse(PlanningResponse planResponse, Parameter[] parameterArray) + { +#pragma warning disable CS0612 // Type or member is obsolete + + if(planResponse.PlanningOutcome == Outcome.Failure) { - var planner = grouping.Key; - var planTransaction = new PlannerTransaction() - { - UniqueId = Guid.NewGuid().ToString(), - PlannerName = planner.Name, - PlannerType = planner.Type, - PlannerVersion = planner.Version, - PlannerId = planner.UniqueId - }; - - try - { - var plannableParameters = grouping.Select(pair => pair.Metadata).ToArray(); - //make metadata thx - planTransaction.TimeRequestSent = DateTime.UtcNow.ToTimestamp(); - - //Create the plan request. Store it in the transaction. - var planRequest = new PlanningRequest(); - planRequest.PlanningParameters.AddRange(plannableParameters.Select(parameter => ConvertToPlanningParameter(parameter, seedExperiments))); - planRequest.AnalysisResults.AddRange(seedAnalysesArr.Select(a => (double)a.Result)); - planRequest.Metadata = metadata; - planTransaction.PlanningRequest = planRequest; - - var planResponse = await planner.Plan(planRequest, cancellationToken); - planTransaction.TimeResponseReceived = DateTime.UtcNow.ToTimestamp(); - planTransaction.PlanningResponse = planResponse; - - if(planResponse.PlanningOutcome == Outcome.Failure) - { - if(string.IsNullOrWhiteSpace(planResponse.ErrorString)) - await _notifier.Notify("Planner Error!", "Planner reported that planning failed, but did not provide any specific error as to why.", NotificationSeverityEnum.Error); + if(string.IsNullOrWhiteSpace(planResponse.ErrorString)) + await _notifier.Notify("Planner Error!", "Planner reported that planning failed, but did not provide any specific error as to why.", NotificationSeverityEnum.Error); - else - await _notifier.Notify($"Planner Reported Error: {planResponse.ErrorString}", "Planner Error!", NotificationSeverityEnum.Error); + else + await _notifier.Notify($"Planner Reported Error: {planResponse.ErrorString}", "Planner Error!", NotificationSeverityEnum.Error); - return false; - } + return false; + } - if(planResponse.PlanningOutcome == Outcome.Warning) - { - if(string.IsNullOrWhiteSpace(planResponse.ErrorString)) - await _notifier.Notify("Planner Warning", "Planner reported a warning, but did not provide specific context for that warning.", NotificationSeverityEnum.Warning); + if(planResponse.PlanningOutcome == Outcome.Warning) + { + if(string.IsNullOrWhiteSpace(planResponse.ErrorString)) + await _notifier.Notify("Planner Warning", "Planner reported a warning, but did not provide specific context for that warning.", NotificationSeverityEnum.Warning); - else - await _notifier.Notify("Planner Warning", $"Planner successfully planned, but reported a warning: {planResponse.ErrorString}", NotificationSeverityEnum.Warning); - } - - if(planResponse.PlanningOutcome == Outcome.Canceled) - await _notifier.Notify("Planning was canceled.", "Planning was canceled.", NotificationSeverityEnum.Info); + else + await _notifier.Notify("Planner Warning", $"Planner successfully planned, but reported a warning: {planResponse.ErrorString}", NotificationSeverityEnum.Warning); + } - if(!planResponse.PlannedParameters.Any()) - { - await _notifier.Notify("Planning Error!", "Tried to plan for experiment, but planning returned no plan results! Campaign will stop.", NotificationSeverityEnum.Error); - return false; - } + if(planResponse.PlanningOutcome == Outcome.Canceled) + await _notifier.Notify("Planning was canceled.", "Planning was canceled.", NotificationSeverityEnum.Info); - foreach(var result in planResponse.PlannedParameters) - { - var parameterPlanTarget = parameterArray.FirstOrDefault(parameter => parameter.GetPlanningMetadata()?.Name == result.ParameterName); + if(!planResponse.PlannedParameters.Any()) + { + await _notifier.Notify("Planning Error!", "Tried to plan for experiment, but planning returned no plan results! Campaign will stop.", NotificationSeverityEnum.Error); + return false; + } - if(parameterPlanTarget is null) - continue; + foreach(var result in planResponse.PlannedParameters) + { + var parameterPlanTarget = parameterArray.FirstOrDefault(parameter => parameter.GetPlanningMetadata()?.Name == result.ParameterName); - parameterPlanTarget.SetResolvedValue(result.ParameterValue); - } - } - catch(Exception e) - { - _logger.LogError("Failed to plan. {}", e.Message); - await _notifier.Notify("Planner Error!", e.Message, NotificationSeverityEnum.Error); - return false; - } + if(parameterPlanTarget is null) + continue; - await LogPlannerTransaction(planTransaction); + parameterPlanTarget.SetResolvedValue(result.ParameterValue); } return true; +#pragma warning restore CS0612 // Type or member is obsolete + } + + /// + /// Takes in a plan and a parameter array and resolves those parameters using the provided plan + /// + /// The plan containing the planned parameters + /// The array of parameters to be resolved + private void ResolveParametersFromPlan(Plan plan, Parameter[] parameterArray) + { + foreach(var result in plan.PlannedParameters) + { + var parameterPlanTarget = parameterArray.FirstOrDefault(parameter => parameter.GetPlanningMetadata()?.Name == result.ParameterName); + + if(parameterPlanTarget is null) + continue; + + parameterPlanTarget.SetResolvedValue(result.ParameterValue); + } } + /// + /// Takes in a piece of parameter metadata and the experiment history to create an ARES planning parameter. + /// + /// + /// + /// An ARES private static PlanningParameter ConvertToPlanningParameter(ParameterMetadata metadata, IEnumerable experimentHistory) { var parameter = new PlanningParameter @@ -172,13 +310,22 @@ private static PlanningParameter ConvertToPlanningParameter(ParameterMetadata me return parameter; } + /// + /// Logs the planner transaction to the database. + /// + /// + /// private async Task LogPlannerTransaction(PlannerTransaction transaction) { - var context = _dbContextFactory.CreateDbContext(); + using var context = _dbContextFactory.CreateDbContext(); await context.PlannerTransactions.AddAsync(transaction); await context.SaveChangesAsync(); } + /// + /// Reseeds the manual planner with it's last provided seed data, essentially returning it to a state as if new seed data had just been provided. + /// + /// public async Task ReseedManualPlanner() { var manualPlanner = _plannerManager.GetManualPlanner(); @@ -186,4 +333,6 @@ public async Task ReseedManualPlanner() if(manualPlanner is not null) await manualPlanner.Reseed(); } + + private ConcurrentDictionary> PlannerQueueDictionary { get; set; } = new ConcurrentDictionary>(); } diff --git a/UI/Components/Layouts/MainLayout.razor b/UI/Components/Layouts/MainLayout.razor index 5211faa6..a948b8d0 100644 --- a/UI/Components/Layouts/MainLayout.razor +++ b/UI/Components/Layouts/MainLayout.razor @@ -258,6 +258,7 @@ Tracker.OnSystemReady -= HandleSystemReady; notificationService.OnNotificationReceived -= HandleBackgroundNotification; _driverSubscription?.Dispose(); + _cts.Cancel(); _cts.Dispose(); } diff --git a/UI/Features/Execution/Components/CampaignConfig.razor b/UI/Features/Execution/Components/CampaignConfig.razor index 0e22f52e..6ac3de3e 100644 --- a/UI/Features/Execution/Components/CampaignConfig.razor +++ b/UI/Features/Execution/Components/CampaignConfig.razor @@ -21,10 +21,18 @@
- +
- - + + +
+
+ +
+ +
+ +
@@ -33,6 +41,7 @@ +
Experiment Tags
Experiment Replication Rate
- + +
+
+ +
+ Planning Batch Size +
+ +
@@ -264,6 +272,10 @@
Replicate Rate
ARES will replicate experiments @ViewModel!.DesiredReplicationRate time(s)
+
+
Batch Size
+
ARES will request @ViewModel!.PlanningBatchSize experiment(s) at a time
+
Tags
@(ViewModel!.SelectedTags.Any() ? string.Join(", ", ViewModel!.SelectedTags.Select(tag => tag.TagName)) : "None")
diff --git a/UI/Features/Execution/ExecutionViewModel.cs b/UI/Features/Execution/ExecutionViewModel.cs index ca594511..20b1d9e4 100644 --- a/UI/Features/Execution/ExecutionViewModel.cs +++ b/UI/Features/Execution/ExecutionViewModel.cs @@ -144,6 +144,11 @@ public Task GetCurrentStopCondition() return _automationClient.GetActiveStopCondition(new Empty(), null); } + public Task GetCurrentPlanningBatchSize() + { + return _automationClient.GetPlanningBatchSize(new Empty(), null); + } + public Task GetCurrentReplicateRate() { return _automationClient.GetReplicateRate(new Empty(), null); @@ -164,11 +169,19 @@ public async Task SetExperimentsToRun() StateChanged?.Invoke(); } - public async Task SetReplanRate() + public async Task SetReplicateRate() { await _automationClient.SetReplicateRate(new ReplicateRate { ReplicateRate_ = DesiredReplicationRate }, null); var replanRateResponse = await GetCurrentReplicateRate(); DesiredReplicationRate = replanRateResponse.ReplicateRate_; + + } + + public async Task SetPlanningBatchSize() + { + await _automationClient.SetPlanningBatchSize(new PlanningBatchSize { BatchSize = PlanningBatchSize }, null); + var planningBatchSizeResponse = await GetCurrentPlanningBatchSize(); + PlanningBatchSize = planningBatchSizeResponse.BatchSize; } public async Task StartCampaign() @@ -176,7 +189,7 @@ public async Task StartCampaign() await ApplyActiveStopCondition(); if(!CampaignActive) - await SetReplanRate(); + await SetReplicateRate(); var executionEligibility = await _automationClient.CheckExecutionEligibility(new Empty(), null); LastExecutionEligibility = executionEligibility; @@ -236,6 +249,7 @@ private async Task RefreshExecutionEligibility() LastExecutionEligibility = await _automationClient.CheckExecutionEligibility(new Empty(), null); } + public async Task ExecutionNotesUploaded(Stream fileStream) { using var reader = new StreamReader(fileStream); @@ -367,39 +381,71 @@ public async Task UpdatePlannerTransactions() if(newestTransaction is null) continue; - OnPlannerTransactionReceived(newestTransaction, transactionList.Count()); + OnPlannerTransactionReceived(newestTransaction, DeterminePlanCount(transactionList.SkipLast(1))); } } } public void OnPlannerTransactionReceived(PlannerTransaction transaction, int currentTurn) { - foreach(var field in transaction.PlanningResponse.PlannedParameters) - { - var metricName = field.ParameterName; - var metricData = field.ParameterValue; - var matchingParam = transaction.PlanningRequest.PlanningParameters.FirstOrDefault(p => p.ParameterName == field.ParameterName); + if(transaction.PlanningResponse.PlannedParameters.Any()) + foreach(var field in transaction.PlanningResponse.PlannedParameters) + ProcessTransactionParameterData(field, transaction, currentTurn); - if(TryGetChartableValue(metricData, out double numericValue) && matchingParam is not null) + else if(transaction.PlanningResponse.Plans.Any()) + { + foreach(var plan in transaction.PlanningResponse.Plans) { - var minBound = matchingParam.MinimumValue; - var maxBound = matchingParam.MaximumValue; - var normalizedValue = 0.0; + foreach(var param in plan.PlannedParameters) + ProcessTransactionParameterData(param, transaction, currentTurn); + + currentTurn++; + } + } + } - if(maxBound > minBound) - normalizedValue = ((numericValue - minBound) / (maxBound - minBound)) * 100; + private int DeterminePlanCount(IEnumerable transactionList) + { + var count = 0; - if(!PlannerMetricsMap.ContainsKey(metricName)) - PlannerMetricsMap[metricName] = new List(); + foreach(var transaction in transactionList) + { + if(transaction.PlanningResponse.PlannedParameters.Any()) + count++; - PlannerMetricsMap[metricName].Add(new ChartMetricPoint - { - ExecutionIndex = currentTurn, - PlotValue = normalizedValue, // Charting Value - RawValue = numericValue // Tooltip Display Value - }); - } + else + count += transaction.PlanningResponse.Plans.Count(); } + + return count; + } + + private void ProcessTransactionParameterData(PlannedParameter field, PlannerTransaction transaction, int currentTurn) + { + var metricName = field.ParameterName; + var metricData = field.ParameterValue; + var matchingParam = transaction.PlanningRequest.PlanningParameters.FirstOrDefault(p => p.ParameterName == field.ParameterName); + + if(TryGetChartableValue(metricData, out double numericValue) && matchingParam is not null) + { + var minBound = matchingParam.MinimumValue; + var maxBound = matchingParam.MaximumValue; + var normalizedValue = 0.0; + + if(maxBound > minBound) + normalizedValue = ((numericValue - minBound) / (maxBound - minBound)) * 100; + + if(!PlannerMetricsMap.ContainsKey(metricName)) + PlannerMetricsMap[metricName] = new List(); + + PlannerMetricsMap[metricName].Add(new ChartMetricPoint + { + ExecutionIndex = currentTurn, + PlotValue = normalizedValue, // Charting Value + RawValue = numericValue // Tooltip Display Value + }); + } + } public void OnAnalyzerTransactionReceived(AnalyzerTransaction transaction, int currentTurn) @@ -530,9 +576,6 @@ private void UpdateCampaignStatus(CampaignExecutionState state) if(CampaignExecutionState == ExecutionState.AwaitingUser) _ = RequestUserConfirmation(); - //if(CampaignActive) - // SelectedExecutionTabIndex = 1; - StateChanged?.Invoke(); } @@ -621,8 +664,10 @@ public async Task RefreshExecutionContext() public async Task RefreshPlannerTransactionData() { + PlannerMetricsMap.Clear(); var plannerTransactions = await _automationClient.GetLatestPlanningTransactions(); + var transactionNumber = 0; foreach(var transactionList in plannerTransactions) { if(transactionList is null) @@ -630,19 +675,24 @@ public async Task RefreshPlannerTransactionData() foreach(var (index, item) in transactionList.Index()) { - OnPlannerTransactionReceived(item, index); + OnPlannerTransactionReceived(item, transactionNumber); + + if(item.PlanningResponse.Plans.Any()) + transactionNumber += item.PlanningResponse.Plans.Count(); + + else + transactionNumber++; } } } public async Task RefreshAnalyzerTransactionData() { + AnalyzerMetrics.Clear(); var analyzerTransactions = await _automationClient.GetLatestAnalyzerTransactions(); foreach(var (index, item) in analyzerTransactions.Index()) - { OnAnalyzerTransactionReceived(item, index); - } } public async Task RefreshCampaignSetup() @@ -722,6 +772,9 @@ public int ActiveExperimentNumber public partial ExperimentStopConditionResponse? CurrentStopCondition { get; set; } public double DesiredResult { get; set; } public double DesiredLeeway { get; set; } + [Reactive] + public partial int PlanningBatchSize { get; set; } = 1; + [Reactive] public int DesiredReplicationRate { get; set; } = 1; [Reactive]