diff --git a/Common/Api/Optimization.cs b/Common/Api/Optimization.cs index d408f5d5e77c..b8f396021260 100644 --- a/Common/Api/Optimization.cs +++ b/Common/Api/Optimization.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; using Newtonsoft.Json; +using QuantConnect.Optimizer; using QuantConnect.Optimizer.Objectives; using QuantConnect.Util; @@ -74,6 +75,15 @@ public class Optimization : BaseOptimization /// [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.ISOShort, DateFormat.UI)] public DateTime Requested { get; set; } + + /// + /// Aggregate diagnostic of the optimization (Sharpe distribution, parameter sensitivity + /// slices, clusters in parameter space, local maxima, zero-order failure breakdown). + /// Populated by the optimization analyzer in LeanOptimizer.TriggerOnEndEvent; + /// omitted on optimizations that haven't run the analyzer or had no usable trials. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public OptimizationAnalysis Analysis { get; set; } } /// diff --git a/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs b/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs new file mode 100644 index 000000000000..deb2623c8723 --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationAnalysisRunParameters.cs @@ -0,0 +1,55 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Bundles the inputs needed by : the full list of + /// completed trials from a finished optimization and the parameter grid spec that + /// drove it. The optimization-side analogue of ResultsAnalysisRunParameters + /// (which serves the backtest analyzer in Engine). + /// + public class OptimizationAnalysisRunParameters + { + /// + /// All completed trials from the optimization (one per backtest), already mapped + /// to the Common-side shape that the analyzer reads. + /// + public IReadOnlyList CompletedTrials { get; } + + /// + /// The optimization parameter grid spec (used for searched-min/max/step bounds and to + /// drive per-parameter slicing). + /// + public IReadOnlyCollection OptimizationParameters { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The completed trials. + /// The parameter grid spec. + public OptimizationAnalysisRunParameters( + IReadOnlyList completedTrials, + IReadOnlyCollection optimizationParameters) + { + CompletedTrials = completedTrials; + OptimizationParameters = optimizationParameters; + } + } +} diff --git a/Common/Optimizer/Analysis/OptimizationAnalyzer.cs b/Common/Optimizer/Analysis/OptimizationAnalyzer.cs new file mode 100644 index 000000000000..6d3fc410bcd0 --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationAnalyzer.cs @@ -0,0 +1,272 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using QuantConnect.Logging; +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + // Types like OptimizationAnalysis / SharpeSummary / Cluster / Mode / etc. live in + // QuantConnect.Optimizer (the parent namespace); they're referenced unqualified + // because the file is inside QuantConnect.Optimizer.Analysis and the C# compiler + // walks outward through parent namespaces when resolving simple names. + /// + /// Builds an aggregate diagnostic () from a completed + /// optimization's compute-job results. Computes the Sharpe distribution, the best trial, + /// per-parameter sensitivity slices, k-means clusters in parameter space, local maxima, + /// and a zero-order failure breakdown. The optimization-side analogue of + /// ; invoked from LeanOptimizer.TriggerOnEndEvent. + /// + public class OptimizationAnalyzer + { + private readonly OptimizationAnalysisRunParameters _parameters; + + /// + /// Initializes a new instance of the class. + /// + /// The inputs to analyze. + public OptimizationAnalyzer(OptimizationAnalysisRunParameters parameters) + { + _parameters = parameters; + } + + /// + /// Runs the full analysis pipeline and returns the aggregate diagnostic. + /// + /// The populated , or null if no usable trials remain. + public OptimizationAnalysis Run() + { + var allTrials = ExtractTrials(_parameters.CompletedTrials); + var trials = allTrials.Where(t => t.HasSharpe).ToList(); + if (trials.Count == 0) + { + Log.Trace("OptimizationAnalyzer.Run(): no completed backtests with parsable Sharpe ratios; skipping analysis"); + return null; + } + + var sharpes = trials.Select(t => t.Sharpe).ToList(); + var overall = new SharpeSummary + { + Mean = sharpes.Average(), + StdDev = StdDev(sharpes), + Min = sharpes.Min(), + Max = sharpes.Max(), + Median = Median(sharpes) + }; + + // Always maximize Sharpe. The optimization's chosen Criterion may be something else + // but the analyzer uses Sharpe as the universal yardstick for the analysis surface. + var best = trials.OrderByDescending(t => t.Sharpe).First(); + var bestSummary = new BestTrialSummary + { + BacktestId = best.BacktestId, + Parameters = new Dictionary(best.Parameters), + SharpeRatio = best.Sharpe + }; + + var paramReports = _parameters.OptimizationParameters + .Select(p => OptimizationSlicing.AnalyzeParameter(p, trials, best)) + .ToList(); + + var clusters = OptimizationClustering.Build(trials, _parameters.OptimizationParameters); + var modes = OptimizationModes.Find(trials, _parameters.OptimizationParameters); + var failed = OptimizationFailedBacktests.Build(allTrials); + + return new OptimizationAnalysis + { + TrialCountTotal = allTrials.Count, + TrialCountUsed = trials.Count, + OverallSharpe = overall, + Best = bestSummary, + Parameters = paramReports, + Clusters = clusters, + Modes = modes, + FailedBacktests = failed + }; + } + + // ── Trial extraction ───────────────────────────────────────────────────── + + /// + /// Parses each completed trial's JSON backtest payload into a typed + /// : parameter values + Sharpe + total orders + the backtest's + /// own diagnostic Analysis tags. + /// + private static List ExtractTrials(IReadOnlyList trialInputs) + { + var trials = new List(); + if (trialInputs == null) return trials; + + foreach (var t in trialInputs) + { + if (t == null || string.IsNullOrEmpty(t.JsonBacktestResult) || t.ParameterSet == null) + { + continue; + } + + Dictionary paramValues; + try + { + paramValues = ParseParameterSet(t.ParameterSet); + } + catch + { + continue; + } + if (paramValues.Count == 0) + { + continue; + } + + ParsedBacktest parsed; + try + { + parsed = JsonConvert.DeserializeObject(t.JsonBacktestResult); + } + catch + { + continue; + } + if (parsed == null) + { + continue; + } + + var hasSharpe = TryReadDouble(parsed.Statistics, "Sharpe Ratio", out var sharpe); + TryReadInt(parsed.Statistics, "Total Orders", out var totalOrders); + + var analysisNames = parsed.Analysis == null + ? new List() + : parsed.Analysis + .Where(a => !string.IsNullOrEmpty(a?.Name)) + .Select(a => a.Name) + .ToList(); + + trials.Add(new TrialRecord( + backtestId: t.BacktestId, + parameters: paramValues, + sharpe: sharpe, + hasSharpe: hasSharpe, + totalOrders: totalOrders, + analysisNames: analysisNames)); + } + return trials; + } + + private static Dictionary ParseParameterSet(ParameterSet parameterSet) + { + var result = new Dictionary(); + if (parameterSet?.Value == null) return result; + foreach (var kv in parameterSet.Value) + { + if (double.TryParse(kv.Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var d)) + { + result[kv.Key] = d; + } + } + return result; + } + + private static bool TryReadDouble(IDictionary statistics, string key, out double value) + { + value = 0; + if (statistics == null) return false; + if (!statistics.TryGetValue(key, out var raw) || string.IsNullOrEmpty(raw)) return false; + // QC statistics often carry trailing units like "%" — strip them before parsing. + var trimmed = raw.TrimEnd('%').Trim(); + return double.TryParse(trimmed, NumberStyles.Any, CultureInfo.InvariantCulture, out value); + } + + private static bool TryReadInt(IDictionary statistics, string key, out int value) + { + value = 0; + if (!TryReadDouble(statistics, key, out var d)) return false; + value = (int)d; + return true; + } + + // ── Aggregate helpers ──────────────────────────────────────────────────── + + private static double StdDev(IReadOnlyCollection values) + { + if (values.Count < 2) return 0; + var mean = values.Average(); + var s = values.Sum(v => (v - mean) * (v - mean)); + return System.Math.Sqrt(s / (values.Count - 1)); + } + + private static double Median(IEnumerable values) + { + var sorted = values.OrderBy(v => v).ToList(); + if (sorted.Count == 0) return 0; + return sorted.Count % 2 == 1 + ? sorted[sorted.Count / 2] + : 0.5 * (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]); + } + + // ── Minimal-shape DTO for JSON deserialization ─────────────────────────── + // + // The JsonBacktestResult string carried by OptimizationResult is the Newtonsoft.Json + // serialization of the in-process backtest Result. We only need a few fields; rather + // than depend on the full Result type (with its many nested dependencies), bind a + // minimal shape with the two properties we read. + + private sealed class ParsedBacktest + { + [JsonProperty("Statistics")] + public IDictionary Statistics { get; set; } + + [JsonProperty("Analysis")] + public List Analysis { get; set; } + } + } + + /// + /// Internal per-trial record used across the optimization-analysis helpers + /// (, , + /// , ). + /// Public visibility kept low: only the helpers in this namespace need it. + /// + internal sealed class TrialRecord + { + public string BacktestId { get; } + public IReadOnlyDictionary Parameters { get; } + public double Sharpe { get; } + public bool HasSharpe { get; } + public int TotalOrders { get; } + public IReadOnlyList AnalysisNames { get; } + + public TrialRecord( + string backtestId, + IReadOnlyDictionary parameters, + double sharpe, + bool hasSharpe, + int totalOrders, + IReadOnlyList analysisNames) + { + BacktestId = backtestId; + Parameters = parameters; + Sharpe = sharpe; + HasSharpe = hasSharpe; + TotalOrders = totalOrders; + AnalysisNames = analysisNames; + } + } +} diff --git a/Common/Optimizer/Analysis/OptimizationClustering.cs b/Common/Optimizer/Analysis/OptimizationClustering.cs new file mode 100644 index 000000000000..1a156ca3310a --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationClustering.cs @@ -0,0 +1,272 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// K-means clustering of trials in standardized parameter space, with k chosen by an + /// elbow heuristic. Centroids are reported in original parameter units so they're + /// directly comparable to trial parameter values. Clusters are ordered by mean Sharpe + /// descending. Deterministic (k-means++ init seeded at 42). + /// + internal static class OptimizationClustering + { + private const int KMin = 2; + private const int KMaxAbsolute = 5; + private const int Seed = 42; + private const int MaxIterations = 100; + private const double PlateauThreshold = 0.7; + + public static IReadOnlyList Build( + IReadOnlyList trials, + IReadOnlyCollection parameters) + { + var output = new List(); + if (trials == null || parameters == null) return output; + if (trials.Count < KMin + 1 || parameters.Count == 0) return output; + + var paramNames = parameters.Select(p => p.Name).ToArray(); + + // Only consider trials carrying values for every parameter. + var usable = trials + .Where(t => paramNames.All(t.Parameters.ContainsKey)) + .ToList(); + if (usable.Count < KMin + 1) return output; + + // Cap k_max at min(absolute, ceil(sqrt(N))) so we don't carve a small N + // into too many tiny clusters. + var sqrtCap = (int)Math.Ceiling(Math.Sqrt(usable.Count)); + var kMaxEffective = Math.Min(KMaxAbsolute, sqrtCap); + var maxK = Math.Min(kMaxEffective, usable.Count - 1); + if (maxK < KMin) return output; + + // Build (N x D) point matrix in original units, then z-score standardize. + var raw = usable + .Select(t => paramNames.Select(n => t.Parameters[n]).ToArray()) + .ToArray(); + var (normalized, means, stds) = Standardize(raw); + + // Sweep k = KMin..maxK and pick by elbow heuristic. + var byK = new Dictionary(); + for (var k = KMin; k <= maxK; k++) + { + byK[k] = KMeans(normalized, k); + } + var bestK = SelectKByElbow(byK); + var pick = byK[bestK]; + + // De-normalize centroids back to the original parameter units so consumers + // can compare them directly to trial parameter values. + var centroidsOriginal = pick.Centroids + .Select(c => Denormalize(c, means, stds)) + .ToArray(); + + for (var c = 0; c < bestK; c++) + { + var memberIndices = Enumerable.Range(0, usable.Count) + .Where(i => pick.Labels[i] == c) + .ToList(); + if (memberIndices.Count == 0) continue; + var sharpes = memberIndices.Select(i => usable[i].Sharpe).ToList(); + + var centroidDict = new Dictionary(paramNames.Length); + for (var d = 0; d < paramNames.Length; d++) + { + centroidDict[paramNames[d]] = centroidsOriginal[c][d]; + } + + output.Add(new Cluster + { + Index = c, + Centroid = centroidDict, + MemberCount = memberIndices.Count, + SharpeMean = sharpes.Average(), + SharpeStdDev = StdDev(sharpes), + SharpeMin = sharpes.Min(), + SharpeMax = sharpes.Max() + }); + } + + // Re-index by Sharpe (highest first) so Cluster 0 is the best-performing region. + var ordered = output.OrderByDescending(x => x.SharpeMean).ToList(); + for (var i = 0; i < ordered.Count; i++) + { + ordered[i].Index = i; + } + return ordered; + } + + private static int SelectKByElbow(Dictionary results) + { + var ks = results.Keys.OrderBy(k => k).ToList(); + if (ks.Count == 1) return ks[0]; + for (var i = 1; i < ks.Count; i++) + { + var prev = results[ks[i - 1]].Wcss; + var curr = results[ks[i]].Wcss; + if (prev > 0 && curr / prev > PlateauThreshold) return ks[i - 1]; + } + return ks[^1]; + } + + private sealed class KMeansResult + { + public int[] Labels { get; } + public double[][] Centroids { get; } + public double Wcss { get; } + + public KMeansResult(int[] labels, double[][] centroids, double wcss) + { + Labels = labels; + Centroids = centroids; + Wcss = wcss; + } + } + + private static KMeansResult KMeans(double[][] points, int k) + { + var n = points.Length; + var d = points[0].Length; + var rng = new Random(Seed); + + // k-means++ initialization. + var centroids = new double[k][]; + centroids[0] = (double[])points[rng.Next(n)].Clone(); + for (var c = 1; c < k; c++) + { + var dists = new double[n]; + for (var i = 0; i < n; i++) + { + var min = double.MaxValue; + for (var j = 0; j < c; j++) + { + var dd = SquaredDistance(points[i], centroids[j]); + if (dd < min) min = dd; + } + dists[i] = min; + } + var sum = dists.Sum(); + var pick = rng.NextDouble() * sum; + double acc = 0; + var chosen = n - 1; + for (var i = 0; i < n; i++) + { + acc += dists[i]; + if (acc >= pick) { chosen = i; break; } + } + centroids[c] = (double[])points[chosen].Clone(); + } + + // Lloyd's iteration. + var labels = new int[n]; + for (var iter = 0; iter < MaxIterations; iter++) + { + var changed = false; + for (var i = 0; i < n; i++) + { + var best = 0; + var bestDist = double.MaxValue; + for (var c = 0; c < k; c++) + { + var dd = SquaredDistance(points[i], centroids[c]); + if (dd < bestDist) { bestDist = dd; best = c; } + } + if (labels[i] != best) { labels[i] = best; changed = true; } + } + if (!changed && iter > 0) break; + + var sums = new double[k][]; + var counts = new int[k]; + for (var c = 0; c < k; c++) sums[c] = new double[d]; + for (var i = 0; i < n; i++) + { + var c = labels[i]; + counts[c]++; + for (var j = 0; j < d; j++) sums[c][j] += points[i][j]; + } + for (var c = 0; c < k; c++) + { + if (counts[c] == 0) continue; + for (var j = 0; j < d; j++) centroids[c][j] = sums[c][j] / counts[c]; + } + } + + double wcss = 0; + for (var i = 0; i < n; i++) wcss += SquaredDistance(points[i], centroids[labels[i]]); + return new KMeansResult(labels, centroids, wcss); + } + + private static (double[][] Normalized, double[] Means, double[] Stds) Standardize(double[][] points) + { + var n = points.Length; + var d = points[0].Length; + var means = new double[d]; + var stds = new double[d]; + for (var j = 0; j < d; j++) + { + double s = 0; + for (var i = 0; i < n; i++) s += points[i][j]; + means[j] = s / n; + } + for (var j = 0; j < d; j++) + { + double s = 0; + for (var i = 0; i < n; i++) + { + var t = points[i][j] - means[j]; + s += t * t; + } + stds[j] = n > 1 ? Math.Sqrt(s / (n - 1)) : 1.0; + if (stds[j] < 1e-12) stds[j] = 1.0; + } + var normalized = new double[n][]; + for (var i = 0; i < n; i++) + { + normalized[i] = new double[d]; + for (var j = 0; j < d; j++) normalized[i][j] = (points[i][j] - means[j]) / stds[j]; + } + return (normalized, means, stds); + } + + private static double[] Denormalize(double[] standardized, double[] means, double[] stds) + { + var d = standardized.Length; + var result = new double[d]; + for (var j = 0; j < d; j++) result[j] = standardized[j] * stds[j] + means[j]; + return result; + } + + private static double SquaredDistance(double[] a, double[] b) + { + double s = 0; + for (var i = 0; i < a.Length; i++) { var d = a[i] - b[i]; s += d * d; } + return s; + } + + private static double StdDev(IReadOnlyCollection values) + { + if (values.Count < 2) return 0; + var mean = values.Average(); + var s = values.Sum(v => (v - mean) * (v - mean)); + return Math.Sqrt(s / (values.Count - 1)); + } + } +} diff --git a/Common/Optimizer/Analysis/OptimizationFailedBacktests.cs b/Common/Optimizer/Analysis/OptimizationFailedBacktests.cs new file mode 100644 index 000000000000..9c6ca5a62324 --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationFailedBacktests.cs @@ -0,0 +1,69 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Aggregates failure-mode signals across backtests that produced zero orders. For up to + /// zero-order trials, counts how many carry each + /// distinct backtest-level analysis tag (e.g. "FlatEquityCurveAnalysis"). Returns null + /// when no trials in the optimization produced zero orders. + /// + internal static class OptimizationFailedBacktests + { + // Cap on how many zero-order trials we look at. We only need a rough tally of the + // common failure modes, not an exhaustive census. + private const int MaxBacktestsToInspect = 10; + + public static FailedBacktestSummary Build(IReadOnlyList trials) + { + if (trials == null) return null; + + var zeroOrder = trials.Where(t => t.TotalOrders == 0).ToList(); + if (zeroOrder.Count == 0) return null; + + var sample = zeroOrder.Take(MaxBacktestsToInspect).ToList(); + var nameCount = new Dictionary(StringComparer.Ordinal); + + foreach (var trial in sample) + { + // De-dupe per-trial so the counts answer "in how many trials did this tag + // appear", not "how many total occurrences across trials". + var seen = new HashSet(StringComparer.Ordinal); + if (trial.AnalysisNames != null) + { + foreach (var name in trial.AnalysisNames) + { + if (string.IsNullOrEmpty(name)) continue; + if (!seen.Add(name)) continue; + nameCount[name] = nameCount.GetValueOrDefault(name, 0) + 1; + } + } + } + + return new FailedBacktestSummary + { + ZeroOrderCount = zeroOrder.Count, + InspectedCount = sample.Count, + AnalysisNameCounts = nameCount + }; + } + } +} diff --git a/Common/Optimizer/Analysis/OptimizationModes.cs b/Common/Optimizer/Analysis/OptimizationModes.cs new file mode 100644 index 000000000000..16d82c44edc0 --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationModes.cs @@ -0,0 +1,113 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Detects local maxima of the Sharpe surface on the parameter grid: trials whose + /// Sharpe is strictly greater than every face-neighbor's Sharpe (face-neighbors differ + /// from the candidate in exactly one parameter by one grid step). Works in 1, 2, 3+ + /// dimensions. Isolated trials with no neighbors are not flagged — there's no + /// distribution to find a mode of. + /// + internal static class OptimizationModes + { + public static IReadOnlyList Find( + IReadOnlyList trials, + IReadOnlyCollection parameters) + { + var modes = new List(); + if (trials == null || parameters == null) return modes; + if (parameters.Count == 0 || trials.Count == 0) return modes; + + var paramNames = parameters.Select(p => p.Name).ToArray(); + + // Sorted distinct values per parameter — these define the grid axes. + var axisValues = new Dictionary>(); + foreach (var name in paramNames) + { + axisValues[name] = trials + .Where(t => t.Parameters.ContainsKey(name)) + .Select(t => t.Parameters[name]) + .Distinct() + .OrderBy(v => v) + .ToList(); + } + + // Map each trial to its grid position (one index per parameter). + var indexed = new List<(TrialRecord Trial, int[] Indices)>(); + foreach (var t in trials) + { + if (!paramNames.All(t.Parameters.ContainsKey)) continue; + var idx = new int[paramNames.Length]; + var ok = true; + for (var d = 0; d < paramNames.Length; d++) + { + idx[d] = axisValues[paramNames[d]].IndexOf(t.Parameters[paramNames[d]]); + if (idx[d] < 0) { ok = false; break; } + } + if (ok) indexed.Add((t, idx)); + } + + // O(1) neighbor lookup by index tuple. + var byTuple = indexed.ToDictionary(p => TupleKey(p.Indices), p => p.Trial); + + foreach (var (trial, idx) in indexed) + { + var totalNeighbors = 0; + var dominatesAll = true; + + for (var d = 0; d < paramNames.Length && dominatesAll; d++) + { + var axisLen = axisValues[paramNames[d]].Count; + foreach (var delta in new[] { -1, 1 }) + { + var ni = idx[d] + delta; + if (ni < 0 || ni >= axisLen) continue; + + var neighborIdx = (int[])idx.Clone(); + neighborIdx[d] = ni; + if (!byTuple.TryGetValue(TupleKey(neighborIdx), out var neighbor)) continue; + + totalNeighbors++; + if (neighbor.Sharpe >= trial.Sharpe) { dominatesAll = false; break; } + } + } + + if (dominatesAll && totalNeighbors > 0) + { + modes.Add(new Mode + { + BacktestId = trial.BacktestId, + Parameters = new Dictionary(trial.Parameters), + SharpeRatio = trial.Sharpe, + NeighborCount = totalNeighbors + }); + } + } + + return modes.OrderByDescending(m => m.SharpeRatio).ToList(); + } + + private static string TupleKey(int[] indices) + => string.Join(",", indices.Select(i => i.ToString(CultureInfo.InvariantCulture))); + } +} diff --git a/Common/Optimizer/Analysis/OptimizationSlicing.cs b/Common/Optimizer/Analysis/OptimizationSlicing.cs new file mode 100644 index 000000000000..5b8246e56389 --- /dev/null +++ b/Common/Optimizer/Analysis/OptimizationSlicing.cs @@ -0,0 +1,194 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Optimizer.Analysis +{ + /// + /// Per-parameter sensitivity analysis. For each optimized parameter, builds a set of + /// 1-D "slices" through the trial cloud (this parameter varies; every other is held + /// constant), fits a piecewise linear interpolant to each slice, and aggregates + /// sensitivity metrics across slices. + /// + internal static class OptimizationSlicing + { + public static ParameterReport AnalyzeParameter( + OptimizationParameter parameter, + IReadOnlyList trials, + TrialRecord best) + { + var name = parameter.Name; + var owning = trials.Where(t => t.Parameters.ContainsKey(name)).ToList(); + + var otherParamNames = owning + .SelectMany(t => t.Parameters.Keys) + .Where(k => k != name) + .Distinct() + .OrderBy(k => k, StringComparer.Ordinal) + .ToList(); + + // Group trials by the values held constant in other parameters — each group is one 1-D slice. + IEnumerable> grouped = otherParamNames.Count == 0 + ? new[] { owning.GroupBy(_ => "").FirstOrDefault() } + .Where(g => g != null) + .Cast>() + : owning.GroupBy(t => SliceKey(t, otherParamNames)); + + var primaryKey = otherParamNames.Count == 0 ? "" : SliceKey(best, otherParamNames); + + var slices = new List(); + foreach (var group in grouped) + { + var isPrimary = group.Key == primaryKey; + var slice = BuildSlice(group.ToList(), name, otherParamNames, isPrimary); + if (slice != null) slices.Add(slice); + } + + var distinctValueCount = owning.Select(t => t.Parameters[name]).Distinct().Count(); + var bestValue = best.Parameters.TryGetValue(name, out var bv) ? bv : double.NaN; + var (searchedMin, searchedMax, step) = ExtractGridSpec(parameter, owning, name); + var bestAtEdge = IsAtSearchedEdge(bestValue, searchedMin, searchedMax, step); + + var meanRange = slices.Count > 0 ? slices.Average(s => s.SharpeRange) : 0; + var maxRange = slices.Count > 0 ? slices.Max(s => s.SharpeRange) : 0; + var maxDerivPerStep = slices.Count > 0 + ? slices.Max(s => s.MaxAbsDerivative) * (step ?? 1.0) + : 0; + + return new ParameterReport + { + Name = name, + SearchedMin = searchedMin, + SearchedMax = searchedMax, + Step = step, + DistinctValueCount = distinctValueCount, + MeanWithinSliceSharpeRange = meanRange, + MaxWithinSliceSharpeRange = maxRange, + MaxAbsDerivativePerStep = maxDerivPerStep, + BestValue = bestValue, + BestAtSearchedEdge = bestAtEdge, + Slices = slices + }; + } + + private static SliceFit BuildSlice( + List trials, + string varyingParamName, + IReadOnlyList otherParamNames, + bool isPrimary) + { + // Collapse duplicate parameter values within a slice (shouldn't happen in a true grid + // sweep, but be defensive — average their Sharpes). + var points = trials + .GroupBy(t => t.Parameters[varyingParamName]) + .Select(g => (X: g.Key, Y: g.Average(t => t.Sharpe))) + .OrderBy(p => p.X) + .ToList(); + + if (points.Count == 0) return null; + + var xs = points.Select(p => p.X).ToList(); + var ys = points.Select(p => p.Y).ToList(); + var sharpeRange = ys.Count >= 2 ? ys.Max() - ys.Min() : 0; + + // Piecewise linear: one segment per adjacent pair; slope IS the sensitivity + // per unit of the parameter for this slice. + var segments = new List(); + double maxAbsDerivative = 0; + for (var i = 0; i < points.Count - 1; i++) + { + var dx = xs[i + 1] - xs[i]; + var slope = (ys[i + 1] - ys[i]) / dx; + segments.Add(new LinearSegment + { + XLo = xs[i], + XHi = xs[i + 1], + A = ys[i], + B = slope + }); + var absSlope = Math.Abs(slope); + if (absSlope > maxAbsDerivative) maxAbsDerivative = absSlope; + } + + var curveType = points.Count >= 2 ? "piecewise linear" : "single point"; + + var fixedParams = new Dictionary(); + if (otherParamNames.Count > 0) + { + var first = trials[0]; + foreach (var p in otherParamNames) + { + if (first.Parameters.TryGetValue(p, out var v)) fixedParams[p] = v; + } + } + + return new SliceFit + { + FixedParameters = fixedParams, + ParameterValues = xs, + SharpeValues = ys, + SharpeRange = sharpeRange, + MaxAbsDerivative = maxAbsDerivative, + CurveType = curveType, + IsPrimary = isPrimary, + Segments = segments + }; + } + + private static (double Min, double Max, double? Step) ExtractGridSpec( + OptimizationParameter parameter, + IReadOnlyList owning, + string name) + { + if (parameter is OptimizationStepParameter step) + { + return ((double)step.MinValue, (double)step.MaxValue, + step.Step.HasValue ? (double)step.Step.Value : (double?)null); + } + + // Fallback when the parameter isn't a step-based one: infer min/max/step from the + // measured distinct values across trials. Step is the smallest gap. + var values = owning.Select(t => t.Parameters[name]).Distinct().OrderBy(v => v).ToList(); + if (values.Count == 0) return (0, 0, null); + if (values.Count == 1) return (values[0], values[0], null); + + var min = values[0]; + var max = values[^1]; + var gaps = new List(); + for (var i = 1; i < values.Count; i++) gaps.Add(values[i] - values[i - 1]); + return (min, max, gaps.Min()); + } + + private static bool IsAtSearchedEdge(double value, double min, double max, double? step) + { + if (double.IsNaN(value)) return false; + var tol = ((step ?? 1.0) / 2) + 1e-9; + return Math.Abs(value - min) <= tol || Math.Abs(value - max) <= tol; + } + + private static string SliceKey(TrialRecord t, IReadOnlyList otherParamNames) + { + return string.Join("|", otherParamNames.Select(p => + (t.Parameters.TryGetValue(p, out var v) ? v : double.NaN) + .ToString("R", CultureInfo.InvariantCulture))); + } + } +} diff --git a/Common/Optimizer/BestTrialSummary.cs b/Common/Optimizer/BestTrialSummary.cs new file mode 100644 index 000000000000..263bf1daf588 --- /dev/null +++ b/Common/Optimizer/BestTrialSummary.cs @@ -0,0 +1,41 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Identifies the single best-performing trial in an optimization (argmax of Sharpe). + /// + public class BestTrialSummary + { + /// + /// The backtest id of the best-performing trial. + /// + public string BacktestId { get; set; } + + /// + /// Parameter values for the best trial (parameter name -> numeric value). + /// + public IReadOnlyDictionary Parameters { get; set; } + + /// + /// Sharpe ratio of the best trial. + /// + public double SharpeRatio { get; set; } + } +} diff --git a/Common/Optimizer/Cluster.cs b/Common/Optimizer/Cluster.cs new file mode 100644 index 000000000000..a50fc6a6713a --- /dev/null +++ b/Common/Optimizer/Cluster.cs @@ -0,0 +1,63 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// One k-means cluster of trials in standardized parameter space. Centroid coordinates + /// are reported in the original parameter units so they are directly comparable to + /// trial parameter values. Clusters are ordered by descending. + /// + public class Cluster + { + /// + /// Cluster index (0 = highest mean Sharpe). + /// + public int Index { get; set; } + + /// + /// Cluster centroid in original parameter units (parameter name -> value). + /// + public IReadOnlyDictionary Centroid { get; set; } + + /// + /// Number of trials assigned to this cluster. + /// + public int MemberCount { get; set; } + + /// + /// Mean Sharpe ratio across the cluster's members. + /// + public double SharpeMean { get; set; } + + /// + /// Sample standard deviation of Sharpe ratios within this cluster. + /// + public double SharpeStdDev { get; set; } + + /// + /// Minimum Sharpe ratio observed within this cluster. + /// + public double SharpeMin { get; set; } + + /// + /// Maximum Sharpe ratio observed within this cluster. + /// + public double SharpeMax { get; set; } + } +} diff --git a/Common/Optimizer/FailedBacktestSummary.cs b/Common/Optimizer/FailedBacktestSummary.cs new file mode 100644 index 000000000000..cfac344f5410 --- /dev/null +++ b/Common/Optimizer/FailedBacktestSummary.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Summary of backtests in an optimization that produced zero orders. Aggregates how many + /// of the inspected backtests carry each backtest-level analysis tag (e.g. + /// "FlatEquityCurveAnalysis"), giving a quick view of common failure modes that aggregate + /// optimization statistics can mask. + /// + public class FailedBacktestSummary + { + /// + /// Total number of backtests in the optimization that produced zero orders. + /// + public int ZeroOrderCount { get; set; } + + /// + /// Number of zero-order backtests actually inspected for analysis tags (capped to + /// keep the summary bounded; may be larger). + /// + public int InspectedCount { get; set; } + + /// + /// Membership counts: analysis-tag name -> number of inspected backtests that carry + /// the tag. Each name appears at most once per backtest in the count. + /// + public IReadOnlyDictionary AnalysisNameCounts { get; set; } + } +} diff --git a/Common/Optimizer/LinearSegment.cs b/Common/Optimizer/LinearSegment.cs new file mode 100644 index 000000000000..70e200204ab3 --- /dev/null +++ b/Common/Optimizer/LinearSegment.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Optimizer +{ + /// + /// One straight-line piece of a piecewise linear interpolant covering [XLo, XHi]. + /// Evaluates as y(x) = A + B * (x - XLo). Passes exactly through (XLo, Sharpe at XLo) + /// and (XHi, Sharpe at XHi). + /// + public class LinearSegment + { + /// + /// Lower bound of this segment (inclusive). + /// + public double XLo { get; set; } + + /// + /// Upper bound of this segment. + /// + public double XHi { get; set; } + + /// + /// Sharpe ratio at XLo (the segment's left endpoint). + /// + public double A { get; set; } + + /// + /// Constant slope through the segment: (Sharpe at XHi - Sharpe at XLo) / (XHi - XLo). + /// + public double B { get; set; } + } +} diff --git a/Common/Optimizer/Mode.cs b/Common/Optimizer/Mode.cs new file mode 100644 index 000000000000..f1dce338c5b8 --- /dev/null +++ b/Common/Optimizer/Mode.cs @@ -0,0 +1,51 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// A local maximum of the Sharpe surface on the parameter grid: a trial whose Sharpe is + /// strictly greater than every face-neighbor's Sharpe (face-neighbors differ from this + /// trial in exactly one parameter by one grid step). Multiple modes indicate a multimodal + /// surface and suggest splitting the next optimization into narrower sweeps around each. + /// + public class Mode + { + /// + /// The backtest id of the mode trial. + /// + public string BacktestId { get; set; } + + /// + /// Parameter values for the mode trial (parameter name -> numeric value). + /// + public IReadOnlyDictionary Parameters { get; set; } + + /// + /// Sharpe ratio of the mode trial. + /// + public double SharpeRatio { get; set; } + + /// + /// Number of face-neighbors this trial was compared against. A higher count means + /// the mode is supported by more surrounding evidence (interior cells have more + /// neighbors than edge or corner cells). + /// + public int NeighborCount { get; set; } + } +} diff --git a/Common/Optimizer/OptimizationAnalysis.cs b/Common/Optimizer/OptimizationAnalysis.cs new file mode 100644 index 000000000000..5f381eb4417b --- /dev/null +++ b/Common/Optimizer/OptimizationAnalysis.cs @@ -0,0 +1,75 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Aggregate diagnostic produced by analyzing a completed optimization. Captures the + /// per-trial Sharpe distribution, the best trial, per-parameter sensitivity slices, + /// k-means clusters in parameter space, local maxima ("modes"), and any zero-order + /// failure breakdown. + /// + public class OptimizationAnalysis + { + /// + /// Total number of trial backtests observed during the optimization (including failures). + /// + public int TrialCountTotal { get; set; } + + /// + /// Number of trial backtests successfully used in the analysis after filtering failed runs. + /// + public int TrialCountUsed { get; set; } + + /// + /// Univariate Sharpe ratio statistics across all used trials. + /// + public SharpeSummary OverallSharpe { get; set; } + + /// + /// The best-performing trial (argmax of Sharpe). + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public BestTrialSummary Best { get; set; } + + /// + /// Per-parameter sensitivity report. One entry per optimized parameter. + /// + public IReadOnlyList Parameters { get; set; } + + /// + /// K-means clusters in standardized parameter space, ordered by mean Sharpe descending. + /// Empty when there are too few trials to cluster meaningfully. + /// + public IReadOnlyList Clusters { get; set; } + + /// + /// Local maxima of the Sharpe surface on the parameter grid (face-neighbor sense), + /// ordered by Sharpe descending. + /// + public IReadOnlyList Modes { get; set; } + + /// + /// Summary of backtests that produced zero orders, with their analysis-tag counts. + /// Null/omitted when no zero-order trials exist. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public FailedBacktestSummary FailedBacktests { get; set; } + } +} diff --git a/Common/Optimizer/OptimizationTrial.cs b/Common/Optimizer/OptimizationTrial.cs new file mode 100644 index 000000000000..cf437fb795b1 --- /dev/null +++ b/Common/Optimizer/OptimizationTrial.cs @@ -0,0 +1,60 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using QuantConnect.Optimizer.Parameters; + +namespace QuantConnect.Optimizer +{ + /// + /// Common-side per-trial input to . Carries + /// only the fields the analyzer reads (backtest id, parameter set, serialized backtest + /// result JSON). Decouples the analyzer from the QuantConnect.Optimizer assembly's + /// OptimizationResult type so analyzer code can live in Common without forcing + /// Common to reference Optimizer. + /// + public class OptimizationTrial + { + /// + /// The backtest id that produced this trial. + /// + public string BacktestId { get; } + + /// + /// The parameter set the trial was run with. + /// + public ParameterSet ParameterSet { get; } + + /// + /// The serialized backtest result. The analyzer reads Statistics + /// (for Sharpe and total orders) and Analysis (for the zero-order + /// failure breakdown) off the deserialized payload. + /// + public string JsonBacktestResult { get; } + + /// + /// Initializes a new instance of the class. + /// + /// The backtest id that produced this trial. + /// The parameter set the trial was run with. + /// The serialized backtest result JSON. + public OptimizationTrial(string backtestId, ParameterSet parameterSet, string jsonBacktestResult) + { + BacktestId = backtestId; + ParameterSet = parameterSet; + JsonBacktestResult = jsonBacktestResult; + } + } +} diff --git a/Common/Optimizer/ParameterReport.cs b/Common/Optimizer/ParameterReport.cs new file mode 100644 index 000000000000..599eea6acbb1 --- /dev/null +++ b/Common/Optimizer/ParameterReport.cs @@ -0,0 +1,88 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// Sensitivity report for a single optimized parameter, computed from one-dimensional + /// cross-sections of the parameter space (slices) where every other parameter is held + /// constant. + /// + public class ParameterReport + { + /// + /// Parameter name as defined in the optimization configuration. + /// + public string Name { get; set; } + + /// + /// Lower bound of the parameter sweep, taken from the optimization configuration. + /// + public double SearchedMin { get; set; } + + /// + /// Upper bound of the parameter sweep, taken from the optimization configuration. + /// + public double SearchedMax { get; set; } + + /// + /// Sweep step size from the optimization configuration, or null if none was provided. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public double? Step { get; set; } + + /// + /// Number of distinct values this parameter took across all completed trials. + /// + public int DistinctValueCount { get; set; } + + /// + /// Average of (max Sharpe - min Sharpe) across every 1-D slice of this parameter. + /// + public double MeanWithinSliceSharpeRange { get; set; } + + /// + /// Maximum of (max Sharpe - min Sharpe) across every 1-D slice of this parameter. + /// + public double MaxWithinSliceSharpeRange { get; set; } + + /// + /// Worst-case Sharpe change between two adjacent grid values of this parameter, + /// taken across all slices and scaled by . + /// + public double MaxAbsDerivativePerStep { get; set; } + + /// + /// This parameter's value at the best trial. + /// + public double BestValue { get; set; } + + /// + /// True when lies within half a step of the searched min or max, + /// signalling that the optimum may lie outside the swept range. + /// + public bool BestAtSearchedEdge { get; set; } + + /// + /// One-dimensional slices of the parameter space used for the sensitivity analysis. + /// Each slice fixes every other parameter and varies only this one. + /// + public IReadOnlyList Slices { get; set; } + } +} diff --git a/Common/Optimizer/SharpeSummary.cs b/Common/Optimizer/SharpeSummary.cs new file mode 100644 index 000000000000..32312ec15afd --- /dev/null +++ b/Common/Optimizer/SharpeSummary.cs @@ -0,0 +1,49 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +namespace QuantConnect.Optimizer +{ + /// + /// Univariate statistics of the Sharpe ratio across all used trials in an optimization. + /// + public class SharpeSummary + { + /// + /// Arithmetic mean of Sharpe ratios across all used trials. + /// + public double Mean { get; set; } + + /// + /// Sample standard deviation of Sharpe ratios across all used trials. + /// + public double StdDev { get; set; } + + /// + /// Minimum Sharpe ratio observed. + /// + public double Min { get; set; } + + /// + /// Maximum Sharpe ratio observed. + /// + public double Max { get; set; } + + /// + /// Median Sharpe ratio across all used trials. + /// + public double Median { get; set; } + } +} diff --git a/Common/Optimizer/SliceFit.cs b/Common/Optimizer/SliceFit.cs new file mode 100644 index 000000000000..da6f0db7043c --- /dev/null +++ b/Common/Optimizer/SliceFit.cs @@ -0,0 +1,73 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; + +namespace QuantConnect.Optimizer +{ + /// + /// One-dimensional cross-section of the parameter space: a single parameter varies while + /// every other parameter is held constant at the values in . + /// The piecewise linear interpolant () passes through every measured + /// (ParameterValue, Sharpe) point exactly. + /// + public class SliceFit + { + /// + /// Values of the other parameters that are held constant for this slice (name -> value). + /// Empty for single-parameter optimizations. + /// + public IReadOnlyDictionary FixedParameters { get; set; } + + /// + /// Measured grid values of the slicing parameter, sorted ascending. + /// + public IReadOnlyList ParameterValues { get; set; } + + /// + /// Sharpe ratio at each entry in (same index). + /// + public IReadOnlyList SharpeValues { get; set; } + + /// + /// max(SharpeValues) - min(SharpeValues) across this slice. Zero for single-point slices. + /// + public double SharpeRange { get; set; } + + /// + /// Maximum |slope| across this slice's linear segments. Equals + /// max(|y[i+1] - y[i]| / (x[i+1] - x[i])). + /// + public double MaxAbsDerivative { get; set; } + + /// + /// Human-readable curve label. Either "piecewise linear" or "single point". + /// + public string CurveType { get; set; } + + /// + /// True for exactly one slice per parameter: the slice whose fixed parameters match + /// the values at the best trial. + /// + public bool IsPrimary { get; set; } + + /// + /// Piecewise linear pieces of the fit. Length = len(ParameterValues) - 1 + /// (empty when there is only one point). + /// + public IReadOnlyList Segments { get; set; } + } +} diff --git a/Optimizer/LeanOptimizer.cs b/Optimizer/LeanOptimizer.cs index 3912a54daaa6..f9d7a73f8cc8 100644 --- a/Optimizer/LeanOptimizer.cs +++ b/Optimizer/LeanOptimizer.cs @@ -21,6 +21,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; +using QuantConnect.Optimizer.Analysis; using QuantConnect.Optimizer.Objectives; using QuantConnect.Optimizer.Parameters; using QuantConnect.Optimizer.Strategies; @@ -41,6 +42,11 @@ public abstract class LeanOptimizer : IDisposable private int _completedBacktest; private volatile bool _disposed; + // Accumulates every completed backtest's result so the optimization-level analyzer + // can run over the full trial history at TriggerOnEndEvent time. IOptimizationStrategy + // only exposes the best Solution, so we maintain our own history here. + private readonly List _completedResults = new(); + /// /// The total completed backtests count /// @@ -185,6 +191,46 @@ protected virtual void TriggerOnEndEvent() CleanUpRunningInstance(); ProcessUpdate(forceSend: true); + // Run the optimization-level analyzer (Sharpe distribution, clusters, modes, + // sensitivity slices, zero-order failure summary) over the full trial history + // and attach the result to the OptimizationResult fired via Ended. Guarded by a + // config flag and a broad try/catch so analyzer failure never breaks the + // optimization itself. Parallels how BacktestingResultHandler.SendFinalResult + // attaches Result.Analysis from ResultsAnalyzer. + OptimizationAnalysis analysis = null; + if (result != null && Config.GetBool("optimization-analysis-enabled", true)) + { + try + { + // Map each in-process OptimizationResult to its Common-side analyzer + // input shape so the analyzer (which lives in Common) doesn't have to + // know about this assembly's OptimizationResult type. + var trials = new List(_completedResults.Count); + foreach (var r in _completedResults) + { + trials.Add(new OptimizationTrial(r.BacktestId, r.ParameterSet, r.JsonBacktestResult)); + } + var parameters = new OptimizationAnalysisRunParameters( + trials, + NodePacket.OptimizationParameters); + analysis = new OptimizationAnalyzer(parameters).Run(); + } + catch (Exception ex) + { + Log.Error(ex, "Error running optimization analysis"); + } + } + + if (result != null && analysis != null) + { + // Re-wrap the solution so the analysis travels with the Ended event. + result = new OptimizationResult( + result.JsonBacktestResult, + result.ParameterSet, + result.BacktestId, + analysis); + } + Ended?.Invoke(this, result); } @@ -246,6 +292,12 @@ protected virtual void NewResult(string jsonBacktestResult, string backtestId) result = new OptimizationResult(jsonBacktestResult, parameterSet, backtestId); } + // Accumulate completed results so the optimization-level analyzer can run over + // the full trial history at TriggerOnEndEvent time. _completedResults is only + // read on TriggerOnEndEvent (under the same lock that wraps NewResult), so a + // plain List is safe here. + _completedResults.Add(result); + // always notify the strategy Strategy.PushNewResults(result); diff --git a/Optimizer/OptimizationResult.cs b/Optimizer/OptimizationResult.cs index 74412b73b3b6..a11cdab7d3a2 100644 --- a/Optimizer/OptimizationResult.cs +++ b/Optimizer/OptimizationResult.cs @@ -47,17 +47,27 @@ public class OptimizationResult /// public ParameterSet ParameterSet { get; } + /// + /// Aggregate diagnostic for the whole optimization. Populated only on the single + /// emitted via LeanOptimizer.Ended at the end + /// of an optimization run; null on per-backtest results pushed during the run and + /// on . + /// + public OptimizationAnalysis Analysis { get; } + /// /// Create an instance of /// /// Optimization target value for this backtest /// Parameter set used in compute job /// The backtest id that generated this result - public OptimizationResult(string jsonBacktestResult, ParameterSet parameterSet, string backtestId) + /// Aggregate optimization-level diagnostic; only populated on the end-of-optimization result. + public OptimizationResult(string jsonBacktestResult, ParameterSet parameterSet, string backtestId, OptimizationAnalysis analysis = null) { JsonBacktestResult = jsonBacktestResult; ParameterSet = parameterSet; BacktestId = backtestId; + Analysis = analysis; } } } diff --git a/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs b/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs new file mode 100644 index 000000000000..04e4fa86904e --- /dev/null +++ b/Tests/Optimizer/Analysis/OptimizationAnalyzerTests.cs @@ -0,0 +1,210 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using Newtonsoft.Json; +using NUnit.Framework; +using QuantConnect.Optimizer; +using QuantConnect.Optimizer.Analysis; +using QuantConnect.Optimizer.Parameters; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace QuantConnect.Tests.Optimizer.Analysis +{ + [TestFixture, Parallelizable(ParallelScope.Self)] + public class OptimizationAnalyzerTests + { + [Test] + public void Run_ProducesOverallSharpeStats() + { + // 3x3 grid of synthetic Sharpe values. + var sharpes = new double[,] + { + { 0.10, 0.20, 0.30 }, + { 0.15, 0.25, 0.35 }, + { 0.18, 0.28, 0.38 } + }; + + var results = BuildGridResults(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analyzer = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)); + + var analysis = analyzer.Run(); + + Assert.NotNull(analysis); + Assert.AreEqual(9, analysis.TrialCountUsed); + Assert.AreEqual(9, analysis.TrialCountTotal); + + // Mean = average of {0.10..0.38} = 0.243333... + Assert.AreEqual(0.2433, analysis.OverallSharpe.Mean, 1e-3); + Assert.AreEqual(0.10, analysis.OverallSharpe.Min, 1e-9); + Assert.AreEqual(0.38, analysis.OverallSharpe.Max, 1e-9); + } + + [Test] + public void Run_BestTrialIsArgmaxSharpe() + { + var sharpes = new double[,] + { + { 0.10, 0.20, 0.30 }, + { 0.15, 0.25, 0.35 }, + { 0.18, 0.28, 0.99 } // peak at (2, 2) + }; + + var results = BuildGridResults(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analysis = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)).Run(); + + Assert.NotNull(analysis.Best); + Assert.AreEqual(0.99, analysis.Best.SharpeRatio, 1e-9); + // Parameters at (xIndex=2, yIndex=2). Grid x: {1,2,3}; y: {10,20,30}. + Assert.AreEqual(3.0, analysis.Best.Parameters["x"], 1e-9); + Assert.AreEqual(30.0, analysis.Best.Parameters["y"], 1e-9); + } + + [Test] + public void Run_FindsInteriorMode() + { + // 3x3 with a single interior peak at (1, 1): should produce one mode with 4 neighbors. + var sharpes = new double[,] + { + { 0.10, 0.20, 0.10 }, + { 0.20, 0.99, 0.20 }, + { 0.10, 0.20, 0.10 } + }; + + var results = BuildGridResults(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 3, yCount: 3); + var analysis = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)).Run(); + + Assert.AreEqual(1, analysis.Modes.Count); + Assert.AreEqual(0.99, analysis.Modes[0].SharpeRatio, 1e-9); + Assert.AreEqual(4, analysis.Modes[0].NeighborCount); + } + + [Test] + public void Run_ClusterCountRespectsSqrtCap() + { + // 4 trials -> ceil(sqrt(4)) = 2 -> max 2 clusters. + var sharpes = new double[,] + { + { 0.10, 0.20 }, + { 0.30, 0.40 } + }; + + var results = BuildGridResults(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)).Run(); + + Assert.LessOrEqual(analysis.Clusters.Count, 2); + } + + [Test] + public void Run_BuildsFailedBacktestSummary_FromZeroOrderTrials() + { + // 2x2 grid; every trial has zero orders and carries known analysis tags. + var sharpes = new double[,] + { + { 0.0, 0.0 }, + { 0.0, 0.0 } + }; + + var results = BuildGridResults( + sharpes, + totalOrders: 0, + analysisNames: new[] { "FlatEquityCurveAnalysis", "ExecutionSpeedAnalysis" }); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)).Run(); + + Assert.NotNull(analysis.FailedBacktests); + Assert.AreEqual(4, analysis.FailedBacktests.ZeroOrderCount); + Assert.AreEqual(4, analysis.FailedBacktests.InspectedCount); + Assert.AreEqual(4, analysis.FailedBacktests.AnalysisNameCounts["FlatEquityCurveAnalysis"]); + Assert.AreEqual(4, analysis.FailedBacktests.AnalysisNameCounts["ExecutionSpeedAnalysis"]); + } + + [Test] + public void Run_OmitsFailedBacktestSummary_WhenAllTrialsTrade() + { + var sharpes = new double[,] + { + { 0.10, 0.20 }, + { 0.30, 0.40 } + }; + + var results = BuildGridResults(sharpes, totalOrders: 5); + var parameters = BuildGridParameters(xCount: 2, yCount: 2); + var analysis = new OptimizationAnalyzer(new OptimizationAnalysisRunParameters(results, parameters)).Run(); + + Assert.IsNull(analysis.FailedBacktests); + } + + // ── helpers ────────────────────────────────────────────────────────────── + + private static List BuildGridResults( + double[,] sharpes, + int totalOrders, + string[] analysisNames = null) + { + var results = new List(); + var xCount = sharpes.GetLength(0); + var yCount = sharpes.GetLength(1); + var id = 0; + for (var i = 0; i < xCount; i++) + { + for (var j = 0; j < yCount; j++) + { + var paramSet = new ParameterSet(id, new Dictionary + { + ["x"] = (i + 1).ToString(CultureInfo.InvariantCulture), + ["y"] = ((j + 1) * 10).ToString(CultureInfo.InvariantCulture) + }); + var json = BuildBacktestJson(sharpes[i, j], totalOrders, analysisNames); + results.Add(new OptimizationTrial($"backtest-{id}", paramSet, json)); + id++; + } + } + return results; + } + + private static string BuildBacktestJson(double sharpe, int totalOrders, string[] analysisNames) + { + var statistics = new Dictionary + { + ["Sharpe Ratio"] = sharpe.ToString("R", CultureInfo.InvariantCulture), + ["Total Orders"] = totalOrders.ToString(CultureInfo.InvariantCulture) + }; + var analyses = (analysisNames ?? System.Array.Empty()) + .Select(n => new QuantConnect.Analysis(n, "issue", null, null, System.Array.Empty())) + .ToList(); + return JsonConvert.SerializeObject(new + { + Statistics = statistics, + Analysis = analyses + }); + } + + private static HashSet BuildGridParameters(int xCount, int yCount) + { + return new HashSet + { + new OptimizationStepParameter("x", 1, xCount, 1), + new OptimizationStepParameter("y", 10, yCount * 10, 10) + }; + } + } +}