diff --git a/OpenUtau.Core/Classic/ClassicRenderer.cs b/OpenUtau.Core/Classic/ClassicRenderer.cs index b4ee80663..650507cf2 100644 --- a/OpenUtau.Core/Classic/ClassicRenderer.cs +++ b/OpenUtau.Core/Classic/ClassicRenderer.cs @@ -48,7 +48,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) { if (phrase.wavtool == SharpWavtool.nameConvergence || phrase.wavtool == SharpWavtool.nameSimple) { return RenderInternal(phrase, progress, trackNo, cancellation, isPreRender); } else { diff --git a/OpenUtau.Core/Classic/WorldlineRenderer.cs b/OpenUtau.Core/Classic/WorldlineRenderer.cs index bd599de10..e747562f1 100644 --- a/OpenUtau.Core/Classic/WorldlineRenderer.cs +++ b/OpenUtau.Core/Classic/WorldlineRenderer.cs @@ -63,7 +63,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) { var resamplerItems = new List(); foreach (var phone in phrase.phones) { resamplerItems.Add(new ResamplerItem(phrase, phone)); diff --git a/OpenUtau.Core/Commands/ExpCommands.cs b/OpenUtau.Core/Commands/ExpCommands.cs index 072e12c71..d28070f2f 100644 --- a/OpenUtau.Core/Commands/ExpCommands.cs +++ b/OpenUtau.Core/Commands/ExpCommands.cs @@ -327,6 +327,7 @@ public override ValidateOptions ValidateOptions public SetCurveCommand(UProject project, UVoicePart part, string abbr, int x, int y, int lastX, int lastY) : base(part) { this.project = project; this.abbr = abbr; + Key = abbr; this.x = x; this.y = y; this.lastX = lastX; @@ -388,6 +389,7 @@ public MergedSetCurveCommand(UProject project, UVoicePart part, string abbr, int[] oldXs, int[] oldYs, int[] newXs, int[] newYs, bool setReal = false) : base(part) { this.project = project; this.abbr = abbr; + Key = setReal ? string.Empty : abbr; this.oldXs = oldXs; this.oldYs = oldYs; this.newXs = newXs; @@ -439,6 +441,7 @@ public class PasteCurveCommand : ExpCommand { public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumerable xs, IEnumerable ys) : base(part) { this.project = project; this.abbr = abbr; + Key = abbr; this.xs = xs.ToArray(); this.ys = ys.ToArray(); var curve = part.curves.FirstOrDefault(c => c.abbr == abbr); @@ -448,6 +451,7 @@ public PasteCurveCommand(UProject project, UVoicePart part, string abbr, IEnumer public PasteCurveCommand(UProject project, UVoicePart part, string abbr, int startX, int startY, int endX, int endY) : base(part) { this.project = project; this.abbr = abbr; + Key = abbr; this.xs = new int[] { startX, endX }; this.ys = new int[] { startY, endY }; var curve = part.curves.FirstOrDefault(c => c.abbr == abbr); @@ -499,6 +503,7 @@ public class ClearCurveCommand : ExpCommand { readonly int[] oldYs; public ClearCurveCommand(UVoicePart part, string abbr) : base(part) { this.abbr = abbr; + Key = abbr; var curve = Part.curves.FirstOrDefault(curve => curve.abbr == abbr); if (curve != null) { oldXs = curve.xs.ToArray(); diff --git a/OpenUtau.Core/Commands/Notifications.cs b/OpenUtau.Core/Commands/Notifications.cs index 95741549e..8a1c8b4f0 100644 --- a/OpenUtau.Core/Commands/Notifications.cs +++ b/OpenUtau.Core/Commands/Notifications.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using OpenUtau.Core.Render; using OpenUtau.Core.Ustx; namespace OpenUtau.Core { @@ -247,6 +249,26 @@ public PartRenderedNotification(UVoicePart part) { public override string ToString() => "Part rendered."; } + public class RealCurvesUpdatedNotification : UNotification { + public readonly IReadOnlyList updates; + public override bool Silent => true; + public RealCurvesUpdatedNotification(UVoicePart part, IReadOnlyList updates) { + this.part = part; + this.updates = updates; + } + public override string ToString() => "Real curves updated."; + } + + public class RealCurveCoverageNotification : UNotification { + public readonly IReadOnlyList<(int start, int end)> ranges; + public override bool Silent => true; + public RealCurveCoverageNotification(UVoicePart part, IReadOnlyList<(int start, int end)> ranges) { + this.part = part; + this.ranges = ranges; + } + public override string ToString() => "Real curve coverage."; + } + public class GotoOtoNotification : UNotification { public readonly USinger? singer; public readonly UOto? oto; diff --git a/OpenUtau.Core/DiffSinger/DiffSingerRealCurveScheduler.cs b/OpenUtau.Core/DiffSinger/DiffSingerRealCurveScheduler.cs new file mode 100644 index 000000000..1db02eab4 --- /dev/null +++ b/OpenUtau.Core/DiffSinger/DiffSingerRealCurveScheduler.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using OpenUtau.Core.Render; +using OpenUtau.Core.Ustx; +using Serilog; + +namespace OpenUtau.Core.DiffSinger { + internal static class DiffSingerRealCurveScheduler { + // Coalesce drag-generated curve commands without adding pointer-level preview logic. + const int DebounceMs = 200; + static readonly object lockObj = new object(); + static readonly Dictionary pending = new Dictionary(); + + public static void TrySchedule(UProject project, UVoicePart part, UCommand command) { + if (command is not ExpCommand expCommand || + !IsCurveEditCommand(command) || + string.IsNullOrEmpty(expCommand.Key) || + expCommand.Part != part || + !CanRefresh(project, part, expCommand.Key)) { + return; + } + Schedule(project, part); + } + + static bool IsCurveEditCommand(UCommand command) { + return command is SetCurveCommand || + command is MergedSetCurveCommand || + command is PasteCurveCommand || + command is ClearCurveCommand; + } + + static bool CanRefresh(UProject project, UVoicePart part, string abbr) { + if (!DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(abbr) || + !project.parts.Contains(part) || + part.trackNo < 0 || + part.trackNo >= project.tracks.Count) { + return false; + } + return project.tracks[part.trackNo].RendererSettings.Renderer is DiffSingerRenderer; + } + + static void Schedule(UProject project, UVoicePart part) { + var cancellation = new CancellationTokenSource(); + lock (lockObj) { + if (pending.TryGetValue(part, out var previous)) { + previous.Cancel(); + } + pending[part] = cancellation; + } + _ = Task.Run(() => RefreshAsync(project, part, cancellation)); + } + + static async Task RefreshAsync(UProject project, UVoicePart part, CancellationTokenSource cancellation) { + try { + await Task.Delay(DebounceMs, cancellation.Token); + var updates = LoadPartUpdates(project, part, cancellation.Token); + if (updates.Count > 0 && !cancellation.IsCancellationRequested) { + DocManager.Inst.ExecuteCmd(new RealCurvesUpdatedNotification(part, updates)); + } + } catch (OperationCanceledException) { + } catch (Exception e) { + Log.Debug(e, "Failed to refresh DiffSinger real curves after curve edit."); + } finally { + lock (lockObj) { + if (pending.TryGetValue(part, out var current) && current == cancellation) { + pending.Remove(part); + } + } + cancellation.Dispose(); + } + } + + static IReadOnlyList LoadPartUpdates( + UProject project, + UVoicePart part, + CancellationToken cancellationToken) { + RenderPhrase[] phrases; + lock (project) { + if (!project.parts.Contains(part) || + part.trackNo < 0 || + part.trackNo >= project.tracks.Count || + project.tracks[part.trackNo].RendererSettings.Renderer is not DiffSingerRenderer) { + return Array.Empty(); + } + phrases = part.renderPhrases.ToArray(); + } + if (phrases.Length == 0) { + return Array.Empty(); + } + var updates = new List(); + foreach (var phrase in phrases) { + if (cancellationToken.IsCancellationRequested) { + return Array.Empty(); + } + updates.AddRange(RealCurveUpdater.LoadPhraseUpdates(part, phrase)); + } + return updates; + } + } +} diff --git a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs index 69d973789..6468c24be 100644 --- a/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs +++ b/OpenUtau.Core/DiffSinger/DiffSingerRenderer.cs @@ -79,7 +79,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) { var task = Task.Run(() => { lock (lockObj) { if (cancellation.IsCancellationRequested) { @@ -114,7 +114,7 @@ public Task Render(RenderPhrase phrase, Progress progress, int tra } } if (result.samples == null) { - result.samples = InvokeDiffsinger(phrase, depth, steps, cancellation); + result.samples = InvokeDiffsinger(phrase, depth, steps, cancellation, renderEvents); if (result.samples != null) { var source = new WaveSource(0, 0, 0, 1); source.SetSamples(result.samples); @@ -135,7 +135,7 @@ public Task Render(RenderPhrase phrase, Progress progress, int tra leadingMs、positionMs、estimatedLengthMs: timeaxis layout in Ms, double */ - float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, CancellationTokenSource cancellation) { + float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, CancellationTokenSource cancellation, RenderPhraseEvents? renderEvents) { var singer = phrase.singer as DiffSingerSinger; //Check if dsconfig.yaml is correct if(String.IsNullOrEmpty(singer.dsConfig.vocoder) || @@ -361,6 +361,7 @@ float[] InvokeDiffsinger(RenderPhrase phrase, double depth, int steps, Cancellat } varianceResult = singer.getVariancePredictor().Process(phrase); } + renderEvents?.ReportRealCurves(BuildRenderedRealCurves(phrase, varianceResult)); //TODO: let user edit variance curves if(singer.dsConfig.useEnergyEmbed){ var energyCurve = phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE); @@ -521,6 +522,10 @@ public RenderPitchResult LoadRenderedPitch(RenderPhrase phrase) { } } + public void ScheduleRealCurveRefresh(UProject project, UVoicePart part, UCommand command) { + DiffSingerRealCurveScheduler.TrySchedule(project, part, command); + } + public List LoadRenderedRealCurves(RenderPhrase phrase) { if (!Preferences.Default.DiffSingerTensorCache) { throw new Exception("Please enable DiffSinger tensor cache and re-render the phrase to display correct base curves."); @@ -532,57 +537,68 @@ public List LoadRenderedRealCurves(RenderPhrase phrase) { var variancePredictor = singer.getVariancePredictor()!; lock (variancePredictor) { var result = variancePredictor.Process(phrase); - var frameMs = result.frameMs; - var headFrames = result.headFrames; - var tailFrames = result.tailFrames; - var startMs = phrase.positionMs - headFrames * frameMs; - var realCurves = new (string, float[], float[], Func)[] { - ( - ENE, result.energy ?? Array.Empty(), - phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE)?.Item2 - ?? Array.Empty(), - x => Math.Clamp(x, -96f, 0f) / 96f + 1f - ), - ( - Format.Ustx.BREC, result.breathiness ?? Array.Empty(), phrase.breathiness, - x => Math.Clamp(x, -96f, 0f) / 96f + 1f - ), - ( - Format.Ustx.VOIC, result.voicing ?? Array.Empty(), phrase.voicing, - x => Math.Clamp(x, -96f, 0f) / 96f + 1f - ), - ( - Format.Ustx.TENC, result.tension ?? Array.Empty(), phrase.tension, - x => Math.Clamp(x, -10f, 10f) / 20f + 0.5f - ), - }.Select(t => { - var abbr = t.Item1; - var realCurve = t.Item2; - if (realCurve.Length == 0) { - return new RenderRealCurveResult { - abbr = abbr, - ticks = Array.Empty(), - values = Array.Empty(), - }; - } - var deltaCurve = DiffSingerUtils.SampleCurve( - phrase, t.Item3, 0, frameMs, realCurve.Length, - headFrames, tailFrames, x => x) - .Select(x => (float)x) - .ToArray(); - var normFunc = t.Item4; + return BuildRenderedRealCurves(phrase, result); + } + } + + internal static bool ShouldRefreshRealCurvesOnCurveEdit(string abbr) { + return abbr == ENE || + abbr == Format.Ustx.BREC || + abbr == Format.Ustx.VOIC || + abbr == Format.Ustx.TENC; + } + + private List BuildRenderedRealCurves(RenderPhrase phrase, VarianceResult result) { + var frameMs = result.frameMs; + var headFrames = result.headFrames; + var tailFrames = result.tailFrames; + var startMs = phrase.positionMs - headFrames * frameMs; + var realCurves = new (string, float[], float[], Func)[] { + ( + ENE, result.energy ?? Array.Empty(), + phrase.curves.FirstOrDefault(curve => curve.Item1 == ENE)?.Item2 + ?? Array.Empty(), + x => Math.Clamp(x, -96f, 0f) / 96f + 1f + ), + ( + Format.Ustx.BREC, result.breathiness ?? Array.Empty(), phrase.breathiness, + x => Math.Clamp(x, -96f, 0f) / 96f + 1f + ), + ( + Format.Ustx.VOIC, result.voicing ?? Array.Empty(), phrase.voicing, + x => Math.Clamp(x, -96f, 0f) / 96f + 1f + ), + ( + Format.Ustx.TENC, result.tension ?? Array.Empty(), phrase.tension, + x => Math.Clamp(x, -10f, 10f) / 20f + 0.5f + ), + }.Select(t => { + var abbr = t.Item1; + var realCurve = t.Item2; + if (realCurve.Length == 0) { return new RenderRealCurveResult { abbr = abbr, - ticks = Enumerable.Range(0, realCurve.Length) - .Select(i => (float)phrase.timeAxis.MsPosToTickPos(startMs + i * frameMs) - phrase.position) - .ToArray(), - values = realCurve.Zip(deltaCurve, varianceDeltaFunctions[abbr]) - .Select(normFunc) - .ToArray() + ticks = Array.Empty(), + values = Array.Empty(), }; - }).ToList(); - return realCurves; - } + } + var deltaCurve = DiffSingerUtils.SampleCurve( + phrase, t.Item3, 0, frameMs, realCurve.Length, + headFrames, tailFrames, x => x) + .Select(x => (float)x) + .ToArray(); + var normFunc = t.Item4; + return new RenderRealCurveResult { + abbr = abbr, + ticks = Enumerable.Range(0, realCurve.Length) + .Select(i => (float)phrase.timeAxis.MsPosToTickPos(startMs + i * frameMs) - phrase.position) + .ToArray(), + values = realCurve.Zip(deltaCurve, varianceDeltaFunctions[abbr]) + .Select(normFunc) + .ToArray() + }; + }).ToList(); + return realCurves; } public UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings) { diff --git a/OpenUtau.Core/DocManager.cs b/OpenUtau.Core/DocManager.cs index 96b50100d..b9ea012bb 100644 --- a/OpenUtau.Core/DocManager.cs +++ b/OpenUtau.Core/DocManager.cs @@ -10,6 +10,7 @@ using OpenUtau.Classic; using OpenUtau.Core.Editing; using OpenUtau.Core.Lib; +using OpenUtau.Core.Render; using OpenUtau.Core.Ustx; using OpenUtau.Core.Util; using Serilog; @@ -222,9 +223,17 @@ public void ExecuteCmd(UCommand cmd) { rangeEndTick = 0; } else if (cmd is SetPlayPosTickNotification setPlayPosTickNotif) { playPosTick = setPlayPosTickNotif.playPosTick; - } else if (cmd is SetRangeSelectionNotification setRange) { +} else if (cmd is SetRangeSelectionNotification setRange) { rangeStartTick = setRange.startTick; rangeEndTick = setRange.endTick; + } else if (cmd is RealCurvesUpdatedNotification realCurvesNotif) { + if (realCurvesNotif.part is UVoicePart voicePart) { + RealCurveUpdater.Apply(Project, voicePart, realCurvesNotif.updates); + } + } else if (cmd is RealCurveCoverageNotification coverageNotif) { + if (coverageNotif.part is UVoicePart coveragePart) { + RealCurveUpdater.TrimToCoverage(Project, coveragePart, coverageNotif.ranges); + } } else if (cmd is SingersChangedNotification) { SingerManager.Inst.SearchAllSingers(); } else if (cmd is ValidateProjectNotification) { @@ -258,6 +267,27 @@ public void ExecuteCmd(UCommand cmd) { Publish(cmd); if (!undoGroup.DeferValidate) { Project.Validate(cmd.ValidateOptions); + ScheduleRealCurveRefresh(cmd); + } + } + + void ScheduleRealCurveRefresh(UCommand cmd) { + if (cmd is not ExpCommand expCommand) { + return; + } + var part = expCommand.Part; + if (!Project.parts.Contains(part) || + part.trackNo < 0 || + part.trackNo >= Project.tracks.Count) { + return; + } + Project.tracks[part.trackNo].RendererSettings.Renderer + ?.ScheduleRealCurveRefresh(Project, part, cmd); + } + + void ScheduleRealCurveRefresh(IEnumerable commands) { + foreach (var cmd in commands) { + ScheduleRealCurveRefresh(cmd); } } @@ -286,6 +316,7 @@ public void EndUndoGroup() { Project.ValidateFull(); } undoGroup.Merge(); + ScheduleRealCurveRefresh(undoGroup.Commands); undoGroup = null; Log.Information("undoGroup ended"); ExecuteCmd(new PreRenderNotification()); @@ -304,7 +335,9 @@ public void RollBackUndoGroup() { } Publish(cmd, true); } + ScheduleRealCurveRefresh(undoGroup.Commands); undoGroup.Commands.Clear(); + ExecuteCmd(new PreRenderNotification()); } public void Undo() { @@ -321,6 +354,7 @@ public void Undo() { Publish(cmd, true); } redoQueue.AddToBack(group); + ScheduleRealCurveRefresh(group.Commands); ExecuteCmd(new PreRenderNotification()); } @@ -338,6 +372,7 @@ public void Redo() { Publish(cmd); } undoQueue.AddToBack(group); + ScheduleRealCurveRefresh(group.Commands); ExecuteCmd(new PreRenderNotification()); } diff --git a/OpenUtau.Core/Editing/NoteBatchEdits.cs b/OpenUtau.Core/Editing/NoteBatchEdits.cs index d8b4248e1..0feaa66d2 100644 --- a/OpenUtau.Core/Editing/NoteBatchEdits.cs +++ b/OpenUtau.Core/Editing/NoteBatchEdits.cs @@ -826,17 +826,17 @@ public void RunAsync( finished += 1; setProgressCallback(finished, part.renderPhrases.Count); } - var commands = curveDict - .Select(kv => new MergedSetCurveCommand( - project, part, kv.Key, - kv.Value?.realXs.ToArray() ?? Array.Empty(), - kv.Value?.realYs.ToArray() ?? Array.Empty(), - newXsDict[kv.Key].ToArray(), - newYsDict[kv.Key].ToArray(), - true)) - .ToList(); DocManager.Inst.PostOnUIThread(() => { + var commands = curveDict + .Select(kv => new MergedSetCurveCommand( + project, part, kv.Key, + kv.Value?.realXs.ToArray() ?? Array.Empty(), + kv.Value?.realYs.ToArray() ?? Array.Empty(), + newXsDict[kv.Key].ToArray(), + newYsDict[kv.Key].ToArray(), + true)) + .ToList(); docManager.StartUndoGroup("command.batch.note", true); commands.ForEach(docManager.ExecuteCmd); docManager.EndUndoGroup(); diff --git a/OpenUtau.Core/Enunu/EnunuRenderer.cs b/OpenUtau.Core/Enunu/EnunuRenderer.cs index fdf634ab7..e19606726 100644 --- a/OpenUtau.Core/Enunu/EnunuRenderer.cs +++ b/OpenUtau.Core/Enunu/EnunuRenderer.cs @@ -73,7 +73,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) { var task = Task.Run(() => { lock (lockObj) { if (cancellation.IsCancellationRequested) { diff --git a/OpenUtau.Core/Properties/AssemblyInfo.cs b/OpenUtau.Core/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..aa105cb38 --- /dev/null +++ b/OpenUtau.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("OpenUtau.Test")] diff --git a/OpenUtau.Core/Render/IRenderer.cs b/OpenUtau.Core/Render/IRenderer.cs index a1d420cc1..71e6d8a9a 100644 --- a/OpenUtau.Core/Render/IRenderer.cs +++ b/OpenUtau.Core/Render/IRenderer.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using OpenUtau.Core; using OpenUtau.Core.Ustx; namespace OpenUtau.Core.Render { @@ -59,6 +60,20 @@ public class RenderRealCurveResult { public float[] values; } + public class RenderPhraseEvents { + readonly Action>? realCurvesCallback; + + public RenderPhraseEvents(Action>? realCurvesCallback = null) { + this.realCurvesCallback = realCurvesCallback; + } + + public void ReportRealCurves(IReadOnlyList realCurves) { + if (realCurves.Count > 0) { + realCurvesCallback?.Invoke(realCurves); + } + } + } + /// /// Interface of phrase-based renderer. /// @@ -68,9 +83,10 @@ public interface IRenderer { bool SupportsRealCurve { get { return false; } } bool SupportsExpression(UExpressionDescriptor descriptor); RenderResult Layout(RenderPhrase phrase); - Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender = false); + Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender = false, RenderPhraseEvents? renderEvents = null); RenderPitchResult LoadRenderedPitch(RenderPhrase phrase); List LoadRenderedRealCurves(RenderPhrase phrase) { return new List(0);} + void ScheduleRealCurveRefresh(UProject project, UVoicePart part, UCommand command) { } UExpressionDescriptor[] GetSuggestedExpressions(USinger singer, URenderSettings renderSettings); } } diff --git a/OpenUtau.Core/Render/RealCurveUpdater.cs b/OpenUtau.Core/Render/RealCurveUpdater.cs new file mode 100644 index 000000000..6846e45ab --- /dev/null +++ b/OpenUtau.Core/Render/RealCurveUpdater.cs @@ -0,0 +1,214 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using OpenUtau.Core.Ustx; + +namespace OpenUtau.Core.Render { + public readonly struct RealCurveUpdate { + public readonly string abbr; + public readonly ulong phraseHash; + public readonly int startTick; + public readonly int endTick; + public readonly int[] xs; + public readonly int[] ys; + + public RealCurveUpdate(string abbr, ulong phraseHash, int startTick, int endTick, int[] xs, int[] ys) { + this.abbr = abbr; + this.phraseHash = phraseHash; + this.startTick = startTick; + this.endTick = endTick; + this.xs = xs; + this.ys = ys; + } + + public bool IsValid => !string.IsNullOrEmpty(abbr) && + endTick >= startTick && xs.Length == ys.Length && xs.Length > 0; + } + + public static class RealCurveUpdater { + public static RealCurveUpdate[] LoadPhraseUpdates(UVoicePart part, RenderPhrase phrase) { + if (!phrase.renderer.SupportsRealCurve) { + return Array.Empty(); + } + return BuildUpdates(part, phrase, phrase.renderer.LoadRenderedRealCurves(phrase)); + } + + internal static RealCurveUpdate[] BuildUpdates( + UVoicePart part, + RenderPhrase phrase, + IEnumerable results) { + return BuildUpdates(part.position, phrase.position, phrase.hash, results); + } + + internal static RealCurveUpdate[] BuildUpdates( + int partPosition, + int phrasePosition, + ulong phraseHash, + IEnumerable results) { + var updates = new List(); + foreach (var result in results) { + if (string.IsNullOrEmpty(result.abbr) || result.ticks == null || result.values == null) { + continue; + } + int count = Math.Min(result.ticks.Length, result.values.Length); + if (count == 0) { + continue; + } + var ticks = result.ticks + .Take(count) + .Select(tick => phrasePosition - partPosition + (int)tick) + .ToArray(); + var values = result.values + .Take(count) + .Select(value => (int)(value * 1000.0)) + .ToArray(); + int startTick = ticks.Min(); + int endTick = ticks.Max(); + var xs = new List(count + 1) { ticks[0] }; + var ys = new List(count + 1) { -1 }; + xs.AddRange(ticks); + ys.AddRange(values); + updates.Add(new RealCurveUpdate( + result.abbr, + phraseHash, + startTick, + endTick, + xs.ToArray(), + ys.ToArray())); + } + return updates.ToArray(); + } + + public static bool Apply(UProject project, UVoicePart part, IReadOnlyList updates) { + if (updates.Count == 0 || !project.parts.Contains(part)) { + return false; + } + var phraseHashes = part.renderPhrases.Select(phrase => phrase.hash).ToHashSet(); + return Apply(project, part, updates, phraseHashes); + } + + // Removes realXs/realYs points outside the union of `ranges` (the tick spans the renderer + // actually produced this pass). Clears ghost real curves left behind when a phrase shrinks, + // moves, splits, or is deleted. realXs is kept sorted, so a single two-pointer walk decides + // each point; a new list is allocated only when a stale point is found (clean case is free). + public static bool TrimToCoverage( + UProject project, UVoicePart part, IReadOnlyList<(int start, int end)> ranges) { + if (!project.parts.Contains(part) || ranges.Count == 0) { + return false; + } + var merged = MergeRanges(ranges); + bool changed = false; + foreach (var curve in part.curves) { + if (TrimCurve(curve, merged)) { + changed = true; + } + } + return changed; + } + + static bool TrimCurve(UCurve curve, List<(int start, int end)> merged) { + var xs = curve.realXs; + var ys = curve.realYs; + if (xs.Count == 0) { + return false; + } + List? newXs = null; + List? newYs = null; + int ri = 0; + for (int i = 0; i < xs.Count; ++i) { + int x = xs[i]; + while (ri < merged.Count && merged[ri].end < x) { + ri++; + } + bool keep = ri < merged.Count && x >= merged[ri].start; + if (keep) { + newXs?.Add(x); + newYs?.Add(ys[i]); + } else if (newXs == null) { + newXs = new List(xs.Count); + newYs = new List(ys.Count); + for (int j = 0; j < i; ++j) { + newXs.Add(xs[j]); + newYs.Add(ys[j]); + } + } + } + if (newXs == null) { + return false; + } + curve.realXs = newXs; + curve.realYs = newYs; + return true; + } + + internal static List<(int start, int end)> MergeRanges(IReadOnlyList<(int start, int end)> ranges) { + var sorted = ranges.ToList(); + sorted.Sort((a, b) => a.start.CompareTo(b.start)); + var merged = new List<(int start, int end)> { sorted[0] }; + for (int i = 1; i < sorted.Count; ++i) { + var last = merged[merged.Count - 1]; + if (sorted[i].start <= last.end) { + merged[merged.Count - 1] = (last.start, Math.Max(last.end, sorted[i].end)); + } else { + merged.Add(sorted[i]); + } + } + return merged; + } + + internal static bool Apply( + UProject project, + UVoicePart part, + IReadOnlyList updates, + IReadOnlySet phraseHashes) { + bool changed = false; + foreach (var update in updates) { + if (!update.IsValid || !phraseHashes.Contains(update.phraseHash)) { + continue; + } + changed |= ApplyUpdate(project, part, update); + } + return changed; + } + + private static bool ApplyUpdate(UProject project, UVoicePart part, RealCurveUpdate update) { + var curve = part.curves.FirstOrDefault(curve => curve.abbr == update.abbr); + if (curve == null) { + var track = project.tracks[part.trackNo]; + if (!track.TryGetExpDescriptor(project, update.abbr, out var descriptor)) { + return false; + } + curve = new UCurve(descriptor); + part.curves.Add(curve); + } + RemoveRange(curve.realXs, curve.realYs, update.startTick, update.endTick); + InsertRange(curve.realXs, curve.realYs, update.xs, update.ys); + return true; + } + + internal static void RemoveRange(List xs, List ys, int startTick, int endTick) { + for (int i = xs.Count - 1; i >= 0; --i) { + if (xs[i] >= startTick && xs[i] <= endTick) { + xs.RemoveAt(i); + ys.RemoveAt(i); + } + } + } + + internal static void InsertRange(List targetXs, List targetYs, int[] xs, int[] ys) { + if (xs.Length == 0) { + return; + } + int insertIndex = targetXs.BinarySearch(xs[0]); + if (insertIndex < 0) { + insertIndex = ~insertIndex; + } else { + while (insertIndex < targetXs.Count && targetXs[insertIndex] <= xs[0]) { + insertIndex++; + } + } + targetXs.InsertRange(insertIndex, xs); + targetYs.InsertRange(insertIndex, ys); + } + } +} diff --git a/OpenUtau.Core/Render/RenderEngine.cs b/OpenUtau.Core/Render/RenderEngine.cs index 75bc3d35b..f31b2cbce 100644 --- a/OpenUtau.Core/Render/RenderEngine.cs +++ b/OpenUtau.Core/Render/RenderEngine.cs @@ -249,24 +249,99 @@ private void RenderRequests( tuples = orderedTuples; } var progress = new Progress(tuples.Sum(t => t.Item1.phones.Length)); + // Only full-project passes (pre-render / export) maintain the real-curve coverage + // invariant. Partial playback passes must not trim curves outside their tick window. + bool maintainCoverage = startTick == 0 && endTick == -1; + var coverageRanges = maintainCoverage + ? new Dictionary>() + : null; foreach (var tuple in tuples) { var phrase = tuple.Item1; var source = tuple.Item2; var request = tuple.Item3; - var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true); + RealCurveUpdate[]? publishedUpdates = null; + var renderEvents = phrase.renderer.SupportsRealCurve + ? new RenderPhraseEvents(realCurves => { + publishedUpdates = PublishRealCurveUpdates(request.part, phrase, realCurves); + }) + : null; + var task = phrase.renderer.Render(phrase, progress, request.trackNo, cancellation, true, renderEvents); task.Wait(); if (cancellation.IsCancellationRequested) { break; } - source.SetSamples(task.Result.samples); + var result = task.Result; + source.SetSamples(result.samples); + if (publishedUpdates == null) { + publishedUpdates = PublishRealCurveUpdates(request.part, phrase); + } + if (coverageRanges != null && publishedUpdates != null) { + AccumulateCoverage(coverageRanges, request.part, publishedUpdates); + } if (request.sources.All(s => s.HasSamples)) { request.part.SetMix(request.mix); + if (coverageRanges != null && + phrase.renderer.SupportsRealCurve && + coverageRanges.TryGetValue(request.part, out var ranges) && + ranges.Count > 0) { + DocManager.Inst.ExecuteCmd(new RealCurveCoverageNotification(request.part, ranges)); + } DocManager.Inst.ExecuteCmd(new PartRenderedNotification(request.part)); } } progress.Clear(); } + private RealCurveUpdate[]? PublishRealCurveUpdates(UVoicePart part, RenderPhrase phrase) { + if (!phrase.renderer.SupportsRealCurve) { + return null; + } + try { + var updates = RealCurveUpdater.LoadPhraseUpdates(part, phrase); + if (updates.Length > 0) { + DocManager.Inst.ExecuteCmd(new RealCurvesUpdatedNotification(part, updates)); + return updates; + } + } catch (Exception e) { + Log.Debug(e, "Failed to refresh rendered real curves."); + } + return null; + } + + private RealCurveUpdate[]? PublishRealCurveUpdates( + UVoicePart part, + RenderPhrase phrase, + IReadOnlyList realCurves) { + if (realCurves.Count == 0) { + return null; + } + try { + var updates = RealCurveUpdater.BuildUpdates(part, phrase, realCurves); + if (updates.Length > 0) { + DocManager.Inst.ExecuteCmd(new RealCurvesUpdatedNotification(part, updates)); + return updates; + } + } catch (Exception e) { + Log.Debug(e, "Failed to publish rendered real curves."); + } + return null; + } + + private static void AccumulateCoverage( + Dictionary> coverage, + UVoicePart part, + RealCurveUpdate[] updates) { + if (!coverage.TryGetValue(part, out var ranges)) { + ranges = new List<(int start, int end)>(); + coverage[part] = ranges; + } + foreach (var update in updates) { + if (update.IsValid) { + ranges.Add((update.startTick, update.endTick)); + } + } + } + public static void ReleaseSourceTemp() { VoicebankFiles.Inst.ReleaseSourceTemp(); } diff --git a/OpenUtau.Core/Vogen/VogenRenderer.cs b/OpenUtau.Core/Vogen/VogenRenderer.cs index 1c3fb51f8..bc817d17b 100644 --- a/OpenUtau.Core/Vogen/VogenRenderer.cs +++ b/OpenUtau.Core/Vogen/VogenRenderer.cs @@ -53,7 +53,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender = false) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender = false, RenderPhraseEvents? renderEvents = null) { var task = Task.Run(() => { lock (lockObj) { if (cancellation.IsCancellationRequested) { diff --git a/OpenUtau.Core/Voicevox/VoicevoxRenderer.cs b/OpenUtau.Core/Voicevox/VoicevoxRenderer.cs index 3f78878f7..c7eb51101 100644 --- a/OpenUtau.Core/Voicevox/VoicevoxRenderer.cs +++ b/OpenUtau.Core/Voicevox/VoicevoxRenderer.cs @@ -57,7 +57,7 @@ public RenderResult Layout(RenderPhrase phrase) { }; } - public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender) { + public Task Render(RenderPhrase phrase, Progress progress, int trackNo, CancellationTokenSource cancellation, bool isPreRender, RenderPhraseEvents? renderEvents = null) { var task = Task.Run(() => { lock (lockObj) { if (cancellation.IsCancellationRequested) { diff --git a/OpenUtau.Test/Core/DiffSinger/DiffSingerRealCurveSchedulerTest.cs b/OpenUtau.Test/Core/DiffSinger/DiffSingerRealCurveSchedulerTest.cs new file mode 100644 index 000000000..ec4bb1ea8 --- /dev/null +++ b/OpenUtau.Test/Core/DiffSinger/DiffSingerRealCurveSchedulerTest.cs @@ -0,0 +1,18 @@ +using OpenUtau.Core.DiffSinger; +using UstxFormat = OpenUtau.Core.Format.Ustx; +using Xunit; + +namespace OpenUtau.Core { + public class DiffSingerRealCurveSchedulerTest { + [Fact] + public void RealCurveRefreshIsLimitedToVarianceOffsetCurves() { + Assert.True(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(DiffSingerUtils.ENE)); + Assert.True(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(UstxFormat.BREC)); + Assert.True(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(UstxFormat.VOIC)); + Assert.True(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(UstxFormat.TENC)); + + Assert.False(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(UstxFormat.PITD)); + Assert.False(DiffSingerRenderer.ShouldRefreshRealCurvesOnCurveEdit(UstxFormat.DYN)); + } + } +} diff --git a/OpenUtau.Test/Core/Render/RealCurveUpdaterTest.cs b/OpenUtau.Test/Core/Render/RealCurveUpdaterTest.cs new file mode 100644 index 000000000..e72168dda --- /dev/null +++ b/OpenUtau.Test/Core/Render/RealCurveUpdaterTest.cs @@ -0,0 +1,155 @@ +using System.Collections.Generic; +using System.Linq; +using OpenUtau.Core.Render; +using OpenUtau.Core.Ustx; +using Xunit; + +namespace OpenUtau.Core { + public class RealCurveUpdaterTest { + const string Ene = "ene"; + + [Fact] + public void RenderPhraseEventsReportsNonEmptyRealCurvesOnly() { + int reports = 0; + var events = new RenderPhraseEvents(_ => reports++); + + events.ReportRealCurves(new List()); + events.ReportRealCurves(new[] { + new RenderRealCurveResult { + abbr = Ene, + ticks = new[] { 0f }, + values = new[] { 0.1f }, + }, + }); + + Assert.Equal(1, reports); + } + + [Fact] + public void BuildUpdatesConvertsTicksToPartLocalCoordinates() { + var result = new RenderRealCurveResult { + abbr = Ene, + ticks = new[] { 0f, 5f, 10f }, + values = new[] { 0.1f, 0.2f, 0.3f }, + }; + + var update = Assert.Single(RealCurveUpdater.BuildUpdates( + partPosition: 100, + phrasePosition: 250, + phraseHash: 42, + results: new[] { result })); + + Assert.Equal(Ene, update.abbr); + Assert.Equal((ulong)42, update.phraseHash); + Assert.Equal(150, update.startTick); + Assert.Equal(160, update.endTick); + Assert.Equal(new[] { 150, 150, 155, 160 }, update.xs); + Assert.Equal(new[] { -1, 100, 200, 300 }, update.ys); + } + + [Fact] + public void ApplyReplacesMatchingPhraseRangeOnly() { + var project = CreateProjectWithCurve(out var part, out var curve); + curve.realXs.AddRange(new[] { 0, 0, 10, 100, 100, 110 }); + curve.realYs.AddRange(new[] { -1, 10, 20, -1, 30, 40 }); + var update = new RealCurveUpdate( + Ene, + phraseHash: 42, + startTick: 100, + endTick: 110, + xs: new[] { 100, 100, 105, 110 }, + ys: new[] { -1, 300, 400, 500 }); + + bool changed = RealCurveUpdater.Apply( + project, + part, + new[] { update }, + new HashSet { 42 }); + + Assert.True(changed); + Assert.Equal(new[] { 0, 0, 10, 100, 100, 105, 110 }, curve.realXs); + Assert.Equal(new[] { -1, 10, 20, -1, 300, 400, 500 }, curve.realYs); + } + + [Fact] + public void ApplySkipsStalePhraseHash() { + var project = CreateProjectWithCurve(out var part, out var curve); + curve.realXs.AddRange(new[] { 0, 0, 10 }); + curve.realYs.AddRange(new[] { -1, 10, 20 }); + var update = new RealCurveUpdate( + Ene, + phraseHash: 42, + startTick: 0, + endTick: 10, + xs: new[] { 0, 0, 5 }, + ys: new[] { -1, 300, 400 }); + + bool changed = RealCurveUpdater.Apply( + project, + part, + new[] { update }, + new HashSet { 43 }); + + Assert.False(changed); + Assert.Equal(new[] { 0, 0, 10 }, curve.realXs); + Assert.Equal(new[] { -1, 10, 20 }, curve.realYs); + } + + [Fact] + public void TrimToCoverageRemovesGhostPointsOutsideCoverage() { + var project = CreateProjectWithCurve(out var part, out var curve); + curve.realXs.AddRange(new[] { 0, 0, 25, 50, 60, 80, 100 }); + curve.realYs.AddRange(new[] { -1, 10, 20, 30, 40, 50, 60 }); + + bool changed = RealCurveUpdater.TrimToCoverage( + project, part, new List<(int start, int end)> { (0, 50) }); + + Assert.True(changed); + Assert.Equal(new[] { 0, 0, 25, 50 }, curve.realXs); + Assert.Equal(new[] { -1, 10, 20, 30 }, curve.realYs); + } + + [Fact] + public void TrimToCoverageKeepsPointsWhenAllInsideCoverage() { + var project = CreateProjectWithCurve(out var part, out var curve); + curve.realXs.AddRange(new[] { 0, 0, 25, 50 }); + curve.realYs.AddRange(new[] { -1, 10, 20, 30 }); + + bool changed = RealCurveUpdater.TrimToCoverage( + project, part, new List<(int start, int end)> { (0, 50) }); + + Assert.False(changed); + Assert.Equal(new[] { 0, 0, 25, 50 }, curve.realXs); + } + + [Fact] + public void TrimToCoverageSkipsWhenNoRanges() { + var project = CreateProjectWithCurve(out var part, out var curve); + curve.realXs.AddRange(new[] { 0, 50, 100 }); + curve.realYs.AddRange(new[] { -1, 30, 60 }); + + bool changed = RealCurveUpdater.TrimToCoverage( + project, part, new List<(int start, int end)>()); + + Assert.False(changed); + Assert.Equal(3, curve.realXs.Count); + } + + static UProject CreateProjectWithCurve(out UVoicePart part, out UCurve curve) { + var project = new UProject(); + var descriptor = new UExpressionDescriptor("energy", Ene, 0, 100, 0) { + type = UExpressionType.Curve, + }; + project.RegisterExpression(descriptor); + part = new UVoicePart { + trackNo = 0, + position = 0, + Duration = 480, + }; + project.parts.Add(part); + curve = new UCurve(descriptor); + part.curves.Add(curve); + return project; + } + } +} diff --git a/OpenUtau/Controls/ExpressionCanvas.cs b/OpenUtau/Controls/ExpressionCanvas.cs index 901ff5def..03817a3f0 100644 --- a/OpenUtau/Controls/ExpressionCanvas.cs +++ b/OpenUtau/Controls/ExpressionCanvas.cs @@ -78,6 +78,12 @@ public ExpressionCanvas() { circleGeometry = new EllipseGeometry(new Rect(-4.5, -4.5, 9, 9)); MessageBus.Current.Listen() .Subscribe(_ => InvalidateVisual()); + MessageBus.Current.Listen() + .Subscribe(_ => { + if (ShowRealCurve) { + InvalidateVisual(); + } + }); MessageBus.Current.Listen() .Subscribe(e => { selectedNotes.Clear(); diff --git a/OpenUtau/ViewModels/NotesViewModel.cs b/OpenUtau/ViewModels/NotesViewModel.cs index c14e9deee..bd337b5cd 100644 --- a/OpenUtau/ViewModels/NotesViewModel.cs +++ b/OpenUtau/ViewModels/NotesViewModel.cs @@ -24,6 +24,7 @@ namespace OpenUtau.App.ViewModels { public class NotesRefreshEvent { } + public class RealCurveRefreshEvent { } public class NotesSelectionEvent { public readonly UNote[] selectedNotes; public readonly UNote[] tempSelectedNotes; @@ -1095,6 +1096,10 @@ public void OnNext(UCommand cmd, bool isUndo) { MessageBus.Current.SendMessage(new NotesRefreshEvent()); } else if (notif is PartRenderedNotification && notif.part == Part) { MessageBus.Current.SendMessage(new WaveformRefreshEvent()); + } else if (notif is RealCurvesUpdatedNotification && notif.part == Part) { + MessageBus.Current.SendMessage(new RealCurveRefreshEvent()); + } else if (notif is RealCurveCoverageNotification && notif.part == Part) { + MessageBus.Current.SendMessage(new RealCurveRefreshEvent()); } } else if (cmd is PartCommand partCommand) { if (cmd is ReplacePartCommand replacePart) {