From 79a1b5b5dbc4579acf72f39c6bad778a3eb2333f Mon Sep 17 00:00:00 2001 From: d-monnet Date: Thu, 23 Oct 2025 16:53:04 +0200 Subject: [PATCH 01/20] Changes to be committed: modified: src/fomo.jl remove useless vector p delete body of find_beta_tilde (TODO in next commit) --- src/fomo.jl | 80 ++++++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 44 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 79bf358b..f40d1040 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -30,7 +30,7 @@ const FOMO_step_backend = DefaultParameter(nlp -> r2_step(), "r2_step()") This structure designed for `fomo` regroups the following parameters: - `η1`, `η2`: step acceptance parameters. - `γ1`, `γ2`: regularization update parameters. - - `γ3` : momentum factor βmax update parameter in case of unsuccessful iteration. + - `γ3` : momentum factor βktilde update parameter in case of unsuccessful iteration. - `αmax`: maximum step parameter for fomo algorithm. - `β ∈ [0,1)`: target decay rate for the momentum. - `θ1`: momentum contribution parameter for convergence condition (1). @@ -110,14 +110,14 @@ A First-Order with MOmentum (FOMO) model-based method for unconstrained optimiza # Algorithm description The step is computed along -d = - (1-βmax) .* ∇f(xk) - βmax .* mk +d = - (1-βktilde) .* ∇f(xk) - βktilde .* mk with mk the memory of past gradients (initialized at 0), and updated at each successful iteration as -mk .= ∇f(xk) .* (1 - βmax) .+ mk .* βmax -and βmax ∈ [0,β] chosen as to ensure d is gradient-related, i.e., the following 2 conditions are satisfied: -(1-βmax) .* ∇f(xk) + βmax .* ∇f(xk)ᵀmk ≥ θ1 * ‖∇f(xk)‖² (1) -‖∇f(xk)‖ ≥ θ2 * ‖(1-βmax) *. ∇f(xk) + βmax .* mk‖ (2) +mk .= ∇f(xk) .* (1 - βktilde) .+ mk .* βktilde +and βktilde ∈ [0,β] chosen as to ensure d is gradient-related, i.e., the following 2 conditions are satisfied: +(1-βktilde) .* ∇f(xk) + βktilde .* ∇f(xk)ᵀmk ≥ θ1 * ‖∇f(xk)‖² (1) +‖∇f(xk)‖ ≥ θ2 * ‖(1-βktilde) *. ∇f(xk) + βktilde .* mk‖ (2) In the nonmonotone case, (1) rewrites -(1-βmax) .* ∇f(xk) + βmax .* ∇f(xk)ᵀmk + (fm - fk)/μk ≥ θ1 * ‖∇f(xk)‖², +(1-βktilde) .* ∇f(xk) + βktilde .* ∇f(xk)ᵀmk + (fm - fk)/μk ≥ θ1 * ‖∇f(xk)‖², with fm the largest objective value over the last M successful iterations, and fk = f(xk). # Advanced usage @@ -140,7 +140,7 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `rtol::T = √eps(T)`: relative tolerance: algorithm stops when ‖∇f(xᵏ)‖ ≤ atol + rtol * ‖∇f(x⁰)‖. - `η1 = $(FOMO_η1)`, `η2 = $(FOMO_η2)`: step acceptance parameters. - `γ1 = $(FOMO_γ1)`, `γ2 = $(FOMO_γ2)`: regularization update parameters. -- `γ3 = $(FOMO_γ3)` : momentum factor βmax update parameter in case of unsuccessful iteration. +- `γ3 = $(FOMO_γ3)` : momentum factor βktilde update parameter in case of unsuccessful iteration. - `αmax = $(FOMO_αmax)`: maximum step parameter for fomo algorithm. - `max_eval::Int = -1`: maximum number of objective evaluations. - `max_time::Float64 = 30.0`: maximum time limit in seconds. @@ -206,7 +206,6 @@ mutable struct FomoSolver{T, V} <: AbstractFirstOrderSolver c::V m::V d::V - p::V o::V α::T params::FOMOParameterSet{T} @@ -219,9 +218,8 @@ function FomoSolver(nlp::AbstractNLPModel{T, V}; M::Int = get(FOMO_M, nlp), kwar c = similar(nlp.meta.x0) m = fill!(similar(nlp.meta.x0), 0) d = fill!(similar(nlp.meta.x0), 0) - p = similar(nlp.meta.x0) o = fill!(Vector{T}(undef, M), -Inf) - return FomoSolver{T, V}(x, g, c, m, d, p, o, T(0), params) + return FomoSolver{T, V}(x, g, c, m, d, o, T(0), params) end @doc (@doc FomoSolver) function fomo( @@ -253,7 +251,7 @@ end M = M, step_backend = step_backend, ) - solver_specific = Dict(:avgβmax => T(0.0)) + solver_specific = Dict(:avgβktilde => T(0.0)) stats = GenericExecutionStats(nlp; solver_specific = solver_specific) return solve!(solver, nlp, stats; kwargs...) end @@ -449,7 +447,6 @@ function SolverCore.solve!( c = solver.c momentum = use_momentum ? solver.m : nothing # not used if no momentum d = use_momentum ? solver.d : solver.g # g = d if no momentum - p = use_momentum ? solver.p : nothing # not used if no momentum set_iter!(stats, 0) f0 = obj(nlp, x) set_objective!(stats, f0) @@ -489,7 +486,7 @@ function SolverCore.solve!( infoline = @sprintf "%5d %9.2e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ' ' else - @info @sprintf "%5s %9s %7s %7s %7s %7s " "iter" "f" "‖∇f‖" step_param_name "ρk" "βmax" + @info @sprintf "%5s %9s %7s %7s %7s %7s " "iter" "f" "‖∇f‖" step_param_name "ρk" "βktilde" infoline = @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ' ' 0 end @@ -516,9 +513,9 @@ function SolverCore.solve!( d .= ∇fk norm_d = norm_∇fk - βmax = T(0) + βktilde = T(0) ρk = T(0) - avgβmax = T(0) + avgβktilde = T(0) siter::Int = 0 oneT = T(1) mdot∇f = T(0) # dot(momentum,∇fk) @@ -526,7 +523,7 @@ function SolverCore.solve!( μk = step_mult(solver.α, norm_d, step_backend) c .= x .- μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = ((oneT - βmax) * norm_∇fk^2 + βmax * mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum + ΔTk = (- norm_∇fk^2 + βktilde * mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) @@ -536,17 +533,14 @@ function SolverCore.solve!( elseif ρk < η1 solver.α = solver.α * γ1 if use_momentum - βmax *= γ3 - d .= ∇fk .* (oneT - βmax) .+ momentum .* βmax + βktilde *= γ3 + d .= -∇fk .+ βktilde .* momentum end end # Acceptance of the new candidate if ρk >= η1 x .= c - if use_momentum - momentum .= ∇fk .* (oneT - β) .+ momentum .* β - end set_objective!(stats, fck) mem_ind = (mem_ind + 1) % M obj_mem[mem_ind + 1] = stats.objective @@ -556,11 +550,11 @@ function SolverCore.solve!( norm_∇fk = norm(∇fk) if use_momentum mdot∇f = dot(momentum, ∇fk) - p .= momentum .- ∇fk - βmax = find_beta(p, mdot∇f, norm_∇fk, μk, stats.objective, max_obj_mem, β, θ1, θ2) - d .= ∇fk .* (oneT - βmax) .+ momentum .* βmax + momentum .= d + βktilde = find_beta_tilde(p, mdot∇f, norm_∇fk, μk, stats.objective, max_obj_mem, β, θ1, θ2) + d .= - ∇fk .+ βktilde .* momentum norm_d = norm(d) - avgβmax += βmax + avgβktilde += βktilde siter += 1 end end @@ -578,7 +572,7 @@ function SolverCore.solve!( @sprintf "%5d %9.2e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk else infoline = - @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βmax + @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βktilde end end @@ -604,38 +598,36 @@ function SolverCore.solve!( done = stats.status != :unknown end if use_momentum - avgβmax /= siter - set_solver_specific!(stats, :avgβmax, avgβmax) + avgβktilde /= siter + set_solver_specific!(stats, :avgβktilde, avgβktilde) end set_solution!(stats, x) return stats end """ - find_beta(m, mdot∇f, norm_∇f, μk, fk, max_obj_mem, β, θ1, θ2) - -Compute βmax which saturates the contribution of the momentum term to the gradient. -`βmax` is computed such that the two gradient-related conditions (first one is relaxed in the nonmonotone case) are ensured: -1. (1-βmax) * ‖∇f(xk)‖² + βmax * ∇f(xk)ᵀm + (max_obj_mem - fk)/μk ≥ θ1 * ‖∇f(xk)‖² -2. ‖∇f(xk)‖ ≥ θ2 * ‖(1-βmax) * ∇f(xk) .+ βmax .* m‖ -with `m` the momentum term and `mdot∇f = ∇f(xk)ᵀm`, `fk` the model at s=0, `max_obj_mem` the largest objective value over the last M successful iterations. + find_beta_tilde(m, mdot∇f, norm_∇f, μk, fk, max_obj_mem, βktilde, θ1, θ2) + +Compute βktilde which saturates the contribution of the momentum term to the gradient. +`βktilde` is computed such that the two gradient-related conditions (first one is relaxed in the nonmonotone case) are ensured: +1. dᵀ∇f(xk) - (max_obj_mem - fk)/μk ≤ - θ1 * ‖∇f(xk)‖² +2. ‖d‖ ≤ θ2 ‖∇f(xk)‖ +In the non monotone case, it is also necessary to ensure (necessarily satisfied in the monotone case) +3. ‖d‖ ≥ θ1 ‖∇f(xk)‖ +with `d` = -∇f(xk) + βktilde `m` the step direction, `fk` the model at s=0, `max_obj_mem` the largest objective value over the last M successful iterations. """ -function find_beta( - p::V, +function find_beta_tilde( mdot∇f::T, norm_∇f::T, + norm_m::T, μk::T, fk::T, max_obj_mem::T, - β::T, + βktilde::T, θ1::T, θ2::T, ) where {T, V} - n1 = norm_∇f^2 - mdot∇f - n2 = norm(p) - β1 = n1 > 0 ? ((1 - θ1) * norm_∇f^2 - (fk - max_obj_mem) / μk) / n1 : β - β2 = n2 != 0 ? (1 - θ2) * norm_∇f / n2 : β - return min(β, min(β1, β2)) + end """ From 94c6f6c913136195649c8cdec0e7ef7bb02380e5 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 24 Oct 2025 09:59:09 +0200 Subject: [PATCH 02/20] find_beta_tilde implemented properly --- src/fomo.jl | 53 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index f40d1040..6ae19b3f 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -217,7 +217,6 @@ function FomoSolver(nlp::AbstractNLPModel{T, V}; M::Int = get(FOMO_M, nlp), kwar g = similar(nlp.meta.x0) c = similar(nlp.meta.x0) m = fill!(similar(nlp.meta.x0), 0) - d = fill!(similar(nlp.meta.x0), 0) o = fill!(Vector{T}(undef, M), -Inf) return FomoSolver{T, V}(x, g, c, m, d, o, T(0), params) end @@ -446,7 +445,6 @@ function SolverCore.solve!( ∇fk = solver.g c = solver.c momentum = use_momentum ? solver.m : nothing # not used if no momentum - d = use_momentum ? solver.d : solver.g # g = d if no momentum set_iter!(stats, 0) f0 = obj(nlp, x) set_objective!(stats, f0) @@ -511,17 +509,14 @@ function SolverCore.solve!( done = stats.status != :unknown - d .= ∇fk - norm_d = norm_∇fk βktilde = T(0) ρk = T(0) avgβktilde = T(0) siter::Int = 0 - oneT = T(1) mdot∇f = T(0) # dot(momentum,∇fk) while !done μk = step_mult(solver.α, norm_d, step_backend) - c .= x .- μk .* d + c .= x .+ μk .* (-∇fk .+ m) step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 ΔTk = (- norm_∇fk^2 + βktilde * mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) @@ -534,7 +529,6 @@ function SolverCore.solve!( solver.α = solver.α * γ1 if use_momentum βktilde *= γ3 - d .= -∇fk .+ βktilde .* momentum end end @@ -546,14 +540,15 @@ function SolverCore.solve!( obj_mem[mem_ind + 1] = stats.objective max_obj_mem = maximum(obj_mem) + if use_momentum + momentum .= -∇fk .+ βktilde .* momentum + norm_m = norm(m) + end grad!(nlp, x, ∇fk) norm_∇fk = norm(∇fk) if use_momentum mdot∇f = dot(momentum, ∇fk) - momentum .= d - βktilde = find_beta_tilde(p, mdot∇f, norm_∇fk, μk, stats.objective, max_obj_mem, β, θ1, θ2) - d .= - ∇fk .+ βktilde .* momentum - norm_d = norm(d) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, β, θ1, θ2) avgβktilde += βktilde siter += 1 end @@ -608,13 +603,15 @@ end """ find_beta_tilde(m, mdot∇f, norm_∇f, μk, fk, max_obj_mem, βktilde, θ1, θ2) -Compute βktilde which saturates the contribution of the momentum term to the gradient. `βktilde` is computed such that the two gradient-related conditions (first one is relaxed in the nonmonotone case) are ensured: -1. dᵀ∇f(xk) - (max_obj_mem - fk)/μk ≤ - θ1 * ‖∇f(xk)‖² +1. dᵀ∇f(xk) - (max_obj_mem - fk)/μk ≤ - θ1 * ‖∇f(xk)‖² 2. ‖d‖ ≤ θ2 ‖∇f(xk)‖ In the non monotone case, it is also necessary to ensure (necessarily satisfied in the monotone case) 3. ‖d‖ ≥ θ1 ‖∇f(xk)‖ with `d` = -∇f(xk) + βktilde `m` the step direction, `fk` the model at s=0, `max_obj_mem` the largest objective value over the last M successful iterations. + +The set satisfying 1 and 2 is an interval, βktilde is chosen as the projection of βk onto that interval. +The set satisfied 1, 2 and 3 is either an interval or the union of two interval, one of them containing 0. If βktilde does not belong to the set, it is chosen as the projection onto the interval containing 0. """ function find_beta_tilde( mdot∇f::T, @@ -623,11 +620,39 @@ function find_beta_tilde( μk::T, fk::T, max_obj_mem::T, - βktilde::T, + βk::T, θ1::T, θ2::T, ) where {T, V} + # Condition 1: βktilde <= β1 if mdot∇f > 0, βktilde <= β1 if mdot∇f < 0, no condition if mdot∇f == 0 + βktilde = βk + β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/mdot∇f + if mdot∇f>0 + βktilde = min(β1,βktilde) + elseif mdot∇fk<0 + βktilde = max(β1,βktilde) + end + # Condition 2: βktilde ∈ [r21,r22], r21 and r22 are roots of P(βktilde) = (1-θ2^2)norm_∇f -2βktilde mdot∇f + βktilde^2 norm_m. + # Roots are necessarily real and r21 ≤ 0 ≤ r22 + Δ2 = 4*(mdot∇f^2- (1-θ2^2)*(norm_∇f*norm_m)^2) + r21 = (mdot∇f-sqrt(Δ2))/((1-θ2^2)*norm_∇f^2) + r22 = (mdot∇f+sqrt(Δ2))/((1-θ2^2)*norm_∇f^2) + βktilde = max(r21,βktilde) + βktilde = min(r22,βktilde) + + # Condition 3: βktilde ∉ [r31,r32], r31 and r32 are roots of Q(βktilde) = (1-θ1^2)norm_∇f -2βktilde mdot∇f + βktilde^2 norm_m + # roots are not necessarily real. If complex condition does not apply. If real, they both have the same sign. + r31 = 0 + r32 = 0 + Δ3= 4*(mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) + if Δ3>0 + r31 = (mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + r32 = (mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + end + if βktilde >r31 && βktilde < r32 + βktilde = sign(r31)*min(abs(r31),abs(r32)) + end end """ From 7ca95dcf13be4a5b69011628f3d6b98cabf8bcf6 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 24 Oct 2025 12:21:33 +0200 Subject: [PATCH 03/20] fixes --- src/fomo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fomo.jl b/src/fomo.jl index 6ae19b3f..aa56b7cc 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -218,7 +218,7 @@ function FomoSolver(nlp::AbstractNLPModel{T, V}; M::Int = get(FOMO_M, nlp), kwar c = similar(nlp.meta.x0) m = fill!(similar(nlp.meta.x0), 0) o = fill!(Vector{T}(undef, M), -Inf) - return FomoSolver{T, V}(x, g, c, m, d, o, T(0), params) + return FomoSolver{T, V}(x, g, c, m, o, T(0), params) end @doc (@doc FomoSolver) function fomo( From f57d9f51338012a90661c08234443ac6c095deec Mon Sep 17 00:00:00 2001 From: d-monnet Date: Mon, 3 Nov 2025 11:48:38 +0100 Subject: [PATCH 04/20] fixes, put back g --- src/fomo.jl | 64 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index aa56b7cc..cf781353 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -20,7 +20,7 @@ const FOMO_αmax = const FOMO_β = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(9 // 10), "T(9/10)") const FOMO_θ1 = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1 // 10), "T(1/10)") const FOMO_θ2 = - DefaultParameter((nlp::AbstractNLPModel) -> eps(eltype(nlp.meta.x0))^(1 // 3), "eps(T)^(1/3)") + DefaultParameter((nlp::AbstractNLPModel) -> 1 / eps(eltype(nlp.meta.x0))^(1 // 3), "1/eps(T)^(1/3)") const FOMO_M = DefaultParameter(1) const FOMO_step_backend = DefaultParameter(nlp -> r2_step(), "r2_step()") @@ -96,7 +96,7 @@ function FOMOParameterSet( Parameter(αmax, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(β, RealInterval(T(0), T(1), upper_open = true)), Parameter(θ1, RealInterval(T(0), T(1))), - Parameter(θ2, RealInterval(T(0), T(1), upper_open = true)), + Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), ) @@ -204,8 +204,8 @@ mutable struct FomoSolver{T, V} <: AbstractFirstOrderSolver x::V g::V c::V - m::V d::V + m::V o::V α::T params::FOMOParameterSet{T} @@ -216,9 +216,10 @@ function FomoSolver(nlp::AbstractNLPModel{T, V}; M::Int = get(FOMO_M, nlp), kwar x = similar(nlp.meta.x0) g = similar(nlp.meta.x0) c = similar(nlp.meta.x0) + d = similar(nlp.meta.x0) m = fill!(similar(nlp.meta.x0), 0) o = fill!(Vector{T}(undef, M), -Inf) - return FomoSolver{T, V}(x, g, c, m, o, T(0), params) + return FomoSolver{T, V}(x, g, c, d, m, o, T(0), params) end @doc (@doc FomoSolver) function fomo( @@ -442,8 +443,9 @@ function SolverCore.solve!( set_time!(stats, 0.0) x = solver.x .= x - ∇fk = solver.g c = solver.c + g = solver.g # step direction, is -∇f if no momentum + d = use_momentum ? solver.d : solver.g # not used if no momentum momentum = use_momentum ? solver.m : nothing # not used if no momentum set_iter!(stats, 0) f0 = obj(nlp, x) @@ -454,12 +456,15 @@ function SolverCore.solve!( obj_mem[mem_ind + 1] = stats.objective max_obj_mem = stats.objective - grad!(nlp, x, ∇fk) - norm_∇fk = norm(∇fk) + grad!(nlp, x, g) + norm_∇fk = norm(g) + d .= -g + norm_d = norm_∇fk set_dual_residual!(stats, norm_∇fk) solver.α = init_alpha(norm_∇fk, step_backend) + # Stopping criterion: fmin = min(-one(T), f0) / eps(T) unbounded = f0 < fmin @@ -514,11 +519,12 @@ function SolverCore.solve!( avgβktilde = T(0) siter::Int = 0 mdot∇f = T(0) # dot(momentum,∇fk) + norm_m = T(0) while !done μk = step_mult(solver.α, norm_d, step_backend) - c .= x .+ μk .* (-∇fk .+ m) + c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = (- norm_∇fk^2 + βktilde * mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum + ΔTk = norm_d^2 * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) @@ -529,6 +535,9 @@ function SolverCore.solve!( solver.α = solver.α * γ1 if use_momentum βktilde *= γ3 + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, βktilde, θ1, θ2) + d .= -g .+ βktilde*momentum # d = -∇fk + γ3*momentumm, avoid storing gradient + norm_d = norm(d) # TODO only needed in TR context, not in R2, end end @@ -541,16 +550,21 @@ function SolverCore.solve!( max_obj_mem = maximum(obj_mem) if use_momentum - momentum .= -∇fk .+ βktilde .* momentum - norm_m = norm(m) + momentum .= -g .+ β*momentum + norm_m = norm(momentum) end - grad!(nlp, x, ∇fk) - norm_∇fk = norm(∇fk) + grad!(nlp, x, g) + norm_∇fk = norm(g) + d .= -g if use_momentum - mdot∇f = dot(momentum, ∇fk) + mdot∇f = dot(momentum, d) βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, β, θ1, θ2) + d .+= βktilde*momentum + norm_d = norm(d) avgβktilde += βktilde siter += 1 + else + norm_d = norm_∇fk end end @@ -567,7 +581,7 @@ function SolverCore.solve!( @sprintf "%5d %9.2e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk else infoline = - @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βktilde + @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βktilde norm_m end end @@ -623,29 +637,30 @@ function find_beta_tilde( βk::T, θ1::T, θ2::T, -) where {T, V} +) where {T} # Condition 1: βktilde <= β1 if mdot∇f > 0, βktilde <= β1 if mdot∇f < 0, no condition if mdot∇f == 0 βktilde = βk β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/mdot∇f if mdot∇f>0 βktilde = min(β1,βktilde) - elseif mdot∇fk<0 + elseif mdot∇f<0 βktilde = max(β1,βktilde) end # Condition 2: βktilde ∈ [r21,r22], r21 and r22 are roots of P(βktilde) = (1-θ2^2)norm_∇f -2βktilde mdot∇f + βktilde^2 norm_m. # Roots are necessarily real and r21 ≤ 0 ≤ r22 - Δ2 = 4*(mdot∇f^2- (1-θ2^2)*(norm_∇f*norm_m)^2) - r21 = (mdot∇f-sqrt(Δ2))/((1-θ2^2)*norm_∇f^2) - r22 = (mdot∇f+sqrt(Δ2))/((1-θ2^2)*norm_∇f^2) - βktilde = max(r21,βktilde) - βktilde = min(r22,βktilde) - + if norm_m ≠ 0 + Δ2 = (mdot∇f^2- (1-θ2^2)*(norm_∇f*norm_m)^2) + r21 = (mdot∇f-sqrt(Δ2))/norm_m + r22 = (mdot∇f+sqrt(Δ2))/norm_m + βktilde = max(r21,βktilde) + βktilde = min(r22,βktilde) + end # Condition 3: βktilde ∉ [r31,r32], r31 and r32 are roots of Q(βktilde) = (1-θ1^2)norm_∇f -2βktilde mdot∇f + βktilde^2 norm_m # roots are not necessarily real. If complex condition does not apply. If real, they both have the same sign. r31 = 0 r32 = 0 - Δ3= 4*(mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) + Δ3= (mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) if Δ3>0 r31 = (mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) r32 = (mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) @@ -653,6 +668,7 @@ function find_beta_tilde( if βktilde >r31 && βktilde < r32 βktilde = sign(r31)*min(abs(r31),abs(r32)) end + βktilde end """ From 453d596bc65b60ac9928c2ed265ec31a472efc9d Mon Sep 17 00:00:00 2001 From: d-monnet Date: Mon, 3 Nov 2025 12:09:58 +0100 Subject: [PATCH 05/20] fix model decrease --- src/fomo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fomo.jl b/src/fomo.jl index cf781353..56ca22be 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -524,7 +524,7 @@ function SolverCore.solve!( μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = norm_d^2 * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum + ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) From f94eb7237726551f55381c0f7469f68476a3616f Mon Sep 17 00:00:00 2001 From: d-monnet Date: Mon, 3 Nov 2025 14:18:31 +0100 Subject: [PATCH 06/20] add betaktilde multiplier to muk --- src/fomo.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 56ca22be..7ebd6a27 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -456,9 +456,11 @@ function SolverCore.solve!( obj_mem[mem_ind + 1] = stats.objective max_obj_mem = stats.objective + oneT = T(1) + grad!(nlp, x, g) norm_∇fk = norm(g) - d .= -g + d .= -g ./ (oneT - β) norm_d = norm_∇fk set_dual_residual!(stats, norm_∇fk) @@ -514,14 +516,14 @@ function SolverCore.solve!( done = stats.status != :unknown - βktilde = T(0) + βktilde = β ρk = T(0) avgβktilde = T(0) siter::Int = 0 mdot∇f = T(0) # dot(momentum,∇fk) norm_m = T(0) while !done - μk = step_mult(solver.α, norm_d, step_backend) + μk = step_mult(solver.α, norm_d, step_backend)*(oneT-βktilde) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum From 18a74cf70e9913a069748a6fd4fa5da1c31068be Mon Sep 17 00:00:00 2001 From: d-monnet Date: Wed, 5 Nov 2025 17:19:49 +0100 Subject: [PATCH 07/20] add momentum backend --- src/fomo.jl | 91 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 71 insertions(+), 20 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 7ebd6a27..66a44cd6 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -1,5 +1,5 @@ export fomo, FomoSolver, FOMOParameterSet, FoSolver, fo, R2, TR -export tr_step, r2_step +export tr_step, r2_step, ia_momentum, nesterov_HB, cg_PR, cg_FR abstract type AbstractFirstOrderSolver <: AbstractOptimizationSolver end @@ -7,6 +7,12 @@ abstract type AbstractFOMethod end struct tr_step <: AbstractFOMethod end struct r2_step <: AbstractFOMethod end +abstract type AbstractMomentumMethod end +struct ia_momentum <: AbstractMomentumMethod end +struct nesterov_HB <: AbstractMomentumMethod end +struct cg_PR <: AbstractMomentumMethod end +struct cg_FR <: AbstractMomentumMethod end + # Default algorithm parameter values const FOMO_η1 = DefaultParameter((nlp::AbstractNLPModel) -> eps(eltype(nlp.meta.x0))^(1 // 4), "eps(T)^(1 // 4)") @@ -23,6 +29,7 @@ const FOMO_θ2 = DefaultParameter((nlp::AbstractNLPModel) -> 1 / eps(eltype(nlp.meta.x0))^(1 // 3), "1/eps(T)^(1/3)") const FOMO_M = DefaultParameter(1) const FOMO_step_backend = DefaultParameter(nlp -> r2_step(), "r2_step()") +const FOMO_momentum_backend = DefaultParameter(nlp -> ia_momentum(), "ia_momentum()") """ FOMOParameterSet{T} <: AbstractParameterSet @@ -37,6 +44,7 @@ This structure designed for `fomo` regroups the following parameters: - `θ2`: momentum contribution parameter for convergence condition (2). - `M` : requires objective decrease over the `M` last iterates (nonmonotone context). `M=1` implies monotone behaviour. - `step_backend`: step computation mode. Options are `r2_step()` for quadratic regulation step and `tr_step()` for first-order trust-region. + - `momentum_backend`: momentum mode for βₖ. Options are `ia_momentum()`, `nesterov_HB()`, `cg_PR()` for Polak-Ribière, `cg_FR()` for Fletcher-Reeves. An additional constructor is @@ -56,6 +64,7 @@ Default values are: - `θ2 = $(FOMO_θ2)` - `M = $(FOMO_M)` - `step_backend = $(FOMO_step_backend) + - `momentum_backend = $(FOMO_momentum_backend)` """ struct FOMOParameterSet{T} <: AbstractParameterSet η1::Parameter{T, RealInterval{T}} @@ -69,6 +78,7 @@ struct FOMOParameterSet{T} <: AbstractParameterSet θ2::Parameter{T, RealInterval{T}} M::Parameter{Int, IntegerRange{Int}} step_backend::Parameter{Union{r2_step, tr_step}, CategoricalSet{Union{r2_step, tr_step}}} + momentum_backend::Parameter{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}} end # add a default constructor @@ -85,6 +95,7 @@ function FOMOParameterSet( θ2::T = get(FOMO_θ2, nlp), M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), + momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend ,nlp) ) where {T} @assert η1 <= η2 FOMOParameterSet( @@ -99,6 +110,7 @@ function FOMOParameterSet( Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), + Parameter(momentum_backend, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}([ia_momentum(); nesterov_HB(); cg_PR(); cg_FR()])), ) end @@ -110,15 +122,20 @@ A First-Order with MOmentum (FOMO) model-based method for unconstrained optimiza # Algorithm description The step is computed along -d = - (1-βktilde) .* ∇f(xk) - βktilde .* mk +dk = - ∇f(xk) - βktilde .* mk with mk the memory of past gradients (initialized at 0), and updated at each successful iteration as -mk .= ∇f(xk) .* (1 - βktilde) .+ mk .* βktilde -and βktilde ∈ [0,β] chosen as to ensure d is gradient-related, i.e., the following 2 conditions are satisfied: -(1-βktilde) .* ∇f(xk) + βktilde .* ∇f(xk)ᵀmk ≥ θ1 * ‖∇f(xk)‖² (1) -‖∇f(xk)‖ ≥ θ2 * ‖(1-βktilde) *. ∇f(xk) + βktilde .* mk‖ (2) -In the nonmonotone case, (1) rewrites -(1-βktilde) .* ∇f(xk) + βktilde .* ∇f(xk)ᵀmk + (fm - fk)/μk ≥ θ1 * ‖∇f(xk)‖², -with fm the largest objective value over the last M successful iterations, and fk = f(xk). +m_k+1 .= dk +and βktilde chosen as to ensure d is gradient-related, i.e., the following 2 conditions are satisfied: +∇f(xk)ᵀdk ≤ - θ1 * ‖∇f(xk)‖² (1) +θ2 * ‖∇f(xk)‖ ≥ ‖dk‖ (2) +In the nonmonotone case, dk must also satisfied (4) to ensure it is non zero: + ‖dk‖ ≥ θ1 ‖∇f(xk)‖, (4) +and in this context, (1) becomes (3) +∇f(xk)ᵀdk - (fm - fk)/μk ≤ - θ1 * ‖∇f(xk)‖² (3) +with fm the largest objective value over the last M successful iterations, and fk = f(xk) + + + # Advanced usage @@ -151,6 +168,7 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `M = $(FOMO_M)` : requires objective decrease over the `M` last iterates (nonmonotone context). `M=1` implies monotone behaviour. - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration. - `step_backend = $(FOMO_step_backend)`: step computation mode. Options are `r2_step()` for quadratic regulation step and `tr_step()` for first-order trust-region. +- `momentum_backend`: momentum mode for βₖ. Options are `ia_momentum()`, `nesterov_HB()`, `cg_PR()` for Polsak-Ribière, `cg_FR()` for Fletcher-Reeves. # Output @@ -235,6 +253,7 @@ end θ2::T = get(FOMO_θ2, nlp), M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), + momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend, nlp), kwargs..., ) where {T, V} solver = FomoSolver( @@ -250,6 +269,7 @@ end θ2 = θ2, M = M, step_backend = step_backend, + momentum_backend = momentum_backend ) solver_specific = Dict(:avgβktilde => T(0.0)) stats = GenericExecutionStats(nlp; solver_specific = solver_specific) @@ -435,9 +455,11 @@ function SolverCore.solve!( M = value(solver.params.M) step_backend = value(solver.params.step_backend) - use_momentum = typeof(solver) <: FomoSolver is_r2 = typeof(step_backend) <: r2_step + use_momentum = typeof(solver) <: FomoSolver + momentum_backend = use_momentum ? value(solver.params.momentum_backend) : nothing # not used if no momentum + SolverCore.reset!(stats) start_time = time() set_time!(stats, 0.0) @@ -460,7 +482,7 @@ function SolverCore.solve!( grad!(nlp, x, g) norm_∇fk = norm(g) - d .= -g ./ (oneT - β) + d .= -g norm_d = norm_∇fk set_dual_residual!(stats, norm_∇fk) @@ -516,14 +538,15 @@ function SolverCore.solve!( done = stats.status != :unknown - βktilde = β + βktilde = T(0) + βk = β ρk = T(0) avgβktilde = T(0) siter::Int = 0 mdot∇f = T(0) # dot(momentum,∇fk) norm_m = T(0) while !done - μk = step_mult(solver.α, norm_d, step_backend)*(oneT-βktilde) + μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum @@ -538,7 +561,7 @@ function SolverCore.solve!( if use_momentum βktilde *= γ3 βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, βktilde, θ1, θ2) - d .= -g .+ βktilde*momentum # d = -∇fk + γ3*momentumm, avoid storing gradient + d .= βktilde*momentum .- g norm_d = norm(d) # TODO only needed in TR context, not in R2, end end @@ -552,20 +575,24 @@ function SolverCore.solve!( max_obj_mem = maximum(obj_mem) if use_momentum - momentum .= -g .+ β*momentum + momentum .= -g .+ βk*momentum norm_m = norm(momentum) + d.=g # temp. storage of ∇fk-1 in d to avoid storage, needed for compute_beta function + norm_∇fk₋ = norm_∇fk end + grad!(nlp, x, g) norm_∇fk = norm(g) - d .= -g if use_momentum - mdot∇f = dot(momentum, d) - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, β, θ1, θ2) - d .+= βktilde*momentum + βk = compute_beta(β,norm_∇fk,norm_∇fk₋,g,d,momentum_backend) + mdot∇f = dot(momentum, g) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, βk, θ1, θ2) + d .= βktilde*momentum .- g norm_d = norm(d) avgβktilde += βktilde siter += 1 else + d .= -g norm_d = norm_∇fk end end @@ -583,7 +610,7 @@ function SolverCore.solve!( @sprintf "%5d %9.2e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk else infoline = - @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βktilde norm_m + @sprintf "%5d %9.2e %7.1e %7.1e %7.1e %7.1e" stats.iter stats.objective norm_∇fk step_param ρk βktilde end end @@ -673,6 +700,30 @@ function find_beta_tilde( βktilde end +""" + compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::ia_momentum) + compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::AbstractMomentumMethod) + compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::cg_PR) + compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::cg_FR) + +Compute β coefficient for the given momentum_backend. +""" +function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::ia_momentum) where {T,V} + β +end + +function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::nesterov_HB) where {T,V} + β +end + +function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::cg_PR) where {T,V} + dp = dot(∇fk,∇fk₋) + (norm_∇fk^2 - dp)/norm_∇fk₋^2 +end + +function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::cg_FR) where {T,V} + norm_∇fk^2/norm_∇fk₋^2 +end """ init_alpha(norm_∇fk::T, ::r2_step) init_alpha(norm_∇fk::T, ::tr_step) From 9a4a7693cdb47f05081f45f31d7b7572c30a0a90 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Wed, 5 Nov 2025 17:24:23 +0100 Subject: [PATCH 08/20] update comment --- src/fomo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fomo.jl b/src/fomo.jl index 66a44cd6..8509fcaa 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -549,7 +549,7 @@ function SolverCore.solve!( μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,d) * μk with momentum, ‖∇fk‖²μk without momentum + ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) From 45aa60a374bd1d7610bd603da6557069aac9cd51 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Wed, 5 Nov 2025 18:08:49 +0100 Subject: [PATCH 09/20] fix model decrease --- src/fomo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fomo.jl b/src/fomo.jl index 8509fcaa..b0b6d004 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -549,7 +549,7 @@ function SolverCore.solve!( μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = (norm_∇fk^2 + βktilde* mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum + ΔTk = (norm_∇fk^2 - βktilde* mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) From 6cf1452c32389b2b109c9ac88ac466348b51da55 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 7 Nov 2025 16:14:30 +0100 Subject: [PATCH 10/20] add kappa parameter --- src/fomo.jl | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index b0b6d004..1f392268 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -545,6 +545,7 @@ function SolverCore.solve!( siter::Int = 0 mdot∇f = T(0) # dot(momentum,∇fk) norm_m = T(0) + κ = oneT while !done μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d @@ -554,17 +555,21 @@ function SolverCore.solve!( unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) # Update regularization parameters + α₋ = solver.α if ρk >= η2 solver.α = min(αmax, γ2 * solver.α) elseif ρk < η1 solver.α = solver.α * γ1 if use_momentum βktilde *= γ3 - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, βktilde, θ1, θ2) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βktilde, θ1, θ2) d .= βktilde*momentum .- g norm_d = norm(d) # TODO only needed in TR context, not in R2, end end + if use_momentum + update_kappa(solver.α, α₋,momentum_backend) + end # Acceptance of the new candidate if ρk >= η1 @@ -586,7 +591,7 @@ function SolverCore.solve!( if use_momentum βk = compute_beta(β,norm_∇fk,norm_∇fk₋,g,d,momentum_backend) mdot∇f = dot(momentum, g) - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, βk, θ1, θ2) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βk, θ1, θ2) d .= βktilde*momentum .- g norm_d = norm(d) avgβktilde += βktilde @@ -643,6 +648,27 @@ function SolverCore.solve!( return stats end +""" + update_kappa(αk, αk₋) + +Returns κ update, needed only for `nesterov_HB` `momentum_backend`. +""" +function update_kappa(αk::T, αk₋::T, ::ia_momentum) where {T} + 1 +end + +function update_kappa(αk::T, αk₋::T, ::nesterov_HB) where {T} + αk/αk₋ +end + +function update_kappa(αk::T, αk₋::T, ::cg_PR) where {T} + 1 +end + +function update_kappa(αk::T, αk₋::T, ::cg_FR) where {T} + 1 +end + """ find_beta_tilde(m, mdot∇f, norm_∇f, μk, fk, max_obj_mem, βktilde, θ1, θ2) @@ -663,13 +689,14 @@ function find_beta_tilde( μk::T, fk::T, max_obj_mem::T, + κ::T, βk::T, θ1::T, θ2::T, ) where {T} # Condition 1: βktilde <= β1 if mdot∇f > 0, βktilde <= β1 if mdot∇f < 0, no condition if mdot∇f == 0 βktilde = βk - β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/mdot∇f + β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/(κ*mdot∇f) if mdot∇f>0 βktilde = min(β1,βktilde) elseif mdot∇f<0 @@ -679,9 +706,9 @@ function find_beta_tilde( # Condition 2: βktilde ∈ [r21,r22], r21 and r22 are roots of P(βktilde) = (1-θ2^2)norm_∇f -2βktilde mdot∇f + βktilde^2 norm_m. # Roots are necessarily real and r21 ≤ 0 ≤ r22 if norm_m ≠ 0 - Δ2 = (mdot∇f^2- (1-θ2^2)*(norm_∇f*norm_m)^2) - r21 = (mdot∇f-sqrt(Δ2))/norm_m - r22 = (mdot∇f+sqrt(Δ2))/norm_m + Δ2 = (κ^2*mdot∇f^2- (1-θ2^2)*(norm_∇f*norm_m)^2) + r21 = (κ*mdot∇f-sqrt(Δ2))/norm_m + r22 = (κ*mdot∇f+sqrt(Δ2))/norm_m βktilde = max(r21,βktilde) βktilde = min(r22,βktilde) end @@ -689,10 +716,10 @@ function find_beta_tilde( # roots are not necessarily real. If complex condition does not apply. If real, they both have the same sign. r31 = 0 r32 = 0 - Δ3= (mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) + Δ3= (κ^2*mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) if Δ3>0 - r31 = (mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) - r32 = (mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + r31 = (κ*mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + r32 = (κ*mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) end if βktilde >r31 && βktilde < r32 βktilde = sign(r31)*min(abs(r31),abs(r32)) From d1a3925682b5ed60d9418f5e530a77ce09423d2b Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 7 Nov 2025 16:42:20 +0100 Subject: [PATCH 11/20] remove ia_momentum backend --- src/fomo.jl | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 1f392268..ea79241a 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -1,5 +1,5 @@ export fomo, FomoSolver, FOMOParameterSet, FoSolver, fo, R2, TR -export tr_step, r2_step, ia_momentum, nesterov_HB, cg_PR, cg_FR +export tr_step, r2_step, nesterov_HB, cg_PR, cg_FR abstract type AbstractFirstOrderSolver <: AbstractOptimizationSolver end @@ -8,7 +8,7 @@ struct tr_step <: AbstractFOMethod end struct r2_step <: AbstractFOMethod end abstract type AbstractMomentumMethod end -struct ia_momentum <: AbstractMomentumMethod end +#struct ia_momentum <: AbstractMomentumMethod end struct nesterov_HB <: AbstractMomentumMethod end struct cg_PR <: AbstractMomentumMethod end struct cg_FR <: AbstractMomentumMethod end @@ -29,7 +29,7 @@ const FOMO_θ2 = DefaultParameter((nlp::AbstractNLPModel) -> 1 / eps(eltype(nlp.meta.x0))^(1 // 3), "1/eps(T)^(1/3)") const FOMO_M = DefaultParameter(1) const FOMO_step_backend = DefaultParameter(nlp -> r2_step(), "r2_step()") -const FOMO_momentum_backend = DefaultParameter(nlp -> ia_momentum(), "ia_momentum()") +const FOMO_momentum_backend = DefaultParameter(nlp -> nesterov_HB(), "nesterov_HB()") """ FOMOParameterSet{T} <: AbstractParameterSet @@ -44,7 +44,7 @@ This structure designed for `fomo` regroups the following parameters: - `θ2`: momentum contribution parameter for convergence condition (2). - `M` : requires objective decrease over the `M` last iterates (nonmonotone context). `M=1` implies monotone behaviour. - `step_backend`: step computation mode. Options are `r2_step()` for quadratic regulation step and `tr_step()` for first-order trust-region. - - `momentum_backend`: momentum mode for βₖ. Options are `ia_momentum()`, `nesterov_HB()`, `cg_PR()` for Polak-Ribière, `cg_FR()` for Fletcher-Reeves. + - `momentum_backend`: momentum mode for βₖ. Options are, `nesterov_HB()`, `cg_PR()` for Polak-Ribière, `cg_FR()` for Fletcher-Reeves. An additional constructor is @@ -78,7 +78,7 @@ struct FOMOParameterSet{T} <: AbstractParameterSet θ2::Parameter{T, RealInterval{T}} M::Parameter{Int, IntegerRange{Int}} step_backend::Parameter{Union{r2_step, tr_step}, CategoricalSet{Union{r2_step, tr_step}}} - momentum_backend::Parameter{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}} + momentum_backend::Parameter{Union{nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{nesterov_HB, cg_PR, cg_FR}}} end # add a default constructor @@ -110,7 +110,7 @@ function FOMOParameterSet( Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), - Parameter(momentum_backend, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}([ia_momentum(); nesterov_HB(); cg_PR(); cg_FR()])), + Parameter(momentum_backend, CategoricalSet{Union{nesterov_HB, cg_PR, cg_FR}}([nesterov_HB(); cg_PR(); cg_FR()])), ) end @@ -168,7 +168,7 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `M = $(FOMO_M)` : requires objective decrease over the `M` last iterates (nonmonotone context). `M=1` implies monotone behaviour. - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration. - `step_backend = $(FOMO_step_backend)`: step computation mode. Options are `r2_step()` for quadratic regulation step and `tr_step()` for first-order trust-region. -- `momentum_backend`: momentum mode for βₖ. Options are `ia_momentum()`, `nesterov_HB()`, `cg_PR()` for Polsak-Ribière, `cg_FR()` for Fletcher-Reeves. +- `momentum_backend`: momentum mode for βₖ. Options are `nesterov_HB()`, `cg_PR()` for Polsak-Ribière, `cg_FR()` for Fletcher-Reeves. # Output @@ -653,9 +653,9 @@ end Returns κ update, needed only for `nesterov_HB` `momentum_backend`. """ -function update_kappa(αk::T, αk₋::T, ::ia_momentum) where {T} - 1 -end +# function update_kappa(αk::T, αk₋::T, ::ia_momentum) where {T} +# 1 +# end function update_kappa(αk::T, αk₋::T, ::nesterov_HB) where {T} αk/αk₋ @@ -694,6 +694,10 @@ function find_beta_tilde( θ1::T, θ2::T, ) where {T} + e = eps(T) + if abs(mdot∇f) 0, βktilde <= β1 if mdot∇f < 0, no condition if mdot∇f == 0 βktilde = βk β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/(κ*mdot∇f) @@ -728,16 +732,15 @@ function find_beta_tilde( end """ - compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::ia_momentum) compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::AbstractMomentumMethod) compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::cg_PR) compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::cg_FR) Compute β coefficient for the given momentum_backend. """ -function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::ia_momentum) where {T,V} - β -end +# function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::ia_momentum) where {T,V} +# β +# end function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::nesterov_HB) where {T,V} β From 56e2c773397dc22952bcd58ddf93a606a8c8eb71 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 7 Nov 2025 16:49:44 +0100 Subject: [PATCH 12/20] update tests --- test/test_solvers.jl | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/test/test_solvers.jl b/test/test_solvers.jl index 30216e4e..a19dc1c9 100644 --- a/test/test_solvers.jl +++ b/test/test_solvers.jl @@ -8,19 +8,33 @@ function tests() ("lbfgs", lbfgs), ("tron", tron), ("R2", R2), - ("fomo_r2", fomo), - ("fomo_tr", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(); kwargs...)), + ("fomo_r2_nesterov_HB", fomo), + ("fomo_tr_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(); kwargs...)), + ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), + ("fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...)), + ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) end @testset "$name : nonmonotone configuration" for (name, solver) in [ ("R2", (nlp; kwargs...) -> R2(nlp, M = 2; kwargs...)), - ("fomo_r2", (nlp; kwargs...) -> fomo(nlp, M = 2; kwargs...)), + ("fomo_r2_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, M = 2; kwargs...)), ( - "fomo_tr", + "fomo_tr_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(); kwargs...), ), + ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), + ( + "fomo_tr_cg_PR", + (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...), + ), + ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ( + "fomo_tr_cg_FR", + (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...), + ), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) From 2dd21659a1b179f6e3bdc7a3a9fa6481f3074354 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Fri, 5 Dec 2025 15:36:49 +0100 Subject: [PATCH 13/20] add default parameters for momentum strategies --- src/fomo.jl | 72 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index ea79241a..94d99bb4 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -26,7 +26,16 @@ const FOMO_αmax = const FOMO_β = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(9 // 10), "T(9/10)") const FOMO_θ1 = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1 // 10), "T(1/10)") const FOMO_θ2 = - DefaultParameter((nlp::AbstractNLPModel) -> 1 / eps(eltype(nlp.meta.x0))^(1 // 3), "1/eps(T)^(1/3)") + DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e3), "T(1e3)") +const FOMO_θ1_HB = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1 // 10), "T(1/10)") +const FOMO_θ2_HB = + DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e3), "T(1e3)") +const FOMO_θ1_PR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e-4), "T(1e-4)") +const FOMO_θ2_PR = + DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e3), "T(1e3)") +const FOMO_θ1_FR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1//2), "T(1/2)") +const FOMO_θ2_FR = + DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e2), "T(1e2)") const FOMO_M = DefaultParameter(1) const FOMO_step_backend = DefaultParameter(nlp -> r2_step(), "r2_step()") const FOMO_momentum_backend = DefaultParameter(nlp -> nesterov_HB(), "nesterov_HB()") @@ -91,8 +100,8 @@ function FOMOParameterSet( γ3::T = get(FOMO_γ3, nlp), αmax::T = get(FOMO_αmax, nlp), β::T = get(FOMO_β, nlp), - θ1::T = get(FOMO_θ1, nlp), - θ2::T = get(FOMO_θ2, nlp), + θ1::T = get_θ1(momentum_backend, nlp), + θ2::T = get_θ2(momentum_backend, nlp), M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend ,nlp) @@ -105,8 +114,8 @@ function FOMOParameterSet( Parameter(γ2, RealInterval(T(1), T(Inf), lower_open = true, upper_open = true)), Parameter(γ3, RealInterval(T(0), T(1))), Parameter(αmax, RealInterval(T(1), T(Inf), upper_open = true)), - Parameter(β, RealInterval(T(0), T(1), upper_open = true)), - Parameter(θ1, RealInterval(T(0), T(1))), + Parameter(β, RealInterval(T(0), T(1), upper_open = true)), + Parameter(θ1, RealInterval(T(0), T(1), lower_open = true)), Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), @@ -551,6 +560,7 @@ function SolverCore.solve!( c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 ΔTk = (norm_∇fk^2 - βktilde* mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum + #ΔTk = norm_d^2 * μk fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) @@ -562,7 +572,7 @@ function SolverCore.solve!( solver.α = solver.α * γ1 if use_momentum βktilde *= γ3 - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βktilde, θ1, θ2) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βktilde, θ1, θ2, M) d .= βktilde*momentum .- g norm_d = norm(d) # TODO only needed in TR context, not in R2, end @@ -591,7 +601,7 @@ function SolverCore.solve!( if use_momentum βk = compute_beta(β,norm_∇fk,norm_∇fk₋,g,d,momentum_backend) mdot∇f = dot(momentum, g) - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βk, θ1, θ2) + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βk, θ1, θ2, M) d .= βktilde*momentum .- g norm_d = norm(d) avgβktilde += βktilde @@ -693,6 +703,7 @@ function find_beta_tilde( βk::T, θ1::T, θ2::T, + M::Int, ) where {T} e = eps(T) if abs(mdot∇f)0 - r31 = (κ*mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) - r32 = (κ*mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) - end - if βktilde >r31 && βktilde < r32 - βktilde = sign(r31)*min(abs(r31),abs(r32)) + if M>1 + r31 = 0 + r32 = 0 + Δ3= (κ^2*mdot∇f^2- (1-θ1^2)*(norm_∇f*norm_m)^2) + if Δ3>0 + r31 = (κ*mdot∇f-sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + r32 = (κ*mdot∇f+sqrt(Δ3))/((1-θ1^2)*norm_∇f^2) + end + if βktilde >r31 && βktilde < r32 + βktilde = sign(r31)*min(abs(r31),abs(r32)) + end end βktilde end @@ -782,3 +796,27 @@ end function step_mult(α::T, norm_∇fk::T, ::tr_step) where {T} α / norm_∇fk end + +function get_θ1(m::AbstractMomentumMethod, nlp::AbstractNLPModel{T, V}) where{T, V} + if typeof(m) == nesterov_HB + θ1 = get(FOMO_θ1_HB, nlp) + elseif typeof(m) == cg_PR + θ1 = get(FOMO_θ1_PR, nlp) + elseif typeof(m) == cg_FR + θ1 = get(FOMO_θ1_FR, nlp) + else + get(FOMO_θ1, nlp) + end +end + +function get_θ2(m::AbstractMomentumMethod, nlp::AbstractNLPModel{T, V}) where{T, V} + if typeof(m) == nesterov_HB + θ2 = get(FOMO_θ2_HB, nlp) + elseif typeof(m) == cg_PR + θ2 = get(FOMO_θ2_PR, nlp) + elseif typeof(m) == cg_FR + θ2 = get(FOMO_θ2_FR, nlp) + else + get(FOMO_θ2, nlp) + end +end From 9eba22d1e322137fa1637b3d05318c0c58b6f28a Mon Sep 17 00:00:00 2001 From: d-monnet Date: Wed, 21 Jan 2026 17:53:59 +0100 Subject: [PATCH 14/20] add ia_momemtum strategy, update tests --- src/fomo.jl | 152 ++++++++++++++++++++++++++++++++++++------- test/test_solvers.jl | 31 +++++---- 2 files changed, 147 insertions(+), 36 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 94d99bb4..cbe6deb5 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -1,5 +1,5 @@ export fomo, FomoSolver, FOMOParameterSet, FoSolver, fo, R2, TR -export tr_step, r2_step, nesterov_HB, cg_PR, cg_FR +export tr_step, r2_step, nesterov_HB, cg_PR, cg_FR, ia_momentum abstract type AbstractFirstOrderSolver <: AbstractOptimizationSolver end @@ -9,6 +9,7 @@ struct r2_step <: AbstractFOMethod end abstract type AbstractMomentumMethod end #struct ia_momentum <: AbstractMomentumMethod end +struct ia_momentum <: AbstractMomentumMethod end struct nesterov_HB <: AbstractMomentumMethod end struct cg_PR <: AbstractMomentumMethod end struct cg_FR <: AbstractMomentumMethod end @@ -87,7 +88,7 @@ struct FOMOParameterSet{T} <: AbstractParameterSet θ2::Parameter{T, RealInterval{T}} M::Parameter{Int, IntegerRange{Int}} step_backend::Parameter{Union{r2_step, tr_step}, CategoricalSet{Union{r2_step, tr_step}}} - momentum_backend::Parameter{Union{nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{nesterov_HB, cg_PR, cg_FR}}} + momentum_backend::Parameter{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}} end # add a default constructor @@ -100,11 +101,11 @@ function FOMOParameterSet( γ3::T = get(FOMO_γ3, nlp), αmax::T = get(FOMO_αmax, nlp), β::T = get(FOMO_β, nlp), - θ1::T = get_θ1(momentum_backend, nlp), - θ2::T = get_θ2(momentum_backend, nlp), M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), - momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend ,nlp) + momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend ,nlp), + θ1::T = get_θ1(momentum_backend, nlp), + θ2::T = get_θ2(momentum_backend, nlp) ) where {T} @assert η1 <= η2 FOMOParameterSet( @@ -119,7 +120,7 @@ function FOMOParameterSet( Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), - Parameter(momentum_backend, CategoricalSet{Union{nesterov_HB, cg_PR, cg_FR}}([nesterov_HB(); cg_PR(); cg_FR()])), + Parameter(momentum_backend, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}([ia_momentum(); nesterov_HB(); cg_PR(); cg_FR()])), ) end @@ -559,8 +560,11 @@ function SolverCore.solve!( μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d step_underflow = x == c # step addition underfow on every dimensions, should happen before solver.α == 0 - ΔTk = (norm_∇fk^2 - βktilde* mdot∇f) * μk # = dot(d,∇fk) * μk with momentum, ‖∇fk‖²μk without momentum - #ΔTk = norm_d^2 * μk + if !use_momentum + ΔTk = norm_∇fk^2*μk + else + ΔTk = compute_model_red(norm_∇fk, mdot∇f, βktilde, μk, momentum_backend) + end fck = obj(nlp, c) unbounded = fck < fmin ρk = (max_obj_mem - fck) / (max_obj_mem - stats.objective + ΔTk) @@ -572,8 +576,8 @@ function SolverCore.solve!( solver.α = solver.α * γ1 if use_momentum βktilde *= γ3 - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βktilde, θ1, θ2, M) - d .= βktilde*momentum .- g + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βktilde, θ1, θ2, M, momentum_backend) + compute_direction!(d, g, momentum, βktilde, momentum_backend) norm_d = norm(d) # TODO only needed in TR context, not in R2, end end @@ -590,7 +594,7 @@ function SolverCore.solve!( max_obj_mem = maximum(obj_mem) if use_momentum - momentum .= -g .+ βk*momentum + update_momentum!(g, momentum, βk, momentum_backend) norm_m = norm(momentum) d.=g # temp. storage of ∇fk-1 in d to avoid storage, needed for compute_beta function norm_∇fk₋ = norm_∇fk @@ -601,8 +605,8 @@ function SolverCore.solve!( if use_momentum βk = compute_beta(β,norm_∇fk,norm_∇fk₋,g,d,momentum_backend) mdot∇f = dot(momentum, g) - βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βk, θ1, θ2, M) - d .= βktilde*momentum .- g + βktilde = find_beta_tilde(mdot∇f, norm_∇fk, norm_m, μk, stats.objective, max_obj_mem, κ, βk, θ1, θ2, M, momentum_backend) + compute_direction!(d, g, momentum, βktilde, momentum_backend) norm_d = norm(d) avgβktilde += βktilde siter += 1 @@ -671,11 +675,7 @@ function update_kappa(αk::T, αk₋::T, ::nesterov_HB) where {T} αk/αk₋ end -function update_kappa(αk::T, αk₋::T, ::cg_PR) where {T} - 1 -end - -function update_kappa(αk::T, αk₋::T, ::cg_FR) where {T} +function update_kappa(αk::T, αk₋::T, ::AbstractMomentumMethod) where {T} 1 end @@ -704,6 +704,7 @@ function find_beta_tilde( θ1::T, θ2::T, M::Int, + momentum_backend::AbstractMomentumMethod ) where {T} e = eps(T) if abs(mdot∇f) 0, βktilde <= β1 if mdot∇f < 0, no condition if mdot∇f == 0 + βktilde = βk + β1 = ((1-θ1)*norm_∇f^2 + (max_obj_mem-fk)/μk)/(norm_∇f^2+mdot∇f) + if p>0 + βktilde = min(β1,βktilde) + elseif p<0 + βktilde = max(β1,βktilde) + end + + # Condition 2: βktilde ∈ [r21,r22], r21 and r22 are roots of P(βktilde) = (1-θ2^2)norm_∇f -2βktilde*(mdot∇f + mdot∇f) + βktilde^2 norm_m. + # Roots are necessarily real and r21 ≤ 0 ≤ r22 + a = norm_∇f^2 + 2*mdot∇f + norm_m^2 + if a1 + c = (1 - θ1^2)*norm_∇f^2 + r31 = 0 + r32 = 0 + Δ3= b^2-4*a*c + if Δ3>0 + r31 = (-b-sqrt(Δ2))/(2*a) + r32 = (-b+sqrt(Δ2))/(2*a) + end + if βktilde >r31 && βktilde < r32 + βktilde = sign(r31)*min(abs(r31),abs(r32)) + end + end + βktilde +end + + + +""" + compute_direction!(d::V, ∇fk::V, momentum::V, β::T, ::AbstractMomentumMethod) + compute_direction!(d::V, ∇fk::V, momentum::V, β::T, ::ia_momentum) + +Compute search direction. +""" + +function compute_direction!(d::V, ∇fk::V, momentum::V, βktilde::T, ::AbstractMomentumMethod) where {T,V} + d .= -∇fk + βktilde*momentum +end + +function compute_direction!(d::V, ∇fk::V, momentum::V, βktilde::T, ::ia_momentum) where {T,V} + d .= -(1-βktilde)*∇fk + βktilde*momentum +end + +""" + compute_model_red(norm_∇f::T, mdot∇f::T, βktilde::T, μk::T, ::AbstractMomentumMethod) + compute_model_red(norm_∇f::T, mdot∇f::T, βktilde::T, μk::T, ::ia_momentum) + +Compute model reduction provided by the step. +""" +function compute_model_red(norm_∇f::T, mdot∇f::T, βktilde::T, μk::T, ::AbstractMomentumMethod) where {T} + (norm_∇f^2 - βktilde* mdot∇f) * μk +end + +function compute_model_red(norm_∇f::T, mdot∇f::T, βktilde::T, μk::T, ::ia_momentum) where {T} + ((1-βktilde) * norm_∇f^2 - βktilde * mdot∇f) * μk +end + +""" + update_momentum!(∇fk::V, momentum::V, β::T, ::AbstractMomentumMethod) + update_momentum!(∇fk::V, momentum::V, β::T, ::ia_momentum) + +Update momentum. +""" + +function update_momentum!(∇fk::V, momentum::V, βk::T, ::AbstractMomentumMethod) where {T,V} + momentum .= -∇fk + βk*momentum +end + +function update_momentum!(∇fk::V, momentum::V, βk::T, ::ia_momentum) where {T,V} + momentum .= -(1-βk)*∇fk + βk*momentum +end + """ compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::AbstractMomentumMethod) compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::, ::cg_PR) @@ -752,11 +859,8 @@ end Compute β coefficient for the given momentum_backend. """ -# function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::ia_momentum) where {T,V} -# β -# end -function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::nesterov_HB) where {T,V} +function compute_beta(β::T, norm_∇fk::T, norm_∇fk₋::T, ∇fk::V, ∇fk₋::V, ::AbstractMomentumMethod) where {T,V} β end @@ -798,7 +902,7 @@ function step_mult(α::T, norm_∇fk::T, ::tr_step) where {T} end function get_θ1(m::AbstractMomentumMethod, nlp::AbstractNLPModel{T, V}) where{T, V} - if typeof(m) == nesterov_HB + if typeof(m) == nesterov_HB θ1 = get(FOMO_θ1_HB, nlp) elseif typeof(m) == cg_PR θ1 = get(FOMO_θ1_PR, nlp) @@ -810,7 +914,7 @@ function get_θ1(m::AbstractMomentumMethod, nlp::AbstractNLPModel{T, V}) where{T end function get_θ2(m::AbstractMomentumMethod, nlp::AbstractNLPModel{T, V}) where{T, V} - if typeof(m) == nesterov_HB + if typeof(m) == nesterov_HB θ2 = get(FOMO_θ2_HB, nlp) elseif typeof(m) == cg_PR θ2 = get(FOMO_θ2_PR, nlp) diff --git a/test/test_solvers.jl b/test/test_solvers.jl index a19dc1c9..80c44b3c 100644 --- a/test/test_solvers.jl +++ b/test/test_solvers.jl @@ -8,33 +8,40 @@ function tests() ("lbfgs", lbfgs), ("tron", tron), ("R2", R2), - ("fomo_r2_nesterov_HB", fomo), - ("fomo_tr_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(); kwargs...)), + ("fomo_r2_ia_momentum", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.ia_momentum(); kwargs...)), + ("fomo_tr_ia_momentum", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.ia_momentum(), step_backend = JSOSolvers.tr_step(); kwargs...)), + ("fomo_r2_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.nesterov_HB(); kwargs...)), + ("fomo_tr_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(), step_backend = JSOSolvers.tr_step(); kwargs...)), ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), - ("fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...)), - ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), - ("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ("fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(), step_backend = JSOSolvers.tr_step(); kwargs...)), + #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + #("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) end @testset "$name : nonmonotone configuration" for (name, solver) in [ ("R2", (nlp; kwargs...) -> R2(nlp, M = 2; kwargs...)), - ("fomo_r2_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, M = 2; kwargs...)), + ("fomo_r2_ia_momentum", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.ia_momentum(), M = 2; kwargs...)), + ( + "fomo_tr_ia_momentum", + (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.ia_momentum(), step_backend = JSOSolvers.tr_step(); kwargs...), + ), + ("fomo_r2_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.nesterov_HB(), M = 2; kwargs...)), ( "fomo_tr_nesterov_HB", - (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(); kwargs...), + (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.nesterov_HB(), step_backend = JSOSolvers.tr_step(); kwargs...), ), ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), ( "fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...), ), - ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), - ( - "fomo_tr_cg_FR", - (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...), - ), + #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + #( + # "fomo_tr_cg_FR", + # (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...), + #), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) From a4d8e6d4ca170333254254880b8dce16923f6667 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Sat, 24 Jan 2026 12:58:08 +0100 Subject: [PATCH 15/20] update docstring --- Project.toml | 8 ++++++++ src/fomo.jl | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index c95dc72e..c4b23046 100644 --- a/Project.toml +++ b/Project.toml @@ -3,24 +3,32 @@ uuid = "10dff2fc-5484-5881-a0e0-c90441020f8a" version = "0.14.3" [deps] +ADNLPModels = "54578032-b7ea-4c30-94aa-7cbd1cce6c9a" Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LinearOperators = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6" NLPModelsModifiers = "e01155f1-5c6f-4375-a9d8-616dd036575f" +NLPModelsTest = "7998695d-6960-4d3a-85c4-e1bceb8cd856" +OptimizationProblems = "5049e819-d29b-5fba-b941-0eee7e64c1c6" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" SolverCore = "ff4d7338-4cf1-434d-91df-b86cb86fb843" SolverParameters = "d076d87d-d1f9-4ea3-a44b-64b4cdd1e470" +SolverTest = "4343dc35-3317-4c6e-8877-f0cc8502c90e" SolverTools = "b5612192-2639-5dc1-abfe-fbedd65fab29" [compat] +ADNLPModels = "0.8.13" Krylov = "0.10.0" LinearOperators = "2.0" NLPModels = "0.21" NLPModelsModifiers = "0.7" +NLPModelsTest = "0.10.6" +OptimizationProblems = "0.9.3" SolverCore = "0.3" SolverParameters = "0.1" +SolverTest = "0.3.17" SolverTools = "0.10" julia = "1.10" diff --git a/src/fomo.jl b/src/fomo.jl index cbe6deb5..e43e0e32 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -73,7 +73,7 @@ Default values are: - `θ1 = $(FOMO_θ1)` - `θ2 = $(FOMO_θ2)` - `M = $(FOMO_M)` - - `step_backend = $(FOMO_step_backend) + - `step_backend = $(FOMO_step_backend)` - `momentum_backend = $(FOMO_momentum_backend)` """ struct FOMOParameterSet{T} <: AbstractParameterSet @@ -328,6 +328,7 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `M = $(FOMO_M)` : algorithm parameter, see [`FOMOParameterSet`](@ref). - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration. - `step_backend = $(FOMO_step_backend)`: algorithm parameter, see [`FOMOParameterSet`](@ref). +- `momentum_backend = $(FOMO_momentum_backend)`: algorithm parameter, see [`FOMOParameterSet`](@ref). # Output From da1db2e1dda78697aab0605176d213bcea7be78a Mon Sep 17 00:00:00 2001 From: d-monnet Date: Sat, 24 Jan 2026 13:12:02 +0100 Subject: [PATCH 16/20] update doc --- src/fomo.jl | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 7e769f1a..dcdd3a2c 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -70,11 +70,11 @@ Default values are: - `γ3::T = $(FOMO_γ3)` - `αmax::T = $(FOMO_αmax)` - `β = $(FOMO_β) ∈ [0,1)` - - `θ1 = $(FOMO_θ1)` - - `θ2 = $(FOMO_θ2)` - `M = $(FOMO_M)` - `step_backend = $(FOMO_step_backend)` - `momentum_backend = $(FOMO_momentum_backend)` + - `θ1 = $(FOMO_θ1)` + - `θ2 = $(FOMO_θ2)` """ struct FOMOParameterSet{T} <: AbstractParameterSet η1::Parameter{T, RealInterval{T}} @@ -84,11 +84,11 @@ struct FOMOParameterSet{T} <: AbstractParameterSet γ3::Parameter{T, RealInterval{T}} αmax::Parameter{T, RealInterval{T}} β::Parameter{T, RealInterval{T}} - θ1::Parameter{T, RealInterval{T}} - θ2::Parameter{T, RealInterval{T}} M::Parameter{Int, IntegerRange{Int}} step_backend::Parameter{Union{r2_step, tr_step}, CategoricalSet{Union{r2_step, tr_step}}} momentum_backend::Parameter{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}} + θ1::Parameter{T, RealInterval{T}} + θ2::Parameter{T, RealInterval{T}} end # add a default constructor @@ -115,12 +115,12 @@ function FOMOParameterSet( Parameter(γ2, RealInterval(T(1), T(Inf), lower_open = true, upper_open = true)), Parameter(γ3, RealInterval(T(0), T(1))), Parameter(αmax, RealInterval(T(1), T(Inf), upper_open = true)), - Parameter(β, RealInterval(T(0), T(1), upper_open = true)), - Parameter(θ1, RealInterval(T(0), T(1), lower_open = true)), - Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), + Parameter(β, RealInterval(T(0), T(1), upper_open = true)), Parameter(M, IntegerRange(Int(1), typemax(Int))), Parameter(step_backend, CategoricalSet{Union{tr_step, r2_step}}([r2_step(); tr_step()])), - Parameter(momentum_backend, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}([ia_momentum(); nesterov_HB(); cg_PR(); cg_FR()])), + Parameter(momentum_backend, CategoricalSet{Union{ia_momentum, nesterov_HB, cg_PR, cg_FR}}([ia_momentum(); nesterov_HB(); cg_PR(); cg_FR()])), + Parameter(θ1, RealInterval(T(0), T(1), lower_open = true)), + Parameter(θ2, RealInterval(T(1), T(Inf), upper_open = true)), ) end @@ -174,12 +174,12 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `max_time::Float64 = 30.0`: maximum time limit in seconds. - `max_iter::Int = typemax(Int)`: maximum number of iterations. - `β = $(FOMO_β) ∈ [0,1)`: target decay rate for the momentum. -- `θ1 = $(FOMO_θ1)`: momentum contribution parameter for convergence condition (1). -- `θ2 = $(FOMO_θ2)`: momentum contribution parameter for convergence condition (2). - `M = $(FOMO_M)` : requires objective decrease over the `M` last iterates (nonmonotone context). `M=1` implies monotone behaviour. - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration. - `step_backend = $(FOMO_step_backend)`: step computation mode. Options are `r2_step()` for quadratic regulation step and `tr_step()` for first-order trust-region. - `momentum_backend`: momentum mode for βₖ. Options are `nesterov_HB()`, `cg_PR()` for Polsak-Ribière, `cg_FR()` for Fletcher-Reeves. +- `θ1 = $(FOMO_θ1)`: momentum contribution parameter for convergence condition (1). +- `θ2 = $(FOMO_θ2)`: momentum contribution parameter for convergence condition (2). # Output @@ -241,11 +241,11 @@ end γ3::T = get(FOMO_γ3, nlp), αmax::T = get(FOMO_αmax, nlp), β::T = get(FOMO_β, nlp), - θ1::T = get(FOMO_θ1, nlp), - θ2::T = get(FOMO_θ2, nlp), M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend, nlp), + θ1::T = get(FOMO_θ1, nlp), + θ2::T = get(FOMO_θ2, nlp), kwargs..., ) where {T, V} solver = FomoSolver( @@ -310,7 +310,6 @@ For advanced usage, first define a `FomoSolver` to preallocate the memory used i - `M = $(FOMO_M)` : algorithm parameter, see [`FOMOParameterSet`](@ref). - `verbose::Int = 0`: if > 0, display iteration details every `verbose` iteration. - `step_backend = $(FOMO_step_backend)`: algorithm parameter, see [`FOMOParameterSet`](@ref). -- `momentum_backend = $(FOMO_momentum_backend)`: algorithm parameter, see [`FOMOParameterSet`](@ref). # Output From 031de3e6f3f2aad3fb39f764e62e1691ac170b93 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Mon, 16 Feb 2026 11:57:40 +0100 Subject: [PATCH 17/20] lower bound Bktilde by 0, improve cg_PR perf --- src/fomo.jl | 6 ++---- test/test_solvers.jl | 10 +++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index dcdd3a2c..40f958b9 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -466,8 +466,6 @@ function SolverCore.solve!( obj_mem[mem_ind + 1] = stats.objective max_obj_mem = stats.objective - oneT = T(1) - grad!(nlp, x, g) norm_∇fk = norm(g) d .= -g @@ -533,7 +531,7 @@ function SolverCore.solve!( siter::Int = 0 mdot∇f = T(0) # dot(momentum,∇fk) norm_m = T(0) - κ = oneT + κ = T(1) while !done μk = step_mult(solver.α, norm_d, step_backend) c .= x .+ μk .* d @@ -721,7 +719,7 @@ function find_beta_tilde( βktilde = sign(r31)*min(abs(r31),abs(r32)) end end - βktilde + max(T(0),βktilde) # PR might return negative value. Doesn't break convergence in theory, but reduces performance in practice. end function find_beta_tilde( diff --git a/test/test_solvers.jl b/test/test_solvers.jl index 1fa7c0fb..bd759285 100644 --- a/test/test_solvers.jl +++ b/test/test_solvers.jl @@ -15,7 +15,7 @@ function tests() ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), ("fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(), step_backend = JSOSolvers.tr_step(); kwargs...)), #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), - #("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) @@ -38,10 +38,10 @@ function tests() (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...), ), #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), - #( - # "fomo_tr_cg_FR", - # (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...), - #), + ( + "fomo_tr_cg_FR", + (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...), + ), ] unconstrained_nlp(solver) multiprecision_nlp(solver, :unc) From 92213e1cf293c84c96490e707b95c06d39de7dd1 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Tue, 17 Feb 2026 12:13:00 +0100 Subject: [PATCH 18/20] fix default parameters init --- src/fomo.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fomo.jl b/src/fomo.jl index 40f958b9..6777a332 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -244,8 +244,8 @@ end M::Int = get(FOMO_M, nlp), step_backend::AbstractFOMethod = get(FOMO_step_backend, nlp), momentum_backend::AbstractMomentumMethod = get(FOMO_momentum_backend, nlp), - θ1::T = get(FOMO_θ1, nlp), - θ2::T = get(FOMO_θ2, nlp), + θ1::T = get_θ1(momentum_backend, nlp), + θ2::T = get_θ2(momentum_backend, nlp), kwargs..., ) where {T, V} solver = FomoSolver( From 8773dbcf02461b8f88b5c8cca9e5a2f54bfa7423 Mon Sep 17 00:00:00 2001 From: d-monnet Date: Tue, 17 Feb 2026 12:19:07 +0100 Subject: [PATCH 19/20] update cg_FR default param, tests now pass --- src/fomo.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fomo.jl b/src/fomo.jl index 6777a332..ab2e6fb1 100644 --- a/src/fomo.jl +++ b/src/fomo.jl @@ -34,7 +34,7 @@ const FOMO_θ2_HB = const FOMO_θ1_PR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e-4), "T(1e-4)") const FOMO_θ2_PR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e3), "T(1e3)") -const FOMO_θ1_FR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1//2), "T(1/2)") +const FOMO_θ1_FR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1//2), "T(2/3)") const FOMO_θ2_FR = DefaultParameter((nlp::AbstractNLPModel) -> eltype(nlp.meta.x0)(1e2), "T(1e2)") const FOMO_M = DefaultParameter(1) From d9f58538d0b270fe6b29c34a127236da889bfbcd Mon Sep 17 00:00:00 2001 From: d-monnet Date: Tue, 17 Feb 2026 14:23:05 +0100 Subject: [PATCH 20/20] add cg_FR tests --- test/test_solvers.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_solvers.jl b/test/test_solvers.jl index bd759285..8f677360 100644 --- a/test/test_solvers.jl +++ b/test/test_solvers.jl @@ -14,7 +14,7 @@ function tests() ("fomo_tr_nesterov_HB", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(), step_backend = JSOSolvers.tr_step(); kwargs...)), ("fomo_r2_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(); kwargs...)), ("fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_PR(), step_backend = JSOSolvers.tr_step(); kwargs...)), - #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ("fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ] unconstrained_nlp(solver) @@ -37,7 +37,7 @@ function tests() "fomo_tr_cg_PR", (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_PR(); kwargs...), ), - #("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), + ("fomo_r2_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, momentum_backend = JSOSolvers.cg_FR(); kwargs...)), ( "fomo_tr_cg_FR", (nlp; kwargs...) -> fomo(nlp, M = 2, step_backend = JSOSolvers.tr_step(), momentum_backend = JSOSolvers.cg_FR(); kwargs...),