From 71065e7ab87cce6f6c1be90ad075c9304b560384 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 24 Sep 2024 14:05:50 -0400 Subject: [PATCH 01/92] refactor: migrate to LineSearch.jl --- src/internal/termination.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/termination.jl b/src/internal/termination.jl index ef3f7c4c0..e09cfcdda 100644 --- a/src/internal/termination.jl +++ b/src/internal/termination.jl @@ -45,7 +45,7 @@ function update_from_termination_cache!(tc_cache, cache, u = get_u(cache)) end function update_from_termination_cache!( - tc_cache, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) + _, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) evaluate_f!(cache, u, cache.p) end From 543f086d2e37b670b522881b8018eb07818115d1 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 26 Sep 2024 05:20:29 -0400 Subject: [PATCH 02/92] fix: forward reinit_cache to SciMLBase.reinit --- src/internal/termination.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/termination.jl b/src/internal/termination.jl index e09cfcdda..ef3f7c4c0 100644 --- a/src/internal/termination.jl +++ b/src/internal/termination.jl @@ -45,7 +45,7 @@ function update_from_termination_cache!(tc_cache, cache, u = get_u(cache)) end function update_from_termination_cache!( - _, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) + tc_cache, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) evaluate_f!(cache, u, cache.p) end From 656053be85de194e06021dddaf16d8dad83ca408 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 24 Sep 2024 14:05:50 -0400 Subject: [PATCH 03/92] refactor: migrate to LineSearch.jl --- src/internal/termination.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/termination.jl b/src/internal/termination.jl index ef3f7c4c0..e09cfcdda 100644 --- a/src/internal/termination.jl +++ b/src/internal/termination.jl @@ -45,7 +45,7 @@ function update_from_termination_cache!(tc_cache, cache, u = get_u(cache)) end function update_from_termination_cache!( - tc_cache, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) + _, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) evaluate_f!(cache, u, cache.p) end From bca5579ec9283003bae1dad6aaed283f48b10a9b Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 17:24:51 -0700 Subject: [PATCH 04/92] feat: setup NonlinearSolveBase --- lib/NonlinearSolveBase/LICENSE | 21 ++ lib/NonlinearSolveBase/Project.toml | 39 ++++ .../ext/NonlinearSolveBaseForwardDiffExt.jl | 9 + .../ext/NonlinearSolveBaseSparseArraysExt.jl | 10 + .../src/NonlinearSolveBase.jl | 28 +++ lib/NonlinearSolveBase/src/autodiff.jl | 0 lib/NonlinearSolveBase/src/common_defaults.jl | 47 +++++ .../src/immutable_problem.jl | 0 lib/NonlinearSolveBase/src/public.jl | 118 ++++++++++++ .../src/termination_conditions.jl | 180 ++++++++++++++++++ lib/NonlinearSolveBase/src/utils.jl | 76 ++++++++ lib/NonlinearSolveBase/test/runtests.jl | 1 + 12 files changed, 529 insertions(+) create mode 100644 lib/NonlinearSolveBase/LICENSE create mode 100644 lib/NonlinearSolveBase/Project.toml create mode 100644 lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl create mode 100644 lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl create mode 100644 lib/NonlinearSolveBase/src/NonlinearSolveBase.jl create mode 100644 lib/NonlinearSolveBase/src/autodiff.jl create mode 100644 lib/NonlinearSolveBase/src/common_defaults.jl create mode 100644 lib/NonlinearSolveBase/src/immutable_problem.jl create mode 100644 lib/NonlinearSolveBase/src/public.jl create mode 100644 lib/NonlinearSolveBase/src/termination_conditions.jl create mode 100644 lib/NonlinearSolveBase/src/utils.jl create mode 100644 lib/NonlinearSolveBase/test/runtests.jl diff --git a/lib/NonlinearSolveBase/LICENSE b/lib/NonlinearSolveBase/LICENSE new file mode 100644 index 000000000..411ad533f --- /dev/null +++ b/lib/NonlinearSolveBase/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Avik Pal and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml new file mode 100644 index 000000000..1c7047ca6 --- /dev/null +++ b/lib/NonlinearSolveBase/Project.toml @@ -0,0 +1,39 @@ +name = "NonlinearSolveBase" +uuid = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" +authors = ["Avik Pal and contributors"] +version = "1.0.0" + +[deps] +ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" +ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" +RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +UnrolledUtilities = "0fe1646c-419e-43be-ac14-22321958931b" + +[weakdeps] +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[extensions] +NonlinearSolveBaseForwardDiffExt = "ForwardDiff" +NonlinearSolveBaseSparseArraysExt = "SparseArrays" + +[compat] +ArrayInterface = "7.9" +Compat = "4.15" +ConcreteStructs = "0.2.3" +FastClosures = "0.3" +ForwardDiff = "0.10.36" +LinearAlgebra = "1.10" +Markdown = "1.10" +RecursiveArrayTools = "3" +SciMLBase = "2.50" +SparseArrays = "1.10" +StaticArraysCore = "1.4" +UnrolledUtilities = "0.1" +julia = "1.10" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl new file mode 100644 index 000000000..c29fea0d1 --- /dev/null +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -0,0 +1,9 @@ +module NonlinearSolveBaseForwardDiffExt + +using ForwardDiff: ForwardDiff, Dual +using NonlinearSolveBase: Utils + +Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V +Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x)) + +end \ No newline at end of file diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl new file mode 100644 index 000000000..d20e9b59b --- /dev/null +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl @@ -0,0 +1,10 @@ +module NonlinearSolveBaseSparseArraysExt + +using NonlinearSolveBase: NonlinearSolveBase +using SparseArrays: AbstractSparseMatrixCSC, nonzeros + +function NonlinearSolveBase.NAN_CHECK(x::AbstractSparseMatrixCSC) + return any(NonlinearSolveBase.NAN_CHECK, nonzeros(x)) +end + +end \ No newline at end of file diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl new file mode 100644 index 000000000..aff03d1e3 --- /dev/null +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -0,0 +1,28 @@ +module NonlinearSolveBase + +using ArrayInterface: ArrayInterface +using Compat: @compat +using ConcreteStructs: @concrete +using FastClosures: @closure +using LinearAlgebra: norm +using Markdown: @doc_str +using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition +using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator +using StaticArraysCore: StaticArray + +include("public.jl") +include("utils.jl") + +include("common_defaults.jl") +include("termination_conditions.jl") +include("autodiff.jl") +include("immutable_problem.jl") + +# Unexported Public API +@compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance)) + +export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTerminationMode, + AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, + RelNormSafeNormTerminationMode, AbsNormSafeNormTerminationMode + +end diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/NonlinearSolveBase/src/common_defaults.jl b/lib/NonlinearSolveBase/src/common_defaults.jl new file mode 100644 index 000000000..4518063a5 --- /dev/null +++ b/lib/NonlinearSolveBase/src/common_defaults.jl @@ -0,0 +1,47 @@ +UNITLESS_ABS2(x::Number) = abs2(x) +function UNITLESS_ABS2(x::AbstractArray) + return mapreduce(UNITLESS_ABS2, Utils.abs2_and_sum, x; init = Utils.zero_init(x)) +end +function UNITLESS_ABS2(x::Union{AbstractVectorOfArray, ArrayPartition}) + return mapreduce(UNITLESS_ABS2, Utils.abs2_and_sum, + Utils.get_internal_array(x); init = Utils.zero_init(x)) +end + +NAN_CHECK(x::Number) = isnan(x) +NAN_CHECK(x::Enum) = false +NAN_CHECK(x::AbstractArray) = any(NAN_CHECK, x) +function NAN_CHECK(x::Union{AbstractVectorOfArray, ArrayPartition}) + return any(NAN_CHECK, Utils.get_internal_array(x)) +end + +L2_NORM(u::Union{AbstractFloat, Complex}) = @fastmath abs(u) +L2_NORM(u::AbstractArray) = @fastmath sqrt(UNITLESS_ABS2(u)) +function L2_NORM(u::AbstractArray{<:Union{AbstractFloat, Complex}}) + if Utils.fast_scalar_indexing(u) + x = zero(eltype(u)) + @simd for i in eachindex(u) + @inbounds @fastmath x += abs2(u[i]) + end + return @fastmath sqrt(real(x)) + end + return @fastmath sqrt(UNITLESS_ABS2(u)) +end +function L2_NORM(u::StaticArray{<:Union{AbstractFloat, Complex}}) + return @fastmath sqrt(real(sum(abs2, u))) +end +L2_NORM(u) = norm(u, 2) + +Linf_NORM(u::Union{AbstractFloat, Complex}) = @fastmath abs(u) +Linf_NORM(u) = maximum(abs, u) + +get_tolerance(η, ::Type{T}) where {T} = Utils.convert_real(T, η) +function get_tolerance(::Nothing, ::Type{T}) where {T} + η = real(oneunit(T)) * (eps(real(one(T)))^(4 // 5)) + return get_tolerance(η, T) +end + +get_tolerance(_, η, ::Type{T}) where {T} = get_tolerance(η, T) +function get_tolerance(::Union{StaticArray, Number}, ::Nothing, ::Type{T}) where {T} + # Rational numbers can throw an error if used inside GPU Kernels + return T(real(oneunit(T)) * (eps(real(one(T)))^(real(T)(0.8)))) +end diff --git a/lib/NonlinearSolveBase/src/immutable_problem.jl b/lib/NonlinearSolveBase/src/immutable_problem.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl new file mode 100644 index 000000000..24f467cbd --- /dev/null +++ b/lib/NonlinearSolveBase/src/public.jl @@ -0,0 +1,118 @@ +# forward declarations of public API +function UNITLESS_ABS2 end +function NAN_CHECK end +function L2_NORM end +function Linf_NORM end +function get_tolerance end + +# Nonlinear Solve Termination Conditions +abstract type AbstractNonlinearTerminationMode end +abstract type AbstractSafeNonlinearTerminationMode <: AbstractNonlinearTerminationMode end +abstract type AbstractSafeBestNonlinearTerminationMode <: + AbstractSafeNonlinearTerminationMode end + +#! format: off +const TERM_DOCS = Dict( + :Norm => doc"``\| \Delta u \| \leq reltol \times \| \Delta u + u \|`` or ``\| \Delta u \| \leq abstol``.", + :Rel => doc"``all \left(| \Delta u | \leq reltol \times | u | \right)``.", + :RelNorm => doc"``\| \Delta u \| \leq reltol \times \| \Delta u + u \|``.", + :Abs => doc"``all \left( | \Delta u | \leq abstol \right)``.", + :AbsNorm => doc"``\| \Delta u \| \leq abstol``." +) + +const TERM_INTERNALNORM_DOCS = """ +where `internalnorm` is the norm to use for the termination condition. Special handling is +done for `norm(_, 2)`, `norm`, `norm(_, Inf)`, and `maximum(abs, _)`.""" +#! format: on + +for name in (:Rel, :Abs) + struct_name = Symbol(name, :TerminationMode) + doctring = TERM_DOCS[name] + + @eval begin + """ + $($struct_name) <: AbstractNonlinearTerminationMode + + Terminates if $($doctring). + + ``\\Delta u`` denotes the increment computed by the nonlinear solver and ``u`` denotes the solution. + """ + struct $(struct_name) <: AbstractNonlinearTerminationMode end + end +end + +for name in (:Norm, :RelNorm, :AbsNorm) + struct_name = Symbol(name, :TerminationMode) + doctring = TERM_DOCS[name] + + @eval begin + """ + $($struct_name) <: AbstractSafeNonlinearTerminationMode + + Terminates if $($doctring). + + ``\\Delta u`` denotes the increment computed by the inner nonlinear solver. + + ## Constructor + + $($struct_name)(internalnorm = nothing) + + $($TERM_INTERNALNORM_DOCS). + """ + struct $(struct_name){F} <: AbstractSafeNonlinearTerminationMode + internalnorm::F + + function $(struct_name)(internalnorm::F) where {F} + norm = Utils.standardize_norm(internalnorm) + return new{typeof(norm)}(norm) + end + end + end +end + +for norm_type in (:RelNorm, :AbsNorm), safety in (:Safe, :SafeBest) + struct_name = Symbol(norm_type, safety, :TerminationMode) + supertype_name = Symbol(:Abstract, safety, :NonlinearTerminationMode) + + doctring = safety == :Safe ? + "Essentially [`$(norm_type)NormTerminationMode`](@ref) + terminate if there \ + has been no improvement for the last `patience_steps` + terminate if the \ + solution blows up (diverges)." : + "Essentially [`$(norm_type)SafeTerminationMode`](@ref), but caches the best\ + solution found so far." + + @eval begin + """ + $($struct_name) <: $($supertype_name) + + $($doctring) + + ## Constructor + + $($struct_name)(internalnorm; protective_threshold = nothing, + patience_steps = 100, patience_objective_multiplier = 3, + min_max_factor = 1.3, max_stalled_steps = nothing) + + $($TERM_INTERNALNORM_DOCS). + """ + @concrete struct $(struct_name) <: $(supertype_name) + internalnorm + protective_threshold + patience_steps::Int + patience_objective_multiplier + min_max_factor + max_stalled_steps <: Union{Nothing, Int} + + function $(struct_name)(internalnorm::F; protective_threshold = nothing, + patience_steps = 100, patience_objective_multiplier = 3, + min_max_factor = 1.3, max_stalled_steps = nothing) where {F} + norm = Utils.standardize_norm(internalnorm) + return new{typeof(norm), typeof(protective_threshold), + typeof(patience_objective_multiplier), + typeof(min_max_factor), typeof(max_stalled_steps)}( + norm, protective_threshold, patience_steps, + patience_objective_multiplier, min_max_factor, max_stalled_steps) + end + end + end +end diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl new file mode 100644 index 000000000..3cbba4b8b --- /dev/null +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -0,0 +1,180 @@ +const RelNormModes = Union{ + RelNormTerminationMode, RelNormSafeTerminationMode, RelNormSafeBestTerminationMode} +const AbsNormModes = Union{ + AbsNormTerminationMode, AbsNormSafeTerminationMode, AbsNormSafeBestTerminationMode} + +# Core Implementation +@concrete mutable struct NonlinearTerminationModeCache{uType, T} + u::uType + retcode::ReturnCode.T + abstol::T + reltol::T + best_objective_value::T + mode + initial_objective + objectives_trace + nsteps::Int + saved_values + u0_norm + step_norm_trace + max_stalled_steps + u_diff_cache::uType +end + +function update_u!!(cache::NonlinearTerminationModeCache, u) + cache.u === nothing && return + if cache.u isa AbstractArray && ArrayInterface.can_setindex(cache.u) + copyto!(cache.u, u) + else + cache.u .= u + end +end + +function SciMLBase.init(du::Union{AbstractArray{T}, T}, u::Union{AbstractArray{T}, T}, + mode::AbstractNonlinearTerminationMode, saved_value_prototype...; + abstol = nothing, reltol = nothing, kwargs...) where {T <: Number} + error("Not yet implemented...") +end + +function SciMLBase.reinit!( + cache::NonlinearTerminationModeCache, du, u, saved_value_prototype...; + abstol = nothing, reltol = nothing, kwargs...) + error("Not yet implemented...") +end + +## This dispatch is needed based on how Terminating Callback works! +function (cache::NonlinearTerminationModeCache)( + integrator::AbstractODEIntegrator, abstol::Number, reltol::Number, min_t) + if min_t === nothing || integrator.t ≥ min_t + return cache(cache.mode, SciMLBase.get_du(integrator), + integrator.u, integrator.uprev, abstol, reltol) + end + return false +end +function (cache::NonlinearTerminationModeCache)(du, u, uprev, args...) + return cache(cache.mode, du, u, uprev, cache.abstol, cache.reltol, args...) +end + +function (cache::NonlinearTerminationModeCache)( + mode::AbstractNonlinearTerminationMode, du, u, uprev, abstol, reltol, args...) + if check_convergence(mode, du, u, uprev, abstol, reltol) + cache.retcode = ReturnCode.Success + return true + end + return false +end + +function (cache::NonlinearTerminationModeCache)( + mode::AbstractSafeNonlinearTerminationMode, du, u, uprev, abstol, reltol, args...) + if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode + objective = Utils.apply_norm(mode.internalnorm, du) + criteria = abstol + else + objective = Utils.apply_norm(mode.internalnorm, du) / + (Utils.apply_norm(mode.internalnorm, du, u) + eps(reltol)) + criteria = reltol + end + + # Protective Break + if !isfinite(objective) + cache.retcode = ReturnCode.Unstable + return true + end + + # By default we turn this off since it have potential for false positives + if mode.protective_threshold !== nothing && + (objective > cache.initial_objective * mode.protective_threshold * length(du)) + cache.retcode = ReturnCode.Unstable + return true + end + + # Check if it is the best solution + if mode isa AbstractSafeBestNonlinearTerminationMode && + objective < cache.best_objective_value + cache.best_objective_value = objective + update_u!!(cache, u) + cache.saved_values !== nothing && length(args) ≥ 1 && (cache.saved_values = args) + end + + # Main Termination Criteria + if objective ≤ criteria + cache.retcode = ReturnCode.Success + return true + end + + # Terminate if we haven't improved for the last `patience_steps` + cache.nsteps += 1 + cache.nsteps == 1 && (cache.initial_objective = objective) + cache.objectives_trace[mod1(cache.nsteps, length(cache.objectives_trace))] = objective + + if objective ≤ mode.patience_objective_multiplier * criteria && + cache.nsteps > mode.patience_steps + if cache.nsteps < length(cache.objectives_trace) + min_obj, max_obj = extrema(@view(cache.objectives_trace[1:(cache.nsteps)])) + else + min_obj, max_obj = extrema(cache.objectives_trace) + end + if min_obj < mode.min_max_factor * max_obj + cache.retcode = ReturnCode.Stalled + return true + end + end + + # Test for stalling if that is enabled + if cache.step_norm_trace !== nothing + if ArrayInterface.can_setindex(cache.u_diff_cache) && !(u isa Number) + @. cache.u_diff_cache = u - uprev + else + cache.u_diff_cache = u .- uprev + end + du_norm = L2_NORM(cache.u_diff_cache) + cache.step_norm_trace[mod1(cache.nsteps, length(cache.step_norm_trace))] = du_norm + if cache.nsteps > mode.max_stalled_steps + max_step_norm = maximum(cache.step_norm_trace) + if mode isa AbsNormSafeTerminationMode || + mode isa AbsNormSafeBestTerminationMode + stalled_step = max_step_norm ≤ abstol + else + stalled_step = max_step_norm ≤ reltol * (max_step_norm + cache.u0_norm) + end + if stalled_step + cache.retcode = ReturnCode.Stalled + return true + end + end + end + + cache.retcode = ReturnCode.Failure + return false +end + +# Check Convergence +function check_convergence(::RelTerminationMode, duₙ, uₙ, __, ___, reltol) + if Utils.fast_scalar_indexing(duₙ) + return all(@closure(xy->begin + x, y = xy + return abs(y) ≤ reltol * abs(x + y) + end), zip(uₙ, duₙ)) + else # using mapreduce here will almost certainly be faster on GPUs + return mapreduce( + @closure((xᵢ, yᵢ)->(abs(yᵢ) ≤ reltol * abs(xᵢ + yᵢ))), *, uₙ, duₙ; init = true) + end +end +function check_convergence(::AbsTerminationMode, duₙ, _, __, abstol, ___) + return all(@closure(x->abs(x) ≤ abstol), duₙ) +end + +function check_convergence(norm::NormTerminationMode, duₙ, uₙ, _, abstol, reltol) + du_norm = Utils.apply_norm(norm.internalnorm, duₙ) + return (du_norm ≤ abstol) || + (du_norm ≤ reltol * Utils.apply_norm(norm.internalnorm, duₙ, uₙ)) +end + +function check_convergence(mode::RelNormModes, duₙ, uₙ, _, __, reltol) + du_norm = Utils.apply_norm(mode.internalnorm, duₙ) + return du_norm ≤ reltol * Utils.apply_norm(mode.internalnorm, duₙ, uₙ) +end + +function check_convergence(mode::AbsNormModes, duₙ, _, __, abstol, ___) + return Utils.apply_norm(mode.internalnorm, duₙ) ≤ abstol +end diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl new file mode 100644 index 000000000..f830e9a17 --- /dev/null +++ b/lib/NonlinearSolveBase/src/utils.jl @@ -0,0 +1,76 @@ +module Utils + +using ArrayInterface: ArrayInterface +using FastClosures: @closure +using LinearAlgebra: norm +using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition +using UnrolledUtilities: unrolled_all + +using ..NonlinearSolveBase: L2_NORM, Linf_NORM + +fast_scalar_indexing(xs...) = unrolled_all(ArrayInterface.fast_scalar_indexing, xs) + +function nonallocating_isapprox(x::Number, y::Number; atol = false, + rtol = atol > 0 ? false : sqrt(eps(promote_type(typeof(x), typeof(y))))) + return isapprox(x, y; atol, rtol) +end +function nonallocating_isapprox(x::AbstractArray, y::AbstractArray; atol = false, + rtol = atol > 0 ? false : sqrt(eps(eltype(x)))) + length(x) == length(y) || return false + d = nonallocating_maximum(-, x, y) + return d ≤ max(atol, rtol * max(maximum(abs, x), maximum(abs, y))) +end + +function nonallocating_maximum(f::F, x, y) where {F} + if fast_scalar_indexing(x, y) + return maximum(@closure((xᵢyᵢ)->begin + xᵢ, yᵢ = xᵢyᵢ + return abs(f(xᵢ, yᵢ)) + end), zip(x, y)) + else + return mapreduce(@closure((xᵢ, yᵢ)->abs(f(xᵢ, yᵢ))), max, x, y) + end +end + +function abs2_and_sum(x, y) + return reduce(Base.add_sum, x, init = zero(real(value(eltype(x))))) + + reduce(Base.add_sum, y, init = zero(real(value(eltype(y))))) +end + +children(x::AbstractVectorOfArray) = x.u +children(x::ArrayPartition) = x.x + +value(::Type{T}) where {T} = T +value(x) = x + +zero_init(x) = zero(real(value(eltype(x)))) +one_init(x) = one(real(value(eltype(x)))) + +standardize_norm(::typeof(Base.Fix1(maximum, abs))) = Linf_NORM +standardize_norm(::typeof(Base.Fix2(norm, Inf))) = Linf_NORM +standardize_norm(::typeof(Base.Fix2(norm, 2))) = L2_NORM +standardize_norm(::typeof(norm)) = L2_NORM +standardize_norm(f::F) where {F} = f + +norm_op(norm::N, op::OP, x, y) where {N, OP} = norm(op.(x, y)) +function norm_op(::typeof(L2_NORM), op::OP, x, y) where {OP} + if fast_scalar_indexing(x, y) + return sqrt(sum(@closure((xᵢ, yᵢ)->begin + xᵢ, yᵢ = xᵢyᵢ + return op(xᵢ, yᵢ)^2 + end), zip(x, y))) + else + return sqrt(mapreduce(@closure((xᵢ, yᵢ)->op(xᵢ, yᵢ)^2), +, x, y)) + end +end +function norm_op(::typeof(Linf_NORM), op::OP, x, y) where {OP} + return nonallocating_maximum(abs ∘ op, x, y) +end + +apply_norm(f::F, x) where {F} = standardize_norm(f)(x) +apply_norm(f::F, x, y) where {F} = norm_op(standardize_norm(f), +, x, y) + +convert_real(::Type{T}, ::Nothing) where {T} = nothing +convert_real(::Type{T}, x) where {T} = real(T(x)) + +end diff --git a/lib/NonlinearSolveBase/test/runtests.jl b/lib/NonlinearSolveBase/test/runtests.jl new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/lib/NonlinearSolveBase/test/runtests.jl @@ -0,0 +1 @@ + From 90ca15997ec874a428f9a9402d4a4ce7dc54e7bb Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 17:38:35 -0700 Subject: [PATCH 05/92] chore: run formatter --- lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl | 2 +- lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl | 2 +- lib/NonlinearSolveBase/src/autodiff.jl | 1 + lib/NonlinearSolveBase/src/immutable_problem.jl | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index c29fea0d1..ae342e4ba 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -6,4 +6,4 @@ using NonlinearSolveBase: Utils Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x)) -end \ No newline at end of file +end diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl index d20e9b59b..be13ebb8f 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseSparseArraysExt.jl @@ -7,4 +7,4 @@ function NonlinearSolveBase.NAN_CHECK(x::AbstractSparseMatrixCSC) return any(NonlinearSolveBase.NAN_CHECK, nonzeros(x)) end -end \ No newline at end of file +end diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index e69de29bb..8b1378917 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -0,0 +1 @@ + diff --git a/lib/NonlinearSolveBase/src/immutable_problem.jl b/lib/NonlinearSolveBase/src/immutable_problem.jl index e69de29bb..8b1378917 100644 --- a/lib/NonlinearSolveBase/src/immutable_problem.jl +++ b/lib/NonlinearSolveBase/src/immutable_problem.jl @@ -0,0 +1 @@ + From 14442d4bac34d8491fd4d770f67a98a2ce3b6954 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 17:45:04 -0700 Subject: [PATCH 06/92] feat: add ImmutableNonlinearProblem --- lib/NonlinearSolveBase/Project.toml | 2 + .../ext/NonlinearSolveBaseDiffEqBaseExt.jl | 14 +++++ .../src/NonlinearSolveBase.jl | 6 +- .../src/immutable_problem.jl | 56 +++++++++++++++++++ 4 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 1c7047ca6..f21e15626 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -16,10 +16,12 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" UnrolledUtilities = "0fe1646c-419e-43be-ac14-22321958931b" [weakdeps] +DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] +NonlinearSolveBaseDiffEqBaseExt = "DiffEqBase" NonlinearSolveBaseForwardDiffExt = "ForwardDiff" NonlinearSolveBaseSparseArraysExt = "SparseArrays" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl new file mode 100644 index 000000000..a3a216cf2 --- /dev/null +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl @@ -0,0 +1,14 @@ +module NonlinearSolveBaseDiffEqBaseExt + +using DiffEqBase: DiffEqBase +using NonlinearSolveBase: ImmutableNonlinearProblem + +function DiffEqBase.get_concrete_problem( + prob::ImmutableNonlinearProblem, isadapt; kwargs...) + u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs) + u0 = DiffEqBase.promote_u0(u0, prob.p, nothing) + p = DiffEqBase.get_concrete_p(prob, kwargs) + return SciMLBase.remake(prob; u0, p) +end + +end diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index aff03d1e3..71c634ef3 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -7,7 +7,9 @@ using FastClosures: @closure using LinearAlgebra: norm using Markdown: @doc_str using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition -using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator +using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinearProblem, + AbstractNonlinearFunction, @add_kwonly, StandardNonlinearProblem, + NullParameters, NonlinearProblem, isinplace using StaticArraysCore: StaticArray include("public.jl") @@ -25,4 +27,6 @@ export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTermi AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, RelNormSafeNormTerminationMode, AbsNormSafeNormTerminationMode +export ImmutableNonlinearProblem + end diff --git a/lib/NonlinearSolveBase/src/immutable_problem.jl b/lib/NonlinearSolveBase/src/immutable_problem.jl index 8b1378917..03b44a9ec 100644 --- a/lib/NonlinearSolveBase/src/immutable_problem.jl +++ b/lib/NonlinearSolveBase/src/immutable_problem.jl @@ -1 +1,57 @@ +struct ImmutableNonlinearProblem{uType, iip, P, F, K, PT} <: + AbstractNonlinearProblem{uType, iip} + f::F + u0::uType + p::P + problem_type::PT + kwargs::K + @add_kwonly function ImmutableNonlinearProblem{iip}( + f::AbstractNonlinearFunction{iip}, u0, p = NullParameters(), + problem_type = StandardNonlinearProblem(); kwargs...) where {iip} + if haskey(kwargs, :p) + error("`p` specified as a keyword argument `p = $(kwargs[:p])` to \ + `NonlinearProblem`. This is not supported.") + end + warn_paramtype(p) + return new{ + typeof(u0), iip, typeof(p), typeof(f), typeof(kwargs), typeof(problem_type)}( + f, u0, p, problem_type, kwargs) + end + + """ + Define a steady state problem using the given function. + `isinplace` optionally sets whether the function is inplace or not. + This is determined automatically, but not inferred. + """ + function ImmutableNonlinearProblem{iip}( + f, u0, p = NullParameters(); kwargs...) where {iip} + return ImmutableNonlinearProblem{iip}(NonlinearFunction{iip}(f), u0, p; kwargs...) + end +end + +""" +Define a nonlinear problem using an instance of +[`AbstractNonlinearFunction`](@ref AbstractNonlinearFunction). +""" +function ImmutableNonlinearProblem( + f::AbstractNonlinearFunction, u0, p = NullParameters(); kwargs...) + return ImmutableNonlinearProblem{isinplace(f)}(f, u0, p; kwargs...) +end + +function ImmutableNonlinearProblem(f, u0, p = NullParameters(); kwargs...) + return ImmutableNonlinearProblem(NonlinearFunction(f), u0, p; kwargs...) +end + +""" +Define a ImmutableNonlinearProblem problem from SteadyStateProblem +""" +function ImmutableNonlinearProblem(prob::AbstractNonlinearProblem) + return ImmutableNonlinearProblem{isinplace(prob)}(prob.f, prob.u0, prob.p) +end + +function Base.convert( + ::Type{ImmutableNonlinearProblem}, prob::T) where {T <: NonlinearProblem} + return ImmutableNonlinearProblem{isinplace(prob)}( + prob.f, prob.u0, prob.p, prob.problem_type; prob.kwargs...) +end From ac7bbae2016bedaec780b5f3751ced17284fcd60 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 18:08:55 -0700 Subject: [PATCH 07/92] fix: finish broken termination conditions --- .../src/termination_conditions.jl | 79 +++++++++++++++++-- 1 file changed, 73 insertions(+), 6 deletions(-) diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 3cbba4b8b..50af54a57 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -30,16 +30,83 @@ function update_u!!(cache::NonlinearTerminationModeCache, u) end end -function SciMLBase.init(du::Union{AbstractArray{T}, T}, u::Union{AbstractArray{T}, T}, - mode::AbstractNonlinearTerminationMode, saved_value_prototype...; - abstol = nothing, reltol = nothing, kwargs...) where {T <: Number} - error("Not yet implemented...") +function SciMLBase.init( + du, u, mode::AbstractNonlinearTerminationMode, saved_value_prototype...; + abstol = nothing, reltol = nothing, kwargs...) + T = promote_type(eltype(du), eltype(u)) + abstol = get_tolerance(abstol, T) + reltol = get_tolerance(reltol, T) + TT = typeof(abstol) + + u_unaliased = mode isa AbstractSafeBestNonlinearTerminationMode ? + (ArrayInterface.can_setindex(u) ? copy(u) : u) : nothing + + if mode isa AbstractSafeNonlinearTerminationMode + if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode + initial_objective = Linf_NORM(du) + u0_norm = nothing + else + initial_objective = Linf_NORM(du) / + (Utils.nonallocating_maximum(+, du, u) + eps(TT)) + u0_norm = mode.max_stalled_steps === nothing ? nothing : L2_NORM(u) + end + objectives_trace = Vector{TT}(undef, mode.patience_steps) + step_norm_trace = mode.max_stalled_steps === nothing ? nothing : + Vector{TT}(undef, mode.max_stalled_steps) + if step_norm_trace !== nothing && + ArrayInterface.can_setindex(u_unaliased) && + !(u_unaliased isa Number) + u_diff_cache = similar(u_unaliased) + else + u_diff_cache = u_unaliased + end + else + initial_objective = nothing + objectives_trace = nothing + u0_norm = nothing + step_norm_trace = nothing + best_value = Utils.convert_real(T, Inf) + max_stalled_steps = nothing + u_diff_cache = u_unaliased + end + + length(saved_value_prototype) == 0 && (saved_value_prototype = nothing) + + return NonlinearTerminationModeCache( + u_unaliased, ReturnCode.Default, abstol, reltol, best_value, mode, + initial_objective, objectives_trace, 0, saved_value_prototype, + u0_norm, step_norm_trace, max_stalled_steps, u_diff_cache) end function SciMLBase.reinit!( cache::NonlinearTerminationModeCache, du, u, saved_value_prototype...; - abstol = nothing, reltol = nothing, kwargs...) - error("Not yet implemented...") + abstol = cache.abstol, reltol = cache.reltol, kwargs...) + T = eltype(cache.abstol) + length(saved_value_prototype) != 0 && (cache.saved_values = saved_value_prototype) + + mode = cache.mode + u_unaliased = mode isa AbstractSafeBestNonlinearTerminationMode ? + (ArrayInterface.can_setindex(u) ? copy(u) : u) : nothing + cache.u = u_unaliased + cache.retcode = ReturnCode.Default + + cache.abstol = get_tolerance(abstol, T) + cache.reltol = get_tolerance(reltol, T) + cache.nsteps = 0 + TT = typeof(cache.abstol) + + if mode isa AbstractSafeNonlinearTerminationMode + if mode isa AbsNormSafeTerminationMode || mode isa AbsNormSafeBestTerminationMode + cache.initial_objective = Linf_NORM(du) + else + cache.initial_objective = Linf_NORM(du) / + (Utils.nonallocating_maximum(+, du, u) + eps(TT)) + cache.max_stalled_steps !== nothing && (cache.u0_norm = L2_NORM(u)) + end + cache.best_objective_value = cache.initial_objective + else + cache.best_objective_value = Utils.convert_real(T, Inf) + end end ## This dispatch is needed based on how Terminating Callback works! From 102c3020d2070855c142d42d0da7eb80361d25db Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 19:32:26 -0700 Subject: [PATCH 08/92] feat: start a SimpleBracketingNonlinearSolve sublibrary --- lib/SimpleBracketingNonlinearSolve/Project.toml | 13 +++++++++++++ .../src/SimpleBracketingNonlinearSolve.jl | 8 ++++++++ lib/SimpleBracketingNonlinearSolve/test/runtests.jl | 0 3 files changed, 21 insertions(+) create mode 100644 lib/SimpleBracketingNonlinearSolve/Project.toml create mode 100644 lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl create mode 100644 lib/SimpleBracketingNonlinearSolve/test/runtests.jl diff --git a/lib/SimpleBracketingNonlinearSolve/Project.toml b/lib/SimpleBracketingNonlinearSolve/Project.toml new file mode 100644 index 000000000..e3462f3bc --- /dev/null +++ b/lib/SimpleBracketingNonlinearSolve/Project.toml @@ -0,0 +1,13 @@ +name = "SimpleBracketingNonlinearSolve" +uuid = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" +authors = ["Avik Pal and contributors"] +version = "1.0.0" + +[deps] +CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" + +[compat] +CommonSolve = "0.2.4" +SciMLBase = "2.50" +julia = "1.10" diff --git a/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl b/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl new file mode 100644 index 000000000..6bac6bdaa --- /dev/null +++ b/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl @@ -0,0 +1,8 @@ +module SimpleBracketingNonlinearSolve + +using CommonSolve: CommonSolve +using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, IntervalNonlinearProblem, ReturnCode + +abstract type AbstractBracketingAlgorithm <: AbstractNonlinearAlgorithm end + +end diff --git a/lib/SimpleBracketingNonlinearSolve/test/runtests.jl b/lib/SimpleBracketingNonlinearSolve/test/runtests.jl new file mode 100644 index 000000000..e69de29bb From 4ed8f11ff1c32e996b24bd0f5753afb7fbe0eeb6 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 21:30:37 -0700 Subject: [PATCH 09/92] feat: implement all bracketing algorithms --- .../Project.toml | 8 +- .../src/BracketingNonlinearSolve.jl | 38 +++++ lib/BracketingNonlinearSolve/src/alefeld.jl | 135 +++++++++++++++++ lib/BracketingNonlinearSolve/src/bisection.jl | 81 ++++++++++ lib/BracketingNonlinearSolve/src/brent.jl | 109 ++++++++++++++ lib/BracketingNonlinearSolve/src/common.jl | 91 +++++++++++ lib/BracketingNonlinearSolve/src/falsi.jl | 71 +++++++++ lib/BracketingNonlinearSolve/src/itp.jl | 141 ++++++++++++++++++ lib/BracketingNonlinearSolve/src/ridder.jl | 83 +++++++++++ .../test/runtests.jl | 0 lib/NonlinearSolveBase/Project.toml | 2 - .../ext/NonlinearSolveBaseDiffEqBaseExt.jl | 14 -- .../src/SimpleBracketingNonlinearSolve.jl | 8 - 13 files changed, 756 insertions(+), 25 deletions(-) rename lib/{SimpleBracketingNonlinearSolve => BracketingNonlinearSolve}/Project.toml (51%) create mode 100644 lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl create mode 100644 lib/BracketingNonlinearSolve/src/alefeld.jl create mode 100644 lib/BracketingNonlinearSolve/src/bisection.jl create mode 100644 lib/BracketingNonlinearSolve/src/brent.jl create mode 100644 lib/BracketingNonlinearSolve/src/common.jl create mode 100644 lib/BracketingNonlinearSolve/src/falsi.jl create mode 100644 lib/BracketingNonlinearSolve/src/itp.jl create mode 100644 lib/BracketingNonlinearSolve/src/ridder.jl rename lib/{SimpleBracketingNonlinearSolve => BracketingNonlinearSolve}/test/runtests.jl (100%) delete mode 100644 lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl delete mode 100644 lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl diff --git a/lib/SimpleBracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml similarity index 51% rename from lib/SimpleBracketingNonlinearSolve/Project.toml rename to lib/BracketingNonlinearSolve/Project.toml index e3462f3bc..31ea28b21 100644 --- a/lib/SimpleBracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -1,13 +1,19 @@ -name = "SimpleBracketingNonlinearSolve" +name = "BracketingNonlinearSolve" uuid = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" authors = ["Avik Pal and contributors"] version = "1.0.0" [deps] CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" [compat] CommonSolve = "0.2.4" +ConcreteStructs = "0.2.3" +NonlinearSolveBase = "1" +PrecompileTools = "1.2.1" SciMLBase = "2.50" julia = "1.10" diff --git a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl new file mode 100644 index 000000000..2d5aed297 --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl @@ -0,0 +1,38 @@ +module BracketingNonlinearSolve + +using ConcreteStructs: @concrete + +using CommonSolve: CommonSolve +using NonlinearSolveBase: NonlinearSolveBase +using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, IntervalNonlinearProblem, ReturnCode + +using PrecompileTools: @compile_workload, @setup_workload + +abstract type AbstractBracketingAlgorithm <: AbstractNonlinearAlgorithm end + +include("common.jl") + +include("alefeld.jl") +include("bisection.jl") +include("brent.jl") +include("falsi.jl") +include("itp.jl") +include("ridder.jl") + +@setup_workload begin + for T in (Float32, Float64) + prob_brack = IntervalNonlinearProblem{false}( + (u, p) -> u^2 - p, T.((0.0, 2.0)), T(2)) + algs = (Alefeld(), Bisection(), Brent(), Falsi(), ITP(), Ridder()) + + @compile_workload begin + for alg in algs + CommonSolve.solve(prob_brack, alg; abstol = 1e-6) + end + end + end +end + +export Alefeld, Bisection, Brent, Falsi, ITP, Ridder + +end diff --git a/lib/BracketingNonlinearSolve/src/alefeld.jl b/lib/BracketingNonlinearSolve/src/alefeld.jl new file mode 100644 index 000000000..2407055ce --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/alefeld.jl @@ -0,0 +1,135 @@ +""" + Alefeld() + +An implementation of algorithm 4.2 from [Alefeld](https://dl.acm.org/doi/10.1145/210089.210111). + +The paper brought up two new algorithms. Here choose to implement algorithm 4.2 rather than +algorithm 4.1 because, in certain sense, the second algorithm(4.2) is an optimal procedure. +""" +struct Alefeld <: AbstractBracketingAlgorithm end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args...; + maxiters = 1000, abstol = nothing, kwargs...) + f = Base.Fix2(prob.f, prob.p) + a, b = prob.tspan + c = a - (b - a) / (f(b) - f(a)) * f(a) + + fc = f(c) + if a == c || b == c + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, left = a, right = b) + end + + if iszero(fc) + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b) + end + + a, b, d = Impl.bracket(f, a, b, c) + e = zero(a) # Set e as 0 before iteration to avoid a non-value f(e) + + for i in 2:maxiters + # The first bracketing block + f₁, f₂, f₃, f₄ = f(a), f(b), f(d), f(e) + if i == 2 || (f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄) + c = Impl.newton_quadratic(f, a, b, d, 2) + else + c = Impl.ipzero(f, a, b, d, e) + if (c - a) * (c - b) ≥ 0 + c = Impl.newton_quadratic(f, a, b, d, 2) + end + end + + ē, fc = d, f(c) + if a == c || b == c + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, + left = a, right = b) + end + + if iszero(fc) + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.Success, left = a, right = b) + end + + ā, b̄, d̄ = Impl.bracket(f, a, b, c) + + # The second bracketing block + f₁, f₂, f₃, f₄ = f(ā), f(b̄), f(d̄), f(ē) + if f₁ == f₂ || f₁ == f₃ || f₁ == f₄ || f₂ == f₃ || f₂ == f₄ || f₃ == f₄ + c = Impl.newton_quadratic(f, ā, b̄, d̄, 3) + else + c = Impl.ipzero(f, ā, b̄, d̄, ē) + if (c - ā) * (c - b̄) ≥ 0 + c = Impl.newton_quadratic(f, ā, b̄, d̄, 3) + end + end + fc = f(c) + + if ā == c || b̄ == c + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, + left = ā, right = b̄) + end + + if iszero(fc) + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄) + end + + ā, b̄, d̄ = Impl.bracket(f, ā, b̄, c) + + # The third bracketing block + u = ifelse(abs(f(ā)) < abs(f(b̄)), ā, b̄) + c = u - 2 * (b̄ - ā) / (f(b̄) - f(ā)) * f(u) + if (abs(c - u)) > 0.5 * (b̄ - ā) + c = 0.5 * (ā + b̄) + end + fc = f(c) + + if ā == c || b̄ == c + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, + left = ā, right = b̄) + end + + if iszero(fc) + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄) + end + + ā, b̄, d = Impl.bracket(f, ā, b̄, c) + + # The last bracketing block + if b̄ - ā < 0.5 * (b - a) + a, b, e = ā, b̄, d̄ + else + e = d + c = 0.5 * (ā + b̄) + fc = f(c) + + if ā == c || b̄ == c + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.FloatingPointLimit, + left = ā, right = b̄) + end + if iszero(fc) + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.Success, left = ā, right = b̄) + end + a, b, d = Impl.bracket(f, ā, b̄, c) + end + end + + # Reassign the value a, b, and c + if b == c + b = d + elseif a == c + a = d + end + fc = f(c) + + # Reuturn solution when run out of max interation + return SciMLBase.build_solution( + prob, alg, c, fc; retcode = ReturnCode.MaxIters, left = a, right = b) +end diff --git a/lib/BracketingNonlinearSolve/src/bisection.jl b/lib/BracketingNonlinearSolve/src/bisection.jl new file mode 100644 index 000000000..e3a38a1fd --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/bisection.jl @@ -0,0 +1,81 @@ +""" + Bisection(; exact_left = false, exact_right = false) + +A common bisection method. + +### Keyword Arguments + + - `exact_left`: whether to enforce whether the left side of the interval must be exactly + zero for the returned result. Defaults to false. + - `exact_right`: whether to enforce whether the right side of the interval must be exactly + zero for the returned result. Defaults to false. + +!!! danger "Keyword Arguments" + + Currently, the keyword arguments are not implemented. +""" +@kwdef struct Bisection <: AbstractBracketingAlgorithm + exact_left::Bool = false + exact_right::Bool = false +end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Bisection, + args...; maxiters = 1000, abstol = nothing, kwargs...) + @assert !SciMLBase.isinplace(prob) "`Bisection` only supports out-of-place problems." + + f = Base.Fix2(prob.f, prob.p) + left, right = prob.tspan + fl, fr = f(left), f(right) + + abstol = NonlinearSolveBase.get_tolerance( + abstol, promote_type(eltype(left), eltype(right))) + + if iszero(fl) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right) + end + + if iszero(fr) + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) + end + + i = 1 + while i ≤ maxiters + mid = (left + right) / 2 + + if mid == left || mid == right + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.FloatingPointLimit, left, right) + end + + fm = f(mid) + if abs((right - left) / 2) < abstol + return SciMLBase.build_solution( + prob, alg, mid, fm; retcode = ReturnCode.Success, left, right) + end + + if iszero(fm) + right = mid + break + end + + if sign(fl) == sign(fm) + fl = fm + left = mid + else + fr = fm + right = mid + end + + i += 1 + end + + sol, i, left, right, fl, fr = Impl.bisection( + left, right, fl, fr, f, abstol, maxiters - i, prob, alg) + + sol !== nothing && return sol + + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) +end diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl new file mode 100644 index 000000000..549cc07a6 --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/brent.jl @@ -0,0 +1,109 @@ +""" + Brent() + +Left non-allocating Brent method. +""" +struct Brent <: AbstractBracketingAlgorithm end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; + maxiters = 1000, abstol = nothing, kwargs...) + @assert !SciMLBase.isinplace(prob) "`Brent` only supports out-of-place problems." + + f = Base.Fix2(prob.f, prob.p) + left, right = prob.tspan + fl, fr = f(left), f(right) + ϵ = eps(convert(typeof(fl), 1)) + + abstol = NonlinearSolveBase.get_tolerance( + abstol, promote_type(eltype(left), eltype(right))) + + if iszero(fl) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right) + end + + if iszero(fr) + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) + end + + if abs(fl) < abs(fr) + left, right = right, left + fl, fr = fr, fl + end + + c = left + d = c + i = 1 + cond = true + + while i < maxiters + fc = f(c) + + if fl != fc && fr != fc + # Inverse quadratic interpolation + s = left * fr * fc / ((fl - fr) * (fl - fc)) + + right * fl * fc / ((fr - fl) * (fr - fc)) + + c * fl * fr / ((fc - fl) * (fc - fr)) + else + # Secant method + s = right - fr * (right - left) / (fr - fl) + end + + if (s < min((3 * left + right) / 4, right) || + s > max((3 * left + right) / 4, right)) || + (cond && abs(s - right) ≥ abs(right - c) / 2) || + (!cond && abs(s - right) ≥ abs(c - d) / 2) || + (cond && abs(right - c) ≤ ϵ) || + (!cond && abs(c - d) ≤ ϵ) + # Bisection method + s = (left + right) / 2 + if s == left || s == right + return SciMLBase.build_solution(prob, alg, left, fl; + retcode = ReturnCode.FloatingPointLimit, left, right) + end + cond = true + else + cond = false + end + + fs = f(s) + if abs((right - left) / 2) < abstol + return SciMLBase.build_solution(prob, alg, s, fs; + retcode = ReturnCode.Success, left, right) + end + + if iszero(fs) + if right < left + left = right + fl = fr + end + right = s + fr = fs + break + end + + if fl * fs < 0 + d, c, right = c, right, s + fr = fs + else + left = s + fl = fs + end + + if abs(fl) < abs(fr) + d = c + c, right, left = right, left, c + fc, fr, fl = fr, fl, fc + end + i += 1 + end + + sol, i, left, right, fl, fr = Impl.bisection( + left, right, fl, fr, f, abstol, maxiters - i, prob, alg) + + sol !== nothing && return sol + + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) +end diff --git a/lib/BracketingNonlinearSolve/src/common.jl b/lib/BracketingNonlinearSolve/src/common.jl new file mode 100644 index 000000000..65239ea63 --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/common.jl @@ -0,0 +1,91 @@ +module Impl + +using SciMLBase: SciMLBase, ReturnCode + +function bisection(left, right, fl, fr, f::F, abstol, maxiters, prob, alg) where {F} + i = 1 + sol = nothing + while i ≤ maxiters + mid = (left + right) / 2 + + if mid == left || mid == right + sol = SciMLBase.build_solution( + prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit) + break + end + + fm = f(mid) + if abs((right - left) / 2) < abstol + sol = SciMLBase.build_solution( + prob, alg, mid, fm; left, right, retcode = ReturnCode.Success) + break + end + + if iszero(fm) + right = mid + fr = fm + else + left = mid + fl = fm + end + + i += 1 + end + + return sol, i, left, right, fl, fr +end + +prevfloat_tdir(x, x0, x1) = ifelse(x1 > x0, prevfloat(x), nextfloat(x)) +nextfloat_tdir(x, x0, x1) = ifelse(x1 > x0, nextfloat(x), prevfloat(x)) +max_tdir(a, b, x0, x1) = ifelse(x1 > x0, max(a, b), min(a, b)) + +function bracket(f::F, a, b, c) where {F} + if iszero(f(c)) + ā, b̄, d = a, b, c + else + if f(a) * f(c) < 0 + ā, b̄, d = a, c, b + elseif f(b) * f(c) < 0 + ā, b̄, d = c, b, a + end + end + return ā, b̄, d +end + +function newton_quadratic(f::F, a, b, d, k) where {F} + A = ((f(d) - f(b)) / (d - b) - (f(b) - f(a)) / (b - a)) / (d - a) + B = (f(b) - f(a)) / (b - a) + + if iszero(A) + return a - (1 / B) * f(a) + elseif A * f(a) > 0 + rᵢ₋₁ = a + else + rᵢ₋₁ = b + end + + for _ in 1:k + rᵢ = rᵢ₋₁ - + (f(a) + B * (rᵢ₋₁ - a) + A * (rᵢ₋₁ - a) * (rᵢ₋₁ - b)) / + (B + A * (2 * rᵢ₋₁ - a - b)) + rᵢ₋₁ = rᵢ + end + + return rᵢ₋₁ +end + +function ipzero(f::F, a, b, c, d) where {F} + Q₁₁ = (c - d) * f(c) / (f(d) - f(c)) + Q₂₁ = (b - c) * f(b) / (f(c) - f(b)) + Q₃₁ = (a - b) * f(a) / (f(b) - f(a)) + D₂₁ = (b - c) * f(c) / (f(c) - f(b)) + D₃₁ = (a - b) * f(b) / (f(b) - f(a)) + Q₂₂ = (D₂₁ - Q₁₁) * f(b) / (f(d) - f(b)) + Q₃₂ = (D₃₁ - Q₂₁) * f(a) / (f(c) - f(a)) + D₃₂ = (D₃₁ - Q₂₁) * f(c) / (f(c) - f(a)) + Q₃₃ = (D₃₂ - Q₂₂) * f(a) / (f(d) - f(a)) + + return a + Q₃₁ + Q₃₂ + Q₃₃ +end + +end diff --git a/lib/BracketingNonlinearSolve/src/falsi.jl b/lib/BracketingNonlinearSolve/src/falsi.jl new file mode 100644 index 000000000..451a4c0b2 --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/falsi.jl @@ -0,0 +1,71 @@ +""" + Falsi() + +A non-allocating regula falsi method. +""" +struct Falsi <: AbstractBracketingAlgorithm end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...; + maxiters = 1000, abstol = nothing, kwargs...) + @assert !SciMLBase.isinplace(prob) "`False` only supports out-of-place problems." + + f = Base.Fix2(prob.f, prob.p) + l, r = prob.tspan # don't reuse these variables + left, right = prob.tspan + fl, fr = f(left), f(right) + + abstol = NonlinearSolveBase.get_tolerance( + abstol, promote_type(eltype(left), eltype(right))) + + if iszero(fl) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right) + end + + if iszero(fr) + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) + end + + i = 1 + while i ≤ maxiters + if Impl.nextfloat_tdir(left, l, r) == right + return SciMLBase.build_solution( + prob, alg, left, fl; left, right, retcode = ReturnCode.FloatingPointLimit) + end + + mid = (fr * left - fl * right) / (fr - fl) + for _ in 1:10 + mid = Impl.max_tdir(left, Impl.prevfloat_tdir(mid, l, r), l, r) + end + + (mid == left || mid == right) && break + + fm = f(mid) + if abs((right - left) / 2) < abstol + return SciMLBase.build_solution( + prob, alg, mid, fm; left, right, retcode = ReturnCode.Success) + end + + if abs(fm) < abstol + right = mid + break + end + + if sign(fl) == sign(fm) + fl, left = fm, mid + else + fr, right = fm, mid + end + + i += 1 + end + + sol, i, left, right, fl, fr = Impl.bisection( + left, right, fl, fr, f, abstol, maxiters - i, prob, alg) + + sol !== nothing && return sol + + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) +end diff --git a/lib/BracketingNonlinearSolve/src/itp.jl b/lib/BracketingNonlinearSolve/src/itp.jl new file mode 100644 index 000000000..dd6ddc23b --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/itp.jl @@ -0,0 +1,141 @@ +""" + ITP(; k1::Real = 0.007, k2::Real = 1.5, n0::Int = 10) + +ITP (Interpolate Truncate & Project) + +Use the [ITP method](https://en.wikipedia.org/wiki/ITP_method) to find a root of a bracketed +function, with a convergence rate between 1 and 1.62. + +This method was introduced in the paper "An Enhancement of the Bisection Method Average +Performance Preserving Minmax Optimality" (https://doi.org/10.1145/3423597) by +I. F. D. Oliveira and R. H. C. Takahashi. + +### Tuning Parameters + +The following keyword parameters are accepted. + + - `n₀::Int = 10`, the 'slack'. Must not be negative. When n₀ = 0 the worst-case is + identical to that of bisection, but increasing n₀ provides greater opportunity for + superlinearity. + - `scaled_κ₁::Float64 = 0.2`. Must not be negative. The recommended value is `0.2`. + Lower values produce tighter asymptotic behaviour, while higher values improve the + steady-state behaviour when truncation is not helpful. + - `κ₂::Real = 2`. Must lie in [1, 1+ϕ ≈ 2.62). Higher values allow for a greater + convergence rate, but also make the method more succeptable to worst-case performance. + In practice, κ₂=1, 2 seems to work well due to the computational simplicity, as κ₂ is + used as an exponent in the method. + +### Computation of κ₁ + +In the current implementation, we compute κ₁ = scaled_κ₁·|Δx₀|^(1 - κ₂); this allows κ₁ to +adapt to the length of the interval and keep the proposed steps proportional to Δx. + +### Worst Case Performance + +n½ + `n₀` iterations, where n½ is the number of iterations using bisection +(n½ = ⌈log2(Δx)/2`tol`⌉). + +### Asymptotic Performance + +If `f` is twice differentiable and the root is simple, then with `n₀` > 0 the convergence +rate is √`κ₂`. +""" +@concrete struct ITP <: AbstractBracketingAlgorithm + scaled_k1 + k2 + n0::Int +end + +function ITP(; scaled_k1::Real = 0.2, k2::Real = 2, n0::Int = 10) + scaled_k1 < 0 && error("Hyper-parameter κ₁ should not be negative") + n0 < 0 && error("Hyper-parameter n₀ should not be negative") + if k2 < 1 || k2 > (1.5 + sqrt(5) / 2) + throw(ArgumentError("Hyper-parameter κ₂ should be between 1 and 1 + ϕ where \ + ϕ ≈ 1.618... is the golden ratio")) + end + return ITP(scaled_k1, k2, n0) +end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::ITP, args...; + maxiters = 1000, abstol = nothing, kwargs...) + @assert !SciMLBase.isinplace(prob) "`ITP` only supports out-of-place problems." + + f = Base.Fix2(prob.f, prob.p) + left, right = prob.tspan + fl, fr = f(left), f(right) + + abstol = NonlinearSolveBase.get_tolerance( + abstol, promote_type(eltype(left), eltype(right))) + + if iszero(fl) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right) + end + + if iszero(fr) + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) + end + + ϵ = abstol + k2 = alg.k2 + k1 = alg.scaled_k1 * abs(right - left)^(1 - k2) + n0 = alg.n0 + n_h = ceil(log2(abs(right - left) / (2 * ϵ))) + mid = (left + right) / 2 + x_f = left + (right - left) * (fl / (fl - fr)) + xt = left + xp = left + r = zero(left) # minmax radius + δ = zero(left) # truncation error + σ = 1.0 + ϵ_s = ϵ * 2^(n_h + n0) + + i = 1 + while i ≤ maxiters + span = abs(right - left) + r = ϵ_s - (span / 2) + δ = k1 * span^k2 + + x_f = left + (right - left) * (fl / (fl - fr)) # Interpolation Step + + diff = mid - x_f + σ = sign(diff) + xt = ifelse(δ ≤ diff, x_f + σ * δ, mid) # Truncation Step + + xp = ifelse(abs(xt - mid) ≤ r, xt, mid - σ * r) # Projection Step + + if abs((left - right) / 2) < ϵ + return SciMLBase.build_solution( + prob, alg, xt, f(xt); retcode = ReturnCode.Success, left, right) + end + + # update + tmin, tmax = minmax(xt, xp) + xp ≥ tmax && (xp = prevfloat(tmax)) + xp ≤ tmin && (xp = nextfloat(tmin)) + yp = f(xp) + yps = yp * sign(fr) + T0 = zero(yps) + if yps > T0 + right, fr = xp, yp + elseif yps < T0 + left, fl = xp, yp + else + return SciMLBase.build_solution( + prob, alg, xp, yps; retcode = ReturnCode.Success, left, right) + end + + i += 1 + mid = (left + right) / 2 + ϵ_s /= 2 + + if Impl.nextfloat_tdir(left, prob.tspan...) == right + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.FloatingPointLimit, left, right) + end + end + + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) +end diff --git a/lib/BracketingNonlinearSolve/src/ridder.jl b/lib/BracketingNonlinearSolve/src/ridder.jl new file mode 100644 index 000000000..785bc2ae7 --- /dev/null +++ b/lib/BracketingNonlinearSolve/src/ridder.jl @@ -0,0 +1,83 @@ +""" + Ridder() + +A non-allocating ridder method. +""" +struct Ridder <: AbstractBracketingAlgorithm end + +function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...; + maxiters = 1000, abstol = nothing, kwargs...) + @assert !SciMLBase.isinplace(prob) "`Ridder` only supports out-of-place problems." + + f = Base.Fix2(prob.f, prob.p) + left, right = prob.tspan + fl, fr = f(left), f(right) + + abstol = NonlinearSolveBase.get_tolerance( + abstol, promote_type(eltype(left), eltype(right))) + + if iszero(fl) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.ExactSolutionLeft, left, right) + end + + if iszero(fr) + return SciMLBase.build_solution( + prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) + end + + xo = oftype(left, Inf) + i = 1 + while i ≤ maxiters + mid = (left + right) / 2 + + if mid == left || mid == right + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.FloatingPointLimit, left, right) + end + + fm = f(mid) + s = sqrt(fm^2 - fl * fr) + if iszero(s) + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.Failure, left, right) + end + + x = mid + (mid - left) * sign(fl - fm) * fm / s + fx = f(x) + xo = x + if abs((right - left) / 2) < abstol + return SciMLBase.build_solution( + prob, alg, mid, fm; retcode = ReturnCode.Success, left, right) + end + + if iszero(fx) + right, fr = x, fx + break + end + + if sign(fx) != sign(fm) + left = mid + fl = fm + right = x + fr = fx + elseif sign(fx) != sign(fl) + right = x + fr = fx + else + @assert sign(fx) != sign(fr) + left = x + fl = fx + end + + i += 1 + end + + sol, i, left, right, fl, fr = Impl.bisection( + left, right, fl, fr, f, abstol, maxiters - i, prob, alg) + + sol !== nothing && return sol + + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) +end \ No newline at end of file diff --git a/lib/SimpleBracketingNonlinearSolve/test/runtests.jl b/lib/BracketingNonlinearSolve/test/runtests.jl similarity index 100% rename from lib/SimpleBracketingNonlinearSolve/test/runtests.jl rename to lib/BracketingNonlinearSolve/test/runtests.jl diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index f21e15626..1c7047ca6 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -16,12 +16,10 @@ StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" UnrolledUtilities = "0fe1646c-419e-43be-ac14-22321958931b" [weakdeps] -DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] -NonlinearSolveBaseDiffEqBaseExt = "DiffEqBase" NonlinearSolveBaseForwardDiffExt = "ForwardDiff" NonlinearSolveBaseSparseArraysExt = "SparseArrays" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl deleted file mode 100644 index a3a216cf2..000000000 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl +++ /dev/null @@ -1,14 +0,0 @@ -module NonlinearSolveBaseDiffEqBaseExt - -using DiffEqBase: DiffEqBase -using NonlinearSolveBase: ImmutableNonlinearProblem - -function DiffEqBase.get_concrete_problem( - prob::ImmutableNonlinearProblem, isadapt; kwargs...) - u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs) - u0 = DiffEqBase.promote_u0(u0, prob.p, nothing) - p = DiffEqBase.get_concrete_p(prob, kwargs) - return SciMLBase.remake(prob; u0, p) -end - -end diff --git a/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl b/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl deleted file mode 100644 index 6bac6bdaa..000000000 --- a/lib/SimpleBracketingNonlinearSolve/src/SimpleBracketingNonlinearSolve.jl +++ /dev/null @@ -1,8 +0,0 @@ -module SimpleBracketingNonlinearSolve - -using CommonSolve: CommonSolve -using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, IntervalNonlinearProblem, ReturnCode - -abstract type AbstractBracketingAlgorithm <: AbstractNonlinearAlgorithm end - -end From 97781f3229d2295d96036ecdbf9ab85882cad800 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 22 Aug 2024 21:55:08 -0700 Subject: [PATCH 10/92] chore: missing license --- lib/BracketingNonlinearSolve/LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/BracketingNonlinearSolve/LICENSE diff --git a/lib/BracketingNonlinearSolve/LICENSE b/lib/BracketingNonlinearSolve/LICENSE new file mode 100644 index 000000000..411ad533f --- /dev/null +++ b/lib/BracketingNonlinearSolve/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Avik Pal and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From a7fb5173b51d8b97e65e18da822a1c5676933ab0 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 23 Aug 2024 11:14:17 -0700 Subject: [PATCH 11/92] chore: qa fixes --- lib/BracketingNonlinearSolve/src/alefeld.jl | 2 +- lib/BracketingNonlinearSolve/src/brent.jl | 4 ++-- lib/BracketingNonlinearSolve/src/ridder.jl | 2 +- lib/BracketingNonlinearSolve/test/runtests.jl | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/BracketingNonlinearSolve/src/alefeld.jl b/lib/BracketingNonlinearSolve/src/alefeld.jl index 2407055ce..a669900dd 100644 --- a/lib/BracketingNonlinearSolve/src/alefeld.jl +++ b/lib/BracketingNonlinearSolve/src/alefeld.jl @@ -129,7 +129,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Alefeld, args... end fc = f(c) - # Reuturn solution when run out of max interation + # Return solution when run out of max iteration return SciMLBase.build_solution( prob, alg, c, fc; retcode = ReturnCode.MaxIters, left = a, right = b) end diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl index 549cc07a6..23ab36df2 100644 --- a/lib/BracketingNonlinearSolve/src/brent.jl +++ b/lib/BracketingNonlinearSolve/src/brent.jl @@ -69,8 +69,8 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; fs = f(s) if abs((right - left) / 2) < abstol - return SciMLBase.build_solution(prob, alg, s, fs; - retcode = ReturnCode.Success, left, right) + return SciMLBase.build_solution( + prob, alg, s, fs; retcode = ReturnCode.Success, left, right) end if iszero(fs) diff --git a/lib/BracketingNonlinearSolve/src/ridder.jl b/lib/BracketingNonlinearSolve/src/ridder.jl index 785bc2ae7..a289bcb05 100644 --- a/lib/BracketingNonlinearSolve/src/ridder.jl +++ b/lib/BracketingNonlinearSolve/src/ridder.jl @@ -80,4 +80,4 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...; return SciMLBase.build_solution( prob, alg, left, fl; retcode = ReturnCode.MaxIters, left, right) -end \ No newline at end of file +end diff --git a/lib/BracketingNonlinearSolve/test/runtests.jl b/lib/BracketingNonlinearSolve/test/runtests.jl index e69de29bb..8b1378917 100644 --- a/lib/BracketingNonlinearSolve/test/runtests.jl +++ b/lib/BracketingNonlinearSolve/test/runtests.jl @@ -0,0 +1 @@ + From d6e868a32d6b6dca872e65d1603bec9544f4d983 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 16 Sep 2024 19:08:26 -0400 Subject: [PATCH 12/92] test: add tests for the bracketing methods --- lib/BracketingNonlinearSolve/Project.toml | 8 ++ .../src/BracketingNonlinearSolve.jl | 5 +- lib/BracketingNonlinearSolve/src/brent.jl | 6 +- .../test/rootfind_tests.jl | 94 +++++++++++++++++++ lib/BracketingNonlinearSolve/test/runtests.jl | 4 + lib/NonlinearSolveBase/Project.toml | 2 - lib/NonlinearSolveBase/src/utils.jl | 3 +- 7 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 lib/BracketingNonlinearSolve/test/rootfind_tests.jl diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index 31ea28b21..6b1a2c9a1 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -17,3 +17,11 @@ NonlinearSolveBase = "1" PrecompileTools = "1.2.1" SciMLBase = "2.50" julia = "1.10" + +[extras] +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" + +[targets] +test = ["InteractiveUtils", "Test", "TestItemRunner"] diff --git a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl index 2d5aed297..491ec4132 100644 --- a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl +++ b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl @@ -2,7 +2,7 @@ module BracketingNonlinearSolve using ConcreteStructs: @concrete -using CommonSolve: CommonSolve +using CommonSolve: CommonSolve, solve using NonlinearSolveBase: NonlinearSolveBase using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, IntervalNonlinearProblem, ReturnCode @@ -33,6 +33,9 @@ include("ridder.jl") end end +export IntervalNonlinearProblem +export solve + export Alefeld, Bisection, Brent, Falsi, ITP, Ridder end diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl index 23ab36df2..5475e675e 100644 --- a/lib/BracketingNonlinearSolve/src/brent.jl +++ b/lib/BracketingNonlinearSolve/src/brent.jl @@ -93,8 +93,10 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; if abs(fl) < abs(fr) d = c - c, right, left = right, left, c - fc, fr, fl = fr, fl, fc + c, right = right, left + left = c + fc, fr = fr, fl + fl = fc end i += 1 end diff --git a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl new file mode 100644 index 000000000..824c10da4 --- /dev/null +++ b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl @@ -0,0 +1,94 @@ +@testsnippet RootfindingTestSnippet begin + using NonlinearSolveBase, BracketingNonlinearSolve + + quadratic_f(u, p) = u .* u .- p + quadratic_f!(du, u, p) = (du .= u .* u .- p) + quadratic_f2(u, p) = @. p[1] * u * u - p[2] +end + +@testitem "Interval Nonlinear Problems" setup=[RootfindingTestSnippet] tags=[:core] begin + @testset for alg in (Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld()) + tspan = (1.0, 20.0) + + function g(p) + probN = IntervalNonlinearProblem{false}(quadratic_f, typeof(p).(tspan), p) + return solve(probN, alg; abstol = 1e-9).left + end + + @testset for p in 1.1:0.1:100.0 + @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3 + # @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3 + end + + t = (p) -> [sqrt(p[2] / p[1])] + p = [0.9, 50.0] + + function g2(p) + probN = IntervalNonlinearProblem{false}(quadratic_f2, tspan, p) + sol = solve(probN, alg; abstol = 1e-9) + return [sol.u] + end + + @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3 + # @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3 + + probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0) + sol = solve(probB, alg; abstol = 1e-9) + @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3 + + if !(alg isa Bisection || alg isa Falsi) + probB = IntervalNonlinearProblem{false}(quadratic_f, (sqrt(2.0), 10.0), 2.0) + sol = solve(probB, alg; abstol = 1e-9) + @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3 + + probB = IntervalNonlinearProblem{false}(quadratic_f, (0.0, sqrt(2.0)), 2.0) + sol = solve(probB, alg; abstol = 1e-9) + @test sol.left≈sqrt(2.0) atol=1e-3 rtol=1e-3 + end + end +end + +@testitem "Tolerance Tests Interval Methods" setup=[RootfindingTestSnippet] tags=[:core] begin + prob = IntervalNonlinearProblem(quadratic_f, (1.0, 20.0), 2.0) + ϵ = eps(Float64) # least possible tol for all methods + + @testset for alg in (Bisection(), Falsi(), ITP()) + @testset for abstol in [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7] + sol = solve(prob, alg; abstol) + result_tol = abs(sol.u - sqrt(2)) + @test result_tol < abstol + # test that the solution is not calculated upto max precision + @test result_tol > ϵ + end + end + + @testset for alg in (Ridder(), Brent()) + # Ridder and Brent converge rapidly so as we lower tolerance below 0.01, it + # converges with max precision to the solution + @testset for abstol in [0.1] + sol = solve(prob, alg; abstol) + result_tol = abs(sol.u - sqrt(2)) + @test result_tol < abstol + # test that the solution is not calculated upto max precision + @test result_tol > ϵ + end + end +end + +@testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTestSnippet] tags=[:core] begin + @testset for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder()) + f1(u, p) = u * u - p + f2(u, p) = p - u * u + + for p in 1:4 + inp1 = IntervalNonlinearProblem(f1, (1.0, 2.0), p) + inp2 = IntervalNonlinearProblem(f2, (1.0, 2.0), p) + inp3 = IntervalNonlinearProblem(f1, (2.0, 1.0), p) + inp4 = IntervalNonlinearProblem(f2, (2.0, 1.0), p) + @test abs.(solve(inp1, alg).u) ≈ sqrt.(p) + @test abs.(solve(inp2, alg).u) ≈ sqrt.(p) + @test abs.(solve(inp3, alg).u) ≈ sqrt.(p) + @test abs.(solve(inp4, alg).u) ≈ sqrt.(p) + end + end +end diff --git a/lib/BracketingNonlinearSolve/test/runtests.jl b/lib/BracketingNonlinearSolve/test/runtests.jl index 8b1378917..6ea6326b0 100644 --- a/lib/BracketingNonlinearSolve/test/runtests.jl +++ b/lib/BracketingNonlinearSolve/test/runtests.jl @@ -1 +1,5 @@ +using TestItemRunner, InteractiveUtils +@info sprint(InteractiveUtils.versioninfo) + +@run_package_tests diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 1c7047ca6..66ae9a35c 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -13,7 +13,6 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" -UnrolledUtilities = "0fe1646c-419e-43be-ac14-22321958931b" [weakdeps] ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" @@ -35,5 +34,4 @@ RecursiveArrayTools = "3" SciMLBase = "2.50" SparseArrays = "1.10" StaticArraysCore = "1.4" -UnrolledUtilities = "0.1" julia = "1.10" diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl index f830e9a17..bd1d799c1 100644 --- a/lib/NonlinearSolveBase/src/utils.jl +++ b/lib/NonlinearSolveBase/src/utils.jl @@ -4,11 +4,10 @@ using ArrayInterface: ArrayInterface using FastClosures: @closure using LinearAlgebra: norm using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition -using UnrolledUtilities: unrolled_all using ..NonlinearSolveBase: L2_NORM, Linf_NORM -fast_scalar_indexing(xs...) = unrolled_all(ArrayInterface.fast_scalar_indexing, xs) +fast_scalar_indexing(xs...) = all(ArrayInterface.fast_scalar_indexing, xs) function nonallocating_isapprox(x::Number, y::Number; atol = false, rtol = atol > 0 ? false : sqrt(eps(promote_type(typeof(x), typeof(y))))) From 1c16330fcc0488cc4b4b446cb317164d297959fa Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 16 Sep 2024 20:30:27 -0400 Subject: [PATCH 13/92] feat: ForwardDiff support in NonlinearSolveBase --- lib/BracketingNonlinearSolve/Project.toml | 10 ++- .../BracketingNonlinearSolveForwardDiffExt.jl | 26 +++++++ .../test/rootfind_tests.jl | 6 +- .../ext/NonlinearSolveBaseForwardDiffExt.jl | 75 ++++++++++++++++++- .../src/NonlinearSolveBase.jl | 2 +- lib/NonlinearSolveBase/src/autodiff.jl | 1 - lib/NonlinearSolveBase/src/public.jl | 4 + lib/NonlinearSolveBase/src/utils.jl | 18 +++++ 8 files changed, 136 insertions(+), 6 deletions(-) create mode 100644 lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl delete mode 100644 lib/NonlinearSolveBase/src/autodiff.jl diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index 6b1a2c9a1..954178d53 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -10,9 +10,16 @@ NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +[weakdeps] +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + +[extensions] +NonlinearSolveBaseForwardDiffExt = "ForwardDiff" + [compat] CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" +ForwardDiff = "0.10.36" NonlinearSolveBase = "1" PrecompileTools = "1.2.1" SciMLBase = "2.50" @@ -20,8 +27,9 @@ julia = "1.10" [extras] InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] -test = ["InteractiveUtils", "Test", "TestItemRunner"] +test = ["InteractiveUtils", "ForwardDiff", "Test", "TestItemRunner"] diff --git a/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl b/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl new file mode 100644 index 000000000..b41a88451 --- /dev/null +++ b/lib/BracketingNonlinearSolve/ext/BracketingNonlinearSolveForwardDiffExt.jl @@ -0,0 +1,26 @@ +module BracketingNonlinearSolveForwardDiffExt + +using CommonSolve: CommonSolve +using ForwardDiff: ForwardDiff, Dual +using NonlinearSolveBase: nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution +using SciMLBase: SciMLBase, IntervalNonlinearProblem + +using BracketingNonlinearSolve: Bisection, Brent, Alefeld, Falsi, ITP, Ridder + +for algT in (Bisection, Brent, Alefeld, Falsi, ITP, Ridder) + @eval function CommonSolve.solve( + prob::IntervalNonlinearProblem{ + uType, iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}}, + alg::$(algT), + args...; + kwargs...) where {uType, iip, T, V, P} + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) + return SciMLBase.build_solution( + prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, + sol.original, left = Dual{T, V, P}(sol.left, partials), + right = Dual{T, V, P}(sol.right, partials)) + end +end + +end diff --git a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl index 824c10da4..e4409bafd 100644 --- a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl +++ b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl @@ -7,6 +7,8 @@ end @testitem "Interval Nonlinear Problems" setup=[RootfindingTestSnippet] tags=[:core] begin + using ForwardDiff + @testset for alg in (Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld()) tspan = (1.0, 20.0) @@ -17,7 +19,7 @@ end @testset for p in 1.1:0.1:100.0 @test g(p)≈sqrt(p) atol=1e-3 rtol=1e-3 - # @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3 + @test ForwardDiff.derivative(g, p)≈1 / (2 * sqrt(p)) atol=1e-3 rtol=1e-3 end t = (p) -> [sqrt(p[2] / p[1])] @@ -30,7 +32,7 @@ end end @test g2(p)≈[sqrt(p[2] / p[1])] atol=1e-3 rtol=1e-3 - # @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3 + @test ForwardDiff.jacobian(g2, p)≈ForwardDiff.jacobian(t, p) atol=1e-3 rtol=1e-3 probB = IntervalNonlinearProblem{false}(quadratic_f, (1.0, 2.0), 2.0) sol = solve(probB, alg; abstol = 1e-9) diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index ae342e4ba..4161e0666 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -1,9 +1,82 @@ module NonlinearSolveBaseForwardDiffExt +using CommonSolve: solve +using FastClosures: @closure using ForwardDiff: ForwardDiff, Dual -using NonlinearSolveBase: Utils +using SciMLBase: SciMLBase, IntervalNonlinearProblem, NonlinearProblem, + NonlinearLeastSquaresProblem, remake + +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, Utils Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x)) +function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( + prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem}, + alg, args...; kwargs...) + p = Utils.value(prob.p) + if prob isa IntervalNonlinearProblem + tspan = Utils.value.(prob.tspan) + newprob = IntervalNonlinearProblem(prob.f, tspan, p; prob.kwargs...) + else + newprob = remake(prob; p, u0 = Utils.value(prob.u0)) + end + + sol = solve(newprob, alg, args...; kwargs...) + + uu = sol.u + Jₚ = nonlinearsolve_∂f_∂p(prob, prob.f, uu, p) + Jᵤ = nonlinearsolve_∂f_∂u(prob, prob.f, uu, p) + z = -Jᵤ \ Jₚ + pp = prob.p + sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z) + + if uu isa Number + partials = sum(sumfun, zip(z, pp)) + elseif p isa Number + partials = sumfun((z, pp)) + else + partials = sum(sumfun, zip(eachcol(z), pp)) + end + + return sol, partials +end + +function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} + if isinplace(prob) + f = @closure p -> begin + du = Utils.safe_similar(u, promote_type(eltype(u), eltype(p))) + f(du, u, p) + return du + end + else + f = Base.Fix1(f, u) + end + if p isa Number + return Utils.safe_reshape(ForwardDiff.derivative(f, p), :, 1) + elseif u isa Number + return Utils.safe_reshape(ForwardDiff.gradient(f, p), 1, :) + else + return ForwardDiff.jacobian(f, p) + end +end + +function nonlinearsolve_∂f_∂u(prob, f::F, u, p) where {F} + if isinplace(prob) + return ForwardDiff.jacobian( + @closure((du, u)->f(du, u, p)), Utils.safe_similar(u), u) + end + return ForwardDiff.jacobian(Base.Fix2(f, p), u) +end + +function NonlinearSolveBase.nonlinearsolve_dual_solution(u::Number, partials, + ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P} + return Dual{T, V, P}(u, partials) +end + +function NonlinearSolveBase.nonlinearsolve_dual_solution(u::AbstractArray, partials, + ::Union{<:AbstractArray{<:Dual{T, V, P}}, Dual{T, V, P}}) where {T, V, P} + return map(((uᵢ, pᵢ),) -> Dual{T, V, P}(uᵢ, pᵢ), zip(u, Utils.restructure(u, partials))) +end + end diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 71c634ef3..f0459447b 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -17,11 +17,11 @@ include("utils.jl") include("common_defaults.jl") include("termination_conditions.jl") -include("autodiff.jl") include("immutable_problem.jl") # Unexported Public API @compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance)) +@compat(public, (nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution)) export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTerminationMode, AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl deleted file mode 100644 index 8b1378917..000000000 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ /dev/null @@ -1 +0,0 @@ - diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl index 24f467cbd..db8c389e2 100644 --- a/lib/NonlinearSolveBase/src/public.jl +++ b/lib/NonlinearSolveBase/src/public.jl @@ -5,6 +5,10 @@ function L2_NORM end function Linf_NORM end function get_tolerance end +# Forward declarations of functions for forward mode AD +function nonlinearsolve_forwarddiff_solve end +function nonlinearsolve_dual_solution end + # Nonlinear Solve Termination Conditions abstract type AbstractNonlinearTerminationMode end abstract type AbstractSafeNonlinearTerminationMode <: AbstractNonlinearTerminationMode end diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl index bd1d799c1..0a8840942 100644 --- a/lib/NonlinearSolveBase/src/utils.jl +++ b/lib/NonlinearSolveBase/src/utils.jl @@ -72,4 +72,22 @@ apply_norm(f::F, x, y) where {F} = norm_op(standardize_norm(f), +, x, y) convert_real(::Type{T}, ::Nothing) where {T} = nothing convert_real(::Type{T}, x) where {T} = real(T(x)) +restructure(::Number, x::Number) = x +restructure(y, x) = ArrayInterface.restructure(y, x) + +function safe_similar(x, args...; kwargs...) + y = similar(x, args...; kwargs...) + return init_bigfloat_array!!(y) +end + +init_bigfloat_array!!(x) = x + +function init_bigfloat_array!!(x::AbstractArray{<:BigFloat}) + ArrayInterface.can_setindex(x) && fill!(x, BigFloat(0)) + return x +end + +safe_reshape(x::Number, args...) = x +safe_reshape(x, args...) = reshape(x, args...) + end From b4d5e1d735eef287acf57326139e5dc2945617e1 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 12:56:21 -0400 Subject: [PATCH 14/92] fix: extension for forward AD support --- lib/BracketingNonlinearSolve/Project.toml | 2 +- lib/BracketingNonlinearSolve/test/rootfind_tests.jl | 2 -- lib/NonlinearSolveBase/Project.toml | 2 ++ .../ext/NonlinearSolveBaseForwardDiffExt.jl | 6 ++++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index 954178d53..b8bf5a806 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -14,7 +14,7 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" [extensions] -NonlinearSolveBaseForwardDiffExt = "ForwardDiff" +BracketingNonlinearSolveForwardDiffExt = "ForwardDiff" [compat] CommonSolve = "0.2.4" diff --git a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl index e4409bafd..1875ede00 100644 --- a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl +++ b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl @@ -1,6 +1,4 @@ @testsnippet RootfindingTestSnippet begin - using NonlinearSolveBase, BracketingNonlinearSolve - quadratic_f(u, p) = u .* u .- p quadratic_f!(du, u, p) = (du .= u .* u .- p) quadratic_f2(u, p) = @. p[1] * u * u - p[2] diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 66ae9a35c..94d12d822 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -5,6 +5,7 @@ version = "1.0.0" [deps] ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" @@ -24,6 +25,7 @@ NonlinearSolveBaseSparseArraysExt = "SparseArrays" [compat] ArrayInterface = "7.9" +CommonSolve = "0.2.4" Compat = "4.15" ConcreteStructs = "0.2.3" FastClosures = "0.3" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index 4161e0666..469c5944b 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -10,6 +10,7 @@ using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, Utils Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x)) +Utils.value(x::AbstractArray{<:Dual}) = Utils.value.(x) function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( prob::Union{IntervalNonlinearProblem, NonlinearProblem, ImmutableNonlinearProblem}, @@ -43,7 +44,7 @@ function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( end function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} - if isinplace(prob) + if SciMLBase.isinplace(prob) f = @closure p -> begin du = Utils.safe_similar(u, promote_type(eltype(u), eltype(p))) f(du, u, p) @@ -62,10 +63,11 @@ function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} end function nonlinearsolve_∂f_∂u(prob, f::F, u, p) where {F} - if isinplace(prob) + if SciMLBase.isinplace(prob) return ForwardDiff.jacobian( @closure((du, u)->f(du, u, p)), Utils.safe_similar(u), u) end + u isa Number && return ForwardDiff.derivative(Base.Fix2(f, p), u) return ForwardDiff.jacobian(Base.Fix2(f, p), u) end From ce2cb880b4afc7e71d8cff2e9d0799812cd41216 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 13:15:41 -0400 Subject: [PATCH 15/92] ci(github-actions): add workflows for subpackages --- .../workflows/CI_BracketingNonlinearSolve.yml | 67 +++++++++++++++++++ .github/workflows/CI_NonlinearSolveBase.yml | 63 +++++++++++++++++ .github/workflows/CompatHelper.yml | 8 ++- 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/CI_BracketingNonlinearSolve.yml create mode 100644 .github/workflows/CI_NonlinearSolveBase.yml diff --git a/.github/workflows/CI_BracketingNonlinearSolve.yml b/.github/workflows/CI_BracketingNonlinearSolve.yml new file mode 100644 index 000000000..d79e69930 --- /dev/null +++ b/.github/workflows/CI_BracketingNonlinearSolve.yml @@ -0,0 +1,67 @@ +name: CI (BracketingNonlinearSolve) + +on: + pull_request: + branches: + - master + paths: + - "lib/BracketingNonlinearSolve/**" + - ".github/workflows/CI_BracketingNonlinearSolve.yml" + push: + branches: + - master + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - "min" + - "1" + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: "Install Dependencies and Run Tests" + run: | + import Pkg + Pkg.Registry.update() + # Install packages present in subdirectories + for path in ("lib/NonlinearSolveBase",) + Pkg.develop(; path) + end + Pkg.instantiate() + Pkg.test(; coverage=true) + shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/BracketingNonlinearSolve {0} + - uses: julia-actions/julia-processcoverage@v1 + with: + directories: lib/BracketingNonlinearSolve/src,lib/BracketingNonlinearSolve/ext + - uses: codecov/codecov-action@v4 + with: + file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/.github/workflows/CI_NonlinearSolveBase.yml b/.github/workflows/CI_NonlinearSolveBase.yml new file mode 100644 index 000000000..f3878acef --- /dev/null +++ b/.github/workflows/CI_NonlinearSolveBase.yml @@ -0,0 +1,63 @@ +name: CI (NonlinearSolveBase) + +on: + pull_request: + branches: + - master + paths: + - "lib/NonlinearSolveBase/**" + - ".github/workflows/CI_NonlinearSolveBase.yml" + push: + branches: + - master + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - "min" + - "1" + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: "Install Dependencies and Run Tests" + run: | + import Pkg + Pkg.Registry.update() + Pkg.instantiate() + Pkg.test(; coverage=true) + shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/NonlinearSolveBase {0} + - uses: julia-actions/julia-processcoverage@v1 + with: + directories: lib/NonlinearSolveBase/src,lib/NonlinearSolveBase/ext + - uses: codecov/codecov-action@v4 + with: + file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 73494545f..805a5fe4c 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -23,4 +23,10 @@ jobs: - name: CompatHelper.main() env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: julia -e 'using CompatHelper; CompatHelper.main(;subdirs=["", "docs"])' + run: | + import CompatHelper + subdirs = ["", "docs"] + append!(subdirs, joinpath.(("lib",), filter(p -> isdir(joinpath("lib", p)), readdir("lib")))) + CompatHelper.main(; subdirs) + shell: julia --color=yes {0} + working-directory: "./" From 5b1dc0f29882e79dfb52b62adc39d5cd1339b26c Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 13:28:46 -0400 Subject: [PATCH 16/92] ci(github-actions): trigger dependent packages on file changes --- .github/workflows/CI_BracketingNonlinearSolve.yml | 3 +++ lib/BracketingNonlinearSolve/Project.toml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI_BracketingNonlinearSolve.yml b/.github/workflows/CI_BracketingNonlinearSolve.yml index d79e69930..5720013e2 100644 --- a/.github/workflows/CI_BracketingNonlinearSolve.yml +++ b/.github/workflows/CI_BracketingNonlinearSolve.yml @@ -7,6 +7,9 @@ on: paths: - "lib/BracketingNonlinearSolve/**" - ".github/workflows/CI_BracketingNonlinearSolve.yml" + - "lib/NonlinearSolveBase/src/**" + - "lib/NonlinearSolveBase/ext/**" + - "lib/NonlinearSolveBase/Project.toml" push: branches: - master diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index b8bf5a806..c71a19dca 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -26,8 +26,8 @@ SciMLBase = "2.50" julia = "1.10" [extras] -InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" From 3ad8c5b63c685d1a46c27754dd941a98f147f667 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 13:49:22 -0400 Subject: [PATCH 17/92] feat: add bracketing default algorithm --- .../src/BracketingNonlinearSolve.jl | 8 ++++++++ lib/BracketingNonlinearSolve/test/rootfind_tests.jl | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl index 491ec4132..62cf7a7b7 100644 --- a/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl +++ b/lib/BracketingNonlinearSolve/src/BracketingNonlinearSolve.jl @@ -19,6 +19,14 @@ include("falsi.jl") include("itp.jl") include("ridder.jl") +# Default Algorithm +function CommonSolve.solve(prob::IntervalNonlinearProblem; kwargs...) + return CommonSolve.solve(prob, ITP(); kwargs...) +end +function CommonSolve.solve(prob::IntervalNonlinearProblem, nothing, args...; kwargs...) + return CommonSolve.solve(prob, ITP(), args...; kwargs...) +end + @setup_workload begin for T in (Float32, Float64) prob_brack = IntervalNonlinearProblem{false}( diff --git a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl index 1875ede00..6a490d6fa 100644 --- a/lib/BracketingNonlinearSolve/test/rootfind_tests.jl +++ b/lib/BracketingNonlinearSolve/test/rootfind_tests.jl @@ -7,7 +7,7 @@ end @testitem "Interval Nonlinear Problems" setup=[RootfindingTestSnippet] tags=[:core] begin using ForwardDiff - @testset for alg in (Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld()) + @testset for alg in (Bisection(), Falsi(), Ridder(), Brent(), ITP(), Alefeld(), nothing) tspan = (1.0, 20.0) function g(p) @@ -52,7 +52,7 @@ end prob = IntervalNonlinearProblem(quadratic_f, (1.0, 20.0), 2.0) ϵ = eps(Float64) # least possible tol for all methods - @testset for alg in (Bisection(), Falsi(), ITP()) + @testset for alg in (Bisection(), Falsi(), ITP(), nothing) @testset for abstol in [0.1, 0.01, 0.001, 0.0001, 1e-5, 1e-6, 1e-7] sol = solve(prob, alg; abstol) result_tol = abs(sol.u - sqrt(2)) @@ -76,7 +76,7 @@ end end @testitem "Flipped Signs and Reversed Tspan" setup=[RootfindingTestSnippet] tags=[:core] begin - @testset for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder()) + @testset for alg in (Alefeld(), Bisection(), Falsi(), Brent(), ITP(), Ridder(), nothing) f1(u, p) = u * u - p f2(u, p) = p - u * u From faf5e7ed0a685f1c088607edb31fdaa340c83a61 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 13:53:34 -0400 Subject: [PATCH 18/92] refactor: setup SimpleNonlinearSolve.jl --- .github/workflows/CI_SimpleNonlinearSolve.yml | 76 +++++++++++++++++++ lib/BracketingNonlinearSolve/Project.toml | 2 +- lib/SimpleNonlinearSolve/LICENSE | 21 +++++ lib/SimpleNonlinearSolve/Project.toml | 23 ++++++ .../src/SimpleNonlinearSolve.jl | 19 +++++ lib/SimpleNonlinearSolve/test/runtests.jl | 0 6 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/CI_SimpleNonlinearSolve.yml create mode 100644 lib/SimpleNonlinearSolve/LICENSE create mode 100644 lib/SimpleNonlinearSolve/Project.toml create mode 100644 lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl create mode 100644 lib/SimpleNonlinearSolve/test/runtests.jl diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml new file mode 100644 index 000000000..e07cb3343 --- /dev/null +++ b/.github/workflows/CI_SimpleNonlinearSolve.yml @@ -0,0 +1,76 @@ +name: CI (SimpleNonlinearSolve) + +on: + pull_request: + branches: + - master + paths: + - "lib/SimpleNonlinearSolve/src/**" + - "lib/SimpleNonlinearSolve/ext/**" + - "lib/SimpleNonlinearSolve/test/**" + - "lib/SimpleNonlinearSolve/Project.toml" + - ".github/workflows/CI_SimpleNonlinearSolve.yml" + - "lib/BracketingNonlinearSolve/src/**" + - "lib/BracketingNonlinearSolve/ext/**" + - "lib/BracketingNonlinearSolve/Project.toml" + - "lib/NonlinearSolveBase/src/**" + - "lib/NonlinearSolveBase/ext/**" + - "lib/NonlinearSolveBase/Project.toml" + push: + branches: + - master + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - "min" + - "1" + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + - uses: actions/cache@v4 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - name: "Install Dependencies and Run Tests" + run: | + import Pkg + Pkg.Registry.update() + # Install packages present in subdirectories + for path in ("lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve") + Pkg.develop(; path) + end + Pkg.instantiate() + Pkg.test(; coverage=true) + shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve {0} + - uses: julia-actions/julia-processcoverage@v1 + with: + directories: lib/SimpleNonlinearSolve/src,lib/SimpleNonlinearSolve/ext + - uses: codecov/codecov-action@v4 + with: + file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index c71a19dca..dc979f660 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -21,7 +21,7 @@ CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" ForwardDiff = "0.10.36" NonlinearSolveBase = "1" -PrecompileTools = "1.2.1" +PrecompileTools = "1.2" SciMLBase = "2.50" julia = "1.10" diff --git a/lib/SimpleNonlinearSolve/LICENSE b/lib/SimpleNonlinearSolve/LICENSE new file mode 100644 index 000000000..8eef16440 --- /dev/null +++ b/lib/SimpleNonlinearSolve/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Julia Computing, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml new file mode 100644 index 000000000..e2c92c016 --- /dev/null +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -0,0 +1,23 @@ +name = "SimpleNonlinearSolve" +uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7" +authors = ["SciML"] +version = "1.13.0" + +[deps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" +NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" +PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Reexport = "189a3867-3050-52da-a836-e630ba90ab69" +SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" + +[compat] +ADTypes = "1.2" +ArrayInterface = "7.16" +BracketingNonlinearSolve = "1" +NonlinearSolveBase = "1" +PrecompileTools = "1.2" +Reexport = "1.2" +SciMLBase = "2.50" +julia = "1.10" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl new file mode 100644 index 000000000..0debfd328 --- /dev/null +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -0,0 +1,19 @@ +module SimpleNonlinearSolve + +using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, + AutoPolyesterForwardDiff +using PrecompileTools: @compile_workload, @setup_workload +using Reexport: @reexport +@reexport using SciMLBase # I don't like this but needed to avoid a breaking change + +using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder + +@setup_workload begin + @compile_workload begin end +end + +export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff + +export Alefeld, Bisection, Brent, Falsi, ITP, Ridder + +end diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl new file mode 100644 index 000000000..e69de29bb From 406607bc919016e0b6542f0ecfd957047732eb14 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 13:56:31 -0400 Subject: [PATCH 19/92] ci(github-actions): fix package dev workflow --- .github/workflows/CI_BracketingNonlinearSolve.yml | 4 +++- .github/workflows/CI_SimpleNonlinearSolve.yml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI_BracketingNonlinearSolve.yml b/.github/workflows/CI_BracketingNonlinearSolve.yml index 5720013e2..fc62f02c6 100644 --- a/.github/workflows/CI_BracketingNonlinearSolve.yml +++ b/.github/workflows/CI_BracketingNonlinearSolve.yml @@ -53,9 +53,11 @@ jobs: import Pkg Pkg.Registry.update() # Install packages present in subdirectories + dev_pks = Pkg.PackageSpec[] for path in ("lib/NonlinearSolveBase",) - Pkg.develop(; path) + push!(dev_pks, Pkg.PackageSpec(; path)) end + Pkg.develop(dev_pks) Pkg.instantiate() Pkg.test(; coverage=true) shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/BracketingNonlinearSolve {0} diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml index e07cb3343..365de3707 100644 --- a/.github/workflows/CI_SimpleNonlinearSolve.yml +++ b/.github/workflows/CI_SimpleNonlinearSolve.yml @@ -59,9 +59,11 @@ jobs: import Pkg Pkg.Registry.update() # Install packages present in subdirectories + dev_pks = Pkg.PackageSpec[] for path in ("lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve") - Pkg.develop(; path) + push!(dev_pks, Pkg.PackageSpec(; path)) end + Pkg.develop(dev_pks) Pkg.instantiate() Pkg.test(; coverage=true) shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve {0} From cbbf8a7519cde8885103e39a2c6e551a349565ac Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 14:59:33 -0400 Subject: [PATCH 20/92] feat: add simplenonlinearsolve AD specific dispatches --- .../src/NonlinearSolveBase.jl | 2 - lib/SimpleNonlinearSolve/Project.toml | 18 ++++++++ .../SimpleNonlinearSolveChainRulesCoreExt.jl | 23 ++++++++++ .../ext/SimpleNonlinearSolveDiffEqBaseExt.jl | 11 +++++ .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 37 ++++++++++++++++ .../ext/SimpleNonlinearSolveTrackerExt.jl | 37 ++++++++++++++++ .../src/SimpleNonlinearSolve.jl | 42 +++++++++++++++++++ 7 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl create mode 100644 lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index f0459447b..25fe50023 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -27,6 +27,4 @@ export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTermi AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, RelNormSafeNormTerminationMode, AbsNormSafeNormTerminationMode -export ImmutableNonlinearProblem - end diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index e2c92c016..5d8817b60 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -7,17 +7,35 @@ version = "1.13.0" ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" +CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +[weakdeps] +ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" +Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[extensions] +SimpleNonlinearSolveChainRulesCoreExt = "ChainRulesCore" +SimpleNonlinearSolveDiffEqBaseExt = "DiffEqBase" +SimpleNonlinearSolveReverseDiffExt = "ReverseDiff" +SimpleNonlinearSolveTrackerExt = "Tracker" + [compat] ADTypes = "1.2" ArrayInterface = "7.16" BracketingNonlinearSolve = "1" +ChainRulesCore = "1.24" +CommonSolve = "0.2.4" +DiffEqBase = "6.155" NonlinearSolveBase = "1" PrecompileTools = "1.2" Reexport = "1.2" +ReverseDiff = "1.15" SciMLBase = "2.50" +Tracker = "0.2.35" julia = "1.10" diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl new file mode 100644 index 000000000..df0bd7573 --- /dev/null +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl @@ -0,0 +1,23 @@ +module SimpleNonlinearSolveChainRulesCoreExt + +using ChainRulesCore: ChainRulesCore, NoTangent +using NonlinearSolveBase: ImmutableNonlinearProblem +using SciMLBase: ChainRulesOriginator, NonlinearLeastSquaresProblem + +using SimpleNonlinearSolve: SimpleNonlinearSolve, simplenonlinearsolve_solve_up, + solve_adjoint + +function ChainRulesCore.rrule(::typeof(simplenonlinearsolve_solve_up), + prob::Union{InternalNonlinearProblem, NonlinearLeastSquaresProblem}, + sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...) + out, ∇internal = solve_adjoint( + prob, sensealg, u0, p, ChainRulesOriginator(), alg, args...; kwargs...) + function ∇simplenonlinearsolve_solve_up(Δ) + ∂f, ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Δ) + return ( + ∂f, ∂prob, ∂sensealg, ∂u0, NoTangent(), ∂p, NoTangent(), NoTangent(), ∂args...) + end + return out, ∇simplenonlinearsolve_solve_up +end + +end diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl new file mode 100644 index 000000000..950a04019 --- /dev/null +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl @@ -0,0 +1,11 @@ +module SimpleNonlinearSolveDiffEqBaseExt + +using DiffEqBase: DiffEqBase + +using SimpleNonlinearSolve: SimpleNonlinearSolve + +function SimpleNonlinearSolve.solve_adjoint_internal(args...; kwargs...) + return DiffEqBase._solve_adjoint(args...; kwargs...) +end + +end diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl new file mode 100644 index 000000000..1357bec83 --- /dev/null +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl @@ -0,0 +1,37 @@ +module SimpleNonlinearSolveReverseDiffExt + +using ArrayInterface: ArrayInterface +using NonlinearSolveBase: ImmutableNonlinearProblem +using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal +using SciMLBase: ReverseDiffOriginator, NonlinearLeastSquaresProblem, remake + +using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint + +for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) + aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any) + for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)] + @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + prob::$(pType), sensealg, u0::$(uT), u0_changed, + p::$(pT), p_changed, alg, args...; kwargs...) + return ReverseDiff.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up, + prob, sensealg, ArrayInterface.aos_to_soa(u0), true, + ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...) + end + end + + @eval ReverseDiff.@grad function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + tprob::$(pType), sensealg, tu0, u0_changed, + tp, p_changed, alg, args...; kwargs...) + u0, p = ReverseDiff.value(tu0), ReverseDiff.value(tp) + prob = remake(tprob; u0, p) + out, ∇internal = solve_adjoint( + prob, sensealg, u0, p, ReverseDiffOriginator(), alg, args...; kwargs...) + + function ∇simplenonlinearsolve_solve_up(Δ...) + ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Δ...) + return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...) + end + end +end + +end diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl new file mode 100644 index 000000000..935484db1 --- /dev/null +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl @@ -0,0 +1,37 @@ +module SimpleNonlinearSolveTrackerExt + +using ArrayInterface: ArrayInterface +using NonlinearSolveBase: ImmutableNonlinearProblem +using SciMLBase: TrackerOriginator, NonlinearLeastSquaresProblem, remake +using Tracker: Tracker, TrackedArray, TrackedReal + +using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint + +for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) + aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any) + for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)] + @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + prob::$(pType), sensealg, u0::$(uT), u0_changed, + p::$(pT), p_changed, alg, args...; kwargs...) + return Tracker.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up, prob, + sensealg, ArrayInterface.aos_to_soa(u0), true, + ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...) + end + end + + @eval Tracker.@grad function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + tprob::$(pType), sensealg, tu0, u0_changed, + tp, p_changed, alg, args...; kwargs...) + u0, p = Tracker.data(tu0), Tracker.data(tp) + prob = remake(tprob; u0, p) + out, ∇internal = solve_adjoint( + prob, sensealg, u0, p, TrackerOriginator(), alg, args...; kwargs...) + + function ∇simplenonlinearsolve_solve_up(Δ) + ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Tracker.data(Δ)) + return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...) + end + end +end + +end diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 0debfd328..99c0be844 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -2,11 +2,53 @@ module SimpleNonlinearSolve using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff +using CommonSolve: CommonSolve, solve using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change +using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder +using NonlinearSolveBase: ImmutableNonlinearProblem + +abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end + +is_extension_loaded(::Val) = false + +# By Pass the highlevel checks for NonlinearProblem for Simple Algorithms +function CommonSolve.solve(prob::NonlinearProblem, + alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) + prob = convert(ImmutableNonlinearProblem, prob) + return solve(prob, alg, args...; kwargs...) +end + +function CommonSolve.solve( + prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, + args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...) + if sensealg === nothing && haskey(prob.kwargs, :sensealg) + sensealg = prob.kwargs[:sensealg] + end + new_u0 = u0 !== nothing ? u0 : prob.u0 + new_p = p !== nothing ? p : prob.p + return simplenonlinearsolve_solve_up(prob, sensealg, new_u0, u0 === nothing, new_p, + p === nothing, alg, args...; prob.kwargs..., kwargs...) +end + +function simplenonlinearsolve_solve_up( + prob::ImmutableNonlinearProblem, sensealg, u0, u0_changed, p, p_changed, + alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) + (u0_changed || p_changed) && (prob = remake(prob; u0, p)) + return SciMLBase.__solve(prob, alg, args...; kwargs...) +end + +# NOTE: This is defined like this so that we don't have to keep have 2 args for the +# extensions +function solve_adjoint(args...; kws...) + is_extension_loaded(Val(:DiffEqBase)) && return solve_adjoint_internal(args...; kws...) + error("Adjoint sensitivity analysis requires `DiffEqBase.jl` to be explicitly loaded.") +end + +function solve_adjoint_internal end @setup_workload begin @compile_workload begin end From d73a1f94891bef67afdf5ba29822681de4d1621e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 15:07:12 -0400 Subject: [PATCH 21/92] feat: add the AD workflows --- lib/SimpleNonlinearSolve/Project.toml | 6 ++++ .../src/SimpleNonlinearSolve.jl | 31 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 5d8817b60..f4ae4e4de 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -8,6 +8,9 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" @@ -32,6 +35,9 @@ BracketingNonlinearSolve = "1" ChainRulesCore = "1.24" CommonSolve = "0.2.4" DiffEqBase = "6.155" +DifferentiationInterface = "0.5.17" +FiniteDiff = "2.24.0" +ForwardDiff = "0.10.36" NonlinearSolveBase = "1" PrecompileTools = "1.2" Reexport = "1.2" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 99c0be844..f1d7b0713 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,16 +1,25 @@ module SimpleNonlinearSolve -using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, - AutoPolyesterForwardDiff using CommonSolve: CommonSolve, solve using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode +# AD Dependencies +using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, + AutoPolyesterForwardDiff +using DifferentiationInterface: DifferentiationInterface +# TODO: move these to extensions in a breaking change. These are not even used in the +# package, but are used to trigger the extension loading in DI.jl +using FiniteDiff: FiniteDiff +using ForwardDiff: ForwardDiff + using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder using NonlinearSolveBase: ImmutableNonlinearProblem +const DI = DifferentiationInterface + abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end is_extension_loaded(::Val) = false @@ -51,7 +60,23 @@ end function solve_adjoint_internal end @setup_workload begin - @compile_workload begin end + for T in (Float32, Float64) + prob_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2)) + prob_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p, ones(T, 3), T(2)) + prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) + + algs = [] + algs_no_iip = [] + + @compile_workload begin + for alg in algs, prob in (prob_scalar, prob_iip, prob_oop) + CommonSolve.solve(prob, alg) + end + for alg in algs_no_iip + CommonSolve.solve(prob_scalar, alg) + end + end + end end export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff From 815ade50973544e29a3648bb0022c80b9d332fac Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 15:19:28 -0400 Subject: [PATCH 22/92] chore: run formatter --- .../ext/SimpleNonlinearSolveTrackerExt.jl | 4 ++-- lib/SimpleNonlinearSolve/test/runtests.jl | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl index 935484db1..a2fa8ff40 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl @@ -13,8 +13,8 @@ for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( prob::$(pType), sensealg, u0::$(uT), u0_changed, p::$(pT), p_changed, alg, args...; kwargs...) - return Tracker.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up, prob, - sensealg, ArrayInterface.aos_to_soa(u0), true, + return Tracker.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up, + prob, sensealg, ArrayInterface.aos_to_soa(u0), true, ArrayInterface.aos_to_soa(p), true, alg, args...; kwargs...) end end diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -0,0 +1 @@ + From d8250eacb9a4d2883683ed80309d35aac84f5619 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 17:44:33 -0400 Subject: [PATCH 23/92] feat: share the termination condition code in NonlinearSolve and SimpleNonlinearSolve --- .../src/NonlinearSolveBase.jl | 7 ++-- .../src/termination_conditions.jl | 34 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 25fe50023..1ba7a0cc3 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -8,16 +8,17 @@ using LinearAlgebra: norm using Markdown: @doc_str using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinearProblem, - AbstractNonlinearFunction, @add_kwonly, StandardNonlinearProblem, - NullParameters, NonlinearProblem, isinplace + NonlinearProblem, NonlinearLeastSquaresProblem, AbstractNonlinearFunction, + @add_kwonly, StandardNonlinearProblem, NullParameters, NonlinearProblem, + isinplace using StaticArraysCore: StaticArray include("public.jl") include("utils.jl") +include("immutable_problem.jl") include("common_defaults.jl") include("termination_conditions.jl") -include("immutable_problem.jl") # Unexported Public API @compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance)) diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 50af54a57..a278861a8 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -245,3 +245,37 @@ end function check_convergence(mode::AbsNormModes, duₙ, _, __, abstol, ___) return Utils.apply_norm(mode.internalnorm, duₙ) ≤ abstol end + +# High-Level API with defaults. +## This is mostly for internal usage in NonlinearSolve and SimpleNonlinearSolve +function default_termination_mode( + ::Union{ImmutableNonlinearProblem, NonlinearProblem}, ::Val{:simple}) + return AbsNormTerminationMode(Base.Fix1(maximum, abs)) +end +function default_termination_mode(::NonlinearLeastSquaresProblem, ::Val{:simple}) + return AbsNormTerminationMode(Base.Fix2(norm, 2)) +end + +function default_termination_mode( + ::Union{ImmutableNonlinearProblem, NonlinearProblem}, ::Val{:regular}) + return AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs); max_stalled_steps = 32) +end + +function default_termination_mode(::NonlinearLeastSquaresProblem, ::Val{:regular}) + return AbsNormSafeBestTerminationMode(Base.Fix2(norm, 2); max_stalled_steps = 32) +end + +function init_termination_cache( + prob::AbstractNonlinearProblem, abstol, reltol, du, u, ::Nothing, callee::Val) + return init_termination_cache( + prob, abstol, reltol, du, u, default_termination_mode(prob, callee), callee) +end + +function init_termination_cache(::AbstractNonlinearProblem, abstol, reltol, du, + u, tc::AbstractNonlinearTerminationMode, ::Val) + T = promote_type(eltype(du), eltype(u)) + abstol = get_tolerance(abstol, T) + reltol = get_tolerance(reltol, T) + cache = init(du, u, tc; abstol, reltol) + return abstol, reltol, cache +end From 1d03fa27fa0f3ad1b1696ffbe3a8795f2c6f223e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 17:44:51 -0400 Subject: [PATCH 24/92] feat: add partial SimpleKlement Implementation --- lib/SimpleNonlinearSolve/Project.toml | 16 +++ .../src/SimpleNonlinearSolve.jl | 10 +- lib/SimpleNonlinearSolve/src/klement.jl | 47 ++++++++ lib/SimpleNonlinearSolve/src/utils.jl | 107 ++++++++++++++++++ lib/SimpleNonlinearSolve/test/runtests.jl | 4 + 5 files changed, 181 insertions(+), 3 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/src/klement.jl create mode 100644 lib/SimpleNonlinearSolve/src/utils.jl diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index f4ae4e4de..7ab2416be 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -9,12 +9,16 @@ ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" +MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" [weakdeps] ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" @@ -36,12 +40,24 @@ ChainRulesCore = "1.24" CommonSolve = "0.2.4" DiffEqBase = "6.155" DifferentiationInterface = "0.5.17" +FastClosures = "0.3.2" FiniteDiff = "2.24.0" ForwardDiff = "0.10.36" +LinearAlgebra = "1.10" +MaybeInplace = "0.1.4" NonlinearSolveBase = "1" PrecompileTools = "1.2" Reexport = "1.2" ReverseDiff = "1.15" SciMLBase = "2.50" +StaticArraysCore = "1.4.3" Tracker = "0.2.35" julia = "1.10" + +[extras] +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" + +[targets] +test = ["InteractiveUtils", "Test", "TestItemRunner"] diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index f1d7b0713..4a9f369f1 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,14 +1,16 @@ module SimpleNonlinearSolve using CommonSolve: CommonSolve, solve +using FastClosures: @closure +using MaybeInplace: @bb using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode +using StaticArraysCore: StaticArray # AD Dependencies -using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, - AutoPolyesterForwardDiff +using ADTypes: AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff using DifferentiationInterface: DifferentiationInterface # TODO: move these to extensions in a breaking change. These are not even used in the # package, but are used to trigger the extension loading in DI.jl @@ -16,7 +18,7 @@ using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder -using NonlinearSolveBase: ImmutableNonlinearProblem +using NonlinearSolveBase: ImmutableNonlinearProblem, get_tolerance const DI = DifferentiationInterface @@ -24,6 +26,8 @@ abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorith is_extension_loaded(::Val) = false +include("utils.jl") + # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms function CommonSolve.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl new file mode 100644 index 000000000..feb93d423 --- /dev/null +++ b/lib/SimpleNonlinearSolve/src/klement.jl @@ -0,0 +1,47 @@ +""" + SimpleKlement() + +A low-overhead implementation of `Klement` [klement2014using](@citep). This +method is non-allocating on scalar and static array problems. +""" +struct SimpleKlement <: AbstractSimpleNonlinearSolveAlgorithm end + +function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, args...; + abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + T = eltype(x) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + @bb δx = copy(x) + @bb fprev = copy(fx) + @bb xo = copy(x) + @bb d = copy(x) + + J = one.(x) + @bb δx² = similar(x) + + for _ in 1:maxiters + any(iszero, J) && (J = Utils.identity_jacobian!!(J)) + + @bb @. δx = fprev / J + + @bb @. x = xo - δx + fx = Utils.eval_f(prob, fx, x) + + # Termination Checks + # tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg) + tc_sol !== nothing && return tc_sol + + @bb δx .*= -1 + @bb @. δx² = δx^2 * J^2 + @bb @. J += (fx - fprev - J * δx) / ifelse(iszero(δx²), T(1e-5), δx²) * δx * (J^2) + + @bb copyto!(fprev, fx) + @bb copyto!(xo, x) + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl new file mode 100644 index 000000000..64845b3c8 --- /dev/null +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -0,0 +1,107 @@ +module Utils + +using ADTypes: AbstractADType, AutoForwardDiff, AutoFiniteDiff, AutoPolyesterForwardDiff +using ArrayInterface: ArrayInterface +using DifferentiationInterface: DifferentiationInterface +using FastClosures: @closure +using LinearAlgebra: LinearAlgebra, I, diagind +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem +using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, + NonlinearFunction +using StaticArraysCore: StaticArray, SArray, SMatrix, SVector + +const DI = DifferentiationInterface + +const safe_similar = NonlinearSolveBase.Utils.safe_similar + +pickchunksize(n::Int) = min(n, 12) + +can_dual(::Type{<:Real}) = true +can_dual(::Type) = false + +maybe_unaliased(x::Union{Number, SArray}, ::Bool) = x +function maybe_unaliased(x::T, alias::Bool) where {T <: AbstractArray} + (alias || !ArrayInterface.can_setindex(T)) && return x + return copy(x) +end + +function get_concrete_autodiff(_, ad::AbstractADType) + DI.check_available(ad) && return ad + error("AD Backend $(ad) is not available. This could be because you haven't loaded the \ + actual backend (See [Differentiation Inferface Docs](https://gdalle.github.io/DifferentiationInterface.jl/DifferentiationInterface/stable/) \ + for more details) or the backend might not be supported by DifferentiationInferface.jl.") +end +function get_concrete_autodiff( + prob, ad::Union{AutoForwardDiff{nothing}, AutoPolyesterForwardDiff{nothing}}) + return get_concrete_autodiff(prob, + ArrayInterface.parameterless_type(ad)(; + chunksize = pickchunksize(length(prob.u0)), ad.tag)) +end +function get_concrete_autodiff(prob, ::Nothing) + if can_dual(eltype(prob.u0)) && DI.check_available(AutoForwardDiff()) + return AutoForwardDiff(; chunksize = pickchunksize(length(prob.u0))) + end + DI.check_available(AutoFiniteDiff()) && return AutoFiniteDiff() + error("Default AD backends are not available. Please load either FiniteDiff or \ + ForwardDiff for default AD selection to work. Else provide a specific AD \ + backend (instead of `nothing`) to the solver.") +end + +# NOTE: This doesn't initialize the `f(x)` but just returns a buffer of the same size +function get_fx(prob::NonlinearLeastSquaresProblem, x) + if SciMLBase.isinplace(prob) && prob.f.resid_prototype === nothing + error("Inplace NonlinearLeastSquaresProblem requires a `resid_prototype` to be \ + specified.") + end + return get_fx(prob.f, x, prob.p) +end +function get_fx(prob::Union{ImmutableNonlinearProblem, NonlinearProblem}, x) + return get_fx(prob.f, x, prob.p) +end +function get_fx(f::NonlinearFunction, x, p) + if SciMLBase.isinplace(f) + f.resid_prototype === nothing && return eltype(x).(f.resid_prototype) + return safe_similar(x) + end + return f(x, p) +end + +function eval_f(prob, fx, x) + SciMLBase.isinplace(prob) || return prob.f(x, prob.p) + prob.f(fx, x, prob.p) + return fx +end + +function fixed_parameter_function(prob::AbstractNonlinearProblem) + SciMLBase.isinplace(prob) && return @closure (du, u) -> prob.f(du, u, prob.p) + return Base.Fix2(prob.f, prob.p) +end + +# __init_identity_jacobian(u::Number, fu, α = true) = oftype(u, α) +# function __init_identity_jacobian(u, fu, α = true) +# J = __similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u)) +# fill!(J, zero(eltype(J))) +# J[diagind(J)] .= eltype(J)(α) +# return J +# end +# function __init_identity_jacobian(u::StaticArray, fu, α = true) +# S1, S2 = length(fu), length(u) +# J = SMatrix{S1, S2, eltype(u)}(I * α) +# return J +# end + +identity_jacobian!!(J::Number) = one(J) +function identity_jacobian!!(J::AbstractVector) + ArrayInterface.can_setindex(J) || return one.(J) + fill!(J, true) + return J +end +function identity_jacobian!!(J::AbstractMatrix) + ArrayInterface.can_setindex(J) || return convert(typeof(J), I) + J[diagind(J)] .= true + return J +end +identity_jacobian!!(::SMatrix{S1, S2, T}) where {S1, S2, T} = SMatrix{S1, S2, T}(I) +identity_jacobian!!(::SVector{S1, T}) where {S1, T} = ones(SVector{S1, T}) + +end diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index 8b1378917..6ea6326b0 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -1 +1,5 @@ +using TestItemRunner, InteractiveUtils +@info sprint(InteractiveUtils.versioninfo) + +@run_package_tests From 00e3ce56d9cc1dd0d2ef270b497fe394e1849552 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 17:45:50 -0400 Subject: [PATCH 25/92] chore: fix typo in error message --- lib/SimpleNonlinearSolve/src/utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 64845b3c8..13aad5655 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -28,8 +28,8 @@ end function get_concrete_autodiff(_, ad::AbstractADType) DI.check_available(ad) && return ad error("AD Backend $(ad) is not available. This could be because you haven't loaded the \ - actual backend (See [Differentiation Inferface Docs](https://gdalle.github.io/DifferentiationInterface.jl/DifferentiationInterface/stable/) \ - for more details) or the backend might not be supported by DifferentiationInferface.jl.") + actual backend (See [Differentiation Interface Docs](https://gdalle.github.io/DifferentiationInterface.jl/DifferentiationInterface/stable/) \ + for more details) or the backend might not be supported by DifferentiationInterface.jl.") end function get_concrete_autodiff( prob, ad::Union{AutoForwardDiff{nothing}, AutoPolyesterForwardDiff{nothing}}) From 317c1412ce97c57d056acfabe4d68d86bde3f555 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 17 Sep 2024 17:50:49 -0400 Subject: [PATCH 26/92] fix: missing import --- lib/SimpleNonlinearSolve/src/utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 13aad5655..a12c90f78 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -6,8 +6,8 @@ using DifferentiationInterface: DifferentiationInterface using FastClosures: @closure using LinearAlgebra: LinearAlgebra, I, diagind using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem -using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, - NonlinearFunction +using SciMLBase: SciMLBase, AbstractNonlinearProblem, NonlinearLeastSquaresProblem, + NonlinearProblem, NonlinearFunction using StaticArraysCore: StaticArray, SArray, SMatrix, SVector const DI = DifferentiationInterface From 0d56e22832bceb188d441abef1ce5919f0ed5e99 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 18 Sep 2024 16:50:29 -0400 Subject: [PATCH 27/92] feat: functional Klement --- .../src/NonlinearSolveBase.jl | 2 +- lib/NonlinearSolveBase/src/public.jl | 4 +- .../src/termination_conditions.jl | 2 +- .../src/SimpleNonlinearSolve.jl | 6 +- lib/SimpleNonlinearSolve/src/klement.jl | 5 +- lib/SimpleNonlinearSolve/src/utils.jl | 57 ++++++++++++++----- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 1ba7a0cc3..63f4b697c 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -10,7 +10,7 @@ using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinearProblem, NonlinearProblem, NonlinearLeastSquaresProblem, AbstractNonlinearFunction, @add_kwonly, StandardNonlinearProblem, NullParameters, NonlinearProblem, - isinplace + isinplace, warn_paramtype using StaticArraysCore: StaticArray include("public.jl") diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl index db8c389e2..d9014d71e 100644 --- a/lib/NonlinearSolveBase/src/public.jl +++ b/lib/NonlinearSolveBase/src/public.jl @@ -51,7 +51,7 @@ for name in (:Norm, :RelNorm, :AbsNorm) @eval begin """ - $($struct_name) <: AbstractSafeNonlinearTerminationMode + $($struct_name) <: AbstractNonlinearTerminationMode Terminates if $($doctring). @@ -63,7 +63,7 @@ for name in (:Norm, :RelNorm, :AbsNorm) $($TERM_INTERNALNORM_DOCS). """ - struct $(struct_name){F} <: AbstractSafeNonlinearTerminationMode + struct $(struct_name){F} <: AbstractNonlinearTerminationMode internalnorm::F function $(struct_name)(internalnorm::F) where {F} diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index a278861a8..4403e12c3 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -276,6 +276,6 @@ function init_termination_cache(::AbstractNonlinearProblem, abstol, reltol, du, T = promote_type(eltype(du), eltype(u)) abstol = get_tolerance(abstol, T) reltol = get_tolerance(reltol, T) - cache = init(du, u, tc; abstol, reltol) + cache = SciMLBase.init(du, u, tc; abstol, reltol) return abstol, reltol, cache end diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 4a9f369f1..4b524e4bf 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -18,7 +18,7 @@ using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder -using NonlinearSolveBase: ImmutableNonlinearProblem, get_tolerance +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, get_tolerance const DI = DifferentiationInterface @@ -28,6 +28,8 @@ is_extension_loaded(::Val) = false include("utils.jl") +include("klement.jl") + # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms function CommonSolve.solve(prob::NonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) @@ -69,7 +71,7 @@ function solve_adjoint_internal end prob_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p, ones(T, 3), T(2)) prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) - algs = [] + algs = [SimpleKlement()] algs_no_iip = [] @compile_workload begin diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl index feb93d423..055f65bc3 100644 --- a/lib/SimpleNonlinearSolve/src/klement.jl +++ b/lib/SimpleNonlinearSolve/src/klement.jl @@ -11,6 +11,7 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, alias_u0 = false, termination_condition = nothing, kwargs...) x = Utils.maybe_unaliased(prob.u0, alias_u0) T = eltype(x) + fx = Utils.get_fx(prob, x) abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) @@ -32,8 +33,8 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, fx = Utils.eval_f(prob, fx, x) # Termination Checks - # tc_sol = check_termination(tc_cache, fx, x, xo, prob, alg) - tc_sol !== nothing && return tc_sol + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) @bb δx .*= -1 @bb @. δx² = δx^2 * J^2 diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index a12c90f78..9008171d3 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -5,9 +5,12 @@ using ArrayInterface: ArrayInterface using DifferentiationInterface: DifferentiationInterface using FastClosures: @closure using LinearAlgebra: LinearAlgebra, I, diagind -using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, + AbstractNonlinearTerminationMode, + AbstractSafeNonlinearTerminationMode, + AbstractSafeBestNonlinearTerminationMode using SciMLBase: SciMLBase, AbstractNonlinearProblem, NonlinearLeastSquaresProblem, - NonlinearProblem, NonlinearFunction + NonlinearProblem, NonlinearFunction, ReturnCode using StaticArraysCore: StaticArray, SArray, SMatrix, SVector const DI = DifferentiationInterface @@ -60,7 +63,7 @@ function get_fx(prob::Union{ImmutableNonlinearProblem, NonlinearProblem}, x) end function get_fx(f::NonlinearFunction, x, p) if SciMLBase.isinplace(f) - f.resid_prototype === nothing && return eltype(x).(f.resid_prototype) + f.resid_prototype === nothing || return eltype(x).(f.resid_prototype) return safe_similar(x) end return f(x, p) @@ -77,18 +80,18 @@ function fixed_parameter_function(prob::AbstractNonlinearProblem) return Base.Fix2(prob.f, prob.p) end -# __init_identity_jacobian(u::Number, fu, α = true) = oftype(u, α) -# function __init_identity_jacobian(u, fu, α = true) -# J = __similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u)) -# fill!(J, zero(eltype(J))) -# J[diagind(J)] .= eltype(J)(α) -# return J -# end -# function __init_identity_jacobian(u::StaticArray, fu, α = true) -# S1, S2 = length(fu), length(u) -# J = SMatrix{S1, S2, eltype(u)}(I * α) -# return J -# end +function identity_jacobian(u::Number, fu::Number, α = true) + return convert(promote_type(eltype(u), eltype(fu)), α) +end +function identity_jacobian(u, fu, α = true) + J = safe_similar(u, promote_type(eltype(u), eltype(fu))) + fill!(J, zero(eltype(J))) + J[diagind(J)] .= eltype(J)(α) + return J +end +function identity_jacobian(u::StaticArray, fu, α = true) + return SMatrix{length(fu), length(u), eltype(u)}(I * α) +end identity_jacobian!!(J::Number) = one(J) function identity_jacobian!!(J::AbstractVector) @@ -104,4 +107,28 @@ end identity_jacobian!!(::SMatrix{S1, S2, T}) where {S1, S2, T} = SMatrix{S1, S2, T}(I) identity_jacobian!!(::SVector{S1, T}) where {S1, T} = ones(SVector{S1, T}) +# Termination Conditions +function check_termination(cache, fx, x, xo, prob) + return check_termination(cache, fx, x, xo, prob, cache.mode) +end + +function check_termination(cache, fx, x, xo, _, ::AbstractNonlinearTerminationMode) + return cache(fx, x, xo), ReturnCode.Success, fx, x +end +function check_termination(cache, fx, x, xo, _, ::AbstractSafeNonlinearTerminationMode) + return cache(fx, x, xo), cache.retcode, fx, x +end +function check_termination(cache, fx, x, xo, prob, ::AbstractSafeBestNonlinearTerminationMode) + if cache(fx, x, xo) + x = cache.u + if SciMLBase.isinplace(prob) + prob.f(fx, x, prob.p) + else + fx = prob.f(x, prob.p) + end + return true, cache.retcode, fx, x + end + return false, ReturnCode.Default, fx, x +end + end From 5d28535bbc9713e66d353d3e552b55401ffecb92 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 18 Sep 2024 16:52:37 -0400 Subject: [PATCH 28/92] chore: apply formatting suggestion Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- lib/SimpleNonlinearSolve/src/utils.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 9008171d3..16cf5142d 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -118,7 +118,8 @@ end function check_termination(cache, fx, x, xo, _, ::AbstractSafeNonlinearTerminationMode) return cache(fx, x, xo), cache.retcode, fx, x end -function check_termination(cache, fx, x, xo, prob, ::AbstractSafeBestNonlinearTerminationMode) +function check_termination( + cache, fx, x, xo, prob, ::AbstractSafeBestNonlinearTerminationMode) if cache(fx, x, xo) x = cache.u if SciMLBase.isinplace(prob) From 0ee06273ea6160802274426263fc0354c616b9f5 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 24 Sep 2024 22:49:53 -0400 Subject: [PATCH 29/92] chore: run formatter --- lib/NonlinearSolveBase/src/termination_conditions.jl | 6 +++--- lib/NonlinearSolveBase/src/utils.jl | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 4403e12c3..8bb58f8eb 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -219,9 +219,9 @@ end function check_convergence(::RelTerminationMode, duₙ, uₙ, __, ___, reltol) if Utils.fast_scalar_indexing(duₙ) return all(@closure(xy->begin - x, y = xy - return abs(y) ≤ reltol * abs(x + y) - end), zip(uₙ, duₙ)) + x, y = xy + return abs(y) ≤ reltol * abs(x + y) + end), zip(uₙ, duₙ)) else # using mapreduce here will almost certainly be faster on GPUs return mapreduce( @closure((xᵢ, yᵢ)->(abs(yᵢ) ≤ reltol * abs(xᵢ + yᵢ))), *, uₙ, duₙ; init = true) diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl index 0a8840942..cb54a6f4c 100644 --- a/lib/NonlinearSolveBase/src/utils.jl +++ b/lib/NonlinearSolveBase/src/utils.jl @@ -23,9 +23,9 @@ end function nonallocating_maximum(f::F, x, y) where {F} if fast_scalar_indexing(x, y) return maximum(@closure((xᵢyᵢ)->begin - xᵢ, yᵢ = xᵢyᵢ - return abs(f(xᵢ, yᵢ)) - end), zip(x, y)) + xᵢ, yᵢ = xᵢyᵢ + return abs(f(xᵢ, yᵢ)) + end), zip(x, y)) else return mapreduce(@closure((xᵢ, yᵢ)->abs(f(xᵢ, yᵢ))), max, x, y) end @@ -55,9 +55,9 @@ norm_op(norm::N, op::OP, x, y) where {N, OP} = norm(op.(x, y)) function norm_op(::typeof(L2_NORM), op::OP, x, y) where {OP} if fast_scalar_indexing(x, y) return sqrt(sum(@closure((xᵢ, yᵢ)->begin - xᵢ, yᵢ = xᵢyᵢ - return op(xᵢ, yᵢ)^2 - end), zip(x, y))) + xᵢ, yᵢ = xᵢyᵢ + return op(xᵢ, yᵢ)^2 + end), zip(x, y))) else return sqrt(mapreduce(@closure((xᵢ, yᵢ)->op(xᵢ, yᵢ)^2), +, x, y)) end From cd798b9470e02cd636bff3b3cd229257abf0a81b Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 24 Sep 2024 22:52:25 -0400 Subject: [PATCH 30/92] ci: make the scripts uniform --- .github/workflows/CI_BracketingNonlinearSolve.yml | 4 +--- .github/workflows/CI_NonlinearSolve.yml | 3 +++ .github/workflows/CI_SimpleNonlinearSolve.yml | 13 +++---------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/CI_BracketingNonlinearSolve.yml b/.github/workflows/CI_BracketingNonlinearSolve.yml index fc62f02c6..6d78e5612 100644 --- a/.github/workflows/CI_BracketingNonlinearSolve.yml +++ b/.github/workflows/CI_BracketingNonlinearSolve.yml @@ -6,10 +6,8 @@ on: - master paths: - "lib/BracketingNonlinearSolve/**" + - "lib/NonlinearSolveBase/**" - ".github/workflows/CI_BracketingNonlinearSolve.yml" - - "lib/NonlinearSolveBase/src/**" - - "lib/NonlinearSolveBase/ext/**" - - "lib/NonlinearSolveBase/Project.toml" push: branches: - master diff --git a/.github/workflows/CI_NonlinearSolve.yml b/.github/workflows/CI_NonlinearSolve.yml index a4c14820e..d684cb3b7 100644 --- a/.github/workflows/CI_NonlinearSolve.yml +++ b/.github/workflows/CI_NonlinearSolve.yml @@ -11,6 +11,9 @@ on: - "Project.toml" - ".github/workflows/CI_NonlinearSolve.yml" - "lib/SciMLNonlinearOperators/**" + - "lib/BracketingNonlinearSolve/**" + - "lib/NonlinearSolveBase/**" + - "lib/SimpleNonlinearSolve/**" push: branches: - master diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml index 365de3707..9854b6d99 100644 --- a/.github/workflows/CI_SimpleNonlinearSolve.yml +++ b/.github/workflows/CI_SimpleNonlinearSolve.yml @@ -5,17 +5,10 @@ on: branches: - master paths: - - "lib/SimpleNonlinearSolve/src/**" - - "lib/SimpleNonlinearSolve/ext/**" - - "lib/SimpleNonlinearSolve/test/**" - - "lib/SimpleNonlinearSolve/Project.toml" + - "lib/SimpleNonlinearSolve/**" + - "lib/BracketingNonlinearSolve/**" + - "lib/NonlinearSolveBase/**" - ".github/workflows/CI_SimpleNonlinearSolve.yml" - - "lib/BracketingNonlinearSolve/src/**" - - "lib/BracketingNonlinearSolve/ext/**" - - "lib/BracketingNonlinearSolve/Project.toml" - - "lib/NonlinearSolveBase/src/**" - - "lib/NonlinearSolveBase/ext/**" - - "lib/NonlinearSolveBase/Project.toml" push: branches: - master From c0ce0524a5341314a188c744bba9947873744625 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 25 Sep 2024 17:28:07 -0400 Subject: [PATCH 31/92] feat: bring in changes from https://github.com/SciML/SimpleNonlinearSolve.jl/pull/158 --- lib/BracketingNonlinearSolve/src/bisection.jl | 10 +++++++++- lib/BracketingNonlinearSolve/src/brent.jl | 10 +++++++++- lib/BracketingNonlinearSolve/src/falsi.jl | 10 +++++++++- lib/BracketingNonlinearSolve/src/itp.jl | 10 +++++++++- lib/BracketingNonlinearSolve/src/ridder.jl | 10 +++++++++- 5 files changed, 45 insertions(+), 5 deletions(-) diff --git a/lib/BracketingNonlinearSolve/src/bisection.jl b/lib/BracketingNonlinearSolve/src/bisection.jl index e3a38a1fd..1611ad34e 100644 --- a/lib/BracketingNonlinearSolve/src/bisection.jl +++ b/lib/BracketingNonlinearSolve/src/bisection.jl @@ -20,7 +20,7 @@ A common bisection method. end function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Bisection, - args...; maxiters = 1000, abstol = nothing, kwargs...) + args...; maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...) @assert !SciMLBase.isinplace(prob) "`Bisection` only supports out-of-place problems." f = Base.Fix2(prob.f, prob.p) @@ -40,6 +40,14 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Bisection, prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) end + if sign(fl) == sign(fr) + verbose && + @warn "The interval is not an enclosing interval, opposite signs at the \ + boundaries are required." + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right) + end + i = 1 while i ≤ maxiters mid = (left + right) / 2 diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl index 5475e675e..fea2ce3f4 100644 --- a/lib/BracketingNonlinearSolve/src/brent.jl +++ b/lib/BracketingNonlinearSolve/src/brent.jl @@ -6,7 +6,7 @@ Left non-allocating Brent method. struct Brent <: AbstractBracketingAlgorithm end function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; - maxiters = 1000, abstol = nothing, kwargs...) + maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...) @assert !SciMLBase.isinplace(prob) "`Brent` only supports out-of-place problems." f = Base.Fix2(prob.f, prob.p) @@ -27,6 +27,14 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) end + if sign(fl) == sign(fr) + verbose && + @warn "The interval is not an enclosing interval, opposite signs at the \ + boundaries are required." + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right) + end + if abs(fl) < abs(fr) left, right = right, left fl, fr = fr, fl diff --git a/lib/BracketingNonlinearSolve/src/falsi.jl b/lib/BracketingNonlinearSolve/src/falsi.jl index 451a4c0b2..8c62b95c6 100644 --- a/lib/BracketingNonlinearSolve/src/falsi.jl +++ b/lib/BracketingNonlinearSolve/src/falsi.jl @@ -6,7 +6,7 @@ A non-allocating regula falsi method. struct Falsi <: AbstractBracketingAlgorithm end function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...; - maxiters = 1000, abstol = nothing, kwargs...) + maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...) @assert !SciMLBase.isinplace(prob) "`False` only supports out-of-place problems." f = Base.Fix2(prob.f, prob.p) @@ -27,6 +27,14 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...; prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) end + if sign(fl) == sign(fr) + verbose && + @warn "The interval is not an enclosing interval, opposite signs at the \ + boundaries are required." + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right) + end + i = 1 while i ≤ maxiters if Impl.nextfloat_tdir(left, l, r) == right diff --git a/lib/BracketingNonlinearSolve/src/itp.jl b/lib/BracketingNonlinearSolve/src/itp.jl index dd6ddc23b..4798f9030 100644 --- a/lib/BracketingNonlinearSolve/src/itp.jl +++ b/lib/BracketingNonlinearSolve/src/itp.jl @@ -57,7 +57,7 @@ function ITP(; scaled_k1::Real = 0.2, k2::Real = 2, n0::Int = 10) end function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::ITP, args...; - maxiters = 1000, abstol = nothing, kwargs...) + maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...) @assert !SciMLBase.isinplace(prob) "`ITP` only supports out-of-place problems." f = Base.Fix2(prob.f, prob.p) @@ -77,6 +77,14 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::ITP, args...; prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) end + if sign(fl) == sign(fr) + verbose && + @warn "The interval is not an enclosing interval, opposite signs at the \ + boundaries are required." + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right) + end + ϵ = abstol k2 = alg.k2 k1 = alg.scaled_k1 * abs(right - left)^(1 - k2) diff --git a/lib/BracketingNonlinearSolve/src/ridder.jl b/lib/BracketingNonlinearSolve/src/ridder.jl index a289bcb05..e4b67a7c7 100644 --- a/lib/BracketingNonlinearSolve/src/ridder.jl +++ b/lib/BracketingNonlinearSolve/src/ridder.jl @@ -6,7 +6,7 @@ A non-allocating ridder method. struct Ridder <: AbstractBracketingAlgorithm end function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...; - maxiters = 1000, abstol = nothing, kwargs...) + maxiters = 1000, abstol = nothing, verbose::Bool = true, kwargs...) @assert !SciMLBase.isinplace(prob) "`Ridder` only supports out-of-place problems." f = Base.Fix2(prob.f, prob.p) @@ -26,6 +26,14 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...; prob, alg, right, fr; retcode = ReturnCode.ExactSolutionRight, left, right) end + if sign(fl) == sign(fr) + verbose && + @warn "The interval is not an enclosing interval, opposite signs at the \ + boundaries are required." + return SciMLBase.build_solution( + prob, alg, left, fl; retcode = ReturnCode.InitialFailure, left, right) + end + xo = oftype(left, Inf) i = 1 while i ≤ maxiters From a17568e4a383e31e9dc528eb5c8f9c3c9300894e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 25 Sep 2024 17:32:36 -0400 Subject: [PATCH 32/92] chore: unnecessary comment --- lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 4b524e4bf..0bb65181a 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -12,8 +12,6 @@ using StaticArraysCore: StaticArray # AD Dependencies using ADTypes: AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff using DifferentiationInterface: DifferentiationInterface -# TODO: move these to extensions in a breaking change. These are not even used in the -# package, but are used to trigger the extension loading in DI.jl using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff From 2d0768ad2c35c026467b34f738892221e5e67cca Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Wed, 25 Sep 2024 18:04:59 -0400 Subject: [PATCH 33/92] feat: automatic backend selection for autodiff --- lib/NonlinearSolveBase/Project.toml | 6 + .../ext/NonlinearSolveBaseForwardDiffExt.jl | 9 +- .../src/NonlinearSolveBase.jl | 9 ++ lib/NonlinearSolveBase/src/autodiff.jl | 109 ++++++++++++++++++ 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 lib/NonlinearSolveBase/src/autodiff.jl diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 94d12d822..819bcc79f 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -4,10 +4,13 @@ authors = ["Avik Pal and contributors"] version = "1.0.0" [deps] +ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" Compat = "34da2185-b29b-5c13-b0c7-acf172513d20" ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" +EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -24,10 +27,13 @@ NonlinearSolveBaseForwardDiffExt = "ForwardDiff" NonlinearSolveBaseSparseArraysExt = "SparseArrays" [compat] +ADTypes = "1.9" ArrayInterface = "7.9" CommonSolve = "0.2.4" Compat = "4.15" ConcreteStructs = "0.2.3" +DifferentiationInterface = "0.6.1" +EnzymeCore = "0.8" FastClosures = "0.3" ForwardDiff = "0.10.36" LinearAlgebra = "1.10" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index 469c5944b..31550da96 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -1,13 +1,20 @@ module NonlinearSolveBaseForwardDiffExt +using ADTypes: ADTypes, AutoForwardDiff, AutoPolyesterForwardDiff using CommonSolve: solve using FastClosures: @closure using ForwardDiff: ForwardDiff, Dual -using SciMLBase: SciMLBase, IntervalNonlinearProblem, NonlinearProblem, +using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem, + NonlinearProblem, NonlinearLeastSquaresProblem, remake using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, Utils +function NonlinearSolveBase.additional_incompatible_backend_check( + prob::AbstractNonlinearProblem, ::Union{AutoForwardDiff, AutoPolyesterForwardDiff}) + return !ForwardDiff.can_dual(eltype(prob.u0)) +end + Utils.value(::Type{Dual{T, V, N}}) where {T, V, N} = V Utils.value(x::Dual) = Utils.value(ForwardDiff.value(x)) Utils.value(x::AbstractArray{<:Dual}) = Utils.value.(x) diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 63f4b697c..4b3eec258 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -1,8 +1,11 @@ module NonlinearSolveBase +using ADTypes: ADTypes, AbstractADType, ForwardMode, ReverseMode using ArrayInterface: ArrayInterface using Compat: @compat using ConcreteStructs: @concrete +using DifferentiationInterface: DifferentiationInterface +using EnzymeCore: EnzymeCore using FastClosures: @closure using LinearAlgebra: norm using Markdown: @doc_str @@ -13,6 +16,8 @@ using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinear isinplace, warn_paramtype using StaticArraysCore: StaticArray +const DI = DifferentiationInterface + include("public.jl") include("utils.jl") @@ -20,9 +25,13 @@ include("immutable_problem.jl") include("common_defaults.jl") include("termination_conditions.jl") +include("autodiff.jl") + # Unexported Public API @compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance)) @compat(public, (nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution)) +@compat(public, (select_forward_mode_autodiff, select_reverse_mode_autodiff, + select_jacobian_autodiff)) export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTerminationMode, AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl new file mode 100644 index 000000000..d2e51389d --- /dev/null +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -0,0 +1,109 @@ +# Here we determine the preferred AD backend. We have a predefined list of ADs and then +# we select the first one that is avialable and would work with the problem. + +# Ordering is important here. We want to select the first one that is compatible with the +# problem. +const ReverseADs = [ + ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), + ADTypes.AutoZygote(), + ADTypes.AutoTracker(), + ADTypes.AutoReverseDiff(), + ADTypes.AutoFiniteDiff() +] + +const ForwardADs = [ + ADTypes.AutoEnzyme(; mode = EnzymeCore.Forward), + ADTypes.AutoPolyesterForwardDiff(), + ADTypes.AutoForwardDiff(), + ADTypes.AutoFiniteDiff() +] + +# TODO: Handle Sparsity + +function select_forward_mode_autodiff( + prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) + if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) + @warn "The chosen AD backend $(ad) is not a forward mode AD. Use with caution." + end + if incompatible_backend_and_problem(prob, ad) + adₙ = select_forward_mode_autodiff(prob, nothing; warn_check_mode) + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + running autodiff selection detected `$(adₙ)` as a potential forward mode \ + backend." + return adₙ + end + return ad +end + +function select_forward_mode_autodiff(prob::AbstractNonlinearProblem, ::Nothing; + warn_check_mode::Bool = true) + idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ForwardADs) + idx !== nothing && return ForwardADs[idx] + throw(ArgumentError("No forward mode AD backend is compatible with the chosen problem. \ + This could be because no forward mode autodiff backend is loaded \ + or the loaded backends don't support the problem.")) +end + +function select_reverse_mode_autodiff( + prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) + if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ReverseMode) + if !is_finite_differences_backend(ad) + @warn "The chosen AD backend $(ad) is not a reverse mode AD. Use with caution." + else + @warn "The chosen AD backend $(ad) is a finite differences backend. This might \ + be slow and inaccurate. Use with caution." + end + end + if incompatible_backend_and_problem(prob, ad) + adₙ = select_reverse_mode_autodiff(prob, nothing; warn_check_mode) + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + running autodiff selection detected `$(adₙ)` as a potential reverse mode \ + backend." + return adₙ + end + return ad +end + +function select_reverse_mode_autodiff(prob::AbstractNonlinearProblem, ::Nothing; + warn_check_mode::Bool = true) + idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ReverseADs) + idx !== nothing && return ReverseADs[idx] + throw(ArgumentError("No reverse mode AD backend is compatible with the chosen problem. \ + This could be because no reverse mode autodiff backend is loaded \ + or the loaded backends don't support the problem.")) +end + +function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ad::AbstractADType) + if incompatible_backend_and_problem(prob, ad) + adₙ = select_jacobian_autodiff(prob, nothing) + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + running autodiff selection detected `$(adₙ)` as a potential jacobian \ + backend." + return adₙ + end + return ad +end + +function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ::Nothing) + idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ForwardADs) + idx !== nothing && !is_finite_differences_backend(ForwardADs[idx]) && + return ForwardADs[idx] + idx = findfirst(!Base.Fix1(incompatible_backend_and_problem, prob), ReverseADs) + idx !== nothing && return ReverseADs[idx] + throw(ArgumentError("No jacobian AD backend is compatible with the chosen problem. \ + This could be because no jacobian autodiff backend is loaded \ + or the loaded backends don't support the problem.")) +end + +function incompatible_backend_and_problem( + prob::AbstractNonlinearProblem, ad::AbstractADType) + !DI.check_available(ad) && return true + SciMLBase.isinplace(prob) && !DI.check_inplace(ad) && return true + return additional_incompatible_backend_check(prob, ad) +end + +additional_incompatible_backend_check(::AbstractNonlinearProblem, ::AbstractADType) = false + +is_finite_differences_backend(ad::AbstractADType) = false +is_finite_differences_backend(::ADTypes.AutoFiniteDiff) = true +is_finite_differences_backend(::ADTypes.AutoFiniteDifferences) = true From 5f2d608397f7318428f2338affd0fcb514848e3d Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Thu, 26 Sep 2024 03:04:53 -0400 Subject: [PATCH 34/92] feat: check for branching for ReverseDiff(compile=true) --- lib/NonlinearSolveBase/Project.toml | 2 ++ lib/NonlinearSolveBase/src/NonlinearSolveBase.jl | 10 ++++++---- lib/NonlinearSolveBase/src/autodiff.jl | 9 +++++++++ 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 819bcc79f..3999de770 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -12,6 +12,7 @@ ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" +FunctionProperties = "f62d2435-5019-4c03-9749-2d4c77af0cbc" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" @@ -36,6 +37,7 @@ DifferentiationInterface = "0.6.1" EnzymeCore = "0.8" FastClosures = "0.3" ForwardDiff = "0.10.36" +FunctionProperties = "0.1.2" LinearAlgebra = "1.10" Markdown = "1.10" RecursiveArrayTools = "3" diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 4b3eec258..5e1a37326 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -7,13 +7,14 @@ using ConcreteStructs: @concrete using DifferentiationInterface: DifferentiationInterface using EnzymeCore: EnzymeCore using FastClosures: @closure +using FunctionProperties: hasbranching using LinearAlgebra: norm using Markdown: @doc_str using RecursiveArrayTools: AbstractVectorOfArray, ArrayPartition using SciMLBase: SciMLBase, ReturnCode, AbstractODEIntegrator, AbstractNonlinearProblem, NonlinearProblem, NonlinearLeastSquaresProblem, AbstractNonlinearFunction, - @add_kwonly, StandardNonlinearProblem, NullParameters, NonlinearProblem, - isinplace, warn_paramtype + @add_kwonly, StandardNonlinearProblem, NullParameters, isinplace, + warn_paramtype using StaticArraysCore: StaticArray const DI = DifferentiationInterface @@ -30,8 +31,9 @@ include("autodiff.jl") # Unexported Public API @compat(public, (L2_NORM, Linf_NORM, NAN_CHECK, UNITLESS_ABS2, get_tolerance)) @compat(public, (nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution)) -@compat(public, (select_forward_mode_autodiff, select_reverse_mode_autodiff, - select_jacobian_autodiff)) +@compat(public, + (select_forward_mode_autodiff, select_reverse_mode_autodiff, + select_jacobian_autodiff)) export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTerminationMode, AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index d2e51389d..f81ce7039 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -7,6 +7,7 @@ const ReverseADs = [ ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), ADTypes.AutoZygote(), ADTypes.AutoTracker(), + ADTypes.AutoReverseDiff(; compile = true), ADTypes.AutoReverseDiff(), ADTypes.AutoFiniteDiff() ] @@ -103,6 +104,14 @@ function incompatible_backend_and_problem( end additional_incompatible_backend_check(::AbstractNonlinearProblem, ::AbstractADType) = false +function additional_incompatible_backend_check(prob::AbstractNonlinearProblem, + ::ADTypes.AutoReverseDiff{true}) + if SciMLBase.isinplace(prob) + fu = prob.f.resid_prototype === nothing ? zero(prob.u0) : prob.f.resid_prototype + return hasbranching(prob.f, fu, prob.u0, prob.p) + end + return hasbranching(prob.f, prob.u0, prob.p) +end is_finite_differences_backend(ad::AbstractADType) = false is_finite_differences_backend(::ADTypes.AutoFiniteDiff) = true From 9efebededbe8e9a8d03963e1ed13f6ee276ea2a7 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 12:51:43 -0400 Subject: [PATCH 35/92] feat: SimpleNewtonRaphson --- docs/Project.toml | 2 +- lib/NonlinearSolveBase/src/autodiff.jl | 17 +++-- lib/SimpleNonlinearSolve/Project.toml | 7 +- .../src/SimpleNonlinearSolve.jl | 10 ++- lib/SimpleNonlinearSolve/src/raphson.jl | 62 ++++++++++++++++ lib/SimpleNonlinearSolve/src/utils.jl | 70 ++++++++++++++++++- 6 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/src/raphson.jl diff --git a/docs/Project.toml b/docs/Project.toml index ab35d42ae..4ad265246 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -30,7 +30,7 @@ AlgebraicMultigrid = "0.5, 0.6" ArrayInterface = "6, 7" BenchmarkTools = "1" DiffEqBase = "6.136" -DifferentiationInterface = "0.6" +DifferentiationInterface = "0.6.1" Documenter = "1" DocumenterCitations = "1" DocumenterInterLinks = "1.0.0" diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index f81ce7039..4c057d25e 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -3,21 +3,21 @@ # Ordering is important here. We want to select the first one that is compatible with the # problem. -const ReverseADs = [ +const ReverseADs = ( ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), ADTypes.AutoZygote(), ADTypes.AutoTracker(), ADTypes.AutoReverseDiff(; compile = true), ADTypes.AutoReverseDiff(), ADTypes.AutoFiniteDiff() -] +) -const ForwardADs = [ +const ForwardADs = ( ADTypes.AutoEnzyme(; mode = EnzymeCore.Forward), ADTypes.AutoPolyesterForwardDiff(), ADTypes.AutoForwardDiff(), ADTypes.AutoFiniteDiff() -] +) # TODO: Handle Sparsity @@ -28,7 +28,8 @@ function select_forward_mode_autodiff( end if incompatible_backend_and_problem(prob, ad) adₙ = select_forward_mode_autodiff(prob, nothing; warn_check_mode) - @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ + could be because the backend package for the choosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential forward mode \ backend." return adₙ @@ -57,7 +58,8 @@ function select_reverse_mode_autodiff( end if incompatible_backend_and_problem(prob, ad) adₙ = select_reverse_mode_autodiff(prob, nothing; warn_check_mode) - @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ + could be because the backend package for the choosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential reverse mode \ backend." return adₙ @@ -77,7 +79,8 @@ end function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ad::AbstractADType) if incompatible_backend_and_problem(prob, ad) adₙ = select_jacobian_autodiff(prob, nothing) - @warn "The chosen AD backend `$(ad)` does not support the chosen problem. After \ + @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ + could be because the backend package for the choosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential jacobian \ backend." return adₙ diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 7ab2416be..e21eb1c5f 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -8,6 +8,7 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" @@ -38,11 +39,13 @@ ArrayInterface = "7.16" BracketingNonlinearSolve = "1" ChainRulesCore = "1.24" CommonSolve = "0.2.4" +ConcreteStructs = "0.2.3" DiffEqBase = "6.155" -DifferentiationInterface = "0.5.17" +DifferentiationInterface = "0.6.1" FastClosures = "0.3.2" FiniteDiff = "2.24.0" ForwardDiff = "0.10.36" +InteractiveUtils = "<0.0.1, 1" LinearAlgebra = "1.10" MaybeInplace = "0.1.4" NonlinearSolveBase = "1" @@ -51,6 +54,8 @@ Reexport = "1.2" ReverseDiff = "1.15" SciMLBase = "2.50" StaticArraysCore = "1.4.3" +Test = "1.10" +TestItemRunner = "1" Tracker = "0.2.35" julia = "1.10" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 0bb65181a..1baa1b50c 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,6 +1,7 @@ module SimpleNonlinearSolve using CommonSolve: CommonSolve, solve +using ConcreteStructs: @concrete using FastClosures: @closure using MaybeInplace: @bb using PrecompileTools: @compile_workload, @setup_workload @@ -27,6 +28,7 @@ is_extension_loaded(::Val) = false include("utils.jl") include("klement.jl") +include("raphson.jl") # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms function CommonSolve.solve(prob::NonlinearProblem, @@ -69,7 +71,10 @@ function solve_adjoint_internal end prob_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p, ones(T, 3), T(2)) prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) - algs = [SimpleKlement()] + algs = [ + SimpleKlement(), + SimpleNewtonRaphson() + ] algs_no_iip = [] @compile_workload begin @@ -87,4 +92,7 @@ export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder +export SimpleKlement +export SimpleGaussNewton, SimpleNewtonRaphson + end diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl new file mode 100644 index 000000000..2af3a825a --- /dev/null +++ b/lib/SimpleNonlinearSolve/src/raphson.jl @@ -0,0 +1,62 @@ +""" + SimpleNewtonRaphson(autodiff) + SimpleNewtonRaphson(; autodiff = nothing) + +A low-overhead implementation of Newton-Raphson. This method is non-allocating on scalar +and static array problems. + +!!! note + + As part of the decreased overhead, this method omits some of the higher level error + catching of the other methods. Thus, to see better error messages, use one of the other + methods like `NewtonRaphson`. + +### Keyword Arguments + + - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e. + automatic backend selection). Valid choices include jacobian backends from + `DifferentiationInterface.jl`. +""" +@kwdef @concrete struct SimpleNewtonRaphson <: AbstractSimpleNonlinearSolveAlgorithm + autodiff = nothing +end + +const SimpleGaussNewton = SimpleNewtonRaphson + +function SciMLBase.__solve( + prob::ImmutableNonlinearProblem, alg::SimpleNewtonRaphson, args...; + abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) + + iszero(fx) && + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + autodiff = SciMLBase.has_jac(prob.f) ? alg.autodiff : + NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff) + + @bb xo = similar(x) + fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? similar(fx) : + nothing + jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) + J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) + + for _ in 1:maxiters + @bb copyto!(xo, x) + δx = Utils.restructure(x, J \ Utils.safe_vec(fx)) + @bb x .-= δx + + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + fx = Utils.eval_f(prob, fx, x) + J = Utils.compute_jacobian!!(J, prob, autodiff, fx_cache, x, jac_cache) + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 16cf5142d..012fc277a 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -2,7 +2,7 @@ module Utils using ADTypes: AbstractADType, AutoForwardDiff, AutoFiniteDiff, AutoPolyesterForwardDiff using ArrayInterface: ArrayInterface -using DifferentiationInterface: DifferentiationInterface +using DifferentiationInterface: DifferentiationInterface, Constant using FastClosures: @closure using LinearAlgebra: LinearAlgebra, I, diagind using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, @@ -132,4 +132,72 @@ function check_termination( return false, ReturnCode.Default, fx, x end +restructure(y, x) = ArrayInterface.restructure(y, x) +restructure(::Number, x::Number) = x + +safe_vec(x::AbstractArray) = vec(x) +safe_vec(x::Number) = x + +function prepare_jacobian(prob, autodiff, _, x::Number) + if SciMLBase.has_jac(prob.f) || SciMLBase.has_vjp(prob.f) || SciMLBase.has_jvp(prob.f) + return nothing + end + return DI.prepare_derivative(prob.f, autodiff, x, Constant(prob.p)) +end +function prepare_jacobian(prob, autodiff, fx, x) + if SciMLBase.has_jac(prob.f) + return nothing + end + if SciMLBase.isinplace(prob.f) + return DI.prepare_jacobian(prob.f, fx, autodiff, x, Constant(prob.p)) + else + return DI.prepare_jacobian(prob.f, autodiff, x, Constant(prob.p)) + end +end + +function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras) + if extras === nothing + if SciMLBase.has_jac(prob.f) + return prob.f.jac(x, prob.p) + elseif SciMLBase.has_vjp(prob.f) + return prob.f.vjp(one(x), x, prob.p) + elseif SciMLBase.has_jvp(prob.f) + return prob.f.jvp(one(x), x, prob.p) + end + end + return DI.derivative(prob.f, extras, autodiff, x, Constant(prob.p)) +end +function compute_jacobian!!(J, prob, autodiff, fx, x, extras) + if J === nothing + if extras === nothing + if SciMLBase.isinplace(prob.f) + J = similar(fx, length(fx), length(x)) + prob.f.jac(J, x, prob.p) + return J + else + return prob.f.jac(x, prob.p) + end + end + if SciMLBase.isinplace(prob) + return DI.jacobian(prob.f, fx, extras, autodiff, x, Constant(prob.p)) + else + return DI.jacobian(prob.f, extras, autodiff, x, Constant(prob.p)) + end + end + if extras === nothing + if SciMLBase.isinplace(prob) + prob.jac(J, x, prob.p) + return J + else + return prob.jac(x, prob.p) + end + end + if SciMLBase.isinplace(prob) + DI.jacobian!(prob.f, J, fx, extras, autodiff, x, Constant(prob.p)) + else + DI.jacobian!(prob.f, J, extras, autodiff, x, Constant(prob.p)) + end + return J +end + end From b094fa63931c5424d2e2fa1596b6fe2e90483fb0 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 12:54:37 -0400 Subject: [PATCH 36/92] fix: typos --- lib/NonlinearSolveBase/src/autodiff.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index 4c057d25e..83b6aa1f7 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -1,5 +1,5 @@ # Here we determine the preferred AD backend. We have a predefined list of ADs and then -# we select the first one that is avialable and would work with the problem. +# we select the first one that is available and would work with the problem. # Ordering is important here. We want to select the first one that is compatible with the # problem. @@ -29,7 +29,7 @@ function select_forward_mode_autodiff( if incompatible_backend_and_problem(prob, ad) adₙ = select_forward_mode_autodiff(prob, nothing; warn_check_mode) @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ - could be because the backend package for the choosen AD isn't loaded. After \ + could be because the backend package for the chosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential forward mode \ backend." return adₙ @@ -59,7 +59,7 @@ function select_reverse_mode_autodiff( if incompatible_backend_and_problem(prob, ad) adₙ = select_reverse_mode_autodiff(prob, nothing; warn_check_mode) @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ - could be because the backend package for the choosen AD isn't loaded. After \ + could be because the backend package for the chosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential reverse mode \ backend." return adₙ @@ -80,7 +80,7 @@ function select_jacobian_autodiff(prob::AbstractNonlinearProblem, ad::AbstractAD if incompatible_backend_and_problem(prob, ad) adₙ = select_jacobian_autodiff(prob, nothing) @warn "The chosen AD backend `$(ad)` does not support the chosen problem. This \ - could be because the backend package for the choosen AD isn't loaded. After \ + could be because the backend package for the chosen AD isn't loaded. After \ running autodiff selection detected `$(adₙ)` as a potential jacobian \ backend." return adₙ From a48a75965f166c2c92eb0dee5b17035f59a47cf8 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 13:05:17 -0400 Subject: [PATCH 37/92] chore: all files --- lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 5 +++++ lib/SimpleNonlinearSolve/src/broyden.jl | 0 lib/SimpleNonlinearSolve/src/dfsane.jl | 0 lib/SimpleNonlinearSolve/src/halley.jl | 0 lib/SimpleNonlinearSolve/src/lbroyden.jl | 0 lib/SimpleNonlinearSolve/src/trust_region.jl | 0 6 files changed, 5 insertions(+) create mode 100644 lib/SimpleNonlinearSolve/src/broyden.jl create mode 100644 lib/SimpleNonlinearSolve/src/dfsane.jl create mode 100644 lib/SimpleNonlinearSolve/src/halley.jl create mode 100644 lib/SimpleNonlinearSolve/src/lbroyden.jl create mode 100644 lib/SimpleNonlinearSolve/src/trust_region.jl diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 1baa1b50c..72f2cd90f 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -27,8 +27,13 @@ is_extension_loaded(::Val) = false include("utils.jl") +include("broyden.jl") +include("dfsane.jl") +include("halley.jl") include("klement.jl") +include("lbroyden.jl") include("raphson.jl") +include("trust_region.jl") # By Pass the highlevel checks for NonlinearProblem for Simple Algorithms function CommonSolve.solve(prob::NonlinearProblem, diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl new file mode 100644 index 000000000..e69de29bb From 3edc3684d9265f992ad406736c3605679e13ac13 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 13:19:37 -0400 Subject: [PATCH 38/92] fix: ordering in jacobian call --- lib/SimpleNonlinearSolve/src/utils.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 012fc277a..2a36ec60f 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -193,7 +193,7 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) end end if SciMLBase.isinplace(prob) - DI.jacobian!(prob.f, J, fx, extras, autodiff, x, Constant(prob.p)) + DI.jacobian!(prob.f, fx, J, extras, autodiff, x, Constant(prob.p)) else DI.jacobian!(prob.f, J, extras, autodiff, x, Constant(prob.p)) end From 8d1d2b4ebf14810caab0396bc6a44b10aa234524 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 13:47:29 -0400 Subject: [PATCH 39/92] chore: run formatter --- lib/SimpleNonlinearSolve/src/broyden.jl | 1 + lib/SimpleNonlinearSolve/src/dfsane.jl | 1 + lib/SimpleNonlinearSolve/src/halley.jl | 1 + lib/SimpleNonlinearSolve/src/lbroyden.jl | 1 + lib/SimpleNonlinearSolve/src/trust_region.jl | 1 + src/internal/termination.jl | 2 +- 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/src/broyden.jl +++ b/lib/SimpleNonlinearSolve/src/broyden.jl @@ -0,0 +1 @@ + diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/src/dfsane.jl +++ b/lib/SimpleNonlinearSolve/src/dfsane.jl @@ -0,0 +1 @@ + diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/src/halley.jl +++ b/lib/SimpleNonlinearSolve/src/halley.jl @@ -0,0 +1 @@ + diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -0,0 +1 @@ + diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -0,0 +1 @@ + diff --git a/src/internal/termination.jl b/src/internal/termination.jl index e09cfcdda..ef3f7c4c0 100644 --- a/src/internal/termination.jl +++ b/src/internal/termination.jl @@ -45,7 +45,7 @@ function update_from_termination_cache!(tc_cache, cache, u = get_u(cache)) end function update_from_termination_cache!( - _, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) + tc_cache, cache, ::AbstractNonlinearTerminationMode, u = get_u(cache)) evaluate_f!(cache, u, cache.p) end From 3bdf737e55e80b34682cd8d613abf8bbd5adbfd2 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 14:16:48 -0400 Subject: [PATCH 40/92] feat: SimpleTrustRegion implementation --- .../src/SimpleNonlinearSolve.jl | 9 +- lib/SimpleNonlinearSolve/src/trust_region.jl | 211 ++++++++++++++++++ lib/SimpleNonlinearSolve/src/utils.jl | 22 -- 3 files changed, 217 insertions(+), 25 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 72f2cd90f..d307e5b99 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -3,6 +3,7 @@ module SimpleNonlinearSolve using CommonSolve: CommonSolve, solve using ConcreteStructs: @concrete using FastClosures: @closure +using LinearAlgebra: dot using MaybeInplace: @bb using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @@ -17,7 +18,8 @@ using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder -using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, get_tolerance +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, get_tolerance, + L2_NORM const DI = DifferentiationInterface @@ -78,7 +80,8 @@ function solve_adjoint_internal end algs = [ SimpleKlement(), - SimpleNewtonRaphson() + SimpleNewtonRaphson(), + SimpleTrustRegion() ] algs_no_iip = [] @@ -98,6 +101,6 @@ export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder export SimpleKlement -export SimpleGaussNewton, SimpleNewtonRaphson +export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion end diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index 8b1378917..37d0ed706 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -1 +1,212 @@ +# """ +# SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius = 0.0, +# initial_trust_radius = 0.0, step_threshold = nothing, +# shrink_threshold = nothing, expand_threshold = nothing, +# shrink_factor = 0.25, expand_factor = 2.0, max_shrink_times::Int = 32, +# nlsolve_update_rule = Val(false)) + +# A low-overhead implementation of a trust-region solver. This method is non-allocating on +# scalar and static array problems. + +# ### Keyword Arguments + +# - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e. +# automatic backend selection). Valid choices include jacobian backends from +# `DifferentiationInterface.jl`. +# - `max_trust_radius`: the maximum radius of the trust region. Defaults to +# `max(norm(f(u0)), maximum(u0) - minimum(u0))`. +# - `initial_trust_radius`: the initial trust region radius. Defaults to +# `max_trust_radius / 11`. +# - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is +# compared with a value `r`, which is the actual reduction in the objective function divided +# by the predicted reduction. If `step_threshold > r` the model is not a good approximation, +# and the step is rejected. Defaults to `0.1`. For more details, see +# [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) +# - `shrink_threshold`: the threshold for shrinking the trust region radius. In every +# iteration, the threshold is compared with a value `r` which is the actual reduction in the +# objective function divided by the predicted reduction. If `shrink_threshold > r` the trust +# region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see +# [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) +# - `expand_threshold`: the threshold for expanding the trust region radius. If a step is +# taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also +# made to see if `expand_threshold < r`. If that is true, the trust region radius is +# expanded by `expand_factor`. Defaults to `0.75`. +# - `shrink_factor`: the factor to shrink the trust region radius with if +# `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`. +# - `expand_factor`: the factor to expand the trust region radius with if +# `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`. +# - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a +# row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`. +# - `nlsolve_update_rule`: If set to `Val(true)`, updates the trust region radius using the +# update rule from NLSolve.jl. Defaults to `Val(false)`. If set to `Val(true)`, few of the +# radius update parameters -- `step_threshold = 0.05`, `expand_threshold = 0.9`, and +# `shrink_factor = 0.5` -- have different defaults. +# """ +@kwdef @concrete struct SimpleTrustRegion <: AbstractSimpleNonlinearSolveAlgorithm + autodiff = nothing + max_trust_radius = 0.0 + initial_trust_radius = 0.0 + step_threshold = 0.0001 + shrink_threshold = nothing + expand_threshold = nothing + shrink_factor = nothing + expand_factor = 2.0 + max_shrink_times::Int = 32 + nlsolve_update_rule = Val(false) +end + +function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegion, + args...; abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + T = eltype(x) + Δₘₐₓ = T(alg.max_trust_radius) + Δ = T(alg.initial_trust_radius) + η₁ = T(alg.step_threshold) + + if alg.shrink_threshold === nothing + η₂ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.05, 0.25)) + else + η₂ = T(alg.shrink_threshold) + end + + if alg.expand_threshold === nothing + η₃ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.9, 0.75)) + else + η₃ = T(alg.expand_threshold) + end + + if alg.shrink_factor === nothing + t₁ = T(ifelse(SciMLBase._unwrap_val(alg.nlsolve_update_rule), 0.5, 0.25)) + else + t₁ = T(alg.shrink_factor) + end + + t₂ = T(alg.expand_factor) + max_shrink_times = alg.max_shrink_times + + autodiff = SciMLBase.has_jac(prob.f) ? alg.autodiff : + NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff) + + fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) + norm_fx = L2_NORM(fx) + + @bb xo = copy(x) + fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? similar(fx) : + nothing + jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) + J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + # Set default trust region radius if not specified by user. + iszero(Δₘₐₓ) && (Δₘₐₓ = max(L2_NORM(fx), maximum(x) - minimum(x))) + if iszero(Δ) + if SciMLBase._unwrap_val(alg.nlsolve_update_rule) + norm_x = L2_NORM(x) + Δ = T(ifelse(norm_x > 0, norm_x, 1)) + else + Δ = T(Δₘₐₓ / 11) + end + end + + fₖ = 0.5 * norm_fx^2 + H = transpose(J) * J + g = Utils.restructure(x, J' * Utils.safe_vec(fx)) + shrink_counter = 0 + + @bb δsd = copy(x) + @bb δN_δsd = copy(x) + @bb δN = copy(x) + @bb Hδ = copy(x) + dogleg_cache = (; δsd, δN_δsd, δN) + + for _ in 1:maxiters + # Solve the trust region subproblem. + δ = dogleg_method!!(dogleg_cache, J, fx, g, Δ) + @bb @. x = xo + δ + + fx = Utils.eval_f(prob, fx, x) + + fₖ₊₁ = L2_NORM(fx)^2 / T(2) + + # Compute the ratio of the actual to predicted reduction. + @bb Hδ = H × vec(δ) + r = (fₖ₊₁ - fₖ) / (dot(δ, g) + (dot(δ, Hδ) / T(2))) + + # Update the trust region radius. + if r ≥ η₂ + shrink_counter = 0 + else + Δ = t₁ * Δ + shrink_counter += 1 + shrink_counter > max_shrink_times && return SciMLBase.build_solution( + prob, alg, x, fx; retcode = ReturnCode.ShrinkThresholdExceeded) + end + + if r ≥ η₁ + # Termination Checks + solved, retcode, fx_sol, x_sol = Utils.check_termination( + tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + # Take the step. + @bb copyto!(xo, x) + + J = Utils.compute_jacobian!!(J, prob, autodiff, fx_cache, x, jac_cache) + fx = Utils.eval_f(prob, fx, x) + + # Update the trust region radius. + if !SciMLBase._unwrap_val(alg.nlsolve_update_rule) && r > η₃ + Δ = min(t₂ * Δ, Δₘₐₓ) + end + fₖ = fₖ₊₁ + + @bb H = transpose(J) × J + @bb g = transpose(J) × vec(fx) + end + + if SciMLBase._unwrap_val(alg.nlsolve_update_rule) + if r > η₃ + Δ = t₂ * L2_NORM(δ) + elseif r > 0.5 + Δ = max(Δ, t₂ * L2_NORM(δ)) + end + end + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end + +function dogleg_method!!(cache, J, f, g, Δ) + (; δsd, δN_δsd, δN) = cache + + # Compute the Newton step + @bb δN .= Utils.restructure(δN, J \ Utils.safe_vec(f)) + @bb δN .*= -1 + # Test if the full step is within the trust region + (L2_NORM(δN) ≤ Δ) && return δN + + # Calcualte Cauchy point, optimum along the steepest descent direction + @bb δsd .= g + @bb @. δsd *= -1 + norm_δsd = L2_NORM(δsd) + + if (norm_δsd ≥ Δ) + @bb @. δsd *= Δ / norm_δsd + return δsd + end + + # Find the intersection point on the boundary + @bb @. δN_δsd = δN - δsd + dot_δN_δsd = dot(δN_δsd, δN_δsd) + dot_δsd_δN_δsd = dot(δsd, δN_δsd) + dot_δsd = dot(δsd, δsd) + fact = dot_δsd_δN_δsd^2 - dot_δN_δsd * (dot_δsd - Δ^2) + tau = (-dot_δsd_δN_δsd + sqrt(fact)) / dot_δN_δsd + @bb @. δsd += tau * δN_δsd + return δsd +end diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 2a36ec60f..29437339e 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -28,28 +28,6 @@ function maybe_unaliased(x::T, alias::Bool) where {T <: AbstractArray} return copy(x) end -function get_concrete_autodiff(_, ad::AbstractADType) - DI.check_available(ad) && return ad - error("AD Backend $(ad) is not available. This could be because you haven't loaded the \ - actual backend (See [Differentiation Interface Docs](https://gdalle.github.io/DifferentiationInterface.jl/DifferentiationInterface/stable/) \ - for more details) or the backend might not be supported by DifferentiationInterface.jl.") -end -function get_concrete_autodiff( - prob, ad::Union{AutoForwardDiff{nothing}, AutoPolyesterForwardDiff{nothing}}) - return get_concrete_autodiff(prob, - ArrayInterface.parameterless_type(ad)(; - chunksize = pickchunksize(length(prob.u0)), ad.tag)) -end -function get_concrete_autodiff(prob, ::Nothing) - if can_dual(eltype(prob.u0)) && DI.check_available(AutoForwardDiff()) - return AutoForwardDiff(; chunksize = pickchunksize(length(prob.u0))) - end - DI.check_available(AutoFiniteDiff()) && return AutoFiniteDiff() - error("Default AD backends are not available. Please load either FiniteDiff or \ - ForwardDiff for default AD selection to work. Else provide a specific AD \ - backend (instead of `nothing`) to the solver.") -end - # NOTE: This doesn't initialize the `f(x)` but just returns a buffer of the same size function get_fx(prob::NonlinearLeastSquaresProblem, x) if SciMLBase.isinplace(prob) && prob.f.resid_prototype === nothing From 6b955012b7743655edbf997306355cc5322c16a7 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 14:53:50 -0400 Subject: [PATCH 41/92] docs: trust region docstring --- lib/SimpleNonlinearSolve/src/trust_region.jl | 89 ++++++++++---------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index 37d0ed706..6bf543220 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -1,48 +1,47 @@ - -# """ -# SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius = 0.0, -# initial_trust_radius = 0.0, step_threshold = nothing, -# shrink_threshold = nothing, expand_threshold = nothing, -# shrink_factor = 0.25, expand_factor = 2.0, max_shrink_times::Int = 32, -# nlsolve_update_rule = Val(false)) - -# A low-overhead implementation of a trust-region solver. This method is non-allocating on -# scalar and static array problems. - -# ### Keyword Arguments - -# - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e. -# automatic backend selection). Valid choices include jacobian backends from -# `DifferentiationInterface.jl`. -# - `max_trust_radius`: the maximum radius of the trust region. Defaults to -# `max(norm(f(u0)), maximum(u0) - minimum(u0))`. -# - `initial_trust_radius`: the initial trust region radius. Defaults to -# `max_trust_radius / 11`. -# - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is -# compared with a value `r`, which is the actual reduction in the objective function divided -# by the predicted reduction. If `step_threshold > r` the model is not a good approximation, -# and the step is rejected. Defaults to `0.1`. For more details, see -# [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) -# - `shrink_threshold`: the threshold for shrinking the trust region radius. In every -# iteration, the threshold is compared with a value `r` which is the actual reduction in the -# objective function divided by the predicted reduction. If `shrink_threshold > r` the trust -# region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see -# [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) -# - `expand_threshold`: the threshold for expanding the trust region radius. If a step is -# taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also -# made to see if `expand_threshold < r`. If that is true, the trust region radius is -# expanded by `expand_factor`. Defaults to `0.75`. -# - `shrink_factor`: the factor to shrink the trust region radius with if -# `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`. -# - `expand_factor`: the factor to expand the trust region radius with if -# `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`. -# - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a -# row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`. -# - `nlsolve_update_rule`: If set to `Val(true)`, updates the trust region radius using the -# update rule from NLSolve.jl. Defaults to `Val(false)`. If set to `Val(true)`, few of the -# radius update parameters -- `step_threshold = 0.05`, `expand_threshold = 0.9`, and -# `shrink_factor = 0.5` -- have different defaults. -# """ +""" + SimpleTrustRegion(; autodiff = AutoForwardDiff(), max_trust_radius = 0.0, + initial_trust_radius = 0.0, step_threshold = nothing, + shrink_threshold = nothing, expand_threshold = nothing, + shrink_factor = 0.25, expand_factor = 2.0, max_shrink_times::Int = 32, + nlsolve_update_rule = Val(false)) + +A low-overhead implementation of a trust-region solver. This method is non-allocating on +scalar and static array problems. + +### Keyword Arguments + + - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e. + automatic backend selection). Valid choices include jacobian backends from + `DifferentiationInterface.jl`. + - `max_trust_radius`: the maximum radius of the trust region. Defaults to + `max(norm(f(u0)), maximum(u0) - minimum(u0))`. + - `initial_trust_radius`: the initial trust region radius. Defaults to + `max_trust_radius / 11`. + - `step_threshold`: the threshold for taking a step. In every iteration, the threshold is + compared with a value `r`, which is the actual reduction in the objective function divided + by the predicted reduction. If `step_threshold > r` the model is not a good approximation, + and the step is rejected. Defaults to `0.1`. For more details, see + [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) + - `shrink_threshold`: the threshold for shrinking the trust region radius. In every + iteration, the threshold is compared with a value `r` which is the actual reduction in the + objective function divided by the predicted reduction. If `shrink_threshold > r` the trust + region radius is shrunk by `shrink_factor`. Defaults to `0.25`. For more details, see + [Rahpeymaii, F.](https://link.springer.com/article/10.1007/s40096-020-00339-4) + - `expand_threshold`: the threshold for expanding the trust region radius. If a step is + taken, i.e `step_threshold < r` (with `r` defined in `shrink_threshold`), a check is also + made to see if `expand_threshold < r`. If that is true, the trust region radius is + expanded by `expand_factor`. Defaults to `0.75`. + - `shrink_factor`: the factor to shrink the trust region radius with if + `shrink_threshold > r` (with `r` defined in `shrink_threshold`). Defaults to `0.25`. + - `expand_factor`: the factor to expand the trust region radius with if + `expand_threshold < r` (with `r` defined in `shrink_threshold`). Defaults to `2.0`. + - `max_shrink_times`: the maximum number of times to shrink the trust region radius in a + row, `max_shrink_times` is exceeded, the algorithm returns. Defaults to `32`. + - `nlsolve_update_rule`: If set to `Val(true)`, updates the trust region radius using the + update rule from NLSolve.jl. Defaults to `Val(false)`. If set to `Val(true)`, few of the + radius update parameters -- `step_threshold = 0.05`, `expand_threshold = 0.9`, and + `shrink_factor = 0.5` -- have different defaults. +""" @kwdef @concrete struct SimpleTrustRegion <: AbstractSimpleNonlinearSolveAlgorithm autodiff = nothing max_trust_radius = 0.0 From 1ea8262ddf696a75605654e2b6a410626dc25a55 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 15:07:00 -0400 Subject: [PATCH 42/92] feat: SimpleBroyden implementation --- lib/SimpleNonlinearSolve/Project.toml | 1 + .../src/SimpleNonlinearSolve.jl | 4 +- lib/SimpleNonlinearSolve/src/broyden.jl | 100 ++++++++++++++++++ lib/SimpleNonlinearSolve/src/utils.jl | 4 +- 4 files changed, 106 insertions(+), 3 deletions(-) diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index e21eb1c5f..62a999325 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -13,6 +13,7 @@ DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +LineSearch = "87fe0de2-c867-4266-b59a-2f0a94fc965b" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index d307e5b99..7b576e5dc 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -3,6 +3,7 @@ module SimpleNonlinearSolve using CommonSolve: CommonSolve, solve using ConcreteStructs: @concrete using FastClosures: @closure +using LineSearch: LiFukushimaLineSearch using LinearAlgebra: dot using MaybeInplace: @bb using PrecompileTools: @compile_workload, @setup_workload @@ -79,6 +80,7 @@ function solve_adjoint_internal end prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) algs = [ + SimpleBroyden(), SimpleKlement(), SimpleNewtonRaphson(), SimpleTrustRegion() @@ -100,7 +102,7 @@ export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder -export SimpleKlement +export SimpleBroyden, SimpleKlement export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion end diff --git a/lib/SimpleNonlinearSolve/src/broyden.jl b/lib/SimpleNonlinearSolve/src/broyden.jl index 8b1378917..6537a4d2d 100644 --- a/lib/SimpleNonlinearSolve/src/broyden.jl +++ b/lib/SimpleNonlinearSolve/src/broyden.jl @@ -1 +1,101 @@ +""" + SimpleBroyden(; linesearch = Val(false), alpha = nothing) +A low-overhead implementation of Broyden. This method is non-allocating on scalar and static +array problems. + +### Keyword Arguments + + - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` + line search else no line search is used. For advanced customization of the line search, + use `Broyden` from `NonlinearSolve.jl`. + - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we + will compute the scaling using `2 * norm(fu) / max(norm(u), true)`. +""" +@concrete struct SimpleBroyden <: AbstractSimpleNonlinearSolveAlgorithm + linesearch <: Union{Val{false}, Val{true}} + alpha +end + +function SimpleBroyden(; + linesearch::Union{Bool, Val{true}, Val{false}} = Val(false), alpha = nothing) + linesearch = linesearch isa Bool ? Val(linesearch) : linesearch + return SimpleBroyden(linesearch, alpha) +end + +function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleBroyden, args...; + abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) + T = promote_type(eltype(fx), eltype(x)) + + iszero(fx) && + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success) + + @bb xo = copy(x) + @bb δx = similar(x) + @bb δf = copy(fx) + @bb fprev = copy(fx) + + if alg.alpha === nothing + fx_norm = L2_NORM(fx) + x_norm = L2_NORM(x) + init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true)) + else + init_α = inv(alg.alpha) + end + + J⁻¹ = Utils.identity_jacobian(fx, x, init_α) + @bb J⁻¹δf = copy(x) + @bb xᵀJ⁻¹ = copy(x) + @bb δJ⁻¹n = copy(x) + @bb δJ⁻¹ = copy(J⁻¹) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + if alg.linesearch === Val(true) + ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing) + ls_cache = init(prob, ls_alg, fx, x) + else + ls_cache = nothing + end + + for _ in 1:maxiters + @bb δx = J⁻¹ × vec(fprev) + @bb δx .*= -1 + + if ls_cache === nothing + α = true + else + ls_sol = solve!(ls_cache, xo, δx) + α = ls_sol.step_size # Ignores the return code for now + end + + @bb @. x = xo + α * δx + fx = Utils.eval_f(prob, fx, x) + @bb @. δf = fx - fprev + + # Termination Checks + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + @bb J⁻¹δf = J⁻¹ × vec(δf) + d = dot(δx, J⁻¹δf) + @bb xᵀJ⁻¹ = transpose(J⁻¹) × vec(δx) + + @bb @. δJ⁻¹n = (δx - J⁻¹δf) / d + + δJ⁻¹n_ = Utils.safe_vec(δJ⁻¹n) + xᵀJ⁻¹_ = Utils.safe_vec(xᵀJ⁻¹) + @bb δJ⁻¹ = δJ⁻¹n_ × transpose(xᵀJ⁻¹_) + @bb J⁻¹ .+= δJ⁻¹ + + @bb copyto!(xo, x) + @bb copyto!(fprev, fx) + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 29437339e..5b4c5a248 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -62,13 +62,13 @@ function identity_jacobian(u::Number, fu::Number, α = true) return convert(promote_type(eltype(u), eltype(fu)), α) end function identity_jacobian(u, fu, α = true) - J = safe_similar(u, promote_type(eltype(u), eltype(fu))) + J = safe_similar(u, promote_type(eltype(u), eltype(fu)), length(fu), length(u)) fill!(J, zero(eltype(J))) J[diagind(J)] .= eltype(J)(α) return J end function identity_jacobian(u::StaticArray, fu, α = true) - return SMatrix{length(fu), length(u), eltype(u)}(I * α) + return SMatrix{length(fu), length(u), promote_type(eltype(fu), eltype(u))}(I * α) end identity_jacobian!!(J::Number) = one(J) From d944b7b4c604870774901da96e915ed4cff89ef6 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 15:53:16 -0400 Subject: [PATCH 43/92] feat: add SimpleHalley method --- .../src/SimpleNonlinearSolve.jl | 10 +-- lib/SimpleNonlinearSolve/src/halley.jl | 83 +++++++++++++++++++ lib/SimpleNonlinearSolve/src/utils.jl | 22 +++++ 3 files changed, 109 insertions(+), 6 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 7b576e5dc..14256ce38 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -4,8 +4,8 @@ using CommonSolve: CommonSolve, solve using ConcreteStructs: @concrete using FastClosures: @closure using LineSearch: LiFukushimaLineSearch -using LinearAlgebra: dot -using MaybeInplace: @bb +using LinearAlgebra: LinearAlgebra, dot +using MaybeInplace: @bb, setindex_trait, CannotSetindex, CanSetindex using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change @@ -82,18 +82,15 @@ function solve_adjoint_internal end algs = [ SimpleBroyden(), SimpleKlement(), + SimpleHalley(), SimpleNewtonRaphson(), SimpleTrustRegion() ] - algs_no_iip = [] @compile_workload begin for alg in algs, prob in (prob_scalar, prob_iip, prob_oop) CommonSolve.solve(prob, alg) end - for alg in algs_no_iip - CommonSolve.solve(prob_scalar, alg) - end end end end @@ -104,5 +101,6 @@ export Alefeld, Bisection, Brent, Falsi, ITP, Ridder export SimpleBroyden, SimpleKlement export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion +export SimpleHalley end diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl index 8b1378917..23ba1c847 100644 --- a/lib/SimpleNonlinearSolve/src/halley.jl +++ b/lib/SimpleNonlinearSolve/src/halley.jl @@ -1 +1,84 @@ +""" + SimpleHalley(autodiff) + SimpleHalley(; autodiff = nothing) +A low-overhead implementation of Halley's Method. + +!!! note + + As part of the decreased overhead, this method omits some of the higher level error + catching of the other methods. Thus, to see better error messages, use one of the other + methods like `NewtonRaphson`. + +### Keyword Arguments + + - `autodiff`: determines the backend used for the Jacobian. Defaults to `nothing` (i.e. + automatic backend selection). Valid choices include jacobian backends from + `DifferentiationInterface.jl`. +""" +@kwdef @concrete struct SimpleHalley <: AbstractSimpleNonlinearSolveAlgorithm + autodiff = nothing +end + +function SciMLBase.__solve( + prob::ImmutableNonlinearProblem, alg::SimpleHalley, args...; + abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) + T = promote_type(eltype(fx), eltype(x)) + + iszero(fx) && + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + autodiff = NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff) + + @bb xo = copy(x) + + strait = setindex_trait(x) + + A = strait isa CanSetindex ? similar(x, length(x), length(x)) : x + Aaᵢ = strait isa CanSetindex ? similar(x, length(x)) : x + cᵢ = strait isa CanSetindex ? similar(x) : x + + for _ in 1:maxiters + fx, J, H = Utils.compute_jacobian_and_hessian(autodiff, prob, fx, x) + + strait isa CannotSetindex && (A = J) + + # Factorize Once and Reuse + J_fact = if J isa Number + J + else + fact = LinearAlgebra.lu(J; check = false) + !LinearAlgebra.issuccess(fact) && return SciMLBase.build_solution( + prob, alg, x, fx; retcode = ReturnCode.Unstable) + fact + end + + aᵢ = J_fact \ Utils.safe_vec(fx) + A_ = Utils.safe_vec(A) + @bb A_ = H × aᵢ + A = Utils.restructure(A, A_) + + @bb Aaᵢ = A × aᵢ + @bb A .*= -1 + bᵢ = J_fact \ Utils.safe_vec(Aaᵢ) + + cᵢ_ = Utils.safe_vec(cᵢ) + @bb @. cᵢ_ = (aᵢ * aᵢ) / (-aᵢ + (T(0.5) * bᵢ)) + cᵢ = Utils.restructure(cᵢ, cᵢ_) + + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + @bb @. x += cᵢ + @bb copyto!(xo, x) + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 5b4c5a248..19d44cf21 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -178,4 +178,26 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) return J end +function compute_jacobian_and_hessian(autodiff, prob, _, x::Number) + H = DI.second_derivative(prob.f, autodiff, x, Constant(prob.p)) + fx, J = DI.value_and_derivative(prob.f, autodiff, x, Constant(prob.p)) + return fx, J, H +end +function compute_jacobian_and_hessian(autodiff, prob, fx, x) + if SciMLBase.isinplace(prob) + jac_fn = @closure (u, p) -> begin + du = similar(fx, promote_type(eltype(fx), eltype(u))) + return DI.jacobian(prob.f, du, autodiff, u, Constant(p)) + end + J, H = DI.value_and_jacobian(jac_fn, autodiff, x, Constant(prob.p)) + fx = Utils.eval_f(prob, fx, x) + return fx, J, H + else + jac_fn = @closure (u, p) -> DI.jacobian(prob.f, autodiff, u, Constant(p)) + J, H = DI.value_and_jacobian(jac_fn, autodiff, x, Constant(prob.p)) + fx = Utils.eval_f(prob, fx, x) + return fx, J, H + end +end + end From ff961b1cabd6510e9ab8735929ab35589d44e932 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 16:20:40 -0400 Subject: [PATCH 44/92] feat: add SimpleDFSane --- lib/SimpleNonlinearSolve/Project.toml | 1 + .../src/SimpleNonlinearSolve.jl | 6 +- lib/SimpleNonlinearSolve/src/dfsane.jl | 169 ++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 62a999325..1962d42cf 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -5,6 +5,7 @@ version = "1.13.0" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +Accessors = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 14256ce38..29adc7131 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,5 +1,6 @@ module SimpleNonlinearSolve +using Accessors: @reset using CommonSolve: CommonSolve, solve using ConcreteStructs: @concrete using FastClosures: @closure @@ -10,7 +11,7 @@ using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode -using StaticArraysCore: StaticArray +using StaticArraysCore: StaticArray, SVector # AD Dependencies using ADTypes: AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff @@ -81,7 +82,9 @@ function solve_adjoint_internal end algs = [ SimpleBroyden(), + # SimpleDFSane(), SimpleKlement(), + # SimpleLimitedMemoryBroyden(), SimpleHalley(), SimpleNewtonRaphson(), SimpleTrustRegion() @@ -100,6 +103,7 @@ export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder export SimpleBroyden, SimpleKlement +export SimpleDFSane export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion export SimpleHalley diff --git a/lib/SimpleNonlinearSolve/src/dfsane.jl b/lib/SimpleNonlinearSolve/src/dfsane.jl index 8b1378917..0d400b0ce 100644 --- a/lib/SimpleNonlinearSolve/src/dfsane.jl +++ b/lib/SimpleNonlinearSolve/src/dfsane.jl @@ -1 +1,170 @@ +""" + SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0, + M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5, + nexp::Int = 2, η_strategy::Function = (f_1, k, x, F) -> f_1 ./ k^2) +A low-overhead implementation of the df-sane method for solving large-scale nonlinear +systems of equations. For in depth information about all the parameters and the algorithm, +see [la2006spectral](@citet). + +### Keyword Arguments + + - `σ_min`: the minimum value of the spectral coefficient `σ_k` which is related to the + step size in the algorithm. Defaults to `1e-10`. + - `σ_max`: the maximum value of the spectral coefficient `σ_k` which is related to the + step size in the algorithm. Defaults to `1e10`. + - `σ_1`: the initial value of the spectral coefficient `σ_k` which is related to the step + size in the algorithm.. Defaults to `1.0`. + - `M`: The monotonicity of the algorithm is determined by a this positive integer. + A value of 1 for `M` would result in strict monotonicity in the decrease of the L2-norm + of the function `f`. However, higher values allow for more flexibility in this + reduction. Despite this, the algorithm still ensures global convergence through the use + of a non-monotone line-search algorithm that adheres to the Grippo-Lampariello-Lucidi + condition. Values in the range of 5 to 20 are usually sufficient, but some cases may call + for a higher value of `M`. The default setting is 10. + - `γ`: a parameter that influences if a proposed step will be accepted. Higher value of + `γ` will make the algorithm more restrictive in accepting steps. Defaults to `1e-4`. + - `τ_min`: if a step is rejected the new step size will get multiplied by factor, and this + parameter is the minimum value of that factor. Defaults to `0.1`. + - `τ_max`: if a step is rejected the new step size will get multiplied by factor, and this + parameter is the maximum value of that factor. Defaults to `0.5`. + - `nexp`: the exponent of the loss, i.e. ``f_k=||F(x_k)||^{nexp}``. The paper uses + `nexp ∈ {1,2}`. Defaults to `2`. + - `η_strategy`: function to determine the parameter `η_k`, which enables growth + of ``||F||^2``. Called as `η_k = η_strategy(f_1, k, x, F)` with `f_1` initialized as + ``f_1=||F(x_1)||^{nexp}``, `k` is the iteration number, `x` is the current `x`-value and + `F` the current residual. Should satisfy ``η_k > 0`` and ``∑ₖ ηₖ < ∞``. Defaults to + ``||F||^2 / k^2``. +""" +@concrete struct SimpleDFSane <: AbstractSimpleNonlinearSolveAlgorithm + σ_min + σ_max + σ_1 + γ + τ_min + τ_max + nexp::Int + η_strategy + M <: Val +end + +# XXX[breaking]: we should change the names to not have unicode +function SimpleDFSane(; σ_min::Real = 1e-10, σ_max::Real = 1e10, σ_1::Real = 1.0, + M::Union{Int, Val} = Val(10), γ::Real = 1e-4, τ_min::Real = 0.1, τ_max::Real = 0.5, + nexp::Int = 2, η_strategy::F = (f_1, k, x, F) -> f_1 ./ k^2) where {F} + M = M isa Int ? Val(M) : M + return SimpleDFSane(σ_min, σ_max, σ_1, γ, τ_min, τ_max, nexp, η_strategy, M) +end + +function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleDFSane, args...; + abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false, + termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) + T = promote_type(eltype(fx), eltype(x)) + + σ_min = T(alg.σ_min) + σ_max = T(alg.σ_max) + σ_k = T(alg.σ_1) + + (; nexp, η_strategy, M) = alg + γ = T(alg.γ) + τ_min = T(alg.τ_min) + τ_max = T(alg.τ_max) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + fx_norm = L2_NORM(fx)^nexp + α_1 = one(T) + f_1 = fx_norm + + history_f_k = dfsane_history_vec(x, fx_norm, alg.M) + + # Generate the cache + @bb x_cache = similar(x) + @bb d = copy(x) + @bb xo = copy(x) + @bb δx = copy(x) + @bb δf = copy(fx) + + k = 0 + while k < maxiters + # Spectral parameter range check + σ_k = sign(σ_k) * clamp(abs(σ_k), σ_min, σ_max) + + # Line search direction + @bb @. d = -σ_k * fx + + η = η_strategy(f_1, k + 1, x, fx) + f_bar = maximum(history_f_k) + α_p = α_1 + α_m = α_1 + + @bb @. x_cache = x + α_p * d + + fx = Utils.eval_f(prob, fx, x_cache) + fx_norm_new = L2_NORM(fx)^nexp + + while k < maxiters + (fx_norm_new ≤ (f_bar + η - γ * α_p^2 * fx_norm)) && break + + α_tp = α_p^2 * fx_norm / (fx_norm_new + (T(2) * α_p - T(1)) * fx_norm) + @bb @. x_cache = x - α_m * d + + fx = Utils.eval_f(prob, fx, x_cache) + fx_norm_new = L2_NORM(fx)^nexp + + (fx_norm_new ≤ (f_bar + η - γ * α_m^2 * fx_norm)) && break + + α_tm = α_m^2 * fx_norm / (fx_norm_new + (T(2) * α_m - T(1)) * fx_norm) + α_p = clamp(α_tp, τ_min * α_p, τ_max * α_p) + α_m = clamp(α_tm, τ_min * α_m, τ_max * α_m) + @bb @. x_cache = x + α_p * d + + fx = Utils.eval_f(prob, fx, x_cache) + fx_norm_new = L2_NORM(fx)^nexp + + k += 1 + end + + @bb copyto!(x, x_cache) + + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + # Update spectral parameter + @bb @. δx = x - xo + @bb @. δf = fx - δf + + σ_k = dot(δx, δx) / dot(δx, δf) + + # Take step + @bb copyto!(xo, x) + @bb copyto!(δf, fx) + fx_norm = fx_norm_new + + # Store function value + idx = mod1(k, SciMLBase._unwrap_val(alg.M)) + if history_f_k isa SVector + history_f_k = Base.setindex(history_f_k, fx_norm_new, idx) + elseif history_f_k isa NTuple + @reset history_f_k[idx] = fx_norm_new + else + history_f_k[idx] = fx_norm_new + end + k += 1 + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end + +function dfsane_history_vec(x::StaticArray, fx_norm, ::Val{M}) where {M} + return ones(SVector{M, eltype(x)}) .* fx_norm +end + +@generated function dfsane_history_vec(x, fx_norm, ::Val{M}) where {M} + M ≥ 11 && return :(fill(fx_norm, M)) # Julia can't specialize here + return :(ntuple(Returns(fx_norm), $(M))) +end From 4febefefb53610adea793b0768ed52ef4f059920 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 16:33:42 -0400 Subject: [PATCH 45/92] fix: only precompile selected workloads for faster loading --- lib/SimpleNonlinearSolve/Project.toml | 1 + .../src/SimpleNonlinearSolve.jl | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 1962d42cf..3a20a18b2 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -49,6 +49,7 @@ FiniteDiff = "2.24.0" ForwardDiff = "0.10.36" InteractiveUtils = "<0.0.1, 1" LinearAlgebra = "1.10" +LineSearch = "0.1.3" MaybeInplace = "0.1.4" NonlinearSolveBase = "1" PrecompileTools = "1.2" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 29adc7131..d772f44d3 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -75,29 +75,33 @@ end function solve_adjoint_internal end @setup_workload begin - for T in (Float32, Float64) + for T in (Float64,) prob_scalar = NonlinearProblem{false}((u, p) -> u .* u .- p, T(0.1), T(2)) prob_iip = NonlinearProblem{true}((du, u, p) -> du .= u .* u .- p, ones(T, 3), T(2)) prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) + # Only compile frequently used algorithms -- mostly from the NonlinearSolve default algs = [ SimpleBroyden(), # SimpleDFSane(), SimpleKlement(), # SimpleLimitedMemoryBroyden(), - SimpleHalley(), - SimpleNewtonRaphson(), - SimpleTrustRegion() + # SimpleHalley(), + SimpleNewtonRaphson(; autodiff = AutoForwardDiff(; chunksize = 1)), + # SimpleTrustRegion() ] @compile_workload begin - for alg in algs, prob in (prob_scalar, prob_iip, prob_oop) - CommonSolve.solve(prob, alg) + @sync for alg in algs + for prob in (prob_scalar, prob_iip, prob_oop) + Threads.@spawn CommonSolve.solve(prob, alg; abstol = 1e-2) + end end end end end + export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder From 6ba6018f44ea15d39173b07c0347f75aace9c9a9 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 16:40:05 -0400 Subject: [PATCH 46/92] chore: run formatter --- lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index d772f44d3..b4fc1d202 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -81,6 +81,7 @@ function solve_adjoint_internal end prob_oop = NonlinearProblem{false}((u, p) -> u .* u .- p, ones(T, 3), T(2)) # Only compile frequently used algorithms -- mostly from the NonlinearSolve default + #!format: off algs = [ SimpleBroyden(), # SimpleDFSane(), @@ -90,6 +91,7 @@ function solve_adjoint_internal end SimpleNewtonRaphson(; autodiff = AutoForwardDiff(; chunksize = 1)), # SimpleTrustRegion() ] + #!format: on @compile_workload begin @sync for alg in algs @@ -101,7 +103,6 @@ function solve_adjoint_internal end end end - export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder From facdb421f52e70d2e693cffb8bd4ce61da3da5be Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Fri, 4 Oct 2024 17:27:09 -0400 Subject: [PATCH 47/92] feat: SimpleLimitedMemoryBroyden impl --- .../src/SimpleNonlinearSolve.jl | 4 +- lib/SimpleNonlinearSolve/src/lbroyden.jl | 327 ++++++++++++++++++ 2 files changed, 329 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index b4fc1d202..c8df31e51 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -11,7 +11,7 @@ using PrecompileTools: @compile_workload, @setup_workload using Reexport: @reexport @reexport using SciMLBase # I don't like this but needed to avoid a breaking change using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode -using StaticArraysCore: StaticArray, SVector +using StaticArraysCore: StaticArray, SArray, SVector, MArray # AD Dependencies using ADTypes: AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff @@ -107,7 +107,7 @@ export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff export Alefeld, Bisection, Brent, Falsi, ITP, Ridder -export SimpleBroyden, SimpleKlement +export SimpleBroyden, SimpleKlement, SimpleLimitedMemoryBroyden export SimpleDFSane export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion export SimpleHalley diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index 8b1378917..2a9ace6dc 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -1 +1,328 @@ +""" + SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27), + linesearch = Val(false), alpha = nothing) +A limited memory implementation of Broyden. This method applies the L-BFGS scheme to +Broyden's method. + +If the threshold is larger than the problem size, then this method will use `SimpleBroyden`. + +### Keyword Arguments: + + - `linesearch`: If `linesearch` is `Val(true)`, then we use the `LiFukushimaLineSearch` + line search else no line search is used. For advanced customization of the line search, + use `Broyden` from `NonlinearSolve.jl`. + - `alpha`: Scale the initial jacobian initialization with `alpha`. If it is `nothing`, we + will compute the scaling using `2 * norm(fu) / max(norm(u), true)`. + +!!! warning + + Currently `alpha` is only used for StaticArray problems. This will be fixed in the + future. +""" +@concrete struct SimpleLimitedMemoryBroyden <: AbstractSimpleNonlinearSolveAlgorithm + linesearch <: Union{Val{false}, Val{true}} + threshold <: Val + alpha +end + +function SimpleLimitedMemoryBroyden(; threshold::Union{Val, Int} = Val(27), + linesearch::Union{Bool, Val{true}, Val{false}} = Val(false), alpha = nothing) + linesearch = linesearch isa Bool ? Val(linesearch) : linesearch + threshold = threshold isa Int ? Val(threshold) : threshold + return SimpleLimitedMemoryBroyden(linesearch, threshold, alpha) +end + +function SciMLBase.__solve( + prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden, + args...; termination_condition = nothing, kwargs...) + if prob.u0 isa SArray + if termination_condition === nothing || + termination_condition isa AbsNormTerminationMode + return internal_static_solve( + prob, alg, args...; termination_condition, kwargs...) + end + @warn "Specifying `termination_condition = $(termination_condition)` for \ + `SimpleLimitedMemoryBroyden` with `SArray` is not non-allocating. Use \ + either `termination_condition = AbsNormTerminationMode()` or \ + `termination_condition = nothing`." maxlog=1 + end + return internal_generic_solve(prob, alg, args...; termination_condition, kwargs...) +end + +@views function internal_generic_solve( + prob::ImmutableNonlinearProblem, alg::SimpleLimitedMemoryBroyden, + args...; abstol = nothing, reltol = nothing, maxiters = 1000, + alias_u0 = false, termination_condition = nothing, kwargs...) + x = Utils.maybe_unaliased(prob.u0, alias_u0) + η = min(SciMLBase._unwrap_val(alg.threshold), maxiters) + + # For scalar problems / if the threshold is larger than problem size just use Broyden + if x isa Number || length(x) ≤ η + return SciMLBase.__solve(prob, SimpleBroyden(; alg.linesearch), args...; abstol, + reltol, maxiters, termination_condition, kwargs...) + end + + fx = Utils.get_fx(prob, x) + + U, Vᵀ = init_low_rank_jacobian(x, fx, x isa StaticArray ? alg.threshold : Val(η)) + + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) + + @bb xo = copy(x) + @bb δx = copy(fx) + @bb δx .*= -1 + @bb fo = copy(fx) + @bb δf = copy(fx) + + @bb vᵀ_cache = copy(x) + Tcache = lbroyden_threshold_cache(x, x isa StaticArray ? alg.threshold : Val(η)) + @bb mat_cache = copy(x) + + if alg.linesearch === Val(true) + ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing) + ls_cache = init(prob, ls_alg, fx, x) + else + ls_cache = nothing + end + + for i in 1:maxiters + if ls_cache === nothing + α = true + else + ls_sol = solve!(ls_cache, xo, δx) + α = ls_sol.step_size # Ignores the return code for now + end + + @bb @. x = xo + α * δx + fx = Utils.eval_f(prob, fx, x) + @bb @. δf = fx - fo + + # Termination Checks + solved, retcode, fx_sol, x_sol = Utils.check_termination(tc_cache, fx, x, xo, prob) + solved && return SciMLBase.build_solution(prob, alg, x_sol, fx_sol; retcode) + + Uₚ = selectdim(U, 2, 1:min(η, i - 1)) + Vᵀₚ = selectdim(Vᵀ, 1, 1:min(η, i - 1)) + + vᵀ = rmatvec!!(vᵀ_cache, Tcache, Uₚ, Vᵀₚ, δx) + mvec = matvec!!(mat_cache, Tcache, Uₚ, Vᵀₚ, δf) + d = dot(vᵀ, δf) + @bb @. δx = (δx - mvec) / d + + selectdim(U, 2, mod1(i, η)) .= Utils.safe_vec(δx) + selectdim(Vᵀ, 1, mod1(i, η)) .= Utils.safe_vec(vᵀ) + + Uₚ = selectdim(U, 2, 1:min(η, i)) + Vᵀₚ = selectdim(Vᵀ, 1, 1:min(η, i)) + δx = matvec!!(δx, Tcache, Uₚ, Vᵀₚ, fx) + @bb @. δx *= -1 + + @bb copyto!(xo, x) + @bb copyto!(fo, fx) + end + + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.MaxIters) +end + +# Non-allocating StaticArrays version of SimpleLimitedMemoryBroyden is actually quite +# finicky, so we'll implement it separately from the generic version +# Ignore termination_condition. Don't pass things into internal functions +function internal_static_solve( + prob::ImmutableNonlinearProblem{<:SArray}, alg::SimpleLimitedMemoryBroyden, + args...; abstol = nothing, maxiters = 1000, kwargs...) + x = prob.u0 + fx = Utils.get_fx(prob, x) + + U, Vᵀ = init_low_rank_jacobian(vec(x), vec(fx), alg.threshold) + + abstol = NonlinearSolveBase.get_tolerance(x, abstol, eltype(x)) + + xo, δx, fo, δf = x, -fx, fx, fx + + if alg.linesearch === Val(true) + ls_alg = LiFukushimaLineSearch(; nan_maxiters = nothing) + ls_cache = init(prob, ls_alg, fx, x) + else + ls_cache = nothing + end + + T = promote_type(eltype(x), eltype(fx)) + if alg.alpha === nothing + fx_norm = L2_NORM(fx) + x_norm = L2_NORM(x) + init_α = ifelse(fx_norm ≥ 1e-5, max(x_norm, T(true)) / (2 * fx_norm), T(true)) + else + init_α = inv(alg.alpha) + end + + converged, res = internal_unrolled_lbroyden_initial_iterations( + prob, xo, fo, δx, abstol, U, Vᵀ, alg.threshold, ls_cache, init_α) + + converged && return SciMLBase.build_solution( + prob, alg, res.x, res.fx; retcode = ReturnCode.Success) + + xo, fo, δx = res.x, res.fx, res.δx + + for i in 1:(maxiters - SciMLBase._unwrap_val(alg.threshold)) + if ls_cache === nothing + α = true + else + ls_sol = solve!(ls_cache, xo, δx) + α = ls_sol.step_size # Ignores the return code for now + end + + x = xo + α * δx + fx = Utils.eval_f(prob, fx, x) + δf = fx - fo + + maximum(abs, fx) ≤ abstol && + return SciMLBase.build_solution(prob, alg, x, fx; retcode = ReturnCode.Success) + + vᵀ = Utils.restructure(x, rmatvec!!(U, Vᵀ, vec(δx), init_α)) + mvec = Utils.restructure(x, matvec!!(U, Vᵀ, vec(δf), init_α)) + + d = dot(vᵀ, δf) + δx = @. (δx - mvec) / d + + U = Base.setindex(U, vec(δx), mod1(i, SciMLBase._unwrap_val(alg.threshold))) + Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), mod1(i, SciMLBase._unwrap_val(alg.threshold))) + + δx = -Utils.restructure(fx, matvec!!(U, Vᵀ, vec(fx), init_α)) + + xo, fo = x, fx + end + + return SciMLBase.build_solution(prob, alg, xo, fo; retcode = ReturnCode.MaxIters) +end + +@generated function internal_unrolled_lbroyden_initial_iterations( + prob, xo, fo, δx, abstol, U, Vᵀ, ::Val{threshold}, + ls_cache, init_α) where {threshold} + calls = [] + for i in 1:threshold + static_idx, static_idx_p1 = Val(i - 1), Val(i) + push!(calls, quote + α = ls_cache === nothing ? true : ls_cache(xo, δx) + x = xo .+ α .* δx + fx = prob.f(x, prob.p) + δf = fx - fo + + maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx) + + Uₚ = first_n_getindex(U, $(static_idx)) + Vᵀₚ = first_n_getindex(Vᵀ, $(static_idx)) + + vᵀ = Utils.restructure(x, rmatvec!!(Uₚ, Vᵀₚ, vec(δx), init_α)) + mvec = Utils.restructure(x, matvec!!(Uₚ, Vᵀₚ, vec(δf), init_α)) + + d = dot(vᵀ, δf) + δx = @. (δx - mvec) / d + + U = Base.setindex(U, vec(δx), $(i)) + Vᵀ = Base.setindex(Vᵀ, vec(vᵀ), $(i)) + + Uₚ = first_n_getindex(U, $(static_idx_p1)) + Vᵀₚ = first_n_getindex(Vᵀ, $(static_idx_p1)) + δx = -Utils.restructure(fx, matvec!!(Uₚ, Vᵀₚ, vec(fx), init_α)) + + x0, fo = x, fx + end) + end + push!(calls, quote + # Termination Check + maximum(abs, fx) ≤ abstol && return true, (; x, fx, δx) + + return false, (; x, fx, δx) + end) + return Expr(:block, calls...) +end + +function rmatvec!!(y, xᵀU, U, Vᵀ, x) + # xᵀ × (-I + UVᵀ) + η = size(U, 2) + if η == 0 + @bb @. y = -x + return y + end + x_ = vec(x) + xᵀU_ = view(xᵀU, 1:η) + @bb xᵀU_ = transpose(U) × x_ + @bb y = transpose(Vᵀ) × vec(xᵀU_) + @bb @. y -= x + return y +end + +rmatvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α +rmatvec!!(U, Vᵀ, x, init_α) = fast_mapTdot(fast_mapdot(x, U), Vᵀ) .- x .* init_α + +function matvec!!(y, Vᵀx, U, Vᵀ, x) + # (-I + UVᵀ) × x + η = size(U, 2) + if η == 0 + @bb @. y = -x + return y + end + x_ = vec(x) + Vᵀx_ = view(Vᵀx, 1:η) + @bb Vᵀx_ = Vᵀ × x_ + @bb y = U × vec(Vᵀx_) + @bb @. y -= x + return y +end + +@inline matvec!!(::Nothing, Vᵀ, x, init_α) = -x .* init_α +@inline matvec!!(U, Vᵀ, x, init_α) = fast_mapTdot(fast_mapdot(x, Vᵀ), U) .- x .* init_α + +function fast_mapdot(x::SVector{S1}, Y::SVector{S2, <:SVector{S1}}) where {S1, S2} + return map(Base.Fix1(dot, x), Y) +end +@generated function fast_mapTdot( + x::SVector{S1}, Y::SVector{S1, <:SVector{S2}}) where {S1, S2} + calls = [] + syms = [gensym("m$(i)") for i in 1:S1] + for i in 1:S1 + push!(calls, :($(syms[i]) = x[$(i)] .* Y[$i])) + end + push!(calls, :(return .+($(syms...)))) + return Expr(:block, calls...) +end + +@generated function first_n_getindex(x::SVector{L, T}, ::Val{N}) where {L, T, N} + @assert N ≤ L + getcalls = ntuple(i -> :(x[$i]), N) + N == 0 && return :(return nothing) + return :(return SVector{$N, $T}(($(getcalls...)))) +end + +lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = similar(x, threshold) +function lbroyden_threshold_cache(x::StaticArray, ::Val{threshold}) where {threshold} + return zeros(MArray{Tuple{threshold}, eltype(x)}) +end +lbroyden_threshold_cache(::SArray, ::Val{threshold}) where {threshold} = nothing + +function init_low_rank_jacobian(u::StaticArray{S1, T1}, fu::StaticArray{S2, T2}, + ::Val{threshold}) where {S1, S2, T1, T2, threshold} + T = promote_type(T1, T2) + fuSize, uSize = Size(fu), Size(u) + Vᵀ = MArray{Tuple{threshold, prod(uSize)}, T}(undef) + U = MArray{Tuple{prod(fuSize), threshold}, T}(undef) + return U, Vᵀ +end +@generated function init_low_rank_jacobian(u::SVector{Lu, T1}, fu::SVector{Lfu, T2}, + ::Val{threshold}) where {Lu, Lfu, T1, T2, threshold} + T = promote_type(T1, T2) + inner_inits_Vᵀ = [:(zeros(SVector{$Lu, $T})) for i in 1:threshold] + inner_inits_U = [:(zeros(SVector{$Lfu, $T})) for i in 1:threshold] + return quote + Vᵀ = SVector($(inner_inits_Vᵀ...)) + U = SVector($(inner_inits_U...)) + return U, Vᵀ + end +end +function init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold} + Vᵀ = similar(u, threshold, length(u)) + U = similar(u, length(fu), threshold) + return U, Vᵀ +end From 5749f2cebfaf43e5f8d9afd8e5810300d2926629 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:04:50 -0400 Subject: [PATCH 48/92] fix: simplenonlinearsolve in cuda kernels --- .buildkite/pipeline.yml | 28 ++++++ lib/BracketingNonlinearSolve/src/bisection.jl | 2 +- lib/BracketingNonlinearSolve/src/brent.jl | 2 +- lib/BracketingNonlinearSolve/src/falsi.jl | 2 +- lib/BracketingNonlinearSolve/src/itp.jl | 2 +- lib/BracketingNonlinearSolve/src/ridder.jl | 2 +- .../src/termination_conditions.jl | 12 +-- lib/SimpleNonlinearSolve/Project.toml | 31 ++++++- lib/SimpleNonlinearSolve/src/lbroyden.jl | 7 +- lib/SimpleNonlinearSolve/src/utils.jl | 61 ++++++++++--- .../test/gpu/cuda_tests.jl | 90 +++++++++++++++++++ lib/SimpleNonlinearSolve/test/runtests.jl | 10 ++- 12 files changed, 219 insertions(+), 30 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8ddf4f4ff..f6b4ed6eb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -27,6 +27,34 @@ steps: # Don't run Buildkite if the commit message includes the text [skip tests] if: build.message !~ /\[skip tests\]/ + - label: "Julia 1 (SimpleNonlinearSolve)" + plugins: + - JuliaCI/julia#v1: + version: "1" + - JuliaCI/julia-coverage#v1: + codecov: true + dirs: + - src + - ext + command: | + julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve -e ' + import Pkg; + Pkg.Registry.update(); + # Install packages present in subdirectories + dev_pks = Pkg.PackageSpec[]; + for path in ("lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve") + push!(dev_pks, Pkg.PackageSpec(; path)) + end + Pkg.develop(dev_pks); + Pkg.instantiate(); + Pkg.test(; coverage=true)' + agents: + queue: "juliagpu" + cuda: "*" + timeout_in_minutes: 60 + # Don't run Buildkite if the commit message includes the text [skip tests] + if: build.message !~ /\[skip tests\]/ + env: GROUP: CUDA JULIA_PKG_SERVER: "" # it often struggles with our large artifacts diff --git a/lib/BracketingNonlinearSolve/src/bisection.jl b/lib/BracketingNonlinearSolve/src/bisection.jl index 1611ad34e..e51416145 100644 --- a/lib/BracketingNonlinearSolve/src/bisection.jl +++ b/lib/BracketingNonlinearSolve/src/bisection.jl @@ -28,7 +28,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Bisection, fl, fr = f(left), f(right) abstol = NonlinearSolveBase.get_tolerance( - abstol, promote_type(eltype(left), eltype(right))) + left, abstol, promote_type(eltype(left), eltype(right))) if iszero(fl) return SciMLBase.build_solution( diff --git a/lib/BracketingNonlinearSolve/src/brent.jl b/lib/BracketingNonlinearSolve/src/brent.jl index fea2ce3f4..fb3740e98 100644 --- a/lib/BracketingNonlinearSolve/src/brent.jl +++ b/lib/BracketingNonlinearSolve/src/brent.jl @@ -15,7 +15,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Brent, args...; ϵ = eps(convert(typeof(fl), 1)) abstol = NonlinearSolveBase.get_tolerance( - abstol, promote_type(eltype(left), eltype(right))) + left, abstol, promote_type(eltype(left), eltype(right))) if iszero(fl) return SciMLBase.build_solution( diff --git a/lib/BracketingNonlinearSolve/src/falsi.jl b/lib/BracketingNonlinearSolve/src/falsi.jl index 8c62b95c6..f56155ef7 100644 --- a/lib/BracketingNonlinearSolve/src/falsi.jl +++ b/lib/BracketingNonlinearSolve/src/falsi.jl @@ -15,7 +15,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Falsi, args...; fl, fr = f(left), f(right) abstol = NonlinearSolveBase.get_tolerance( - abstol, promote_type(eltype(left), eltype(right))) + left, abstol, promote_type(eltype(left), eltype(right))) if iszero(fl) return SciMLBase.build_solution( diff --git a/lib/BracketingNonlinearSolve/src/itp.jl b/lib/BracketingNonlinearSolve/src/itp.jl index 4798f9030..821047a5a 100644 --- a/lib/BracketingNonlinearSolve/src/itp.jl +++ b/lib/BracketingNonlinearSolve/src/itp.jl @@ -65,7 +65,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::ITP, args...; fl, fr = f(left), f(right) abstol = NonlinearSolveBase.get_tolerance( - abstol, promote_type(eltype(left), eltype(right))) + left, abstol, promote_type(eltype(left), eltype(right))) if iszero(fl) return SciMLBase.build_solution( diff --git a/lib/BracketingNonlinearSolve/src/ridder.jl b/lib/BracketingNonlinearSolve/src/ridder.jl index e4b67a7c7..d988c9dc5 100644 --- a/lib/BracketingNonlinearSolve/src/ridder.jl +++ b/lib/BracketingNonlinearSolve/src/ridder.jl @@ -14,7 +14,7 @@ function CommonSolve.solve(prob::IntervalNonlinearProblem, alg::Ridder, args...; fl, fr = f(left), f(right) abstol = NonlinearSolveBase.get_tolerance( - abstol, promote_type(eltype(left), eltype(right))) + left, abstol, promote_type(eltype(left), eltype(right))) if iszero(fl) return SciMLBase.build_solution( diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 8bb58f8eb..7978c19b8 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -34,8 +34,8 @@ function SciMLBase.init( du, u, mode::AbstractNonlinearTerminationMode, saved_value_prototype...; abstol = nothing, reltol = nothing, kwargs...) T = promote_type(eltype(du), eltype(u)) - abstol = get_tolerance(abstol, T) - reltol = get_tolerance(reltol, T) + abstol = get_tolerance(u, abstol, T) + reltol = get_tolerance(u, reltol, T) TT = typeof(abstol) u_unaliased = mode isa AbstractSafeBestNonlinearTerminationMode ? @@ -90,8 +90,8 @@ function SciMLBase.reinit!( cache.u = u_unaliased cache.retcode = ReturnCode.Default - cache.abstol = get_tolerance(abstol, T) - cache.reltol = get_tolerance(reltol, T) + cache.abstol = get_tolerance(u, abstol, T) + cache.reltol = get_tolerance(u, reltol, T) cache.nsteps = 0 TT = typeof(cache.abstol) @@ -274,8 +274,8 @@ end function init_termination_cache(::AbstractNonlinearProblem, abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode, ::Val) T = promote_type(eltype(du), eltype(u)) - abstol = get_tolerance(abstol, T) - reltol = get_tolerance(reltol, T) + abstol = get_tolerance(u, abstol, T) + reltol = get_tolerance(u, reltol, T) cache = SciMLBase.init(du, u, tc; abstol, reltol) return abstol, reltol, cache end diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 3a20a18b2..fae63544a 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -37,35 +37,62 @@ SimpleNonlinearSolveTrackerExt = "Tracker" [compat] ADTypes = "1.2" +Accessors = "0.1" +AllocCheck = "0.1.1" +Aqua = "0.8.7" ArrayInterface = "7.16" BracketingNonlinearSolve = "1" +CUDA = "5.3" ChainRulesCore = "1.24" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" DiffEqBase = "6.155" DifferentiationInterface = "0.6.1" +Enzyme = "0.13" +ExplicitImports = "1.9" FastClosures = "0.3.2" FiniteDiff = "2.24.0" ForwardDiff = "0.10.36" InteractiveUtils = "<0.0.1, 1" -LinearAlgebra = "1.10" LineSearch = "0.1.3" +LinearAlgebra = "1.10" MaybeInplace = "0.1.4" +NonlinearProblemLibrary = "0.1.2" NonlinearSolveBase = "1" +Pkg = "1.10" +PolyesterForwardDiff = "0.1" PrecompileTools = "1.2" +Random = "1.10" Reexport = "1.2" ReverseDiff = "1.15" SciMLBase = "2.50" +SciMLSensitivity = "7.68" +StaticArrays = "1.9" StaticArraysCore = "1.4.3" Test = "1.10" TestItemRunner = "1" Tracker = "0.2.35" +Zygote = "0.6.70" julia = "1.10" [extras] +AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" +ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +NonlinearProblemLibrary = "b7050fa9-e91f-4b37-bcee-a89a063da141" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" +SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" +StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" +Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" +Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["InteractiveUtils", "Test", "TestItemRunner"] +test = ["AllocCheck", "Aqua", "CUDA", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index 2a9ace6dc..1ab200f74 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -204,7 +204,12 @@ end for i in 1:threshold static_idx, static_idx_p1 = Val(i - 1), Val(i) push!(calls, quote - α = ls_cache === nothing ? true : ls_cache(xo, δx) + if ls_cache === nothing + α = true + else + ls_sol = solve!(ls_cache, xo, δx) + α = ls_sol.step_size # Ignores the return code for now + end x = xo .+ α .* δx fx = prob.f(x, prob.p) δf = fx - fo diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 19d44cf21..011788a1c 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -2,6 +2,7 @@ module Utils using ADTypes: AbstractADType, AutoForwardDiff, AutoFiniteDiff, AutoPolyesterForwardDiff using ArrayInterface: ArrayInterface +using ConcreteStructs: @concrete using DifferentiationInterface: DifferentiationInterface, Constant using FastClosures: @closure using LinearAlgebra: LinearAlgebra, I, diagind @@ -116,25 +117,35 @@ restructure(::Number, x::Number) = x safe_vec(x::AbstractArray) = vec(x) safe_vec(x::Number) = x +abstract type AbstractJacobianMode end + +struct AnalyticJacobian <: AbstractJacobianMode end +@concrete struct DIExtras <: AbstractJacobianMode + prep +end +struct DINoPreparation <: AbstractJacobianMode end + +# While we could run prep in other cases, we don't since we need it completely +# non-allocating for running inside GPU kernels function prepare_jacobian(prob, autodiff, _, x::Number) if SciMLBase.has_jac(prob.f) || SciMLBase.has_vjp(prob.f) || SciMLBase.has_jvp(prob.f) - return nothing + return AnalyticJacobian() end - return DI.prepare_derivative(prob.f, autodiff, x, Constant(prob.p)) + # return DI.prepare_derivative(prob.f, autodiff, x, Constant(prob.p)) + return DINoPreparation() end function prepare_jacobian(prob, autodiff, fx, x) - if SciMLBase.has_jac(prob.f) - return nothing - end + SciMLBase.has_jac(prob.f) && return AnalyticJacobian() if SciMLBase.isinplace(prob.f) - return DI.prepare_jacobian(prob.f, fx, autodiff, x, Constant(prob.p)) + return DIExtras(DI.prepare_jacobian(prob.f, fx, autodiff, x, Constant(prob.p))) else + x isa SArray && return DINoPreparation() return DI.prepare_jacobian(prob.f, autodiff, x, Constant(prob.p)) end end function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras) - if extras === nothing + if extras isa AnalyticJacobian if SciMLBase.has_jac(prob.f) return prob.f.jac(x, prob.p) elseif SciMLBase.has_vjp(prob.f) @@ -143,11 +154,15 @@ function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras) return prob.f.jvp(one(x), x, prob.p) end end - return DI.derivative(prob.f, extras, autodiff, x, Constant(prob.p)) + if extras isa DIExtras + return DI.derivative(prob.f, extras.prep, autodiff, x, Constant(prob.p)) + else + return DI.derivative(prob.f, autodiff, x, Constant(prob.p)) + end end function compute_jacobian!!(J, prob, autodiff, fx, x, extras) if J === nothing - if extras === nothing + if extras isa AnalyticJacobian if SciMLBase.isinplace(prob.f) J = similar(fx, length(fx), length(x)) prob.f.jac(J, x, prob.p) @@ -157,12 +172,17 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) end end if SciMLBase.isinplace(prob) - return DI.jacobian(prob.f, fx, extras, autodiff, x, Constant(prob.p)) + @assert extras isa DIExtras + return DI.jacobian(prob.f, fx, extras.prep, autodiff, x, Constant(prob.p)) else - return DI.jacobian(prob.f, extras, autodiff, x, Constant(prob.p)) + if extras isa DIExtras + return DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) + else + return DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) + end end end - if extras === nothing + if extras isa AnalyticJacobian if SciMLBase.isinplace(prob) prob.jac(J, x, prob.p) return J @@ -171,9 +191,22 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) end end if SciMLBase.isinplace(prob) - DI.jacobian!(prob.f, fx, J, extras, autodiff, x, Constant(prob.p)) + @assert extras isa DIExtras + DI.jacobian!(prob.f, fx, J, extras.prep, autodiff, x, Constant(prob.p)) else - DI.jacobian!(prob.f, J, extras, autodiff, x, Constant(prob.p)) + if ArrayInterface.can_setindex(J) + if extras isa DIExtras + DI.jacobian!(prob.f, J, extras.prep, autodiff, x, Constant(prob.p)) + else + DI.jacobian!(prob.f, J, autodiff, x, Constant(prob.p)) + end + else + if extras isa DIExtras + J = DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) + else + J = DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) + end + end end return J end diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl new file mode 100644 index 000000000..8ecee2fed --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl @@ -0,0 +1,90 @@ +@testitem "Solving on CUDA" tags=[:cuda] begin + using StaticArrays, CUDA, SimpleNonlinearSolve + + if CUDA.functional() + CUDA.allowscalar(false) + + f(u, p) = u .* u .- 2 + f!(du, u, p) = (du .= u .* u .- 2) + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleNewtonRaphson(), + SimpleDFSane(), + SimpleTrustRegion(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleBroyden(), + SimpleLimitedMemoryBroyden(), + SimpleKlement(), + SimpleHalley(), + SimpleBroyden(; linesearch = Val(true)), + SimpleLimitedMemoryBroyden(; linesearch = Val(true)) + ) + # Static Arrays + u0 = @SVector[1.0f0, 1.0f0] + probN = NonlinearProblem{false}(f, u0) + sol = solve(probN, alg; abstol = 1.0f-6) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, sol.resid) ≤ 1.0f-6 + + # Regular Arrays + u0 = [1.0, 1.0] + probN = NonlinearProblem{false}(f, u0) + sol = solve(probN, alg; abstol = 1.0f-6) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, sol.resid) ≤ 1.0f-6 + + # Regular Arrays Inplace + if !(alg isa SimpleHalley) + u0 = [1.0, 1.0] + probN = NonlinearProblem{true}(f!, u0) + sol = solve(probN, alg; abstol = 1.0f-6) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, sol.resid) ≤ 1.0f-6 + end + end + end +end + +@testitem "CUDA Kernel Launch Test" tags=[:cuda] begin + using StaticArrays, CUDA, SimpleNonlinearSolve + using NonlinearSolveBase: ImmutableNonlinearProblem + + if CUDA.functional() + CUDA.allowscalar(false) + + f(u, p) = u .* u .- p + + function kernel_function(prob, alg) + solve(prob, alg) + return nothing + end + + @testset for u0 in (1.0f0, @SVector[1.0f0, 1.0f0]) + prob = convert(ImmutableNonlinearProblem, NonlinearProblem{false}(f, u0, 2.0f0)) + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleNewtonRaphson(), + SimpleDFSane(), + SimpleTrustRegion(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleBroyden(), + SimpleLimitedMemoryBroyden(), + SimpleKlement(), + SimpleHalley(), + SimpleBroyden(; linesearch = Val(true)), + SimpleLimitedMemoryBroyden(; linesearch = Val(true)) + ) + @test begin + try + @cuda kernel_function(prob, alg) + @info "Successfully launched kernel for $(alg)." + true + catch err + @error "Kernel Launch failed for $(alg)." + false + end + end broken=(alg isa SimpleHalley && u0 isa StaticArray) + end + end + end +end diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index 6ea6326b0..dde4bacf4 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -1,5 +1,11 @@ -using TestItemRunner, InteractiveUtils +using TestItemRunner, InteractiveUtils, Pkg @info sprint(InteractiveUtils.versioninfo) -@run_package_tests +const GROUP = lowercase(get(ENV, "GROUP", "All")) + +if GROUP == "all" + @run_package_tests +else + @run_package_tests filter=ti->(Symbol(GROUP) in ti.tags) +end From 73d960cd4b7f7ce501f017c63b72383ed1d2668c Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:34:58 -0400 Subject: [PATCH 49/92] fix: exotic types --- .../SimpleNonlinearSolveChainRulesCoreExt.jl | 2 +- .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 7 +++-- .../ext/SimpleNonlinearSolveTrackerExt.jl | 2 +- .../src/SimpleNonlinearSolve.jl | 7 +++-- lib/SimpleNonlinearSolve/src/halley.jl | 6 ++-- lib/SimpleNonlinearSolve/src/lbroyden.jl | 6 ++-- lib/SimpleNonlinearSolve/src/raphson.jl | 4 +-- lib/SimpleNonlinearSolve/src/trust_region.jl | 4 +-- lib/SimpleNonlinearSolve/src/utils.jl | 5 ++-- .../test/core/exotic_type_tests.jl | 30 +++++++++++++++++++ .../test/core/qa_tests.jl | 19 ++++++++++++ 11 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl create mode 100644 lib/SimpleNonlinearSolve/test/core/qa_tests.jl diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl index df0bd7573..f56dee537 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveChainRulesCoreExt.jl @@ -8,7 +8,7 @@ using SimpleNonlinearSolve: SimpleNonlinearSolve, simplenonlinearsolve_solve_up, solve_adjoint function ChainRulesCore.rrule(::typeof(simplenonlinearsolve_solve_up), - prob::Union{InternalNonlinearProblem, NonlinearLeastSquaresProblem}, + prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...) out, ∇internal = solve_adjoint( prob, sensealg, u0, p, ChainRulesOriginator(), alg, args...; kwargs...) diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl index 1357bec83..7b476b3c5 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl @@ -6,11 +6,12 @@ using ReverseDiff: ReverseDiff, TrackedArray, TrackedReal using SciMLBase: ReverseDiffOriginator, NonlinearLeastSquaresProblem, remake using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint +import SimpleNonlinearSolve: simplenonlinearsolve_solve_up -for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) +for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem) aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any) for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)] - @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + @eval function simplenonlinearsolve_solve_up( prob::$(pType), sensealg, u0::$(uT), u0_changed, p::$(pT), p_changed, alg, args...; kwargs...) return ReverseDiff.track(SimpleNonlinearSolve.simplenonlinearsolve_solve_up, @@ -19,7 +20,7 @@ for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) end end - @eval ReverseDiff.@grad function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( + @eval ReverseDiff.@grad function simplenonlinearsolve_solve_up( tprob::$(pType), sensealg, tu0, u0_changed, tp, p_changed, alg, args...; kwargs...) u0, p = ReverseDiff.value(tu0), ReverseDiff.value(tp) diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl index a2fa8ff40..ead5a8e29 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl @@ -7,7 +7,7 @@ using Tracker: Tracker, TrackedArray, TrackedReal using SimpleNonlinearSolve: SimpleNonlinearSolve, solve_adjoint -for pType in (InternalNonlinearProblem, NonlinearLeastSquaresProblem) +for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem) aTypes = (TrackedArray, AbstractArray{<:TrackedReal}, Any) for (uT, pT) in collect(Iterators.product(aTypes, aTypes))[1:(end - 1)] @eval function SimpleNonlinearSolve.simplenonlinearsolve_solve_up( diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index c8df31e51..0b33c923e 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -14,19 +14,20 @@ using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode using StaticArraysCore: StaticArray, SArray, SVector, MArray # AD Dependencies -using ADTypes: AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff +using ADTypes: AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff using DifferentiationInterface: DifferentiationInterface using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder -using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, get_tolerance, - L2_NORM +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, L2_NORM const DI = DifferentiationInterface abstract type AbstractSimpleNonlinearSolveAlgorithm <: AbstractNonlinearAlgorithm end +const safe_similar = NonlinearSolveBase.Utils.safe_similar + is_extension_loaded(::Val) = false include("utils.jl") diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl index 23ba1c847..6b8948248 100644 --- a/lib/SimpleNonlinearSolve/src/halley.jl +++ b/lib/SimpleNonlinearSolve/src/halley.jl @@ -41,9 +41,9 @@ function SciMLBase.__solve( strait = setindex_trait(x) - A = strait isa CanSetindex ? similar(x, length(x), length(x)) : x - Aaᵢ = strait isa CanSetindex ? similar(x, length(x)) : x - cᵢ = strait isa CanSetindex ? similar(x) : x + A = strait isa CanSetindex ? safe_similar(x, length(x), length(x)) : x + Aaᵢ = strait isa CanSetindex ? safe_similar(x, length(x)) : x + cᵢ = strait isa CanSetindex ? safe_similar(x) : x for _ in 1:maxiters fx, J, H = Utils.compute_jacobian_and_hessian(autodiff, prob, fx, x) diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index 1ab200f74..ce39ca10d 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -301,7 +301,7 @@ end return :(return SVector{$N, $T}(($(getcalls...)))) end -lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = similar(x, threshold) +lbroyden_threshold_cache(x, ::Val{threshold}) where {threshold} = safe_similar(x, threshold) function lbroyden_threshold_cache(x::StaticArray, ::Val{threshold}) where {threshold} return zeros(MArray{Tuple{threshold}, eltype(x)}) end @@ -327,7 +327,7 @@ end end end function init_low_rank_jacobian(u, fu, ::Val{threshold}) where {threshold} - Vᵀ = similar(u, threshold, length(u)) - U = similar(u, length(fu), threshold) + Vᵀ = safe_similar(u, threshold, length(u)) + U = safe_similar(u, length(fu), threshold) return U, Vᵀ end diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl index 2af3a825a..763fcf32a 100644 --- a/lib/SimpleNonlinearSolve/src/raphson.jl +++ b/lib/SimpleNonlinearSolve/src/raphson.jl @@ -41,8 +41,8 @@ function SciMLBase.__solve( NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff) @bb xo = similar(x) - fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? similar(fx) : - nothing + fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? + safe_similar(fx) : nothing jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index 6bf543220..27b210d65 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -93,8 +93,8 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegi norm_fx = L2_NORM(fx) @bb xo = copy(x) - fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? similar(fx) : - nothing + fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? + safe_similar(fx) : nothing jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 011788a1c..946c10529 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -1,6 +1,5 @@ module Utils -using ADTypes: AbstractADType, AutoForwardDiff, AutoFiniteDiff, AutoPolyesterForwardDiff using ArrayInterface: ArrayInterface using ConcreteStructs: @concrete using DifferentiationInterface: DifferentiationInterface, Constant @@ -164,7 +163,7 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) if J === nothing if extras isa AnalyticJacobian if SciMLBase.isinplace(prob.f) - J = similar(fx, length(fx), length(x)) + J = safe_similar(fx, length(fx), length(x)) prob.f.jac(J, x, prob.p) return J else @@ -219,7 +218,7 @@ end function compute_jacobian_and_hessian(autodiff, prob, fx, x) if SciMLBase.isinplace(prob) jac_fn = @closure (u, p) -> begin - du = similar(fx, promote_type(eltype(fx), eltype(u))) + du = safe_similar(fx, promote_type(eltype(fx), eltype(u))) return DI.jacobian(prob.f, du, autodiff, u, Constant(p)) end J, H = DI.value_and_jacobian(jac_fn, autodiff, x, Constant(prob.p)) diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl new file mode 100644 index 000000000..e19a3d32e --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl @@ -0,0 +1,30 @@ +@testitem "BigFloat Support" tags=[:core] begin + using SimpleNonlinearSolve, LinearAlgebra + + fn_iip = NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p) + fn_oop = NonlinearFunction{false}((u, p) -> u .* u .- p) + + u0 = BigFloat[1.0, 1.0, 1.0] + prob_iip_bf = NonlinearProblem{true}(fn_iip, u0, BigFloat(2)) + prob_oop_bf = NonlinearProblem{false}(fn_oop, u0, BigFloat(2)) + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleNewtonRaphson(), + SimpleBroyden(), + SimpleKlement(), + SimpleDFSane(), + SimpleTrustRegion(), + SimpleLimitedMemoryBroyden(), + SimpleHalley() + ) + sol = solve(prob_oop_bf, alg) + @test maximum(abs, sol.resid) < 1e-6 + @test SciMLBase.successful_retcode(sol.retcode) + + alg isa SimpleHalley && continue + + sol = solve(prob_iip_bf, alg) + @test maximum(abs, sol.resid) < 1e-6 + @test SciMLBase.successful_retcode(sol.retcode) + end +end diff --git a/lib/SimpleNonlinearSolve/test/core/qa_tests.jl b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl new file mode 100644 index 000000000..d6a5a9b8e --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl @@ -0,0 +1,19 @@ +@testitem "Aqua" tags=[:core] begin + using Aqua, SimpleNonlinearSolve + + Aqua.test_all(SimpleNonlinearSolve; piracies = false, ambiguities = false) + Aqua.test_piracies(SimpleNonlinearSolve; + treat_as_own = [ + NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem]) + Aqua.test_ambiguities(SimpleNonlinearSolve; recursive = false) +end + +@testitem "Explicit Imports" tags=[:core] begin + import ReverseDiff, Tracker, StaticArrays, Zygote + using ExplicitImports, SimpleNonlinearSolve + + @test check_no_implicit_imports( + SimpleNonlinearSolve; skip = (Base, Core, SciMLBase)) === nothing + @test check_no_stale_explicit_imports(SimpleNonlinearSolve) === nothing + @test check_all_qualified_accesses_via_owners(SimpleNonlinearSolve) === nothing +end From 51b24f9760c987385db8f597981fb2219a55f919 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:35:39 -0400 Subject: [PATCH 50/92] chore: run formatter --- lib/SimpleNonlinearSolve/test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index dde4bacf4..a22783e59 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -7,5 +7,5 @@ const GROUP = lowercase(get(ENV, "GROUP", "All")) if GROUP == "all" @run_package_tests else - @run_package_tests filter=ti->(Symbol(GROUP) in ti.tags) + @run_package_tests filter = ti -> (Symbol(GROUP) in ti.tags) end From 54a678a6a7aff1aea802026ee1d75baacfae5834 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:37:35 -0400 Subject: [PATCH 51/92] fix: typos --- .typos.toml | 3 ++- lib/SimpleNonlinearSolve/src/trust_region.jl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.typos.toml b/.typos.toml index d78350bfb..4bde74f7c 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,2 +1,3 @@ [default.extend-words] -SER = "SER" \ No newline at end of file +SER = "SER" +fo = "fo" diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index 27b210d65..47acc5437 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -189,7 +189,7 @@ function dogleg_method!!(cache, J, f, g, Δ) # Test if the full step is within the trust region (L2_NORM(δN) ≤ Δ) && return δN - # Calcualte Cauchy point, optimum along the steepest descent direction + # Calculate Cauchy point, optimum along the steepest descent direction @bb δsd .= g @bb @. δsd *= -1 norm_δsd = L2_NORM(δsd) From c5884d3f8cca1674858e7e25a46fcfef0620eb2a Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:56:09 -0400 Subject: [PATCH 52/92] test: bring over more tests --- .github/workflows/CI_SimpleNonlinearSolve.yml | 5 +++ .../test/core/allocation_tests.jl | 40 +++++++++++++++++++ .../test/core/matrix_resizing_tests.jl | 19 +++++++++ 3 files changed, 64 insertions(+) create mode 100644 lib/SimpleNonlinearSolve/test/core/allocation_tests.jl create mode 100644 lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml index 9854b6d99..1f8306a8c 100644 --- a/.github/workflows/CI_SimpleNonlinearSolve.yml +++ b/.github/workflows/CI_SimpleNonlinearSolve.yml @@ -32,6 +32,9 @@ jobs: - ubuntu-latest - macos-latest - windows-latest + group: + - core + - adjoint steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 @@ -60,6 +63,8 @@ jobs: Pkg.instantiate() Pkg.test(; coverage=true) shell: julia --color=yes --code-coverage=user --depwarn=yes --project=lib/SimpleNonlinearSolve {0} + env: + GROUP: ${{ matrix.group }} - uses: julia-actions/julia-processcoverage@v1 with: directories: lib/SimpleNonlinearSolve/src,lib/SimpleNonlinearSolve/ext diff --git a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl new file mode 100644 index 000000000..5da872b71 --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl @@ -0,0 +1,40 @@ +@itesitem "Allocation Tests" tags=[:core] begin + using SimpleNonlinearSolve, StaticArrays, AllocCheck + + quadratic_f(u, p) = u .* u .- p + quadratic_f!(du, u, p) = (du .= u .* u .- p) + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleNewtonRaphson(), + SimpleTrustRegion(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleBroyden(), + SimpleLimitedMemoryBroyden(), + SimpleKlement(), + SimpleHalley(), + SimpleBroyden(; linesearch = Val(true)), + SimpleLimitedMemoryBroyden(; linesearch = Val(true)) + ) + @check_allocs nlsolve(prob, alg) = SciMLBase.solve(prob, alg; abstol = 1e-9) + + nlprob_scalar = NonlinearProblem{false}(quadratic_f, 1.0, 2.0) + nlprob_sa = NonlinearProblem{false}(quadratic_f, @SVector[1.0, 1.0], 2.0) + + try + nlsolve(nlprob_scalar, alg) + @test true + catch e + @error e + @test false + end + + # ForwardDiff allocates for hessian since we don't propagate the chunksize + try + nlsolve(nlprob_sa, alg) + @test true + catch e + @error e + @test false broken = (alg isa SimpleHalley) + end + end +end diff --git a/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl new file mode 100644 index 000000000..17d9a6f42 --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/matrix_resizing_tests.jl @@ -0,0 +1,19 @@ +@testitem "Matrix Resizing" tags=[:core] begin + ff(u, p) = u .* u .- p + u0 = ones(2, 3) + p = 2.0 + vecprob = NonlinearProblem(ff, vec(u0), p) + prob = NonlinearProblem(ff, u0, p) + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleKlement(), + SimpleBroyden(), + SimpleNewtonRaphson(), + SimpleDFSane(), + SimpleLimitedMemoryBroyden(; threshold = Val(2)), + SimpleTrustRegion(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)) + ) + @test vec(solve(prob, alg).u) ≈ solve(vecprob, alg).u + end +end From e7cb5c261e882f31d36ae265ee16af12cc44be74 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 21:58:15 -0400 Subject: [PATCH 53/92] test: adjoints --- .../test/core/adjoint_tests.jl | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl new file mode 100644 index 000000000..c56850eb5 --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl @@ -0,0 +1,21 @@ +@testitem "Simple Adjoint Test" tags=[:adjoint] begin + using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote + + ff(u, p) = u .^ 2 .- p + + function solve_nlprob(p) + prob = NonlinearProblem{false}(ff, [1.0, 2.0], p) + sol = solve(prob, SimpleNewtonRaphson()) + res = sol isa AbstractArray ? sol : sol.u + return sum(abs2, res) + end + + p = [3.0, 2.0] + + ∂p_zygote = only(Zygote.gradient(solve_nlprob, p)) + ∂p_forwarddiff = ForwardDiff.gradient(solve_nlprob, p) + ∂p_tracker = Tracker.data(only(Tracker.gradient(solve_nlprob, p))) + ∂p_reversediff = ReverseDiff.gradient(solve_nlprob, p) + @test ∂p_zygote ≈ ∂p_tracker ≈ ∂p_reversediff + @test ∂p_zygote ≈ ∂p_forwarddiff ≈ ∂p_tracker ≈ ∂p_reversediff +end From dcf62eb2190469df56f4c2ade23dbf5880d08f4a Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 22:17:17 -0400 Subject: [PATCH 54/92] fix: minor fixes to support adjoints --- lib/NonlinearSolveBase/Project.toml | 3 +++ .../ext/NonlinearSolveBaseDiffEqBaseExt.jl | 16 ++++++++++++++ lib/SimpleNonlinearSolve/Project.toml | 5 +++-- .../ext/SimpleNonlinearSolveDiffEqBaseExt.jl | 2 ++ .../ext/SimpleNonlinearSolveReverseDiffExt.jl | 2 ++ .../ext/SimpleNonlinearSolveTrackerExt.jl | 2 ++ .../src/SimpleNonlinearSolve.jl | 22 +++++++++++++++---- .../test/core/adjoint_tests.jl | 3 ++- .../test/core/allocation_tests.jl | 2 +- .../test/core/rootfind_tests.jl | 0 10 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl create mode 100644 lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 3999de770..a89d035cf 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -20,10 +20,12 @@ SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" [weakdeps] +DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [extensions] +NonlinearSolveBaseDiffEqBaseExt = "DiffEqBase" NonlinearSolveBaseForwardDiffExt = "ForwardDiff" NonlinearSolveBaseSparseArraysExt = "SparseArrays" @@ -33,6 +35,7 @@ ArrayInterface = "7.9" CommonSolve = "0.2.4" Compat = "4.15" ConcreteStructs = "0.2.3" +DiffEqBase = "6.149" DifferentiationInterface = "0.6.1" EnzymeCore = "0.8" FastClosures = "0.3" diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl new file mode 100644 index 000000000..346a5ee55 --- /dev/null +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseDiffEqBaseExt.jl @@ -0,0 +1,16 @@ +module NonlinearSolveBaseDiffEqBaseExt + +using DiffEqBase: DiffEqBase +using SciMLBase: remake + +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem + +function DiffEqBase.get_concrete_problem( + prob::ImmutableNonlinearProblem, isadapt; kwargs...) + u0 = DiffEqBase.get_concrete_u0(prob, isadapt, nothing, kwargs) + u0 = DiffEqBase.promote_u0(u0, prob.p, nothing) + p = DiffEqBase.get_concrete_p(prob, kwargs) + return remake(prob; u0 = u0, p = p) +end + +end diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index fae63544a..8c6322730 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -46,7 +46,7 @@ CUDA = "5.3" ChainRulesCore = "1.24" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" -DiffEqBase = "6.155" +DiffEqBase = "6.149" DifferentiationInterface = "0.6.1" Enzyme = "0.13" ExplicitImports = "1.9" @@ -79,6 +79,7 @@ julia = "1.10" AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -95,4 +96,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["AllocCheck", "Aqua", "CUDA", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] +test = ["AllocCheck", "Aqua", "CUDA", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl index 950a04019..4954ffb26 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveDiffEqBaseExt.jl @@ -4,6 +4,8 @@ using DiffEqBase: DiffEqBase using SimpleNonlinearSolve: SimpleNonlinearSolve +SimpleNonlinearSolve.is_extension_loaded(::Val{:DiffEqBase}) = true + function SimpleNonlinearSolve.solve_adjoint_internal(args...; kwargs...) return DiffEqBase._solve_adjoint(args...; kwargs...) end diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl index 7b476b3c5..0a407986e 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveReverseDiffExt.jl @@ -32,6 +32,8 @@ for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem) ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Δ...) return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...) end + + return Array(out), ∇simplenonlinearsolve_solve_up end end diff --git a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl index ead5a8e29..d29c2ac61 100644 --- a/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl +++ b/lib/SimpleNonlinearSolve/ext/SimpleNonlinearSolveTrackerExt.jl @@ -31,6 +31,8 @@ for pType in (ImmutableNonlinearProblem, NonlinearLeastSquaresProblem) ∂prob, ∂sensealg, ∂u0, ∂p, _, ∂args... = ∇internal(Tracker.data(Δ)) return (∂prob, ∂sensealg, ∂u0, nothing, ∂p, nothing, nothing, ∂args...) end + + return out, ∇simplenonlinearsolve_solve_up end end diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 0b33c923e..23de7dbc6 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -20,7 +20,8 @@ using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder -using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, L2_NORM +using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, L2_NORM, + nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution const DI = DifferentiationInterface @@ -47,6 +48,20 @@ function CommonSolve.solve(prob::NonlinearProblem, return solve(prob, alg, args...; kwargs...) end +function CommonSolve.solve( + prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip, + <:Union{ + <:ForwardDiff.Dual{T, V, P}, <:AbstractArray{<:ForwardDiff.Dual{T, V, P}}}}, + alg::AbstractSimpleNonlinearSolveAlgorithm, + args...; + kwargs...) where {T, V, P, iip} + prob = convert(ImmutableNonlinearProblem, prob) + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) + return SciMLBase.build_solution( + prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) +end + function CommonSolve.solve( prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...) @@ -59,9 +74,8 @@ function CommonSolve.solve( p === nothing, alg, args...; prob.kwargs..., kwargs...) end -function simplenonlinearsolve_solve_up( - prob::ImmutableNonlinearProblem, sensealg, u0, u0_changed, p, p_changed, - alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) +function simplenonlinearsolve_solve_up(prob::ImmutableNonlinearProblem, sensealg, u0, + u0_changed, p, p_changed, alg, args...; kwargs...) (u0_changed || p_changed) && (prob = remake(prob; u0, p)) return SciMLBase.__solve(prob, alg, args...; kwargs...) end diff --git a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl index c56850eb5..449801ad7 100644 --- a/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/adjoint_tests.jl @@ -1,5 +1,6 @@ @testitem "Simple Adjoint Test" tags=[:adjoint] begin - using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote + using ForwardDiff, ReverseDiff, SciMLSensitivity, Tracker, Zygote, DiffEqBase, + SimpleNonlinearSolve ff(u, p) = u .^ 2 .- p diff --git a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl index 5da872b71..67cee39c0 100644 --- a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl @@ -34,7 +34,7 @@ @test true catch e @error e - @test false broken = (alg isa SimpleHalley) + @test false broken=(alg isa SimpleHalley) end end end diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl new file mode 100644 index 000000000..e69de29bb From a51f8e711264d4532b5f28364bce9aeb6b2d71ad Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 6 Oct 2024 23:00:18 -0400 Subject: [PATCH 55/92] test: 23 test problems --- .../test/core/23_test_problems_tests.jl | 103 ++++++++++++++++++ .../test/core/exotic_type_tests.jl | 6 +- .../test/core/forward_diff_tests.jl | 0 .../test/gpu/cuda_tests.jl | 18 +-- 4 files changed, 116 insertions(+), 11 deletions(-) create mode 100644 lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl create mode 100644 lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl new file mode 100644 index 000000000..7bb28c649 --- /dev/null +++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl @@ -0,0 +1,103 @@ +@testsnippet RobustnessTestSnippet begin + using NonlinearProblemLibrary, NonlinearSolveBase, LinearAlgebra + + problems = NonlinearProblemLibrary.problems + dicts = NonlinearProblemLibrary.dicts + + function test_on_library( + problems, dicts, alg_ops, broken_tests, ϵ = 1e-4; skip_tests = nothing) + for (idx, (problem, dict)) in enumerate(zip(problems, dicts)) + x = dict["start"] + res = similar(x) + nlprob = NonlinearProblem(problem, copy(x)) + @testset "$idx: $(dict["title"])" begin + for alg in alg_ops + try + sol = solve(nlprob, alg; + termination_condition = AbsNormTerminationMode(norm)) + problem(res, sol.u, nothing) + + skip = skip_tests !== nothing && idx in skip_tests[alg] + if skip + @test_skip norm(res) ≤ ϵ + continue + end + broken = idx in broken_tests[alg] ? true : false + @test norm(res)≤ϵ broken=broken + catch e + @error e + broken = idx in broken_tests[alg] ? true : false + if broken + @test false broken=true + else + @test 1 == 2 + end + end + end + end + end + end +end + +@testitem "23 Test Problems: SimpleNewtonRaphson" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleNewtonRaphson(),) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + broken_tests[alg_ops[1]] = [] + + test_on_library(problems, dicts, alg_ops, broken_tests) +end + +@testitem "23 Test Problems: SimpleHalley" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleHalley(),) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + if Sys.isapple() + broken_tests[alg_ops[1]] = [1, 5, 11, 15, 16, 18] + else + broken_tests[alg_ops[1]] = [1, 5, 15, 16, 18] + end + + test_on_library(problems, dicts, alg_ops, broken_tests) +end + +@testitem "23 Test Problems: SimpleTrustRegion" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true))) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + broken_tests[alg_ops[1]] = [3, 15, 16, 21] + broken_tests[alg_ops[2]] = [15, 16] + + test_on_library(problems, dicts, alg_ops, broken_tests) +end + +@testitem "23 Test Problems: SimpleDFSane" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleDFSane(),) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + if Sys.isapple() + broken_tests[alg_ops[1]] = [1, 2, 3, 5, 6, 21] + else + broken_tests[alg_ops[1]] = [1, 2, 3, 4, 5, 6, 11, 21] + end + + test_on_library(problems, dicts, alg_ops, broken_tests) +end + +@testitem "23 Test Problems: SimpleBroyden" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleBroyden(),) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + broken_tests[alg_ops[1]] = [1, 5, 11] + + test_on_library(problems, dicts, alg_ops, broken_tests) +end + +@testitem "23 Test Problems: SimpleKlement" setup=[RobustnessTestSnippet] tags=[:core] begin + alg_ops = (SimpleKlement(),) + + broken_tests = Dict(alg => Int[] for alg in alg_ops) + broken_tests[alg_ops[1]] = [1, 2, 4, 5, 11, 12, 22] + + test_on_library(problems, dicts, alg_ops, broken_tests) +end diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl index e19a3d32e..7e7223ea0 100644 --- a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl @@ -9,13 +9,13 @@ prob_oop_bf = NonlinearProblem{false}(fn_oop, u0, BigFloat(2)) @testset "$(nameof(typeof(alg)))" for alg in ( - SimpleNewtonRaphson(), + SimpleNewtonRaphson(; autodiff = AutoForwardDiff()), SimpleBroyden(), SimpleKlement(), SimpleDFSane(), - SimpleTrustRegion(), + SimpleTrustRegion(; autodiff = AutoForwardDiff()), SimpleLimitedMemoryBroyden(), - SimpleHalley() + SimpleHalley(; autodiff = AutoForwardDiff()) ) sol = solve(prob_oop_bf, alg) @test maximum(abs, sol.resid) < 1e-6 diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl new file mode 100644 index 000000000..e69de29bb diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl index 8ecee2fed..d51ce353f 100644 --- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl +++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl @@ -8,14 +8,15 @@ f!(du, u, p) = (du .= u .* u .- 2) @testset "$(nameof(typeof(alg)))" for alg in ( - SimpleNewtonRaphson(), + SimpleNewtonRaphson(; autodiff = AutoForwardDiff()), SimpleDFSane(), - SimpleTrustRegion(), - SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleTrustRegion(; autodiff = AutoForwardDiff()), + SimpleTrustRegion(; + nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff()), SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(), - SimpleHalley(), + SimpleHalley(; autodiff = AutoForwardDiff()), SimpleBroyden(; linesearch = Val(true)), SimpleLimitedMemoryBroyden(; linesearch = Val(true)) ) @@ -63,14 +64,15 @@ end prob = convert(ImmutableNonlinearProblem, NonlinearProblem{false}(f, u0, 2.0f0)) @testset "$(nameof(typeof(alg)))" for alg in ( - SimpleNewtonRaphson(), + SimpleNewtonRaphson(; autodiff = AutoForwardDiff()), SimpleDFSane(), - SimpleTrustRegion(), - SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleTrustRegion(; autodiff = AutoForwardDiff()), + SimpleTrustRegion(; + nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff()), SimpleBroyden(), SimpleLimitedMemoryBroyden(), SimpleKlement(), - SimpleHalley(), + SimpleHalley(; autodiff = AutoForwardDiff()), SimpleBroyden(; linesearch = Val(true)), SimpleLimitedMemoryBroyden(; linesearch = Val(true)) ) From 5059e0d2da41164528fe7a2dd81efc34c9e81c1f Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 11:46:01 -0400 Subject: [PATCH 56/92] fix: simple klement implementation --- lib/SimpleNonlinearSolve/src/klement.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/SimpleNonlinearSolve/src/klement.jl b/lib/SimpleNonlinearSolve/src/klement.jl index 055f65bc3..31c4cca96 100644 --- a/lib/SimpleNonlinearSolve/src/klement.jl +++ b/lib/SimpleNonlinearSolve/src/klement.jl @@ -12,6 +12,7 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleKlement, x = Utils.maybe_unaliased(prob.u0, alias_u0) T = eltype(x) fx = Utils.get_fx(prob, x) + fx = Utils.eval_f(prob, fx, x) abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) From 59a87ed452d5c266dca980dec27cadb74abb8af7 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 11:52:31 -0400 Subject: [PATCH 57/92] chore: run the formatter --- lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl | 1 + lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl @@ -0,0 +1 @@ + diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl index e69de29bb..8b1378917 100644 --- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl @@ -0,0 +1 @@ + From 052050c161df997e08f39649ff001357d4ebe382 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 15:41:40 -0400 Subject: [PATCH 58/92] feat: bump major version of SimpleNonlinearSolve --- lib/NonlinearSolveBase/src/NonlinearSolveBase.jl | 7 ++++--- lib/SimpleNonlinearSolve/Project.toml | 4 +--- .../src/SimpleNonlinearSolve.jl | 13 ++++++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 5e1a37326..645b84959 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -35,8 +35,9 @@ include("autodiff.jl") (select_forward_mode_autodiff, select_reverse_mode_autodiff, select_jacobian_autodiff)) -export RelTerminationMode, AbsTerminationMode, NormTerminationMode, RelNormTerminationMode, - AbsNormTerminationMode, RelNormSafeTerminationMode, AbsNormSafeTerminationMode, - RelNormSafeNormTerminationMode, AbsNormSafeNormTerminationMode +export RelTerminationMode, AbsTerminationMode, + NormTerminationMode, RelNormTerminationMode, AbsNormTerminationMode, + RelNormSafeTerminationMode, AbsNormSafeTerminationMode, + RelNormSafeBestTerminationMode, AbsNormSafeBestTerminationMode end diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 8c6322730..5f708e564 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -1,7 +1,7 @@ name = "SimpleNonlinearSolve" uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7" authors = ["SciML"] -version = "1.13.0" +version = "2.0.0" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" @@ -19,7 +19,6 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" -Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" @@ -63,7 +62,6 @@ Pkg = "1.10" PolyesterForwardDiff = "0.1" PrecompileTools = "1.2" Random = "1.10" -Reexport = "1.2" ReverseDiff = "1.15" SciMLBase = "2.50" SciMLSensitivity = "7.68" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 23de7dbc6..4ed53295e 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -8,13 +8,12 @@ using LineSearch: LiFukushimaLineSearch using LinearAlgebra: LinearAlgebra, dot using MaybeInplace: @bb, setindex_trait, CannotSetindex, CanSetindex using PrecompileTools: @compile_workload, @setup_workload -using Reexport: @reexport -@reexport using SciMLBase # I don't like this but needed to avoid a breaking change -using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, ReturnCode +using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, NonlinearLeastSquaresProblem, + IntervalNonlinearProblem, ReturnCode using StaticArraysCore: StaticArray, SArray, SVector, MArray # AD Dependencies -using ADTypes: AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff +using ADTypes: ADTypes using DifferentiationInterface: DifferentiationInterface using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff @@ -118,13 +117,17 @@ function solve_adjoint_internal end end end -export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff +export IntervalNonlinearProblem export Alefeld, Bisection, Brent, Falsi, ITP, Ridder +export NonlinearProblem, NonlinearLeastSquaresProblem + export SimpleBroyden, SimpleKlement, SimpleLimitedMemoryBroyden export SimpleDFSane export SimpleGaussNewton, SimpleNewtonRaphson, SimpleTrustRegion export SimpleHalley +export solve + end From 58e6c006ab2c84c9f63713438a5844c9d851b6ed Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 17:13:19 -0400 Subject: [PATCH 59/92] test: comprehensive testing of root finding --- docs/src/release_notes.md | 22 ++- .../src/termination_conditions.jl | 4 +- .../src/SimpleNonlinearSolve.jl | 10 +- lib/SimpleNonlinearSolve/src/halley.jl | 3 +- lib/SimpleNonlinearSolve/src/lbroyden.jl | 2 +- .../test/core/23_test_problems_tests.jl | 11 +- .../test/core/exotic_type_tests.jl | 2 +- .../test/core/rootfind_tests.jl | 154 ++++++++++++++++++ .../test/gpu/cuda_tests.jl | 4 +- 9 files changed, 195 insertions(+), 17 deletions(-) diff --git a/docs/src/release_notes.md b/docs/src/release_notes.md index 9d517267e..0208b1e66 100644 --- a/docs/src/release_notes.md +++ b/docs/src/release_notes.md @@ -1,6 +1,24 @@ # Release Notes -## Breaking Changes in `NonlinearSolve.jl` v3 +## Oct '24 + +### Breaking Changes in `NonlinearSolve.jl` v4 + +### Breaking Changes in `SimpleNonlinearSolve.jl` v2 + + - `Auto*` structs are no longer exported. Load `ADTypes` to access them. + - Use of termination conditions from `DiffEqBase` has been removed. Use the termination + conditions from `NonlinearSolveBase` instead. + - We no longer export the entire `SciMLBase`. Instead selected functionality relevant to + `SimpleNonlinearSolve` has been exported. + - If no autodiff is provided, we now choose from a list of autodiffs based on the packages + loaded. For example, if `Enzyme` is loaded, we will default to that. In general, we + don't guarantee the exact autodiff selected if `autodiff` is not provided (i.e. + `nothing`). + +## Dec '23 + +### Breaking Changes in `NonlinearSolve.jl` v3 - `GeneralBroyden` and `GeneralKlement` have been renamed to `Broyden` and `Klement` respectively. @@ -8,7 +26,7 @@ - The old style of specifying autodiff with `chunksize`, `standardtag`, etc. has been deprecated in favor of directly specifying the autodiff type, like `AutoForwardDiff`. -## Breaking Changes in `SimpleNonlinearSolve.jl` v1 +### Breaking Changes in `SimpleNonlinearSolve.jl` v1 - Batched solvers have been removed in favor of `BatchedArrays.jl`. Stay tuned for detailed tutorials on how to use `BatchedArrays.jl` with `NonlinearSolve` & `SimpleNonlinearSolve` diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 7978c19b8..3e957f139 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -26,7 +26,7 @@ function update_u!!(cache::NonlinearTerminationModeCache, u) if cache.u isa AbstractArray && ArrayInterface.can_setindex(cache.u) copyto!(cache.u, u) else - cache.u .= u + cache.u = u end end @@ -60,6 +60,8 @@ function SciMLBase.init( else u_diff_cache = u_unaliased end + best_value = initial_objective + max_stalled_steps = mode.max_stalled_steps else initial_objective = nothing objectives_trace = nothing diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 4ed53295e..081201a24 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,19 +1,19 @@ module SimpleNonlinearSolve using Accessors: @reset -using CommonSolve: CommonSolve, solve +using CommonSolve: CommonSolve, solve, init, solve! using ConcreteStructs: @concrete using FastClosures: @closure using LineSearch: LiFukushimaLineSearch using LinearAlgebra: LinearAlgebra, dot using MaybeInplace: @bb, setindex_trait, CannotSetindex, CanSetindex using PrecompileTools: @compile_workload, @setup_workload -using SciMLBase: AbstractNonlinearAlgorithm, NonlinearProblem, NonlinearLeastSquaresProblem, - IntervalNonlinearProblem, ReturnCode +using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, NonlinearFunction, NonlinearProblem, + NonlinearLeastSquaresProblem, IntervalNonlinearProblem, ReturnCode, remake using StaticArraysCore: StaticArray, SArray, SVector, MArray # AD Dependencies -using ADTypes: ADTypes +using ADTypes: ADTypes, AutoForwardDiff using DifferentiationInterface: DifferentiationInterface using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff @@ -121,7 +121,7 @@ export IntervalNonlinearProblem export Alefeld, Bisection, Brent, Falsi, ITP, Ridder -export NonlinearProblem, NonlinearLeastSquaresProblem +export NonlinearFunction, NonlinearProblem, NonlinearLeastSquaresProblem export SimpleBroyden, SimpleKlement, SimpleLimitedMemoryBroyden export SimpleDFSane diff --git a/lib/SimpleNonlinearSolve/src/halley.jl b/lib/SimpleNonlinearSolve/src/halley.jl index 6b8948248..30eb1a821 100644 --- a/lib/SimpleNonlinearSolve/src/halley.jl +++ b/lib/SimpleNonlinearSolve/src/halley.jl @@ -35,7 +35,8 @@ function SciMLBase.__solve( abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( prob, abstol, reltol, fx, x, termination_condition, Val(:simple)) - autodiff = NonlinearSolveBase.select_jacobian_autodiff(prob, alg.autodiff) + # The way we write the 2nd order derivatives, we know Enzyme won't work there + autodiff = alg.autodiff === nothing ? AutoForwardDiff() : alg.autodiff @bb xo = copy(x) diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index ce39ca10d..3a17e0936 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -38,7 +38,7 @@ function SciMLBase.__solve( args...; termination_condition = nothing, kwargs...) if prob.u0 isa SArray if termination_condition === nothing || - termination_condition isa AbsNormTerminationMode + termination_condition isa NonlinearSolveBase.AbsNormTerminationMode return internal_static_solve( prob, alg, args...; termination_condition, kwargs...) end diff --git a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl index 7bb28c649..6dd85b94b 100644 --- a/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/23_test_problems_tests.jl @@ -1,5 +1,5 @@ @testsnippet RobustnessTestSnippet begin - using NonlinearProblemLibrary, NonlinearSolveBase, LinearAlgebra + using NonlinearProblemLibrary, NonlinearSolveBase, LinearAlgebra, ADTypes problems = NonlinearProblemLibrary.problems dicts = NonlinearProblemLibrary.dicts @@ -40,7 +40,7 @@ end @testitem "23 Test Problems: SimpleNewtonRaphson" setup=[RobustnessTestSnippet] tags=[:core] begin - alg_ops = (SimpleNewtonRaphson(),) + alg_ops = (SimpleNewtonRaphson(; autodiff = AutoForwardDiff()),) broken_tests = Dict(alg => Int[] for alg in alg_ops) broken_tests[alg_ops[1]] = [] @@ -49,7 +49,7 @@ end end @testitem "23 Test Problems: SimpleHalley" setup=[RobustnessTestSnippet] tags=[:core] begin - alg_ops = (SimpleHalley(),) + alg_ops = (SimpleHalley(; autodiff = AutoForwardDiff()),) broken_tests = Dict(alg => Int[] for alg in alg_ops) if Sys.isapple() @@ -62,7 +62,10 @@ end end @testitem "23 Test Problems: SimpleTrustRegion" setup=[RobustnessTestSnippet] tags=[:core] begin - alg_ops = (SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true))) + alg_ops = ( + SimpleTrustRegion(; autodiff = AutoForwardDiff()), + SimpleTrustRegion(; nlsolve_update_rule = Val(true), autodiff = AutoForwardDiff()) + ) broken_tests = Dict(alg => Int[] for alg in alg_ops) broken_tests[alg_ops[1]] = [3, 15, 16, 21] diff --git a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl index 7e7223ea0..6227348a2 100644 --- a/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/exotic_type_tests.jl @@ -1,5 +1,5 @@ @testitem "BigFloat Support" tags=[:core] begin - using SimpleNonlinearSolve, LinearAlgebra + using SimpleNonlinearSolve, LinearAlgebra, ADTypes, SciMLBase fn_iip = NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p) fn_oop = NonlinearFunction{false}((u, p) -> u .* u .- p) diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl index 8b1378917..c99b670cb 100644 --- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl @@ -1 +1,155 @@ +@testsnippet RootfindTestSnippet begin + using StaticArrays, Random, LinearAlgebra, ForwardDiff, NonlinearSolveBase, SciMLBase + using ADTypes, PolyesterForwardDiff, Enzyme, ReverseDiff + quadratic_f(u, p) = u .* u .- p + quadratic_f!(du, u, p) = (du .= u .* u .- p) + + function newton_fails(u, p) + return 0.010000000000000002 .+ + 10.000000000000002 ./ (1 .+ + (0.21640425613334457 .+ + 216.40425613334457 ./ (1 .+ + (0.21640425613334457 .+ + 216.40425613334457 ./ (1 .+ 0.0006250000000000001(u .^ 2.0))) .^ 2.0)) .^ + 2.0) .- 0.0011552453009332421u .- p + end + + const TERMINATION_CONDITIONS = [ + NormTerminationMode(Base.Fix1(maximum, abs)), + RelTerminationMode(), + RelNormTerminationMode(Base.Fix1(maximum, abs)), + RelNormSafeTerminationMode(Base.Fix1(maximum, abs)), + RelNormSafeBestTerminationMode(Base.Fix1(maximum, abs)), + AbsTerminationMode(), + AbsNormTerminationMode(Base.Fix1(maximum, abs)), + AbsNormSafeTerminationMode(Base.Fix1(maximum, abs)), + AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs)) + ] + + function run_nlsolve_oop(f::F, u0, p = 2.0; solver) where {F} + return solve(NonlinearProblem{false}(f, u0, p), solver; abstol = 1e-9) + end + function run_nlsolve_iip(f!::F, u0, p = 2.0; solver) where {F} + return solve(NonlinearProblem{true}(f!, u0, p), solver; abstol = 1e-9) + end +end + +@testitem "First Order Methods" setup=[RootfindTestSnippet] tags=[:core] begin + @testset for alg in ( + SimpleNewtonRaphson, + SimpleTrustRegion, + (; kwargs...) -> SimpleTrustRegion(; kwargs..., nlsolve_update_rule = Val(true)) + ) + @testset for autodiff in ( + AutoForwardDiff(), + AutoFiniteDiff(), + AutoReverseDiff(), + AutoEnzyme(), + nothing + ) + @testset "[OOP] u0: $(typeof(u0))" for u0 in ( + [1.0, 1.0], @SVector[1.0, 1.0], 1.0) + sol = run_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff)) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9 + end + + @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],) + sol = run_nlsolve_iip(quadratic_f!, u0; solver = alg(; autodiff)) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9 + end + + @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS, + u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0]) + + probN = NonlinearProblem(quadratic_f, u0, 2.0) + @test all(solve( + probN, alg(; autodiff = AutoForwardDiff()); termination_condition).u .≈ + sqrt(2.0)) + end + end + end +end + +@testitem "Second Order Methods" setup=[RootfindTestSnippet] tags=[:core] begin + @testset for alg in ( + SimpleHalley, + ) + @testset for autodiff in ( + AutoForwardDiff(), + AutoFiniteDiff(), + AutoReverseDiff(), + nothing + ) + @testset "[OOP] u0: $(typeof(u0))" for u0 in ( + [1.0, 1.0], @SVector[1.0, 1.0], 1.0) + sol = run_nlsolve_oop(quadratic_f, u0; solver = alg(; autodiff)) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9 + end + end + + @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS, + u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0]) + + probN = NonlinearProblem(quadratic_f, u0, 2.0) + @test all(solve( + probN, alg(; autodiff = AutoForwardDiff()); termination_condition).u .≈ + sqrt(2.0)) + end + end +end + +@testitem "Derivative Free Methods" setup=[RootfindTestSnippet] tags=[:core] begin + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleBroyden(), + SimpleKlement(), + SimpleDFSane(), + SimpleLimitedMemoryBroyden(), + SimpleBroyden(; linesearch = Val(true)), + SimpleLimitedMemoryBroyden(; linesearch = Val(true)) + ) + @testset "[OOP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0], @SVector[1.0, 1.0], 1.0) + sol = run_nlsolve_oop(quadratic_f, u0; solver = alg) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9 + end + + @testset "[IIP] u0: $(typeof(u0))" for u0 in ([1.0, 1.0],) + sol = run_nlsolve_iip(quadratic_f!, u0; solver = alg) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, quadratic_f(sol.u, 2.0)) < 1e-9 + end + + @testset "Termination Condition: $(nameof(typeof(termination_condition))) u0: $(nameof(typeof(u0)))" for termination_condition in TERMINATION_CONDITIONS, + u0 in (1.0, [1.0, 1.0], @SVector[1.0, 1.0]) + + probN = NonlinearProblem(quadratic_f, u0, 2.0) + @test all(solve(probN, alg; termination_condition).u .≈ sqrt(2.0)) + end + end +end + +@testitem "Newton Fails" setup=[RootfindTestSnippet] tags=[:core] begin + u0 = [-10.0, -1.0, 1.0, 2.0, 3.0, 4.0, 10.0] + p = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] + + @testset "$(nameof(typeof(alg)))" for alg in ( + SimpleDFSane(), + SimpleTrustRegion(), + SimpleHalley(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)) + ) + sol = run_nlsolve_oop(newton_fails, u0, p; solver = alg) + @test SciMLBase.successful_retcode(sol) + @test maximum(abs, newton_fails(sol.u, p)) < 1e-9 + end +end + +@testitem "Kwargs Propagation" setup=[RootfindTestSnippet] tags=[:core] begin + prob = NonlinearProblem(quadratic_f, ones(4), 2.0; maxiters = 2) + sol = solve(prob, SimpleNewtonRaphson()) + @test sol.retcode === ReturnCode.MaxIters +end diff --git a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl index d51ce353f..09fa52304 100644 --- a/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl +++ b/lib/SimpleNonlinearSolve/test/gpu/cuda_tests.jl @@ -1,5 +1,5 @@ @testitem "Solving on CUDA" tags=[:cuda] begin - using StaticArrays, CUDA, SimpleNonlinearSolve + using StaticArrays, CUDA, SimpleNonlinearSolve, ADTypes if CUDA.functional() CUDA.allowscalar(false) @@ -47,7 +47,7 @@ end @testitem "CUDA Kernel Launch Test" tags=[:cuda] begin - using StaticArrays, CUDA, SimpleNonlinearSolve + using StaticArrays, CUDA, SimpleNonlinearSolve, ADTypes using NonlinearSolveBase: ImmutableNonlinearProblem if CUDA.functional() From a48b0c26aad8fa4da9e603337ac186ea74f6d628 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 18:05:59 -0400 Subject: [PATCH 60/92] feat: support NLLS forward AD --- .../ext/NonlinearSolveBaseForwardDiffExt.jl | 104 ++++++++++++++++-- .../src/SimpleNonlinearSolve.jl | 13 +++ lib/SimpleNonlinearSolve/src/raphson.jl | 3 +- 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index 31550da96..7f9aec5a5 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -1,15 +1,19 @@ module NonlinearSolveBaseForwardDiffExt using ADTypes: ADTypes, AutoForwardDiff, AutoPolyesterForwardDiff +using ArrayInterface: ArrayInterface using CommonSolve: solve +using DifferentiationInterface: DifferentiationInterface, Constant using FastClosures: @closure using ForwardDiff: ForwardDiff, Dual +using LinearAlgebra: mul! using SciMLBase: SciMLBase, AbstractNonlinearProblem, IntervalNonlinearProblem, - NonlinearProblem, - NonlinearLeastSquaresProblem, remake + NonlinearProblem, NonlinearLeastSquaresProblem, remake using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, Utils +const DI = DifferentiationInterface + function NonlinearSolveBase.additional_incompatible_backend_check( prob::AbstractNonlinearProblem, ::Union{AutoForwardDiff, AutoPolyesterForwardDiff}) return !ForwardDiff.can_dual(eltype(prob.u0)) @@ -50,22 +54,108 @@ function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( return sol, partials end +function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( + prob::NonlinearLeastSquaresProblem, alg, args...; kwargs...) + p = Utils.value(prob.p) + newprob = remake(prob; p, u0 = Utils.value(prob.u0)) + sol = solve(newprob, alg, args...; kwargs...) + uu = sol.u + + # First check for custom `vjp` then custom `Jacobian` and if nothing is provided use + # nested autodiff as the last resort + if SciMLBase.has_vjp(prob.f) + if SciMLBase.isinplace(prob) + vjp_fn = @closure (du, u, p) -> begin + resid = Utils.safe_similar(du, length(sol.resid)) + prob.f(resid, u, p) + prob.f.vjp(du, resid, u, p) + du .*= 2 + return nothing + end + else + vjp_fn = @closure (u, p) -> begin + resid = prob.f(u, p) + return reshape(2 .* prob.f.vjp(resid, u, p), size(u)) + end + end + elseif SciMLBase.has_jac(prob.f) + if SciMLBase.isinplace(prob) + vjp_fn = @closure (du, u, p) -> begin + J = Utils.safe_similar(du, length(sol.resid), length(u)) + prob.f.jac(J, u, p) + resid = Utils.safe_similar(du, length(sol.resid)) + prob.f(resid, u, p) + mul!(reshape(du, 1, :), vec(resid)', J, 2, false) + return nothing + end + else + vjp_fn = @closure (u, p) -> begin + return reshape(2 .* vec(prob.f(u, p))' * prob.f.jac(u, p), size(u)) + end + end + else + # For small problems, nesting ForwardDiff is actually quite fast + autodiff = length(uu) + length(sol.resid) ≥ 50 ? + NonlinearSolveBase.select_reverse_mode_autodiff(prob, nothing) : + AutoForwardDiff() + + if SciMLBase.isinplace(prob) + vjp_fn = @closure (du, u, p) -> begin + resid = Utils.safe_similar(du, length(sol.resid)) + prob.f(resid, u, p) + # Using `Constant` lead to dual ordering issues + ff = @closure (du, u) -> prob.f(du, u, p) + resid2 = copy(resid) + DI.pullback!(ff, resid2, (du,), autodiff, u, (resid,)) + @. du *= 2 + return nothing + end + else + vjp_fn = @closure (u, p) -> begin + v = prob.f(u, p) + # Using `Constant` lead to dual ordering issues + ff = Base.Fix2(prob.f, p) + res = only(DI.pullback(ff, autodiff, u, (v,))) + ArrayInterface.can_setindex(res) || return 2 .* res + @. res *= 2 + return res + end + end + end + + Jₚ = nonlinearsolve_∂f_∂p(prob, vjp_fn, uu, newprob.p) + Jᵤ = nonlinearsolve_∂f_∂u(prob, vjp_fn, uu, newprob.p) + z = -Jᵤ \ Jₚ + pp = prob.p + sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z) + + if uu isa Number + partials = sum(sumfun, zip(z, pp)) + elseif p isa Number + partials = sumfun((z, pp)) + else + partials = sum(sumfun, zip(eachcol(z), pp)) + end + + return sol, partials +end + function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} if SciMLBase.isinplace(prob) - f = @closure p -> begin + f2 = @closure p -> begin du = Utils.safe_similar(u, promote_type(eltype(u), eltype(p))) f(du, u, p) return du end else - f = Base.Fix1(f, u) + f2 = Base.Fix1(f, u) end if p isa Number - return Utils.safe_reshape(ForwardDiff.derivative(f, p), :, 1) + return Utils.safe_reshape(ForwardDiff.derivative(f2, p), :, 1) elseif u isa Number - return Utils.safe_reshape(ForwardDiff.gradient(f, p), 1, :) + return Utils.safe_reshape(ForwardDiff.gradient(f2, p), 1, :) else - return ForwardDiff.jacobian(f, p) + return ForwardDiff.jacobian(f2, p) end end diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 081201a24..e83de5e39 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -61,6 +61,19 @@ function CommonSolve.solve( prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) end +function CommonSolve.solve( + prob::NonlinearLeastSquaresProblem{<:Union{Number, <:AbstractArray}, iip, + <:Union{ + <:ForwardDiff.Dual{T, V, P}, <:AbstractArray{<:ForwardDiff.Dual{T, V, P}}}}, + alg::AbstractSimpleNonlinearSolveAlgorithm, + args...; + kwargs...) where {T, V, P, iip} + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) + return SciMLBase.build_solution( + prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) +end + function CommonSolve.solve( prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...) diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl index 763fcf32a..a18a1b6be 100644 --- a/lib/SimpleNonlinearSolve/src/raphson.jl +++ b/lib/SimpleNonlinearSolve/src/raphson.jl @@ -24,7 +24,8 @@ end const SimpleGaussNewton = SimpleNewtonRaphson function SciMLBase.__solve( - prob::ImmutableNonlinearProblem, alg::SimpleNewtonRaphson, args...; + prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem}, + alg::SimpleNewtonRaphson, args...; abstol = nothing, reltol = nothing, maxiters = 1000, alias_u0 = false, termination_condition = nothing, kwargs...) x = Utils.maybe_unaliased(prob.u0, alias_u0) From 186dc8656bb9220823d9b35425eea772b1dff1e3 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 18:07:41 -0400 Subject: [PATCH 61/92] ci: other internal deps for NonlinearSolve --- .buildkite/pipeline.yml | 4 ++-- .github/workflows/CI_NonlinearSolve.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f6b4ed6eb..9eb146557 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,8 +14,8 @@ steps: Pkg.Registry.update(); # Install packages present in subdirectories dev_pks = Pkg.PackageSpec[]; - for path in ("lib/SciMLJacobianOperators",) - push!(dev_pks, Pkg.PackageSpec(; path)); + for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve") + push!(dev_pks, Pkg.PackageSpec(; path)); end Pkg.develop(dev_pks); Pkg.instantiate(); diff --git a/.github/workflows/CI_NonlinearSolve.yml b/.github/workflows/CI_NonlinearSolve.yml index d684cb3b7..843a04b54 100644 --- a/.github/workflows/CI_NonlinearSolve.yml +++ b/.github/workflows/CI_NonlinearSolve.yml @@ -63,7 +63,7 @@ jobs: Pkg.Registry.update() # Install packages present in subdirectories dev_pks = Pkg.PackageSpec[] - for path in ("lib/SciMLJacobianOperators",) + for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve") push!(dev_pks, Pkg.PackageSpec(; path)) end Pkg.develop(dev_pks) From f66818e667b0d879648e8fe1cbb65fa1905d424a Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 19:03:03 -0400 Subject: [PATCH 62/92] test: NLLS forwarddiff rules testing --- .../src/SimpleNonlinearSolve.jl | 6 +- lib/SimpleNonlinearSolve/src/raphson.jl | 2 +- lib/SimpleNonlinearSolve/src/trust_region.jl | 2 +- lib/SimpleNonlinearSolve/src/utils.jl | 4 +- .../test/core/forward_diff_tests.jl | 114 ++++++++++++++++++ 5 files changed, 122 insertions(+), 6 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index e83de5e39..b4e641832 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -75,7 +75,8 @@ function CommonSolve.solve( end function CommonSolve.solve( - prob::ImmutableNonlinearProblem, alg::AbstractSimpleNonlinearSolveAlgorithm, + prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem}, + alg::AbstractSimpleNonlinearSolveAlgorithm, args...; sensealg = nothing, u0 = nothing, p = nothing, kwargs...) if sensealg === nothing && haskey(prob.kwargs, :sensealg) sensealg = prob.kwargs[:sensealg] @@ -86,7 +87,8 @@ function CommonSolve.solve( p === nothing, alg, args...; prob.kwargs..., kwargs...) end -function simplenonlinearsolve_solve_up(prob::ImmutableNonlinearProblem, sensealg, u0, +function simplenonlinearsolve_solve_up( + prob::Union{ImmutableNonlinearProblem, NonlinearLeastSquaresProblem}, sensealg, u0, u0_changed, p, p_changed, alg, args...; kwargs...) (u0_changed || p_changed) && (prob = remake(prob; u0, p)) return SciMLBase.__solve(prob, alg, args...; kwargs...) diff --git a/lib/SimpleNonlinearSolve/src/raphson.jl b/lib/SimpleNonlinearSolve/src/raphson.jl index a18a1b6be..ebbb5f9f9 100644 --- a/lib/SimpleNonlinearSolve/src/raphson.jl +++ b/lib/SimpleNonlinearSolve/src/raphson.jl @@ -43,7 +43,7 @@ function SciMLBase.__solve( @bb xo = similar(x) fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? - safe_similar(fx) : nothing + safe_similar(fx) : fx jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) diff --git a/lib/SimpleNonlinearSolve/src/trust_region.jl b/lib/SimpleNonlinearSolve/src/trust_region.jl index 47acc5437..32e7a6219 100644 --- a/lib/SimpleNonlinearSolve/src/trust_region.jl +++ b/lib/SimpleNonlinearSolve/src/trust_region.jl @@ -94,7 +94,7 @@ function SciMLBase.__solve(prob::ImmutableNonlinearProblem, alg::SimpleTrustRegi @bb xo = copy(x) fx_cache = (SciMLBase.isinplace(prob) && !SciMLBase.has_jac(prob.f)) ? - safe_similar(fx) : nothing + safe_similar(fx) : fx jac_cache = Utils.prepare_jacobian(prob, autodiff, fx_cache, x) J = Utils.compute_jacobian!!(nothing, prob, autodiff, fx_cache, x, jac_cache) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 946c10529..fbc3d3c23 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -183,10 +183,10 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras) end if extras isa AnalyticJacobian if SciMLBase.isinplace(prob) - prob.jac(J, x, prob.p) + prob.f.jac(J, x, prob.p) return J else - return prob.jac(x, prob.p) + return prob.f.jac(x, prob.p) end end if SciMLBase.isinplace(prob) diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl index 8b1378917..0005796f9 100644 --- a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl @@ -1 +1,115 @@ +@testitem "ForwardDiff.jl Integration NonlinearLeastSquaresProblem" tags=[:core] begin + using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra, + Zygote, ReverseDiff + using DifferentiationInterface + const DI = DifferentiationInterface + + true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4]) + + θ_true = [1.0, 0.1, 2.0, 0.5] + x = [-1.0, -0.5, 0.0, 0.5, 1.0] + y_target = true_function(x, θ_true) + + loss_function(θ, p) = true_function(p, θ) .- y_target + + loss_function_jac(θ, p) = ForwardDiff.jacobian(Base.Fix2(loss_function, p), θ) + + loss_function_vjp(v, θ, p) = reshape(vec(v)' * loss_function_jac(θ, p), size(θ)) + + function loss_function!(resid, θ, p) + ŷ = true_function(p, θ) + @. resid = ŷ - y_target + return + end + + function loss_function_jac!(J, θ, p) + J .= ForwardDiff.jacobian(θ -> loss_function(θ, p), θ) + return + end + + function loss_function_vjp!(vJ, v, θ, p) + vec(vJ) .= reshape(vec(v)' * loss_function_jac(θ, p), size(θ)) + return + end + + θ_init = θ_true .+ 0.1 + + @testset for alg in ( + SimpleGaussNewton(), + SimpleGaussNewton(; autodiff = AutoForwardDiff()), + SimpleGaussNewton(; autodiff = AutoFiniteDiff()), + SimpleGaussNewton(; autodiff = AutoReverseDiff()) + ) + function obj_1(p) + prob_oop = NonlinearLeastSquaresProblem{false}(loss_function, θ_init, p) + sol = solve(prob_oop, alg) + return sum(abs2, sol.u) + end + + function obj_2(p) + ff = NonlinearFunction{false}( + loss_function; resid_prototype = zeros(length(y_target))) + prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p) + sol = solve(prob_oop, alg) + return sum(abs2, sol.u) + end + + function obj_3(p) + ff = NonlinearFunction{false}(loss_function; vjp = loss_function_vjp) + prob_oop = NonlinearLeastSquaresProblem{false}(ff, θ_init, p) + sol = solve(prob_oop, alg) + return sum(abs2, sol.u) + end + + finitediff = DI.gradient(obj_1, AutoFiniteDiff(), x) + + fdiff1 = DI.gradient(obj_1, AutoForwardDiff(), x) + fdiff2 = DI.gradient(obj_2, AutoForwardDiff(), x) + fdiff3 = DI.gradient(obj_3, AutoForwardDiff(), x) + + @test finitediff≈fdiff1 atol=1e-5 + @test finitediff≈fdiff2 atol=1e-5 + @test finitediff≈fdiff3 atol=1e-5 + @test fdiff1 ≈ fdiff2 ≈ fdiff3 + + function obj_4(p) + prob_iip = NonlinearLeastSquaresProblem( + NonlinearFunction{true}( + loss_function!; resid_prototype = zeros(length(y_target))), + θ_init, + p) + sol = solve(prob_iip, alg) + return sum(abs2, sol.u) + end + + function obj_5(p) + ff = NonlinearFunction{true}( + loss_function!; resid_prototype = zeros(length(y_target)), + jac = loss_function_jac!) + prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p) + sol = solve(prob_iip, alg) + return sum(abs2, sol.u) + end + + function obj_6(p) + ff = NonlinearFunction{true}( + loss_function!; resid_prototype = zeros(length(y_target)), + vjp = loss_function_vjp!) + prob_iip = NonlinearLeastSquaresProblem(ff, θ_init, p) + sol = solve(prob_iip, alg) + return sum(abs2, sol.u) + end + + finitediff = DI.gradient(obj_4, AutoFiniteDiff(), x) + + fdiff4 = DI.gradient(obj_4, AutoForwardDiff(), x) + fdiff5 = DI.gradient(obj_5, AutoForwardDiff(), x) + fdiff6 = DI.gradient(obj_6, AutoForwardDiff(), x) + + @test finitediff≈fdiff4 atol=1e-5 + @test finitediff≈fdiff5 atol=1e-5 + @test finitediff≈fdiff6 atol=1e-5 + @test fdiff4 ≈ fdiff5 ≈ fdiff6 + end +end From e96cb9ac7fcde90c422c7a51c76f76bde12bd52e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 19:49:53 -0400 Subject: [PATCH 63/92] test: NonlinearProblem forward diff testing --- .../test/core/forward_diff_tests.jl | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl index 0005796f9..2b392e39c 100644 --- a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl @@ -1,3 +1,86 @@ +@testitem "ForwardDiff.jl Integration: NonlinearProblem" tags=[:core] begin + using ArrayInterface + using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra, + Zygote, ReverseDiff, SciMLBase + using DifferentiationInterface + + const DI = DifferentiationInterface + + test_f!(du, u, p) = (@. du = u^2 - p) + test_f(u, p) = u .^ 2 .- p + + jacobian_f(::Number, p) = 1 / (2 * √p) + jacobian_f(::Number, p::Number) = 1 / (2 * √p) + jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p)) + jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p))) + + @testset for alg in ( + SimpleNewtonRaphson(), + SimpleTrustRegion(), + SimpleTrustRegion(; nlsolve_update_rule = Val(true)), + SimpleHalley(), + SimpleBroyden(), + SimpleKlement(), + SimpleDFSane() + ) + us = ( + 2.0, + @SVector([1.0, 1.0]), + [1.0, 1.0], + ones(2, 2), + @SArray(ones(2, 2)) + ) + + @testset "Scalar AD" begin + for p in 1.0:0.1:100.0, u0 in us + sol = solve(NonlinearProblem{false}(test_f, u0, p), alg) + if SciMLBase.successful_retcode(sol) + gs = abs.(ForwardDiff.derivative(p) do pᵢ + solve(NonlinearProblem{false}(test_f, u0, pᵢ), alg).u + end) + gs_true = abs.(jacobian_f(u0, p)) + + if !(isapprox(gs, gs_true, atol = 1e-5)) + @show sol.retcode, sol.u + @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_gradient=gs true_gradient=gs_true + else + @test abs.(gs)≈abs.(gs_true) atol=1e-5 + end + end + end + end + + @testset "Jacobian" begin + @testset "$(typeof(u0))" for u0 in us[2:end], + p in ([2.0, 1.0], [2.0 1.0; 3.0 4.0]) + + if u0 isa AbstractArray && p isa AbstractArray + size(u0) != size(p) && continue + end + + @testset for (iip, fn) in ((false, test_f), (true, test_f!)) + iip && (u0 isa Number || !ArrayInterface.can_setindex(u0)) && continue + + sol = solve(NonlinearProblem{iip}(fn, u0, p), alg) + if SciMLBase.successful_retcode(sol) + gs = abs.(ForwardDiff.jacobian(p) do pᵢ + solve(NonlinearProblem{iip}(fn, u0, pᵢ), alg).u + end) + gs_true = abs.(jacobian_f(u0, p)) + + if !(isapprox(gs, gs_true, atol = 1e-5)) + @show sol.retcode, sol.u + @error "ForwardDiff Failed for u0=$(u0) and p=$(p) with $(alg)" forwardiff_jacobian=gs true_jacobian=gs_true + else + @test abs.(gs)≈abs.(gs_true) atol=1e-5 + end + end + end + end + end + end +end + @testitem "ForwardDiff.jl Integration NonlinearLeastSquaresProblem" tags=[:core] begin using ForwardDiff, FiniteDiff, SimpleNonlinearSolve, StaticArrays, LinearAlgebra, Zygote, ReverseDiff From 36227c9d61aad92e3b202ec42659c85500b66b57 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 19:51:55 -0400 Subject: [PATCH 64/92] test: lazy install packages --- lib/SimpleNonlinearSolve/Project.toml | 6 +----- lib/SimpleNonlinearSolve/test/runtests.jl | 3 +++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 5f708e564..27dd69fed 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -41,7 +41,6 @@ AllocCheck = "0.1.1" Aqua = "0.8.7" ArrayInterface = "7.16" BracketingNonlinearSolve = "1" -CUDA = "5.3" ChainRulesCore = "1.24" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" @@ -64,7 +63,6 @@ PrecompileTools = "1.2" Random = "1.10" ReverseDiff = "1.15" SciMLBase = "2.50" -SciMLSensitivity = "7.68" StaticArrays = "1.9" StaticArraysCore = "1.4.3" Test = "1.10" @@ -76,7 +74,6 @@ julia = "1.10" [extras] AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" @@ -86,7 +83,6 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" -SciMLSensitivity = "1ed8b502-d754-442c-8d5d-10ac956f44a1" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" @@ -94,4 +90,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["AllocCheck", "Aqua", "CUDA", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "SciMLSensitivity", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] +test = ["AllocCheck", "Aqua", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index a22783e59..cab77902c 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -4,6 +4,9 @@ using TestItemRunner, InteractiveUtils, Pkg const GROUP = lowercase(get(ENV, "GROUP", "All")) +(GROUP == "all" || GROUP == "cuda") && Pkg.add(["CUDA"]) +(GROUP == "all" || GROUP == "adjoint") && Pkg.add(["SciMLSensitivity"]) + if GROUP == "all" @run_package_tests else From 15c342e50017067db5d2d04a43f4490f36ea0432 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 20:10:58 -0400 Subject: [PATCH 65/92] fix: auto-set autodiff for ForwardDiff if trying to propagate Duals --- lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 6 ++++++ lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl | 4 ++-- lib/SimpleNonlinearSolve/test/core/qa_tests.jl | 3 +-- lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl | 2 +- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index b4e641832..6d1187668 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -54,6 +54,9 @@ function CommonSolve.solve( alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip} + if hasfield(typeof(alg), :autodiff) && alg.autodiff === nothing + @reset alg.autodiff = AutoForwardDiff() + end prob = convert(ImmutableNonlinearProblem, prob) sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) @@ -68,6 +71,9 @@ function CommonSolve.solve( alg::AbstractSimpleNonlinearSolveAlgorithm, args...; kwargs...) where {T, V, P, iip} + if hasfield(typeof(alg), :autodiff) && alg.autodiff === nothing + @reset alg.autodiff = AutoForwardDiff() + end sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) return SciMLBase.build_solution( diff --git a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl index 2b392e39c..a857e7a17 100644 --- a/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/forward_diff_tests.jl @@ -14,7 +14,7 @@ jacobian_f(u, p::Number) = one.(u) .* (1 / (2 * √p)) jacobian_f(u, p::AbstractArray) = diagm(vec(@. 1 / (2 * √p))) - @testset for alg in ( + @testset "#(nameof(typeof(alg)))" for alg in ( SimpleNewtonRaphson(), SimpleTrustRegion(), SimpleTrustRegion(; nlsolve_update_rule = Val(true)), @@ -118,7 +118,7 @@ end θ_init = θ_true .+ 0.1 - @testset for alg in ( + for alg in ( SimpleGaussNewton(), SimpleGaussNewton(; autodiff = AutoForwardDiff()), SimpleGaussNewton(; autodiff = AutoFiniteDiff()), diff --git a/lib/SimpleNonlinearSolve/test/core/qa_tests.jl b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl index d6a5a9b8e..cef74ac38 100644 --- a/lib/SimpleNonlinearSolve/test/core/qa_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/qa_tests.jl @@ -12,8 +12,7 @@ end import ReverseDiff, Tracker, StaticArrays, Zygote using ExplicitImports, SimpleNonlinearSolve - @test check_no_implicit_imports( - SimpleNonlinearSolve; skip = (Base, Core, SciMLBase)) === nothing + @test check_no_implicit_imports(SimpleNonlinearSolve; skip = (Base, Core)) === nothing @test check_no_stale_explicit_imports(SimpleNonlinearSolve) === nothing @test check_all_qualified_accesses_via_owners(SimpleNonlinearSolve) === nothing end diff --git a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl index c99b670cb..b6a39a9e9 100644 --- a/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/rootfind_tests.jl @@ -36,7 +36,7 @@ end @testitem "First Order Methods" setup=[RootfindTestSnippet] tags=[:core] begin - @testset for alg in ( + for alg in ( SimpleNewtonRaphson, SimpleTrustRegion, (; kwargs...) -> SimpleTrustRegion(; kwargs..., nlsolve_update_rule = Val(true)) From 6a1737d54af39be0f79156859b748a6cc4115be9 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 20:26:47 -0400 Subject: [PATCH 66/92] fix: QA for NonlinearSolveBase --- lib/NonlinearSolveBase/Project.toml | 16 +++++++++++++ .../ext/NonlinearSolveBaseForwardDiffExt.jl | 2 +- .../src/NonlinearSolveBase.jl | 3 ++- .../src/termination_conditions.jl | 10 ++++---- lib/NonlinearSolveBase/test/runtests.jl | 23 +++++++++++++++++++ 5 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index a89d035cf..162ce2742 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -31,6 +31,7 @@ NonlinearSolveBaseSparseArraysExt = "SparseArrays" [compat] ADTypes = "1.9" +Aqua = "0.8.7" ArrayInterface = "7.9" CommonSolve = "0.2.4" Compat = "4.15" @@ -38,13 +39,28 @@ ConcreteStructs = "0.2.3" DiffEqBase = "6.149" DifferentiationInterface = "0.6.1" EnzymeCore = "0.8" +ExplicitImports = "1.10.1" FastClosures = "0.3" ForwardDiff = "0.10.36" FunctionProperties = "0.1.2" +InteractiveUtils = "<0.0.1, 1" LinearAlgebra = "1.10" Markdown = "1.10" RecursiveArrayTools = "3" SciMLBase = "2.50" SparseArrays = "1.10" StaticArraysCore = "1.4" +Test = "1.10" julia = "1.10" + +[extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" +ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Aqua", "DiffEqBase", "ExplicitImports", "ForwardDiff", "InteractiveUtils", "SparseArrays", "Test"] diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index 7f9aec5a5..e50be6c47 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -3,7 +3,7 @@ module NonlinearSolveBaseForwardDiffExt using ADTypes: ADTypes, AutoForwardDiff, AutoPolyesterForwardDiff using ArrayInterface: ArrayInterface using CommonSolve: solve -using DifferentiationInterface: DifferentiationInterface, Constant +using DifferentiationInterface: DifferentiationInterface using FastClosures: @closure using ForwardDiff: ForwardDiff, Dual using LinearAlgebra: mul! diff --git a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl index 645b84959..b07b4b168 100644 --- a/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl +++ b/lib/NonlinearSolveBase/src/NonlinearSolveBase.jl @@ -1,7 +1,8 @@ module NonlinearSolveBase -using ADTypes: ADTypes, AbstractADType, ForwardMode, ReverseMode +using ADTypes: ADTypes, AbstractADType using ArrayInterface: ArrayInterface +using CommonSolve: CommonSolve using Compat: @compat using ConcreteStructs: @concrete using DifferentiationInterface: DifferentiationInterface diff --git a/lib/NonlinearSolveBase/src/termination_conditions.jl b/lib/NonlinearSolveBase/src/termination_conditions.jl index 3e957f139..9f20e46bc 100644 --- a/lib/NonlinearSolveBase/src/termination_conditions.jl +++ b/lib/NonlinearSolveBase/src/termination_conditions.jl @@ -30,9 +30,9 @@ function update_u!!(cache::NonlinearTerminationModeCache, u) end end -function SciMLBase.init( - du, u, mode::AbstractNonlinearTerminationMode, saved_value_prototype...; - abstol = nothing, reltol = nothing, kwargs...) +function CommonSolve.init( + ::AbstractNonlinearProblem, mode::AbstractNonlinearTerminationMode, du, u, + saved_value_prototype...; abstol = nothing, reltol = nothing, kwargs...) T = promote_type(eltype(du), eltype(u)) abstol = get_tolerance(u, abstol, T) reltol = get_tolerance(u, reltol, T) @@ -273,11 +273,11 @@ function init_termination_cache( prob, abstol, reltol, du, u, default_termination_mode(prob, callee), callee) end -function init_termination_cache(::AbstractNonlinearProblem, abstol, reltol, du, +function init_termination_cache(prob::AbstractNonlinearProblem, abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode, ::Val) T = promote_type(eltype(du), eltype(u)) abstol = get_tolerance(u, abstol, T) reltol = get_tolerance(u, reltol, T) - cache = SciMLBase.init(du, u, tc; abstol, reltol) + cache = CommonSolve.init(prob, tc, du, u; abstol, reltol) return abstol, reltol, cache end diff --git a/lib/NonlinearSolveBase/test/runtests.jl b/lib/NonlinearSolveBase/test/runtests.jl index 8b1378917..07c0f14c6 100644 --- a/lib/NonlinearSolveBase/test/runtests.jl +++ b/lib/NonlinearSolveBase/test/runtests.jl @@ -1 +1,24 @@ +using InteractiveUtils, Test +@info sprint(InteractiveUtils.versioninfo) + +# Changing any code here triggers all the other tests to be run. So we intentionally +# keep the tests here minimal. +@testset "NonlinearSolveBase.jl" begin + @testset "Aqua" begin + using Aqua, NonlinearSolveBase + + Aqua.test_all(NonlinearSolveBase; piracies = false, ambiguities = false) + Aqua.test_piracies(NonlinearSolveBase) + Aqua.test_ambiguities(NonlinearSolveBase; recursive = false) + end + + @testset "Explicit Imports" begin + import ForwardDiff, SparseArrays, DiffEqBase + using ExplicitImports, NonlinearSolveBase + + @test check_no_implicit_imports(NonlinearSolveBase; skip = (Base, Core)) === nothing + @test check_no_stale_explicit_imports(NonlinearSolveBase) === nothing + @test check_all_qualified_accesses_via_owners(NonlinearSolveBase) === nothing + end +end From 9a4a2758b7a2ecddaf3775cf4b513f8a186f5831 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 7 Oct 2024 20:27:50 -0400 Subject: [PATCH 67/92] test: wrap in a global testset --- lib/BracketingNonlinearSolve/test/runtests.jl | 6 ++++-- lib/SimpleNonlinearSolve/test/runtests.jl | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/BracketingNonlinearSolve/test/runtests.jl b/lib/BracketingNonlinearSolve/test/runtests.jl index 6ea6326b0..d2c6f80a4 100644 --- a/lib/BracketingNonlinearSolve/test/runtests.jl +++ b/lib/BracketingNonlinearSolve/test/runtests.jl @@ -1,5 +1,7 @@ -using TestItemRunner, InteractiveUtils +using TestItemRunner, InteractiveUtils, Test @info sprint(InteractiveUtils.versioninfo) -@run_package_tests +@testset "BracketingNonlinearSolve.jl" begin + @run_package_tests +end diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index cab77902c..c3f02030d 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -1,4 +1,4 @@ -using TestItemRunner, InteractiveUtils, Pkg +using TestItemRunner, InteractiveUtils, Pkg, Test @info sprint(InteractiveUtils.versioninfo) @@ -7,8 +7,10 @@ const GROUP = lowercase(get(ENV, "GROUP", "All")) (GROUP == "all" || GROUP == "cuda") && Pkg.add(["CUDA"]) (GROUP == "all" || GROUP == "adjoint") && Pkg.add(["SciMLSensitivity"]) -if GROUP == "all" - @run_package_tests -else - @run_package_tests filter = ti -> (Symbol(GROUP) in ti.tags) +@testset "SimpleNonlinearSolve.jl" begin + if GROUP == "all" + @run_package_tests + else + @run_package_tests filter = ti -> (Symbol(GROUP) in ti.tags) + end end From b71192f606bc3c9f7ef94f8069482c24e698e43e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 8 Oct 2024 12:02:21 -0400 Subject: [PATCH 68/92] fix: write out the AD as dispatches --- lib/SimpleNonlinearSolve/src/utils.jl | 93 +++++++++++++-------------- 1 file changed, 45 insertions(+), 48 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index fbc3d3c23..27988026e 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -143,72 +143,69 @@ function prepare_jacobian(prob, autodiff, fx, x) end end -function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras) - if extras isa AnalyticJacobian - if SciMLBase.has_jac(prob.f) +function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::AnalyticJacobian) + if SciMLBase.has_jac(prob.f) + return prob.f.jac(x, prob.p) + elseif SciMLBase.has_vjp(prob.f) + return prob.f.vjp(one(x), x, prob.p) + elseif SciMLBase.has_jvp(prob.f) + return prob.f.jvp(one(x), x, prob.p) + end +end +function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::DIExtras) + return DI.derivative(prob.f, extras.prep, autodiff, x, Constant(prob.p)) +end +function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::DINoPreparation) + return DI.derivative(prob.f, autodiff, x, Constant(prob.p)) +end + +function compute_jacobian!!(J, prob, autodiff, fx, x, ::AnalyticJacobian) + if J === nothing + if SciMLBase.isinplace(prob.f) + J = safe_similar(fx, length(fx), length(x)) + prob.f.jac(J, x, prob.p) + return J + else return prob.f.jac(x, prob.p) - elseif SciMLBase.has_vjp(prob.f) - return prob.f.vjp(one(x), x, prob.p) - elseif SciMLBase.has_jvp(prob.f) - return prob.f.jvp(one(x), x, prob.p) end end - if extras isa DIExtras - return DI.derivative(prob.f, extras.prep, autodiff, x, Constant(prob.p)) + if SciMLBase.isinplace(prob.f) + prob.f.jac(J, x, prob.p) + return J else - return DI.derivative(prob.f, autodiff, x, Constant(prob.p)) + return prob.f.jac(x, prob.p) end end -function compute_jacobian!!(J, prob, autodiff, fx, x, extras) + +function compute_jacobian!!(J, prob, autodiff, fx, x, ::DIExtras) if J === nothing - if extras isa AnalyticJacobian - if SciMLBase.isinplace(prob.f) - J = safe_similar(fx, length(fx), length(x)) - prob.f.jac(J, x, prob.p) - return J - else - return prob.f.jac(x, prob.p) - end - end - if SciMLBase.isinplace(prob) - @assert extras isa DIExtras + if SciMLBase.isinplace(prob.f) return DI.jacobian(prob.f, fx, extras.prep, autodiff, x, Constant(prob.p)) else - if extras isa DIExtras - return DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) - else - return DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) - end + return DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) end end - if extras isa AnalyticJacobian - if SciMLBase.isinplace(prob) - prob.f.jac(J, x, prob.p) - return J - else - return prob.f.jac(x, prob.p) - end - end - if SciMLBase.isinplace(prob) - @assert extras isa DIExtras - DI.jacobian!(prob.f, fx, J, extras.prep, autodiff, x, Constant(prob.p)) + if SciMLBase.isinplace(prob.f) + DI.jacobian!(prob.f, J, fx, extras.prep, autodiff, x, Constant(prob.p)) else if ArrayInterface.can_setindex(J) - if extras isa DIExtras - DI.jacobian!(prob.f, J, extras.prep, autodiff, x, Constant(prob.p)) - else - DI.jacobian!(prob.f, J, autodiff, x, Constant(prob.p)) - end + DI.jacobian!(prob.f, J, extras.prep, autodiff, x, Constant(prob.p)) else - if extras isa DIExtras - J = DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) - else - J = DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) - end + J = DI.jacobian(prob.f, extras.prep, autodiff, x, Constant(prob.p)) end end return J end +function compute_jacobian!!(J, prob, autodiff, fx, x, ::DINoPreparation) + @assert !SciMLBase.isinplace(prob.f) "This shouldn't happen. Open an issue." + J === nothing && return DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) + if ArrayInterface.can_setindex(J) + DI.jacobian!(prob.f, J, autodiff, x, Constant(prob.p)) + else + J = DI.jacobian(prob.f, autodiff, x, Constant(prob.p)) + end + return J +end function compute_jacobian_and_hessian(autodiff, prob, _, x::Number) H = DI.second_derivative(prob.f, autodiff, x, Constant(prob.p)) From 4842fc6f339cce7790b4a019c36bfdea3a8fae59 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 8 Oct 2024 12:04:12 -0400 Subject: [PATCH 69/92] test: install AllocCheck if needed --- .github/workflows/CI_SimpleNonlinearSolve.yml | 1 + lib/SimpleNonlinearSolve/Project.toml | 4 +--- lib/SimpleNonlinearSolve/test/core/allocation_tests.jl | 2 +- lib/SimpleNonlinearSolve/test/runtests.jl | 1 + 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI_SimpleNonlinearSolve.yml b/.github/workflows/CI_SimpleNonlinearSolve.yml index 1f8306a8c..11c3ef7c2 100644 --- a/.github/workflows/CI_SimpleNonlinearSolve.yml +++ b/.github/workflows/CI_SimpleNonlinearSolve.yml @@ -35,6 +35,7 @@ jobs: group: - core - adjoint + - alloc_check steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 27dd69fed..82f69c6ad 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -37,7 +37,6 @@ SimpleNonlinearSolveTrackerExt = "Tracker" [compat] ADTypes = "1.2" Accessors = "0.1" -AllocCheck = "0.1.1" Aqua = "0.8.7" ArrayInterface = "7.16" BracketingNonlinearSolve = "1" @@ -72,7 +71,6 @@ Zygote = "0.6.70" julia = "1.10" [extras] -AllocCheck = "9b6a8646-10ed-4001-bbdc-1d2f46dfbb1a" Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" @@ -90,4 +88,4 @@ Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["AllocCheck", "Aqua", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] +test = ["Aqua", "DiffEqBase", "Enzyme", "ExplicitImports", "InteractiveUtils", "NonlinearProblemLibrary", "Pkg", "PolyesterForwardDiff", "Random", "ReverseDiff", "StaticArrays", "Test", "TestItemRunner", "Tracker", "Zygote"] diff --git a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl index 67cee39c0..1f8472cf3 100644 --- a/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl +++ b/lib/SimpleNonlinearSolve/test/core/allocation_tests.jl @@ -1,4 +1,4 @@ -@itesitem "Allocation Tests" tags=[:core] begin +@itesitem "Allocation Tests" tags=[:alloc_check] begin using SimpleNonlinearSolve, StaticArrays, AllocCheck quadratic_f(u, p) = u .* u .- p diff --git a/lib/SimpleNonlinearSolve/test/runtests.jl b/lib/SimpleNonlinearSolve/test/runtests.jl index c3f02030d..a76760dc8 100644 --- a/lib/SimpleNonlinearSolve/test/runtests.jl +++ b/lib/SimpleNonlinearSolve/test/runtests.jl @@ -6,6 +6,7 @@ const GROUP = lowercase(get(ENV, "GROUP", "All")) (GROUP == "all" || GROUP == "cuda") && Pkg.add(["CUDA"]) (GROUP == "all" || GROUP == "adjoint") && Pkg.add(["SciMLSensitivity"]) +(GROUP == "all" || GROUP == "alloc_check") && Pkg.add(["AllocCheck"]) @testset "SimpleNonlinearSolve.jl" begin if GROUP == "all" From 06e9c03da1ee16949811d3f028f5210a03ef0419 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 8 Oct 2024 14:12:14 -0400 Subject: [PATCH 70/92] fix: missing extras --- lib/SimpleNonlinearSolve/src/utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 27988026e..9a1c36508 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -152,7 +152,7 @@ function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::AnalyticJacobian return prob.f.jvp(one(x), x, prob.p) end end -function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::DIExtras) +function compute_jacobian!!(_, prob, autodiff, fx, x::Number, extras::DIExtras) return DI.derivative(prob.f, extras.prep, autodiff, x, Constant(prob.p)) end function compute_jacobian!!(_, prob, autodiff, fx, x::Number, ::DINoPreparation) @@ -177,7 +177,7 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, ::AnalyticJacobian) end end -function compute_jacobian!!(J, prob, autodiff, fx, x, ::DIExtras) +function compute_jacobian!!(J, prob, autodiff, fx, x, extras::DIExtras) if J === nothing if SciMLBase.isinplace(prob.f) return DI.jacobian(prob.f, fx, extras.prep, autodiff, x, Constant(prob.p)) From 6c11fb6f871f678f1b24a5dc5b61284e3b83da17 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 20 Oct 2024 17:14:23 -0400 Subject: [PATCH 71/92] test(BracketingNonlinearSolve): qq using Aqua and ExplicitImports --- lib/BracketingNonlinearSolve/Project.toml | 9 ++++++++- lib/BracketingNonlinearSolve/test/qa_tests.jl | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 lib/BracketingNonlinearSolve/test/qa_tests.jl diff --git a/lib/BracketingNonlinearSolve/Project.toml b/lib/BracketingNonlinearSolve/Project.toml index dc979f660..6fc241d7d 100644 --- a/lib/BracketingNonlinearSolve/Project.toml +++ b/lib/BracketingNonlinearSolve/Project.toml @@ -17,19 +17,26 @@ ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" BracketingNonlinearSolveForwardDiffExt = "ForwardDiff" [compat] +Aqua = "0.8.9" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" +ExplicitImports = "1.10.1" ForwardDiff = "0.10.36" +InteractiveUtils = "<0.0.1, 1" NonlinearSolveBase = "1" PrecompileTools = "1.2" SciMLBase = "2.50" +Test = "1.10" +TestItemRunner = "1" julia = "1.10" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" +ExplicitImports = "7d51a73a-1435-4ff3-83d9-f097790105c7" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a" [targets] -test = ["InteractiveUtils", "ForwardDiff", "Test", "TestItemRunner"] +test = ["Aqua", "ExplicitImports", "ForwardDiff", "InteractiveUtils", "Test", "TestItemRunner"] diff --git a/lib/BracketingNonlinearSolve/test/qa_tests.jl b/lib/BracketingNonlinearSolve/test/qa_tests.jl new file mode 100644 index 000000000..de27e33a4 --- /dev/null +++ b/lib/BracketingNonlinearSolve/test/qa_tests.jl @@ -0,0 +1,16 @@ +@testitem "Aqua" tags=[:core] begin + using Aqua, BracketingNonlinearSolve + + Aqua.test_all(BracketingNonlinearSolve; piracies = false, ambiguities = false) + Aqua.test_piracies(BracketingNonlinearSolve; treat_as_own = [IntervalNonlinearProblem]) + Aqua.test_ambiguities(BracketingNonlinearSolve; recursive = false) +end + +@testitem "Explicit Imports" tags=[:core] begin + import ForwardDiff + using ExplicitImports, BracketingNonlinearSolve + + @test check_no_implicit_imports(BracketingNonlinearSolve; skip = (Base, Core)) === nothing + @test check_no_stale_explicit_imports(BracketingNonlinearSolve) === nothing + @test check_all_qualified_accesses_via_owners(BracketingNonlinearSolve) === nothing +end From 5b63767ee5d11db0751636994bcdb741bbc8e741 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Sun, 20 Oct 2024 17:45:00 -0400 Subject: [PATCH 72/92] fix(SimpleNonlinearSolve): incorrect argument ordering --- lib/SimpleNonlinearSolve/src/utils.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 9a1c36508..311762429 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -139,7 +139,7 @@ function prepare_jacobian(prob, autodiff, fx, x) return DIExtras(DI.prepare_jacobian(prob.f, fx, autodiff, x, Constant(prob.p))) else x isa SArray && return DINoPreparation() - return DI.prepare_jacobian(prob.f, autodiff, x, Constant(prob.p)) + return DIExtras(DI.prepare_jacobian(prob.f, autodiff, x, Constant(prob.p))) end end @@ -186,7 +186,7 @@ function compute_jacobian!!(J, prob, autodiff, fx, x, extras::DIExtras) end end if SciMLBase.isinplace(prob.f) - DI.jacobian!(prob.f, J, fx, extras.prep, autodiff, x, Constant(prob.p)) + DI.jacobian!(prob.f, fx, J, extras.prep, autodiff, x, Constant(prob.p)) else if ArrayInterface.can_setindex(J) DI.jacobian!(prob.f, J, extras.prep, autodiff, x, Constant(prob.p)) From 3620e6c243155110a437635d6720a015f682aa35 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 12:03:33 -0400 Subject: [PATCH 73/92] feat: remove LineSearches.jl dependency --- Project.toml | 9 +++++---- docs/src/native/solvers.md | 5 +---- ext/NonlinearSolveNLsolveExt.jl | 7 +++++-- src/NonlinearSolve.jl | 7 ++----- src/algorithms/extension_algs.jl | 8 +++++++- src/algorithms/klement.jl | 7 ------- src/core/approximate_jacobian.jl | 6 ------ src/core/generalized_first_order.jl | 7 ------- src/default.jl | 10 +++++----- src/globalization/line_search.jl | 26 -------------------------- 10 files changed, 25 insertions(+), 67 deletions(-) diff --git a/Project.toml b/Project.toml index 725aa5c9d..6bf13d3a3 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "NonlinearSolve" uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" authors = ["SciML"] -version = "3.15.2" +version = "4.0.0" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" @@ -15,7 +15,6 @@ FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" LineSearch = "87fe0de2-c867-4266-b59a-2f0a94fc965b" -LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" @@ -41,6 +40,7 @@ BandedMatrices = "aae01518-5342-5314-be14-df237901396f" FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce" FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176" LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" +LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9" NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba" NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" @@ -54,7 +54,7 @@ NonlinearSolveFixedPointAccelerationExt = "FixedPointAcceleration" NonlinearSolveLeastSquaresOptimExt = "LeastSquaresOptim" NonlinearSolveMINPACKExt = "MINPACK" NonlinearSolveNLSolversExt = "NLSolvers" -NonlinearSolveNLsolveExt = "NLsolve" +NonlinearSolveNLsolveExt = ["NLsolve", "LineSearches"] NonlinearSolveSIAMFANLEquationsExt = "SIAMFANLEquations" NonlinearSolveSpeedMappingExt = "SpeedMapping" @@ -132,6 +132,7 @@ FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176" Hwloc = "0e44f5e4-bd66-52a0-8798-143a42290a1d" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" +LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9" ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba" @@ -151,4 +152,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] +test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] diff --git a/docs/src/native/solvers.md b/docs/src/native/solvers.md index a5deca141..aebaee379 100644 --- a/docs/src/native/solvers.md +++ b/docs/src/native/solvers.md @@ -23,10 +23,7 @@ documentation. algorithms, consult the [LinearSolve.jl documentation](https://docs.sciml.ai/LinearSolve/stable/). - `linesearch`: the line search algorithm to use. Defaults to [`NoLineSearch()`](@extref LineSearch.NoLineSearch), - which means that no line search is performed. Algorithms from - [`LineSearches.jl`](https://github.com/JuliaNLSolvers/LineSearches.jl/) must be - wrapped in [`LineSearchesJL`](@ref) before being supplied. For a detailed documentation - refer to [Line Search Algorithms](@ref line-search). + which means that no line search is performed. - `autodiff`/`jacobian_ad`: etermines the backend used for the Jacobian. Note that this argument is ignored if an analytical Jacobian is passed, as that will be used instead. Defaults to `nothing` which means that a default is selected according to the problem diff --git a/ext/NonlinearSolveNLsolveExt.jl b/ext/NonlinearSolveNLsolveExt.jl index 77ed4a56f..95de37055 100644 --- a/ext/NonlinearSolveNLsolveExt.jl +++ b/ext/NonlinearSolveNLsolveExt.jl @@ -1,5 +1,6 @@ module NonlinearSolveNLsolveExt +using LineSearches: Static using NonlinearSolve: NonlinearSolve, NLsolveJL, TraceMinimal using NLsolve: NLsolve, OnceDifferentiable, nlsolve using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode @@ -31,9 +32,11 @@ function SciMLBase.__solve( store_trace = StT || alg.store_trace extended_trace = !(trace_level isa TraceMinimal) || alg.extended_trace + linesearch = alg.linesearch === missing ? Static() : alg.linesearch + original = nlsolve(df, vec(u0); ftol = abstol, iterations = maxiters, alg.method, - store_trace, extended_trace, alg.linesearch, alg.linsolve, - alg.factor, alg.autoscale, alg.m, alg.beta, show_trace) + store_trace, extended_trace, linesearch, alg.linsolve, alg.factor, + alg.autoscale, alg.m, alg.beta, show_trace) f!(vec(resid), original.zero) u = prob.u0 isa Number ? original.zero[1] : reshape(original.zero, size(prob.u0)) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 5e2dc5555..1f48371e6 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -23,8 +23,7 @@ using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Sy UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril, istriu, lu, mul!, norm, pinv, tril!, triu! using LineSearch: LineSearch, AbstractLineSearchAlgorithm, AbstractLineSearchCache, - NoLineSearch, RobustNonMonotoneLineSearch -using LineSearches: LineSearches + NoLineSearch, RobustNonMonotoneLineSearch, BackTracking using LinearSolve: LinearSolve, LUFactorization, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver @@ -174,9 +173,7 @@ export NewtonDescent, SteepestDescent, Dogleg, DampedNewtonDescent, GeodesicAcce # Globalization ## Line Search Algorithms -export LineSearchesJL, LiFukushimaLineSearch # FIXME: deprecated. use LineSearch.jl directly -export Static, HagerZhang, MoreThuente, StrongWolfe, BackTracking # FIXME: deprecated -export LineSearch, NoLineSearch, RobustNonMonotoneLineSearch +export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch, LineSearchesJL ## Trust Region Algorithms export RadiusUpdateSchemes diff --git a/src/algorithms/extension_algs.jl b/src/algorithms/extension_algs.jl index 46ebc8dae..9c08c5202 100644 --- a/src/algorithms/extension_algs.jl +++ b/src/algorithms/extension_algs.jl @@ -203,6 +203,12 @@ end acceleration of the fixed-point iteration xₙ₊₁ = xₙ + beta*f(xₙ), where by default beta = 1. +!!! warning + + Line Search Algorithms from [`LineSearch.jl`](https://github.com/SciML/LineSearch.jl) + aren't supported by `NLsolveJL`. Instead, use the line search algorithms from + [`LineSearches.jl`](https://github.com/JuliaNLSolvers/LineSearches.jl). + ### Submethod Choice Choices for methods in `NLsolveJL`: @@ -234,7 +240,7 @@ For more information on these arguments, consult the end function NLsolveJL(; method = :trust_region, autodiff = :central, store_trace = missing, - extended_trace = missing, linesearch = LineSearches.Static(), + extended_trace = missing, linesearch = missing, linsolve = (x, A, b) -> copyto!(x, A \ b), factor = 1.0, autoscale = true, m = 10, beta = one(Float64), show_trace = missing) if Base.get_extension(@__MODULE__, :NonlinearSolveNLsolveExt) === nothing diff --git a/src/algorithms/klement.jl b/src/algorithms/klement.jl index 5e911d8c0..f5d3edf3d 100644 --- a/src/algorithms/klement.jl +++ b/src/algorithms/klement.jl @@ -27,13 +27,6 @@ over this. function Klement(; max_resets::Int = 100, linsolve = nothing, alpha = nothing, linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing, init_jacobian::Val{IJ} = Val(:identity)) where {IJ} - if !(linesearch isa AbstractLineSearchAlgorithm) - Base.depwarn( - "Passing in a `LineSearches.jl` algorithm directly is deprecated. \ - Please use `LineSearchesJL` instead.", :Klement) - linesearch = LineSearchesJL(; method = linesearch) - end - if IJ === :identity initialization = IdentityInitialization(alpha, DiagonalStructure()) elseif IJ === :true_jacobian diff --git a/src/core/approximate_jacobian.jl b/src/core/approximate_jacobian.jl index 6484c0408..2e0c64a82 100644 --- a/src/core/approximate_jacobian.jl +++ b/src/core/approximate_jacobian.jl @@ -59,12 +59,6 @@ function ApproximateJacobianSolveAlgorithm{concrete_jac, name}(; linesearch = missing, trustregion = missing, descent, update_rule, reinit_rule, initialization, max_resets::Int = typemax(Int), max_shrink_times::Int = typemax(Int)) where {concrete_jac, name} - if linesearch !== missing && !(linesearch isa AbstractLineSearchAlgorithm) - Base.depwarn("Passing in a `LineSearches.jl` algorithm directly is deprecated. \ - Please use `LineSearchesJL` instead.", - :GeneralizedFirstOrderAlgorithm) - linesearch = LineSearchesJL(; method = linesearch) - end return ApproximateJacobianSolveAlgorithm{concrete_jac, name}( linesearch, trustregion, descent, update_rule, reinit_rule, max_resets, max_shrink_times, initialization) diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl index a485c7c65..ebdaf1fc8 100644 --- a/src/core/generalized_first_order.jl +++ b/src/core/generalized_first_order.jl @@ -66,13 +66,6 @@ function GeneralizedFirstOrderAlgorithm{concrete_jac, name}(; jacobian_ad !== nothing && ADTypes.mode(jacobian_ad) isa ADTypes.ReverseMode, jacobian_ad, nothing)) - if linesearch !== missing && !(linesearch isa AbstractLineSearchAlgorithm) - Base.depwarn("Passing in a `LineSearches.jl` algorithm directly is deprecated. \ - Please use `LineSearchesJL` instead.", - :GeneralizedFirstOrderAlgorithm) - linesearch = LineSearchesJL(; method = linesearch) - end - return GeneralizedFirstOrderAlgorithm{concrete_jac, name}( linesearch, trustregion, descent, max_shrink_times, jacobian_ad, forward_ad, reverse_ad) diff --git a/src/default.jl b/src/default.jl index c0924e2ff..967b2e0e8 100644 --- a/src/default.jl +++ b/src/default.jl @@ -364,7 +364,7 @@ function RobustMultiNewton(::Type{T} = Float64; concrete_jac = nothing, linsolve radius_update_scheme = RadiusUpdateSchemes.Bastin), NewtonRaphson(; concrete_jac, linsolve, precs, autodiff), NewtonRaphson(; concrete_jac, linsolve, precs, - linesearch = LineSearch.BackTracking(), autodiff), + linesearch = BackTracking(), autodiff), TrustRegion(; concrete_jac, linsolve, precs, radius_update_scheme = RadiusUpdateSchemes.NLsolve, autodiff), TrustRegion(; concrete_jac, linsolve, precs, @@ -405,7 +405,7 @@ function FastShortcutNonlinearPolyalg( else algs = (NewtonRaphson(; concrete_jac, linsolve, precs, autodiff), NewtonRaphson(; concrete_jac, linsolve, precs, - linesearch = LineSearch.BackTracking(), autodiff), + linesearch = BackTracking(), autodiff), TrustRegion(; concrete_jac, linsolve, precs, autodiff), TrustRegion(; concrete_jac, linsolve, precs, radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff)) @@ -426,7 +426,7 @@ function FastShortcutNonlinearPolyalg( SimpleKlement(), NewtonRaphson(; concrete_jac, linsolve, precs, autodiff), NewtonRaphson(; concrete_jac, linsolve, precs, - linesearch = LineSearch.BackTracking(), autodiff), + linesearch = BackTracking(), autodiff), TrustRegion(; concrete_jac, linsolve, precs, radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff)) end @@ -445,7 +445,7 @@ function FastShortcutNonlinearPolyalg( Klement(; linsolve, precs, autodiff), NewtonRaphson(; concrete_jac, linsolve, precs, autodiff), NewtonRaphson(; concrete_jac, linsolve, precs, - linesearch = LineSearch.BackTracking(), autodiff), + linesearch = BackTracking(), autodiff), TrustRegion(; concrete_jac, linsolve, precs, autodiff), TrustRegion(; concrete_jac, linsolve, precs, radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff)) @@ -481,7 +481,7 @@ function FastShortcutNLLSPolyalg( linsolve, precs, disable_geodesic = Val(true), autodiff, kwargs...), TrustRegion(; concrete_jac, linsolve, precs, autodiff, kwargs...), GaussNewton(; concrete_jac, linsolve, precs, - linesearch = LineSearch.BackTracking(), autodiff, kwargs...), + linesearch = BackTracking(), autodiff, kwargs...), TrustRegion(; concrete_jac, linsolve, precs, radius_update_scheme = RadiusUpdateSchemes.Bastin, autodiff, kwargs...), LevenbergMarquardt(; linsolve, precs, autodiff, kwargs...)) diff --git a/src/globalization/line_search.jl b/src/globalization/line_search.jl index c7c342bee..7549f1f9d 100644 --- a/src/globalization/line_search.jl +++ b/src/globalization/line_search.jl @@ -1,29 +1,3 @@ -LineSearchesJL(method; kwargs...) = LineSearchesJL(; method, kwargs...) -function LineSearchesJL(; method = LineSearches.Static(), autodiff = nothing, α = true) - Base.depwarn("`LineSearchesJL(...)` is deprecated. Please use `LineSearchesJL` from \ - LineSearch.jl instead.", - :LineSearchesJL) - - # Prevent breaking old code - method isa LineSearch.LineSearchesJL && - return LineSearch.LineSearchesJL(method.method, α, autodiff) - method isa AbstractLineSearchAlgorithm && return method - return LineSearch.LineSearchesJL(method, α, autodiff) -end - -for alg in (:Static, :HagerZhang, :MoreThuente, :BackTracking, :StrongWolfe) - depmsg = "`$(alg)(args...; kwargs...)` is deprecated. Please use `LineSearchesJL(; \ - method = $(alg)(args...; kwargs...))` instead." - @eval function $(alg)(args...; autodiff = nothing, initial_alpha = true, kwargs...) - Base.depwarn($(depmsg), $(Meta.quot(alg))) - return LineSearch.LineSearchesJL(; - method = LineSearches.$(alg)(args...; kwargs...), autodiff, initial_alpha) - end -end - -Base.@deprecate LiFukushimaLineSearch(; nan_max_iter::Int = 5, kwargs...) LineSearch.LiFukushimaLineSearch(; - nan_maxiters = nan_max_iter, kwargs...) - function callback_into_cache!(topcache, cache::AbstractLineSearchCache, args...) LineSearch.callback_into_cache!(cache, get_fu(topcache)) end From 3b47dbf0d94664ba2d412d44df5713f24d6b493e Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 12:13:35 -0400 Subject: [PATCH 74/92] chore: remove deprecated functionalities --- ext/NonlinearSolveMINPACKExt.jl | 4 +- ext/NonlinearSolveNLsolveExt.jl | 6 +- ext/NonlinearSolveSpeedMappingExt.jl | 2 +- src/algorithms/extension_algs.jl | 85 +++------------------------ src/algorithms/levenberg_marquardt.jl | 18 ++---- src/internal/jacobian.jl | 16 ++--- 6 files changed, 23 insertions(+), 108 deletions(-) diff --git a/ext/NonlinearSolveMINPACKExt.jl b/ext/NonlinearSolveMINPACKExt.jl index a7be409d4..8299b0b45 100644 --- a/ext/NonlinearSolveMINPACKExt.jl +++ b/ext/NonlinearSolveMINPACKExt.jl @@ -19,8 +19,8 @@ function SciMLBase.__solve( method = ifelse(alg.method === :auto, ifelse(prob isa NonlinearLeastSquaresProblem, :lm, :hybr), alg.method) - show_trace = alg.show_trace || ShT - tracing = alg.tracing || StT + show_trace = ShT + tracing = StT tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0)) if alg.autodiff === missing && prob.f.jac === nothing diff --git a/ext/NonlinearSolveNLsolveExt.jl b/ext/NonlinearSolveNLsolveExt.jl index 95de37055..9872c7953 100644 --- a/ext/NonlinearSolveNLsolveExt.jl +++ b/ext/NonlinearSolveNLsolveExt.jl @@ -28,9 +28,9 @@ function SciMLBase.__solve( end abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0)) - show_trace = ShT || alg.show_trace - store_trace = StT || alg.store_trace - extended_trace = !(trace_level isa TraceMinimal) || alg.extended_trace + show_trace = ShT + store_trace = StT + extended_trace = !(trace_level isa TraceMinimal) linesearch = alg.linesearch === missing ? Static() : alg.linesearch diff --git a/ext/NonlinearSolveSpeedMappingExt.jl b/ext/NonlinearSolveSpeedMappingExt.jl index 2813e3e58..b39394a3b 100644 --- a/ext/NonlinearSolveSpeedMappingExt.jl +++ b/ext/NonlinearSolveSpeedMappingExt.jl @@ -14,7 +14,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SpeedMappingJL, args...; prob; alias_u0, make_fixed_point = Val(true)) tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u)) - time_limit = ifelse(maxtime === nothing, alg.time_limit, maxtime) + time_limit = ifelse(maxtime === nothing, 1000, maxtime) sol = speedmapping(u; m!, tol, Lp = Inf, maps_limit = maxiters, alg.orders, alg.check_obj, store_info, alg.σ_min, alg.stabilize, time_limit) diff --git a/src/algorithms/extension_algs.jl b/src/algorithms/extension_algs.jl index 9c08c5202..3cf36ca9f 100644 --- a/src/algorithms/extension_algs.jl +++ b/src/algorithms/extension_algs.jl @@ -142,39 +142,15 @@ NonlinearLeastSquaresProblem. This algorithm is only available if `MINPACK.jl` is installed. """ @concrete struct CMINPACK <: AbstractNonlinearSolveExtensionAlgorithm - show_trace::Bool - tracing::Bool method::Symbol autodiff end -function CMINPACK(; show_trace = missing, tracing = missing, - method::Symbol = :auto, autodiff = missing) +function CMINPACK(; method::Symbol = :auto, autodiff = missing) if Base.get_extension(@__MODULE__, :NonlinearSolveMINPACKExt) === nothing error("CMINPACK requires MINPACK.jl to be loaded") end - - if show_trace !== missing - Base.depwarn( - "`show_trace` for CMINPACK has been deprecated and will be removed \ - in v4. Use the `show_trace` keyword argument via the logging API \ - https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \ - instead.", :CMINPACK) - else - show_trace = false - end - - if tracing !== missing - Base.depwarn( - "`tracing` for CMINPACK has been deprecated and will be removed \ - in v4. Use the `store_trace` keyword argument via the logging API \ - https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \ - instead.", :CMINPACK) - else - tracing = false - end - - return CMINPACK(show_trace, tracing, method, autodiff) + return CMINPACK(method, autodiff) end """ @@ -228,63 +204,26 @@ For more information on these arguments, consult the @concrete struct NLsolveJL <: AbstractNonlinearSolveExtensionAlgorithm method::Symbol autodiff - store_trace::Bool - extended_trace::Bool linesearch linsolve factor autoscale::Bool m::Int beta - show_trace::Bool end -function NLsolveJL(; method = :trust_region, autodiff = :central, store_trace = missing, - extended_trace = missing, linesearch = missing, +function NLsolveJL(; method = :trust_region, autodiff = :central, linesearch = missing, linsolve = (x, A, b) -> copyto!(x, A \ b), factor = 1.0, - autoscale = true, m = 10, beta = one(Float64), show_trace = missing) + autoscale = true, m = 10, beta = one(Float64)) if Base.get_extension(@__MODULE__, :NonlinearSolveNLsolveExt) === nothing error("NLsolveJL requires NLsolve.jl to be loaded") end - if show_trace !== missing - Base.depwarn("`show_trace` for NLsolveJL has been deprecated and will be removed \ - in v4. Use the `show_trace` keyword argument via the logging API \ - https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \ - instead.", - :NLsolveJL) - else - show_trace = false - end - - if store_trace !== missing - Base.depwarn( - "`store_trace` for NLsolveJL has been deprecated and will be removed \ - in v4. Use the `store_trace` keyword argument via the logging API \ - https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ \ - instead.", - :NLsolveJL) - else - store_trace = false - end - - if extended_trace !== missing - Base.depwarn( - "`extended_trace` for NLsolveJL has been deprecated and will be \ - removed in v4. Use the `trace_level = TraceAll()` keyword argument \ - via the logging API \ - https://docs.sciml.ai/NonlinearSolve/stable/basics/Logging/ instead.", - :NLsolveJL) - else - extended_trace = false - end - if autodiff isa Symbol && autodiff !== :central && autodiff !== :forward error("`autodiff` must be `:central` or `:forward`.") end - return NLsolveJL(method, autodiff, store_trace, extended_trace, linesearch, - linsolve, factor, autoscale, m, beta, show_trace) + return NLsolveJL(method, autodiff, linesearch, linsolve, factor, autoscale, m, beta) end """ @@ -349,25 +288,15 @@ Fixed Point Problems. We allow using this algorithm to solve root finding proble stabilize::Bool check_obj::Bool orders::Vector{Int} - time_limit end function SpeedMappingJL(; σ_min = 0.0, stabilize::Bool = false, check_obj::Bool = false, - orders::Vector{Int} = [3, 3, 2], time_limit = missing) + orders::Vector{Int} = [3, 3, 2]) if Base.get_extension(@__MODULE__, :NonlinearSolveSpeedMappingExt) === nothing error("SpeedMappingJL requires SpeedMapping.jl to be loaded") end - if time_limit !== missing - Base.depwarn("`time_limit` keyword argument to `SpeedMappingJL` has been \ - deprecated and will be removed in v4. Pass `maxtime = ` to \ - `SciMLBase.solve`.", - :SpeedMappingJL) - else - time_limit = 1000 - end - - return SpeedMappingJL(σ_min, stabilize, check_obj, orders, time_limit) + return SpeedMappingJL(σ_min, stabilize, check_obj, orders) end """ diff --git a/src/algorithms/levenberg_marquardt.jl b/src/algorithms/levenberg_marquardt.jl index 66554ee2d..e8a2fd0d7 100644 --- a/src/algorithms/levenberg_marquardt.jl +++ b/src/algorithms/levenberg_marquardt.jl @@ -31,19 +31,11 @@ For the remaining arguments, see [`GeodesicAcceleration`](@ref) and [`NonlinearSolve.LevenbergMarquardtTrustRegion`](@ref) documentations. """ function LevenbergMarquardt(; - concrete_jac = missing, linsolve = nothing, precs = DEFAULT_PRECS, - damping_initial::Real = 1.0, α_geodesic::Real = 0.75, - damping_increase_factor::Real = 2.0, damping_decrease_factor::Real = 3.0, - finite_diff_step_geodesic = 0.1, b_uphill::Real = 1.0, - autodiff = nothing, min_damping_D::Real = 1e-8, disable_geodesic = False) - if concrete_jac !== missing - Base.depwarn("The `concrete_jac` keyword argument is deprecated and will be \ - removed in v0.4. This kwarg doesn't make sense (and is currently \ - ignored) for LM since it needs to materialize the Jacobian to \ - compute the Damping Term", - :LevenbergMarquardt) - end - + linsolve = nothing, precs = DEFAULT_PRECS, damping_initial::Real = 1.0, + α_geodesic::Real = 0.75, damping_increase_factor::Real = 2.0, + damping_decrease_factor::Real = 3.0, finite_diff_step_geodesic = 0.1, + b_uphill::Real = 1.0, autodiff = nothing, + min_damping_D::Real = 1e-8, disable_geodesic = False) descent = DampedNewtonDescent(; linsolve, precs, initial_damping = damping_initial, diff --git a/src/internal/jacobian.jl b/src/internal/jacobian.jl index b78eb7383..358bca792 100644 --- a/src/internal/jacobian.jl +++ b/src/internal/jacobian.jl @@ -243,17 +243,11 @@ function select_fastest_coloring_algorithm( return GreedyColoringAlgorithm(LargestFirst()) end -function construct_concrete_adtype(f::NonlinearFunction, ad::AutoSparse) - Base.depwarn( - "Specifying a sparse AD type for Nonlinear Problems is deprecated. \ - Instead use the `sparsity`, `jac_prototype`, and `colorvec` to specify \ - the right sparsity pattern and coloring algorithm. Ignoring the sparsity \ - detection algorithm and coloring algorithm present in $(ad).", - :NonlinearSolve) - if f.sparsity === nothing && f.jac_prototype === nothing - @set! f.sparsity = TracerSparsityDetector() - end - return construct_concrete_adtype(f, get_dense_ad(ad)) +function construct_concrete_adtype(::NonlinearFunction, ad::AutoSparse) + error("Specifying a sparse AD type for Nonlinear Problems was removed in v4. \ + Instead use the `sparsity`, `jac_prototype`, and `colorvec` to specify \ + the right sparsity pattern and coloring algorithm. Ignoring the sparsity \ + detection algorithm and coloring algorithm present in $(ad).") end get_dense_ad(ad) = ad From 6fdbe654cc131b361c6e9358a8a0b454212f1f39 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 12:38:06 -0400 Subject: [PATCH 75/92] chore: remove more deprecations --- Project.toml | 4 +--- src/NonlinearSolve.jl | 48 +++++++++++++++++++++---------------- test/core/nlls_tests.jl | 10 ++++++-- test/core/rootfind_tests.jl | 41 +++++++++++++++++++++---------- 4 files changed, 64 insertions(+), 39 deletions(-) diff --git a/Project.toml b/Project.toml index 6bf13d3a3..246b935a7 100644 --- a/Project.toml +++ b/Project.toml @@ -26,7 +26,6 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" SciMLJacobianOperators = "19f34311-ddf3-4b8b-af20-060888a46c0e" SciMLOperators = "c0aeaf25-5076-4817-a8d5-81caf7dfa961" -Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46" SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" @@ -104,8 +103,7 @@ SIAMFANLEquations = "1.0.1" SciMLBase = "2.54.0" SciMLJacobianOperators = "0.1" SciMLOperators = "0.3.10" -Setfield = "1.1.1" -SimpleNonlinearSolve = "1.12.3" +SimpleNonlinearSolve = "2" SparseArrays = "1.10" SparseConnectivityTracer = "0.6.5" SparseMatrixColorings = "0.4.2" diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 1f48371e6..a41027531 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -1,9 +1,5 @@ module NonlinearSolve -if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@max_methods")) - @eval Base.Experimental.@max_methods 1 -end - using Reexport: @reexport using PrecompileTools: @compile_workload, @setup_workload @@ -23,7 +19,7 @@ using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Sy UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril, istriu, lu, mul!, norm, pinv, tril!, triu! using LineSearch: LineSearch, AbstractLineSearchAlgorithm, AbstractLineSearchCache, - NoLineSearch, RobustNonMonotoneLineSearch, BackTracking + NoLineSearch, RobustNonMonotoneLineSearch, BackTracking, LineSearchesJL using LinearSolve: LinearSolve, LUFactorization, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver @@ -34,7 +30,6 @@ using RecursiveArrayTools: recursivecopy! using SciMLBase: AbstractNonlinearAlgorithm, AbstractNonlinearProblem, _unwrap_val, isinplace, NLStats using SciMLOperators: AbstractSciMLOperator -using Setfield: @set! using StaticArraysCore: StaticArray, SVector, SArray, MArray, Size, SMatrix using SymbolicIndexingInterface: SymbolicIndexingInterface, ParameterIndexingProxy, symbolic_container, parameter_values, state_values, getu, @@ -44,8 +39,6 @@ using SymbolicIndexingInterface: SymbolicIndexingInterface, ParameterIndexingPro using ADTypes: ADTypes, AbstractADType, AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff, AutoZygote, AutoEnzyme, AutoSparse, NoSparsityDetector, KnownJacobianSparsityDetector -using ADTypes: AutoSparseFiniteDiff, AutoSparseForwardDiff, AutoSparsePolyesterForwardDiff, - AutoSparseZygote # FIXME: deprecated, remove in future using DifferentiationInterface: DifferentiationInterface, Constant using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff, Dual @@ -114,11 +107,20 @@ include("default.jl") push!(probs_nls, NonlinearProblem(fn, u0, 2.0)) end - nls_algs = (NewtonRaphson(), TrustRegion(), LevenbergMarquardt(), - PseudoTransient(), Broyden(), Klement(), DFSane(), nothing) + nls_algs = ( + NewtonRaphson(), + TrustRegion(), + LevenbergMarquardt(), + # PseudoTransient(), + Broyden(), + Klement(), + # DFSane(), + nothing + ) probs_nlls = NonlinearLeastSquaresProblem[] - nlfuncs = ((NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), + nlfuncs = ( + (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]), ( NonlinearFunction{true}( @@ -127,15 +129,22 @@ include("default.jl") ( NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p), resid_prototype = zeros(4)), - [0.1, 0.1])) + [0.1, 0.1] + ) + ) for (fn, u0) in nlfuncs push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0)) end - nlls_algs = (LevenbergMarquardt(), GaussNewton(), TrustRegion(), - LevenbergMarquardt(; linsolve = LUFactorization()), - GaussNewton(; linsolve = LUFactorization()), - TrustRegion(; linsolve = LUFactorization()), nothing) + nlls_algs = ( + LevenbergMarquardt(), + GaussNewton(), + TrustRegion(), + # LevenbergMarquardt(; linsolve = LUFactorization()), + # GaussNewton(; linsolve = LUFactorization()), + # TrustRegion(; linsolve = LUFactorization()), + nothing + ) @compile_workload begin @sync begin @@ -177,7 +186,7 @@ export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch, Line ## Trust Region Algorithms export RadiusUpdateSchemes -# Export the termination conditions from DiffEqBase +# Export the termination conditions from NonlinearSolveBase export SteadyStateDiffEqTerminationMode, SimpleNonlinearSolveTerminationMode, NormTerminationMode, RelTerminationMode, RelNormTerminationMode, AbsTerminationMode, AbsNormTerminationMode, RelSafeTerminationMode, AbsSafeTerminationMode, @@ -189,8 +198,5 @@ export TraceAll, TraceMinimal, TraceWithJacobianConditionNumber # Reexport ADTypes export AutoFiniteDiff, AutoForwardDiff, AutoPolyesterForwardDiff, AutoZygote, AutoEnzyme, AutoSparse -# FIXME: deprecated, remove in future -export AutoSparseFiniteDiff, AutoSparseForwardDiff, AutoSparsePolyesterForwardDiff, - AutoSparseZygote -end # module +end diff --git a/test/core/nlls_tests.jl b/test/core/nlls_tests.jl index 483107f69..361c5ba20 100644 --- a/test/core/nlls_tests.jl +++ b/test/core/nlls_tests.jl @@ -2,6 +2,13 @@ using Reexport @reexport using NonlinearSolve, LinearSolve, LinearAlgebra, StableRNGs, Random, ForwardDiff, Zygote +using LineSearches: LineSearches, Static, HagerZhang, MoreThuente, StrongWolfe + +linesearches = [] +for ls in (Static(), HagerZhang(), MoreThuente(), StrongWolfe(), LineSearches.BackTracking()) + push!(linesearches, LineSearchesJL(; method = ls)) +end +push!(linesearches, BackTracking()) true_function(x, θ) = @. θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4]) true_function(y, x, θ) = (@. y = θ[1] * exp(θ[2] * x) * cos(θ[3] * x + θ[4])) @@ -29,8 +36,7 @@ solvers = [] for linsolve in [nothing, LUFactorization(), KrylovJL_GMRES(), KrylovJL_LSMR()] vjp_autodiffs = linsolve isa KrylovJL ? [nothing, AutoZygote(), AutoFiniteDiff()] : [nothing] - for linesearch in [Static(), BackTracking(), HagerZhang(), StrongWolfe(), MoreThuente()], - vjp_autodiff in vjp_autodiffs + for linesearch in linesearches, vjp_autodiff in vjp_autodiffs push!(solvers, GaussNewton(; linsolve, linesearch, vjp_autodiff)) end diff --git a/test/core/rootfind_tests.jl b/test/core/rootfind_tests.jl index cb8d62e99..36c3e335b 100644 --- a/test/core/rootfind_tests.jl +++ b/test/core/rootfind_tests.jl @@ -2,6 +2,7 @@ using Reexport @reexport using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random, LinearAlgebra, ForwardDiff, Zygote, Enzyme, DiffEqBase +using LineSearches: LineSearches _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x)) @@ -46,8 +47,19 @@ function nlprob_iterator_interface(f, p_range, ::Val{iip}, solver) where {iip} return sols end +for alg in (:Static, :StrongWolfe, :BackTracking, :MoreThuente, :HagerZhang) + algname = Symbol(:LineSearches, alg) + @eval function $(algname)(args...; autodiff = nothing, initial_alpha = true, kwargs...) + return LineSearch.LineSearchesJL(; + method = LineSearches.$(alg)(args...; kwargs...), autodiff, initial_alpha) + end +end + export nlprob_iterator_interface, benchmark_nlsolve_oop, benchmark_nlsolve_iip, TERMINATION_CONDITIONS, _nameof, newton_fails, quadratic_f, quadratic_f! +export LineSearchesStatic, LineSearchesStrongWolfe, LineSearchesBackTracking, + LineSearchesMoreThuente, LineSearchesHagerZhang + end # --- NewtonRaphson tests --- @@ -57,9 +69,10 @@ end AutoForwardDiff(), AutoZygote(), AutoFiniteDiff() ), linesearch in ( - Static(; autodiff = ad), StrongWolfe(; autodiff = ad), - BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad), - HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad) + LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad), + LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad), + LineSearchesHagerZhang(; autodiff = ad), + LineSearchesMoreThuente(; autodiff = ad) ) u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0) @@ -471,9 +484,10 @@ end AutoForwardDiff(), AutoZygote(), AutoFiniteDiff() ), linesearch in ( - Static(; autodiff = ad), StrongWolfe(; autodiff = ad), - BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad), - HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad) + LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad), + LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad), + LineSearchesHagerZhang(; autodiff = ad), + LineSearchesMoreThuente(; autodiff = ad) ), init_jacobian in (Val(:identity), Val(:true_jacobian)), update_rule in (Val(:good_broyden), Val(:bad_broyden), Val(:diagonal)) @@ -524,9 +538,10 @@ end AutoForwardDiff(), AutoZygote(), AutoFiniteDiff() ), linesearch in ( - Static(; autodiff = ad), StrongWolfe(; autodiff = ad), - BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad), - HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad) + LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad), + LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad), + LineSearchesHagerZhang(; autodiff = ad), + LineSearchesMoreThuente(; autodiff = ad) ), init_jacobian in (Val(:identity), Val(:true_jacobian), Val(:true_jacobian_diagonal)) @@ -577,10 +592,10 @@ end AutoForwardDiff(), AutoZygote(), AutoFiniteDiff() ), linesearch in ( - Static(; autodiff = ad), StrongWolfe(; autodiff = ad), - BackTracking(; autodiff = ad), LineSearch.BackTracking(; autodiff = ad), - HagerZhang(; autodiff = ad), MoreThuente(; autodiff = ad), - LiFukushimaLineSearch() + LineSearchesStatic(; autodiff = ad), LineSearchesStrongWolfe(; autodiff = ad), + LineSearchesBackTracking(; autodiff = ad), BackTracking(; autodiff = ad), + LineSearchesHagerZhang(; autodiff = ad), + LineSearchesMoreThuente(; autodiff = ad), LiFukushimaLineSearch() ) u0s = ([1.0, 1.0], @SVector[1.0, 1.0], 1.0) From 16ecd16f51358210ee0c6f78bd6d25f21b07d582 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 12:48:14 -0400 Subject: [PATCH 76/92] chore: run formatter --- test/core/nlls_tests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/core/nlls_tests.jl b/test/core/nlls_tests.jl index 361c5ba20..040627c00 100644 --- a/test/core/nlls_tests.jl +++ b/test/core/nlls_tests.jl @@ -5,7 +5,8 @@ using Reexport using LineSearches: LineSearches, Static, HagerZhang, MoreThuente, StrongWolfe linesearches = [] -for ls in (Static(), HagerZhang(), MoreThuente(), StrongWolfe(), LineSearches.BackTracking()) +for ls in ( + Static(), HagerZhang(), MoreThuente(), StrongWolfe(), LineSearches.BackTracking()) push!(linesearches, LineSearchesJL(; method = ls)) end push!(linesearches, BackTracking()) @@ -37,7 +38,6 @@ for linsolve in [nothing, LUFactorization(), KrylovJL_GMRES(), KrylovJL_LSMR()] vjp_autodiffs = linsolve isa KrylovJL ? [nothing, AutoZygote(), AutoFiniteDiff()] : [nothing] for linesearch in linesearches, vjp_autodiff in vjp_autodiffs - push!(solvers, GaussNewton(; linsolve, linesearch, vjp_autodiff)) end end From 91c5636eb9b6c81c4549cf6e70f06851813b0aba Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 12:55:28 -0400 Subject: [PATCH 77/92] test: try fixing circular deps --- .github/workflows/Documentation.yml | 2 +- Project.toml | 3 +-- docs/Project.toml | 8 ++++---- test/runtests.jl | 7 ++++++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index 008cd511e..84f38404f 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -22,7 +22,7 @@ jobs: Pkg.Registry.update() # Install packages present in subdirectories dev_pks = Pkg.PackageSpec[] - for path in ("lib/SciMLJacobianOperators", ".") + for path in ("lib/SciMLJacobianOperators", ".", "lib/SimpleNonlinearSolve", "lib/NonlinearSolveBase", "lib/BracketingNonlinearSolve") push!(dev_pks, Pkg.PackageSpec(; path)) end Pkg.develop(dev_pks) diff --git a/Project.toml b/Project.toml index 246b935a7..184aed113 100644 --- a/Project.toml +++ b/Project.toml @@ -132,7 +132,6 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9" -ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba" NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" NaNMath = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" @@ -150,4 +149,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "ModelingToolkit", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] +test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] diff --git a/docs/Project.toml b/docs/Project.toml index 4ad265246..d01ae32a1 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -37,14 +37,14 @@ DocumenterInterLinks = "1.0.0" IncompleteLU = "0.2" InteractiveUtils = "<0.0.1, 1" LinearSolve = "2" -ModelingToolkit = "8, 9" -NonlinearSolve = "3" +ModelingToolkit = "9" +NonlinearSolve = "4" OrdinaryDiffEqTsit5 = "1.1.0" Plots = "1" -Random = "<0.0.1, 1" +Random = "1.10" SciMLBase = "2.4" SciMLJacobianOperators = "0.1" -SimpleNonlinearSolve = "1" +SimpleNonlinearSolve = "2" SparseConnectivityTracer = "0.6.5" StaticArrays = "1" SteadyStateDiffEq = "2" diff --git a/test/runtests.jl b/test/runtests.jl index 74676f3ad..bf2a8ecdb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,9 +1,14 @@ -using ReTestItems, NonlinearSolve, Hwloc, InteractiveUtils +using ReTestItems, NonlinearSolve, Hwloc, InteractiveUtils, Pkg @info sprint(InteractiveUtils.versioninfo) const GROUP = lowercase(get(ENV, "GROUP", "All")) +const EXTRA_PKGS = Pkg.PackageSpec[] +(GROUP == "all" || GROUP == "downstream") && + push!(EXTRA_PKGS, Pkg.PackageSpec("ModelingToolkit")) +Pkg.add(EXTRA_PKGS) + const RETESTITEMS_NWORKERS = parse( Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4)))) const RETESTITEMS_NWORKER_THREADS = parse(Int, From c6a517e06062cd36a4c27b2cd520c3bdf2bd79cc Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 13:04:37 -0400 Subject: [PATCH 78/92] chore: remove unnecessary deps --- Project.toml | 9 ++++----- src/NonlinearSolve.jl | 8 ++++---- src/algorithms/levenberg_marquardt.jl | 10 ++++------ src/descent/damped_newton.jl | 12 ++++++------ src/internal/approximate_initialization.jl | 6 +++--- test/runtests.jl | 4 ++-- 6 files changed, 23 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index 184aed113..572dbb6fe 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" -FastBroadcast = "7034ab61-46d4-4ed7-9d0f-46aef9175898" FastClosures = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" FiniteDiff = "6a86dc24-6348-571c-b903-95158fe2bd41" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" @@ -18,6 +17,7 @@ LineSearch = "87fe0de2-c867-4266-b59a-2f0a94fc965b" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" +NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" Preferences = "21216c6a-2e73-6563-6e65-726566657250" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" @@ -28,7 +28,6 @@ SciMLJacobianOperators = "19f34311-ddf3-4b8b-af20-060888a46c0e" SciMLOperators = "c0aeaf25-5076-4817-a8d5-81caf7dfa961" SimpleNonlinearSolve = "727e6d20-b764-4bd8-a329-72de5adea6c7" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" -SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SparseMatrixColorings = "0a514795-09f3-496d-8182-132a7b665d35" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" SymbolicIndexingInterface = "2efcf032-c050-4f8e-a9bb-153293bab1f5" @@ -69,7 +68,6 @@ DiffEqBase = "6.155.3" DifferentiationInterface = "0.6.1" Enzyme = "0.13.2" ExplicitImports = "1.5" -FastBroadcast = "0.3.5" FastClosures = "0.3.2" FastLevenbergMarquardt = "0.1" FiniteDiff = "2.24" @@ -85,11 +83,11 @@ LinearAlgebra = "1.10" LinearSolve = "2.35" MINPACK = "1.2" MaybeInplace = "0.1.4" -ModelingToolkit = "9.41.0" NLSolvers = "0.5" NLsolve = "4.5" NaNMath = "1" NonlinearProblemLibrary = "0.1.2" +NonlinearSolveBase = "1" OrdinaryDiffEqTsit5 = "1.1.0" Pkg = "1.10" PrecompileTools = "1.2" @@ -141,6 +139,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4" +SparseConnectivityTracer = "9f842d2f-2579-4b1d-911e-f412cf18a3f5" SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412" StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" @@ -149,4 +148,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" [targets] -test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] +test = ["Aqua", "BandedMatrices", "BenchmarkTools", "CUDA", "Enzyme", "ExplicitImports", "FastLevenbergMarquardt", "FixedPointAcceleration", "Hwloc", "InteractiveUtils", "LeastSquaresOptim", "LineSearches", "MINPACK", "NLSolvers", "NLsolve", "NaNMath", "NonlinearProblemLibrary", "OrdinaryDiffEqTsit5", "Pkg", "Random", "ReTestItems", "SIAMFANLEquations", "SparseConnectivityTracer", "SpeedMapping", "StableRNGs", "StaticArrays", "Sundials", "Test", "Zygote"] diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index a41027531..15e1ee3a9 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -12,14 +12,14 @@ using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode, NormTerminationMode, RelNormTerminationMode, RelSafeBestTerminationMode, RelSafeTerminationMode, RelTerminationMode, SimpleNonlinearSolveTerminationMode, SteadyStateDiffEqTerminationMode -using FastBroadcast: @.. using FastClosures: @closure using LazyArrays: LazyArrays, ApplyArray, cache using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Symmetric, UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril, istriu, lu, mul!, norm, pinv, tril!, triu! using LineSearch: LineSearch, AbstractLineSearchAlgorithm, AbstractLineSearchCache, - NoLineSearch, RobustNonMonotoneLineSearch, BackTracking, LineSearchesJL + NoLineSearch, RobustNonMonotoneLineSearch, BackTracking, LineSearchesJL, + LiFukushimaLineSearch using LinearSolve: LinearSolve, LUFactorization, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver @@ -47,7 +47,6 @@ using SciMLJacobianOperators: AbstractJacobianOperator, JacobianOperator, VecJac ## Sparse AD Support using SparseArrays: AbstractSparseMatrix, SparseMatrixCSC -using SparseConnectivityTracer: TracerSparsityDetector # This can be dropped in the next release using SparseMatrixColorings: ConstantColoringAlgorithm, GreedyColoringAlgorithm, LargestFirst @@ -182,7 +181,8 @@ export NewtonDescent, SteepestDescent, Dogleg, DampedNewtonDescent, GeodesicAcce # Globalization ## Line Search Algorithms -export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch, LineSearchesJL +export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch, + LiFukushimaLineSearch, LineSearchesJL ## Trust Region Algorithms export RadiusUpdateSchemes diff --git a/src/algorithms/levenberg_marquardt.jl b/src/algorithms/levenberg_marquardt.jl index e8a2fd0d7..501a5dd29 100644 --- a/src/algorithms/levenberg_marquardt.jl +++ b/src/algorithms/levenberg_marquardt.jl @@ -156,18 +156,16 @@ end @inline function __update_LM_diagonal!!(y::Diagonal, x::AbstractMatrix) if __can_setindex(y.diag) if fast_scalar_indexing(y.diag) - @inbounds for i in axes(x, 1) - y.diag[i] = max(y.diag[i], x[i, i]) + @simd for i in axes(x, 1) + @inbounds y.diag[i] = max(y.diag[i], x[i, i]) end return y else - idxs = diagind(x) - @.. broadcast=false y.diag=max(y.diag, @view(x[idxs])) + y .= max.(y.diag, @view(x[diagind(x)])) return y end else - idxs = diagind(x) - return Diagonal(@.. broadcast=false max(y.diag, @view(x[idxs]))) + return Diagonal(max.(y.diag, @view(x[diagind(x)]))) end end diff --git a/src/descent/damped_newton.jl b/src/descent/damped_newton.jl index 77b204b13..ba3e1d028 100644 --- a/src/descent/damped_newton.jl +++ b/src/descent/damped_newton.jl @@ -226,12 +226,12 @@ end if __can_setindex(J_cache) copyto!(J_cache, J) if fast_scalar_indexing(J_cache) - @inbounds for i in axes(J_cache, 1) - J_cache[i, i] += D[i, i] + @simd for i in axes(J_cache, 1) + @inbounds J_cache[i, i] += D[i, i] end else idxs = diagind(J_cache) - @.. broadcast=false @view(J_cache[idxs])=@view(J[idxs]) + @view(D[idxs]) + J_cache[idxs] .= @view(J[idxs]) .+ @view(D[idxs]) end return J_cache else @@ -242,12 +242,12 @@ end if __can_setindex(J_cache) copyto!(J_cache, J) if fast_scalar_indexing(J_cache) - @inbounds for i in axes(J_cache, 1) - J_cache[i, i] += D + @simd for i in axes(J_cache, 1) + @inbounds J_cache[i, i] += D end else idxs = diagind(J_cache) - @.. broadcast=false @view(J_cache[idxs])=@view(J[idxs]) + D + J_cache[idxs] .= @view(J[idxs]) .+ D end return J_cache else diff --git a/src/internal/approximate_initialization.jl b/src/internal/approximate_initialization.jl index 5ce348ae3..985234ac4 100644 --- a/src/internal/approximate_initialization.jl +++ b/src/internal/approximate_initialization.jl @@ -20,11 +20,11 @@ end function (::DiagonalStructure)(J::AbstractVector, J_new::AbstractMatrix) if __can_setindex(J) if fast_scalar_indexing(J) - @inbounds for i in eachindex(J) - J[i] = J_new[i, i] + @simd for i in eachindex(J) + @inbounds J[i] = J_new[i, i] end else - @.. broadcast=false J=@view(J_new[diagind(J_new)]) + J .= @view(J_new[diagind(J_new)]) end return J end diff --git a/test/runtests.jl b/test/runtests.jl index bf2a8ecdb..33ca5da7a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,7 @@ const GROUP = lowercase(get(ENV, "GROUP", "All")) const EXTRA_PKGS = Pkg.PackageSpec[] (GROUP == "all" || GROUP == "downstream") && push!(EXTRA_PKGS, Pkg.PackageSpec("ModelingToolkit")) -Pkg.add(EXTRA_PKGS) +length(EXTRA_PKGS) ≥ 1 && Pkg.add(EXTRA_PKGS) const RETESTITEMS_NWORKERS = parse( Int, get(ENV, "RETESTITEMS_NWORKERS", string(min(Hwloc.num_physical_cores(), 4)))) @@ -19,4 +19,4 @@ const RETESTITEMS_NWORKER_THREADS = parse(Int, ReTestItems.runtests(NonlinearSolve; tags = (GROUP == "all" ? nothing : [Symbol(GROUP)]), nworkers = RETESTITEMS_NWORKERS, nworker_threads = RETESTITEMS_NWORKER_THREADS, - testitem_timeout = 3600, retries = 4) + testitem_timeout = 3600) From 88140cbdd68f21211eeacf1cdcf32402f2264f33 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 16:48:58 -0400 Subject: [PATCH 79/92] fix: `misc` test group --- src/NonlinearSolve.jl | 13 +++---------- test/misc/bruss_tests.jl | 5 ----- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 15e1ee3a9..903a1f33d 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -17,11 +17,9 @@ using LazyArrays: LazyArrays, ApplyArray, cache using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Symmetric, UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril, istriu, lu, mul!, norm, pinv, tril!, triu! -using LineSearch: LineSearch, AbstractLineSearchAlgorithm, AbstractLineSearchCache, - NoLineSearch, RobustNonMonotoneLineSearch, BackTracking, LineSearchesJL, - LiFukushimaLineSearch -using LinearSolve: LinearSolve, LUFactorization, QRFactorization, - needs_concrete_A, AbstractFactorization, +using LineSearch: LineSearch, AbstractLineSearchCache, LineSearchesJL, NoLineSearch, + RobustNonMonotoneLineSearch, BackTracking, LiFukushimaLineSearch +using LinearSolve: LinearSolve, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver using MaybeInplace: @bb using Printf: @printf @@ -110,10 +108,8 @@ include("default.jl") NewtonRaphson(), TrustRegion(), LevenbergMarquardt(), - # PseudoTransient(), Broyden(), Klement(), - # DFSane(), nothing ) @@ -139,9 +135,6 @@ include("default.jl") LevenbergMarquardt(), GaussNewton(), TrustRegion(), - # LevenbergMarquardt(; linsolve = LUFactorization()), - # GaussNewton(; linsolve = LUFactorization()), - # TrustRegion(; linsolve = LUFactorization()), nothing ) diff --git a/test/misc/bruss_tests.jl b/test/misc/bruss_tests.jl index 80b1bd540..32a3fff68 100644 --- a/test/misc/bruss_tests.jl +++ b/test/misc/bruss_tests.jl @@ -52,11 +52,6 @@ sol = solve(prob_brusselator_2d_sparse, NewtonRaphson(); abstol = 1e-8) @test norm(sol.resid, Inf) < 1e-8 - # Deprecated - sol = solve(prob_brusselator_2d, - NewtonRaphson(autodiff = AutoSparse(AutoFiniteDiff())); abstol = 1e-8) - @test norm(sol.resid, Inf) < 1e-8 - f! = (du, u) -> brusselator_2d_loop(du, u, p) du0 = similar(u0) jac_prototype = ADTypes.jacobian_sparsity(f!, du0, u0, TracerSparsityDetector()) From ca9c835da9f64f74236c86be519615162b37b184 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 17:14:12 -0400 Subject: [PATCH 80/92] fix: forwarddiff support --- .../ext/NonlinearSolveBaseForwardDiffExt.jl | 12 ++++++------ lib/NonlinearSolveBase/src/public.jl | 2 ++ src/NonlinearSolve.jl | 3 +++ src/internal/forward_diff.jl | 17 +++++++---------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl index e50be6c47..c4f1dc901 100644 --- a/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl +++ b/lib/NonlinearSolveBase/ext/NonlinearSolveBaseForwardDiffExt.jl @@ -37,8 +37,8 @@ function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( sol = solve(newprob, alg, args...; kwargs...) uu = sol.u - Jₚ = nonlinearsolve_∂f_∂p(prob, prob.f, uu, p) - Jᵤ = nonlinearsolve_∂f_∂u(prob, prob.f, uu, p) + Jₚ = NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, prob.f, uu, p) + Jᵤ = NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, prob.f, uu, p) z = -Jᵤ \ Jₚ pp = prob.p sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z) @@ -123,8 +123,8 @@ function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( end end - Jₚ = nonlinearsolve_∂f_∂p(prob, vjp_fn, uu, newprob.p) - Jᵤ = nonlinearsolve_∂f_∂u(prob, vjp_fn, uu, newprob.p) + Jₚ = NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, vjp_fn, uu, newprob.p) + Jᵤ = NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, vjp_fn, uu, newprob.p) z = -Jᵤ \ Jₚ pp = prob.p sumfun = ((z, p),) -> map(Base.Fix2(*, ForwardDiff.partials(p)), z) @@ -140,7 +140,7 @@ function NonlinearSolveBase.nonlinearsolve_forwarddiff_solve( return sol, partials end -function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} +function NonlinearSolveBase.nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} if SciMLBase.isinplace(prob) f2 = @closure p -> begin du = Utils.safe_similar(u, promote_type(eltype(u), eltype(p))) @@ -159,7 +159,7 @@ function nonlinearsolve_∂f_∂p(prob, f::F, u, p) where {F} end end -function nonlinearsolve_∂f_∂u(prob, f::F, u, p) where {F} +function NonlinearSolveBase.nonlinearsolve_∂f_∂u(prob, f::F, u, p) where {F} if SciMLBase.isinplace(prob) return ForwardDiff.jacobian( @closure((du, u)->f(du, u, p)), Utils.safe_similar(u), u) diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl index d9014d71e..eceea6d75 100644 --- a/lib/NonlinearSolveBase/src/public.jl +++ b/lib/NonlinearSolveBase/src/public.jl @@ -8,6 +8,8 @@ function get_tolerance end # Forward declarations of functions for forward mode AD function nonlinearsolve_forwarddiff_solve end function nonlinearsolve_dual_solution end +function nonlinearsolve_∂f_∂p end +function nonlinearsolve_∂f_∂u end # Nonlinear Solve Termination Conditions abstract type AbstractNonlinearTerminationMode end diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 903a1f33d..853cbdf19 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -22,6 +22,9 @@ using LineSearch: LineSearch, AbstractLineSearchCache, LineSearchesJL, NoLineSea using LinearSolve: LinearSolve, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver using MaybeInplace: @bb +using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve, + nonlinearsolve_dual_solution, nonlinearsolve_∂f_∂p, + nonlinearsolve_∂f_∂u using Printf: @printf using Preferences: Preferences, @load_preference, @set_preferences! using RecursiveArrayTools: recursivecopy! diff --git a/src/internal/forward_diff.jl b/src/internal/forward_diff.jl index 190c80645..a4238674e 100644 --- a/src/internal/forward_diff.jl +++ b/src/internal/forward_diff.jl @@ -1,15 +1,12 @@ -# Not part of public API but helps reduce code duplication -import SimpleNonlinearSolve: __nlsolve_ad, __nlsolve_dual_soln, __nlsolve_∂f_∂p, - __nlsolve_∂f_∂u - +# XXX: dispatch on `__solve` & `__init` function SciMLBase.solve( prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip, <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}}, alg::Union{Nothing, AbstractNonlinearAlgorithm}, args...; kwargs...) where {T, V, P, iip} - sol, partials = __nlsolve_ad(prob, alg, args...; kwargs...) - dual_soln = __nlsolve_dual_soln(sol.u, partials, prob.p) + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) return SciMLBase.build_solution( prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) end @@ -53,10 +50,10 @@ function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache) prob = cache.prob uu = sol.u - f_p = __nlsolve_∂f_∂p(prob, prob.f, uu, cache.values_p) - f_x = __nlsolve_∂f_∂u(prob, prob.f, uu, cache.values_p) + Jₚ = nonlinearsolve_∂f_∂p(prob, prob.f, uu, cache.values_p) + Jᵤ = nonlinearsolve_∂f_∂u(prob, prob.f, uu, cache.values_p) - z_arr = -f_x \ f_p + z_arr = -Jᵤ \ Jₚ sumfun = ((z, p),) -> map(zᵢ -> zᵢ * ForwardDiff.partials(p), z) if cache.p isa Number @@ -65,7 +62,7 @@ function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache) partials = sum(sumfun, zip(eachcol(z_arr), cache.p)) end - dual_soln = __nlsolve_dual_soln(sol.u, partials, cache.p) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, cache.p) return SciMLBase.build_solution( prob, cache.alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) end From f6041afb6ba0da1f2b1a3a73fa718e65541ae514 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 17:35:20 -0400 Subject: [PATCH 81/92] fix: remove deprecated APIs --- docs/src/basics/sparsity_detection.md | 19 ---------------- docs/src/basics/termination_condition.md | 15 ------------ test/core/rootfind_tests.jl | 29 ++++++++++++------------ 3 files changed, 15 insertions(+), 48 deletions(-) diff --git a/docs/src/basics/sparsity_detection.md b/docs/src/basics/sparsity_detection.md index de924c398..b7782c437 100644 --- a/docs/src/basics/sparsity_detection.md +++ b/docs/src/basics/sparsity_detection.md @@ -76,22 +76,3 @@ prob = NonlinearProblem( Refer to the documentation of DifferentiationInterface.jl and SparseConnectivityTracer.jl for more information on sparsity detection algorithms. - -## Case III: Sparse AD Type is being Used - -!!! warning - - This is now deprecated. Please use the previous two cases instead. - -If you constructed a Nonlinear Solver with a sparse AD type, for example - -```julia -NewtonRaphson(; autodiff = AutoSparse(AutoForwardDiff())) -# OR -TrustRegion(; autodiff = AutoSparse(AutoZygote())) -``` - -then NonlinearSolve will automatically perform matrix coloring and use sparse -differentiation if none of `sparsity` or `jac_prototype` is provided. We default to using -`TracerSparsityDetector()`. `Case I/II` take precedence for sparsity detection and we -perform sparse AD based on those options if those are provided. diff --git a/docs/src/basics/termination_condition.md b/docs/src/basics/termination_condition.md index a87f157aa..fd01a06ad 100644 --- a/docs/src/basics/termination_condition.md +++ b/docs/src/basics/termination_condition.md @@ -54,18 +54,3 @@ not used as a default anywhere. ```@docs SimpleNonlinearSolveTerminationMode ``` - -### Return Codes (Deprecated) - -These are deprecated and will be removed in a future release. Use the -`use_deprecated_retcodes = Val(false)` option to `SciMLBase.init` to use the new return -`ReturnCode` versions. - -```@docs -DiffEqBase.NonlinearSafeTerminationReturnCode -DiffEqBase.NonlinearSafeTerminationReturnCode.Success -DiffEqBase.NonlinearSafeTerminationReturnCode.Default -DiffEqBase.NonlinearSafeTerminationReturnCode.Failure -DiffEqBase.NonlinearSafeTerminationReturnCode.PatienceTermination -DiffEqBase.NonlinearSafeTerminationReturnCode.ProtectiveTermination -``` diff --git a/test/core/rootfind_tests.jl b/test/core/rootfind_tests.jl index 36c3e335b..e3f92fd26 100644 --- a/test/core/rootfind_tests.jl +++ b/test/core/rootfind_tests.jl @@ -1,7 +1,8 @@ @testsetup module CoreRootfindTesting using Reexport @reexport using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random, - LinearAlgebra, ForwardDiff, Zygote, Enzyme, DiffEqBase + LinearAlgebra, ForwardDiff, Zygote, Enzyme, DiffEqBase, + SparseConnectivityTracer using LineSearches: LineSearches _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x)) @@ -116,12 +117,12 @@ end @test nlprob_iterator_interface(quadratic_f, p, Val(false), NewtonRaphson()) ≈ sqrt.(p) @test nlprob_iterator_interface(quadratic_f!, p, Val(true), NewtonRaphson()) ≈ sqrt.(p) - @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in ( - AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()), - AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())), + @testset "Sparsity ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in ( + AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()), u0 in (1.0, [1.0, 1.0]) - probN = NonlinearProblem(quadratic_f, u0, 2.0) + probN = NonlinearProblem( + NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0) @test all(solve(probN, NewtonRaphson(; autodiff)).u .≈ sqrt(2.0)) end @@ -180,12 +181,12 @@ end @test nlprob_iterator_interface(quadratic_f!, p, Val(true), TrustRegion()) ≈ sqrt.(p) @testset "$(_nameof(autodiff)) u0: $(_nameof(u0)) $(radius_update_scheme)" for autodiff in ( - AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()), - AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())), + AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()), u0 in (1.0, [1.0, 1.0]), radius_update_scheme in radius_update_schemes - probN = NonlinearProblem(quadratic_f, u0, 2.0) + probN = NonlinearProblem( + NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0) @test all(solve(probN, TrustRegion(; autodiff, radius_update_scheme)).u .≈ sqrt(2.0)) end @@ -276,11 +277,11 @@ end end @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in ( - AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()), - AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())), + AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()), u0 in (1.0, [1.0, 1.0]) - probN = NonlinearProblem(quadratic_f, u0, 2.0) + probN = NonlinearProblem( + NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0) @test all(solve( probN, LevenbergMarquardt(; autodiff); abstol = 1e-9, reltol = 1e-9).u .≈ sqrt(2.0)) @@ -458,11 +459,11 @@ end quadratic_f!, p, Val(true), PseudoTransient(; alpha_initial = 10.0)) ≈ sqrt.(p) @testset "ADType: $(autodiff) u0: $(_nameof(u0))" for autodiff in ( - AutoSparse(AutoForwardDiff()), AutoSparse(AutoFiniteDiff()), - AutoZygote(), AutoSparse(AutoZygote()), AutoSparse(AutoEnzyme())), + AutoForwardDiff(), AutoFiniteDiff(), AutoZygote(), AutoEnzyme()), u0 in (1.0, [1.0, 1.0]) - probN = NonlinearProblem(quadratic_f, u0, 2.0) + probN = NonlinearProblem( + NonlinearFunction(quadratic_f; sparsity = TracerSparsityDetector()), u0, 2.0) @test all(solve(probN, PseudoTransient(; alpha_initial = 10.0, autodiff)).u .≈ sqrt(2.0)) end From 0b91de727e87355a42c28c21d078781790e3ca04 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 20:01:16 -0400 Subject: [PATCH 82/92] refactor: use functionality from `NonlinearSolveBase` instead of `DiffEqBase` --- Project.toml | 2 +- docs/src/basics/solve.md | 2 +- docs/src/basics/termination_condition.md | 25 +++------------- ...NonlinearSolveFastLevenbergMarquardtExt.jl | 5 ++-- ...NonlinearSolveFixedPointAccelerationExt.jl | 3 +- ext/NonlinearSolveLeastSquaresOptimExt.jl | 5 ++-- ext/NonlinearSolveMINPACKExt.jl | 3 +- ext/NonlinearSolveNLSolversExt.jl | 5 ++-- ext/NonlinearSolveNLsolveExt.jl | 3 +- ext/NonlinearSolveSIAMFANLEquationsExt.jl | 5 ++-- ext/NonlinearSolveSpeedMappingExt.jl | 3 +- lib/NonlinearSolveBase/src/utils.jl | 2 +- src/NonlinearSolve.jl | 19 ++++-------- src/abstract_types.jl | 15 ++++------ src/algorithms/broyden.jl | 2 +- src/algorithms/lbroyden.jl | 2 +- src/algorithms/levenberg_marquardt.jl | 2 +- src/algorithms/pseudo_transient.jl | 2 +- src/core/approximate_jacobian.jl | 6 ++-- src/core/generalized_first_order.jl | 6 ++-- src/core/spectral_methods.jl | 4 +-- src/default.jl | 4 +-- src/descent/dogleg.jl | 2 +- src/descent/geodesic_acceleration.jl | 2 +- src/globalization/trust_region.jl | 4 +-- src/internal/approximate_initialization.jl | 8 ++--- src/internal/termination.jl | 30 ++----------------- src/utils.jl | 16 ++-------- 28 files changed, 65 insertions(+), 122 deletions(-) diff --git a/Project.toml b/Project.toml index 572dbb6fe..8841eca18 100644 --- a/Project.toml +++ b/Project.toml @@ -64,7 +64,7 @@ BandedMatrices = "1.5" BenchmarkTools = "1.4" CUDA = "5.5" ConcreteStructs = "0.2.3" -DiffEqBase = "6.155.3" +DiffEqBase = "6.158.3" DifferentiationInterface = "0.6.1" Enzyme = "0.13.2" ExplicitImports = "1.5" diff --git a/docs/src/basics/solve.md b/docs/src/basics/solve.md index 8ceeaa5de..f7c238966 100644 --- a/docs/src/basics/solve.md +++ b/docs/src/basics/solve.md @@ -21,7 +21,7 @@ solve(prob::SciMLBase.NonlinearProblem, args...; kwargs...) `real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)`. - `reltol::Number`: The relative tolerance. Defaults to `real(oneunit(T)) * (eps(real(one(T))))^(4 // 5)`. - - `termination_condition`: Termination Condition from DiffEqBase. Defaults to + - `termination_condition`: Termination Condition from NonlinearSolveBase. Defaults to `AbsSafeBestTerminationMode()` for `NonlinearSolve.jl` and `AbsTerminateMode()` for `SimpleNonlinearSolve.jl`. diff --git a/docs/src/basics/termination_condition.md b/docs/src/basics/termination_condition.md index fd01a06ad..9a7c718ec 100644 --- a/docs/src/basics/termination_condition.md +++ b/docs/src/basics/termination_condition.md @@ -14,9 +14,6 @@ cache = init(du, u, AbsSafeBestTerminationMode(); abstol = 1e-9, reltol = 1e-9) If `abstol` and `reltol` are not supplied, then we choose a default based on the element types of `du` and `u`. -We can query the `cache` using `DiffEqBase.get_termination_mode`, `DiffEqBase.get_abstol` -and `DiffEqBase.get_reltol`. - To test for termination simply call the `cache`: ```julia @@ -28,8 +25,8 @@ terminated = cache(du, u, uprev) ```@docs AbsTerminationMode AbsNormTerminationMode -AbsSafeTerminationMode -AbsSafeBestTerminationMode +AbsNormSafeTerminationMode +AbsNormSafeBestTerminationMode ``` ### Relative Tolerance @@ -37,20 +34,6 @@ AbsSafeBestTerminationMode ```@docs RelTerminationMode RelNormTerminationMode -RelSafeTerminationMode -RelSafeBestTerminationMode -``` - -### Both Absolute and Relative Tolerance - -```@docs -NormTerminationMode -SteadyStateDiffEqTerminationMode -``` - -The following was named to match an older version of SimpleNonlinearSolve. It is currently -not used as a default anywhere. - -```@docs -SimpleNonlinearSolveTerminationMode +RelNormSafeTerminationMode +RelNormSafeBestTerminationMode ``` diff --git a/ext/NonlinearSolveFastLevenbergMarquardtExt.jl b/ext/NonlinearSolveFastLevenbergMarquardtExt.jl index 262f811a7..e772043b3 100644 --- a/ext/NonlinearSolveFastLevenbergMarquardtExt.jl +++ b/ext/NonlinearSolveFastLevenbergMarquardtExt.jl @@ -3,6 +3,7 @@ module NonlinearSolveFastLevenbergMarquardtExt using ArrayInterface: ArrayInterface using FastClosures: @closure using FastLevenbergMarquardt: FastLevenbergMarquardt +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, FastLevenbergMarquardtJL using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode using StaticArraysCore: SArray @@ -33,8 +34,8 @@ function SciMLBase.__solve(prob::Union{NonlinearLeastSquaresProblem, NonlinearPr else @closure (du, u, p) -> fn(du, u) end - abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u)) - reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(u)) + abstol = get_tolerance(abstol, eltype(u)) + reltol = get_tolerance(reltol, eltype(u)) _jac_fn = NonlinearSolve.__construct_extension_jac( prob, alg, u, resid; alg.autodiff, can_handle_oop = Val(prob.u0 isa SArray)) diff --git a/ext/NonlinearSolveFixedPointAccelerationExt.jl b/ext/NonlinearSolveFixedPointAccelerationExt.jl index 6e26e5351..2d36d7f56 100644 --- a/ext/NonlinearSolveFixedPointAccelerationExt.jl +++ b/ext/NonlinearSolveFixedPointAccelerationExt.jl @@ -1,5 +1,6 @@ module NonlinearSolveFixedPointAccelerationExt +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, FixedPointAccelerationJL using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode using FixedPointAcceleration: FixedPointAcceleration, fixed_point @@ -13,7 +14,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::FixedPointAccelerationJL f, u0, resid = NonlinearSolve.__construct_extension_f( prob; alias_u0, make_fixed_point = Val(true), force_oop = Val(true)) - tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0)) + tol = get_tolerance(abstol, eltype(u0)) sol = fixed_point(f, u0; Algorithm = alg.algorithm, MaxIter = maxiters, MaxM = alg.m, ConvergenceMetricThreshold = tol, ExtrapolationPeriod = alg.extrapolation_period, diff --git a/ext/NonlinearSolveLeastSquaresOptimExt.jl b/ext/NonlinearSolveLeastSquaresOptimExt.jl index 6abe13a9c..20dac092d 100644 --- a/ext/NonlinearSolveLeastSquaresOptimExt.jl +++ b/ext/NonlinearSolveLeastSquaresOptimExt.jl @@ -2,6 +2,7 @@ module NonlinearSolveLeastSquaresOptimExt using ConcreteStructs: @concrete using LeastSquaresOptim: LeastSquaresOptim +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, LeastSquaresOptimJL, TraceMinimal using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode @@ -42,8 +43,8 @@ function SciMLBase.__init(prob::Union{NonlinearLeastSquaresProblem, NonlinearPro NonlinearSolve.__test_termination_condition(termination_condition, :LeastSquaresOptim) f!, u, resid = NonlinearSolve.__construct_extension_f(prob; alias_u0) - abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u)) - reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(u)) + abstol = get_tolerance(abstol, eltype(u)) + reltol = get_tolerance(reltol, eltype(u)) if prob.f.jac === nothing && alg.autodiff isa Symbol lsoprob = LSO.LeastSquaresProblem(; x = u, f!, y = resid, alg.autodiff, diff --git a/ext/NonlinearSolveMINPACKExt.jl b/ext/NonlinearSolveMINPACKExt.jl index 8299b0b45..3761a817f 100644 --- a/ext/NonlinearSolveMINPACKExt.jl +++ b/ext/NonlinearSolveMINPACKExt.jl @@ -1,6 +1,7 @@ module NonlinearSolveMINPACKExt using MINPACK: MINPACK +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, CMINPACK using SciMLBase: SciMLBase, NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode using FastClosures: @closure @@ -21,7 +22,7 @@ function SciMLBase.__solve( show_trace = ShT tracing = StT - tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0)) + tol = get_tolerance(abstol, eltype(u0)) if alg.autodiff === missing && prob.f.jac === nothing original = MINPACK.fsolve( diff --git a/ext/NonlinearSolveNLSolversExt.jl b/ext/NonlinearSolveNLSolversExt.jl index e78dab947..a9f41c87c 100644 --- a/ext/NonlinearSolveNLSolversExt.jl +++ b/ext/NonlinearSolveNLSolversExt.jl @@ -6,6 +6,7 @@ using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff using LinearAlgebra: norm using NLSolvers: NLSolvers, NEqOptions, NEqProblem +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, NLSolversJL using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode @@ -14,8 +15,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::NLSolversJL, args...; alias_u0::Bool = false, termination_condition = nothing, kwargs...) NonlinearSolve.__test_termination_condition(termination_condition, :NLSolversJL) - abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(prob.u0)) - reltol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, eltype(prob.u0)) + abstol = get_tolerance(abstol, eltype(prob.u0)) + reltol = get_tolerance(reltol, eltype(prob.u0)) options = NEqOptions(; maxiter = maxiters, f_abstol = abstol, f_reltol = reltol) diff --git a/ext/NonlinearSolveNLsolveExt.jl b/ext/NonlinearSolveNLsolveExt.jl index 9872c7953..301ed7137 100644 --- a/ext/NonlinearSolveNLsolveExt.jl +++ b/ext/NonlinearSolveNLsolveExt.jl @@ -1,6 +1,7 @@ module NonlinearSolveNLsolveExt using LineSearches: Static +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, NLsolveJL, TraceMinimal using NLsolve: NLsolve, OnceDifferentiable, nlsolve using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode @@ -27,7 +28,7 @@ function SciMLBase.__solve( df = OnceDifferentiable(f!, jac!, vec(u0), vec(resid), J) end - abstol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u0)) + abstol = get_tolerance(abstol, eltype(u0)) show_trace = ShT store_trace = StT extended_trace = !(trace_level isa TraceMinimal) diff --git a/ext/NonlinearSolveSIAMFANLEquationsExt.jl b/ext/NonlinearSolveSIAMFANLEquationsExt.jl index 6ec3e8393..65154b63a 100644 --- a/ext/NonlinearSolveSIAMFANLEquationsExt.jl +++ b/ext/NonlinearSolveSIAMFANLEquationsExt.jl @@ -1,6 +1,7 @@ module NonlinearSolveSIAMFANLEquationsExt using FastClosures: @closure +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, SIAMFANLEquationsJL using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode using SIAMFANLEquations: SIAMFANLEquations, aasol, nsol, nsoli, nsolsc, ptcsol, ptcsoli, @@ -40,8 +41,8 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SIAMFANLEquationsJL, arg (; method, delta, linsolve, m, beta) = alg T = eltype(prob.u0) - atol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, T) - rtol = NonlinearSolve.DEFAULT_TOLERANCE(reltol, T) + atol = get_tolerance(abstol, T) + rtol = get_tolerance(reltol, T) if prob.u0 isa Number f = @closure u -> prob.f(u, prob.p) diff --git a/ext/NonlinearSolveSpeedMappingExt.jl b/ext/NonlinearSolveSpeedMappingExt.jl index b39394a3b..ff9b4683b 100644 --- a/ext/NonlinearSolveSpeedMappingExt.jl +++ b/ext/NonlinearSolveSpeedMappingExt.jl @@ -1,5 +1,6 @@ module NonlinearSolveSpeedMappingExt +using NonlinearSolveBase: NonlinearSolveBase, get_tolerance using NonlinearSolve: NonlinearSolve, SpeedMappingJL using SciMLBase: SciMLBase, NonlinearProblem, ReturnCode using SpeedMapping: speedmapping @@ -12,7 +13,7 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SpeedMappingJL, args...; m!, u, resid = NonlinearSolve.__construct_extension_f( prob; alias_u0, make_fixed_point = Val(true)) - tol = NonlinearSolve.DEFAULT_TOLERANCE(abstol, eltype(u)) + tol = get_tolerance(abstol, eltype(u)) time_limit = ifelse(maxtime === nothing, 1000, maxtime) diff --git a/lib/NonlinearSolveBase/src/utils.jl b/lib/NonlinearSolveBase/src/utils.jl index cb54a6f4c..825f63767 100644 --- a/lib/NonlinearSolveBase/src/utils.jl +++ b/lib/NonlinearSolveBase/src/utils.jl @@ -54,7 +54,7 @@ standardize_norm(f::F) where {F} = f norm_op(norm::N, op::OP, x, y) where {N, OP} = norm(op.(x, y)) function norm_op(::typeof(L2_NORM), op::OP, x, y) where {OP} if fast_scalar_indexing(x, y) - return sqrt(sum(@closure((xᵢ, yᵢ)->begin + return sqrt(sum(@closure((xᵢyᵢ)->begin xᵢ, yᵢ = xᵢyᵢ return op(xᵢ, yᵢ)^2 end), zip(x, y))) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 853cbdf19..fd2c0bc3d 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -6,12 +6,7 @@ using PrecompileTools: @compile_workload, @setup_workload using ArrayInterface: ArrayInterface, can_setindex, restructure, fast_scalar_indexing, ismutable using ConcreteStructs: @concrete -using DiffEqBase: DiffEqBase, AbstractNonlinearTerminationMode, - AbstractSafeBestNonlinearTerminationMode, AbsNormTerminationMode, - AbsSafeBestTerminationMode, AbsSafeTerminationMode, AbsTerminationMode, - NormTerminationMode, RelNormTerminationMode, RelSafeBestTerminationMode, - RelSafeTerminationMode, RelTerminationMode, - SimpleNonlinearSolveTerminationMode, SteadyStateDiffEqTerminationMode +using DiffEqBase: DiffEqBase # Needed for `init` / `solve` dispatches using FastClosures: @closure using LazyArrays: LazyArrays, ApplyArray, cache using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Symmetric, @@ -24,7 +19,9 @@ using LinearSolve: LinearSolve, QRFactorization, needs_concrete_A, AbstractFacto using MaybeInplace: @bb using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution, nonlinearsolve_∂f_∂p, - nonlinearsolve_∂f_∂u + nonlinearsolve_∂f_∂u, L2_NORM, AbstractNonlinearTerminationMode, + AbstractSafeNonlinearTerminationMode, + AbstractSafeBestNonlinearTerminationMode using Printf: @printf using Preferences: Preferences, @load_preference, @set_preferences! using RecursiveArrayTools: recursivecopy! @@ -51,7 +48,7 @@ using SparseArrays: AbstractSparseMatrix, SparseMatrixCSC using SparseMatrixColorings: ConstantColoringAlgorithm, GreedyColoringAlgorithm, LargestFirst -@reexport using SciMLBase, SimpleNonlinearSolve +@reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase const DI = DifferentiationInterface @@ -182,12 +179,6 @@ export LineSearch, BackTracking, NoLineSearch, RobustNonMonotoneLineSearch, ## Trust Region Algorithms export RadiusUpdateSchemes -# Export the termination conditions from NonlinearSolveBase -export SteadyStateDiffEqTerminationMode, SimpleNonlinearSolveTerminationMode, - NormTerminationMode, RelTerminationMode, RelNormTerminationMode, AbsTerminationMode, - AbsNormTerminationMode, RelSafeTerminationMode, AbsSafeTerminationMode, - RelSafeBestTerminationMode, AbsSafeBestTerminationMode - # Tracing Functionality export TraceAll, TraceMinimal, TraceWithJacobianConditionNumber diff --git a/src/abstract_types.jl b/src/abstract_types.jl index 0bb2e13d2..415dc5797 100644 --- a/src/abstract_types.jl +++ b/src/abstract_types.jl @@ -263,8 +263,7 @@ Abstract Type for Damping Functions in DampedNewton. ```julia __internal_init( prob::AbstractNonlinearProblem, f::AbstractDampingFunction, initial_damping, - J, fu, u, args...; internal_norm = DEFAULT_NORM, kwargs...) --> -AbstractDampingFunctionCache + J, fu, u, args...; internal_norm = L2_NORM, kwargs...) --> AbstractDampingFunctionCache ``` Returns a [`AbstractDampingFunctionCache`](@ref). @@ -348,7 +347,7 @@ Abstract Type for all Jacobian Initialization Algorithms used in NonlinearSolve. ```julia __internal_init( prob::AbstractNonlinearProblem, alg::AbstractJacobianInitialization, solver, - f::F, fu, u, p; linsolve = missing, internalnorm::IN = DEFAULT_NORM, kwargs...) + f::F, fu, u, p; linsolve = missing, internalnorm::IN = L2_NORM, kwargs...) ``` Returns a [`NonlinearSolve.InitializedApproximateJacobianCache`](@ref). @@ -382,9 +381,8 @@ Abstract Type for all Approximate Jacobian Update Rules used in NonlinearSolve.j ```julia __internal_init( - prob::AbstractNonlinearProblem, alg::AbstractApproximateJacobianUpdateRule, J, - fu, u, du, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F} --> -AbstractApproximateJacobianUpdateRuleCache{INV} + prob::AbstractNonlinearProblem, alg::AbstractApproximateJacobianUpdateRule, J, fu, u, + du, args...; internalnorm::F = L2_NORM, kwargs...) where {F} --> AbstractApproximateJacobianUpdateRuleCache{INV} ``` """ abstract type AbstractApproximateJacobianUpdateRule{INV} end @@ -440,9 +438,8 @@ Abstract Type for all Trust Region Methods used in NonlinearSolve.jl. ```julia __internal_init( - prob::AbstractNonlinearProblem, alg::AbstractTrustRegionMethod, f::F, fu, u, - p, args...; internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF} --> -AbstractTrustRegionMethodCache + prob::AbstractNonlinearProblem, alg::AbstractTrustRegionMethod, f::F, fu, u, p, args...; + internalnorm::IF = L2_NORM, kwargs...) where {F, IF} --> AbstractTrustRegionMethodCache ``` """ abstract type AbstractTrustRegionMethod end diff --git a/src/algorithms/broyden.jl b/src/algorithms/broyden.jl index 77d2a06c4..4b4b3df9d 100644 --- a/src/algorithms/broyden.jl +++ b/src/algorithms/broyden.jl @@ -171,7 +171,7 @@ end function __internal_init(prob::AbstractNonlinearProblem, alg::Union{GoodBroydenUpdateRule, BadBroydenUpdateRule}, J, fu, u, - du, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F} + du, args...; internalnorm::F = L2_NORM, kwargs...) where {F} @bb J⁻¹dfu = similar(u) @bb dfu = copy(fu) if alg isa GoodBroydenUpdateRule || J isa Diagonal diff --git a/src/algorithms/lbroyden.jl b/src/algorithms/lbroyden.jl index bead1a057..8fb0db748 100644 --- a/src/algorithms/lbroyden.jl +++ b/src/algorithms/lbroyden.jl @@ -44,7 +44,7 @@ jacobian_initialized_preinverted(::BroydenLowRankInitialization) = true function __internal_init( prob::AbstractNonlinearProblem, alg::BroydenLowRankInitialization{T}, solver, f::F, fu, u, p; maxiters = 1000, - internalnorm::IN = DEFAULT_NORM, kwargs...) where {T, F, IN} + internalnorm::IN = L2_NORM, kwargs...) where {T, F, IN} if u isa Number # Use the standard broyden return __internal_init(prob, IdentityInitialization(true, FullStructure()), solver, f, fu, u, p; maxiters, kwargs...) diff --git a/src/algorithms/levenberg_marquardt.jl b/src/algorithms/levenberg_marquardt.jl index 501a5dd29..2b7c08f95 100644 --- a/src/algorithms/levenberg_marquardt.jl +++ b/src/algorithms/levenberg_marquardt.jl @@ -98,7 +98,7 @@ end function __internal_init( prob::AbstractNonlinearProblem, f::LevenbergMarquardtDampingFunction, initial_damping, J, fu, u, ::Val{NF}; - internalnorm::F = DEFAULT_NORM, kwargs...) where {F, NF} + internalnorm::F = L2_NORM, kwargs...) where {F, NF} T = promote_type(eltype(u), eltype(fu)) DᵀD = __init_diagonal(u, T(f.min_damping)) if NF diff --git a/src/algorithms/pseudo_transient.jl b/src/algorithms/pseudo_transient.jl index 0da85dd94..1266d0702 100644 --- a/src/algorithms/pseudo_transient.jl +++ b/src/algorithms/pseudo_transient.jl @@ -53,7 +53,7 @@ end function __internal_init( prob::AbstractNonlinearProblem, f::SwitchedEvolutionRelaxation, initial_damping, - J, fu, u, args...; internalnorm::F = DEFAULT_NORM, kwargs...) where {F} + J, fu, u, args...; internalnorm::F = L2_NORM, kwargs...) where {F} T = promote_type(eltype(u), eltype(fu)) return SwitchedEvolutionRelaxationCache( internalnorm(fu), T(1 / initial_damping), internalnorm) diff --git a/src/core/approximate_jacobian.jl b/src/core/approximate_jacobian.jl index 2e0c64a82..3522a39f4 100644 --- a/src/core/approximate_jacobian.jl +++ b/src/core/approximate_jacobian.jl @@ -148,7 +148,7 @@ function SciMLBase.__init( args...; stats = empty_nlstats(), alias_u0 = false, maxtime = nothing, maxiters = 1000, abstol = nothing, reltol = nothing, linsolve_kwargs = (;), termination_condition = nothing, - internalnorm::F = DEFAULT_NORM, kwargs...) where {uType, iip, F} + internalnorm::F = L2_NORM, kwargs...) where {uType, iip, F} timer = get_timer_output() @static_timeit timer "cache construction" begin (; f, u0, p) = prob @@ -162,8 +162,8 @@ function SciMLBase.__init( initialization_cache = __internal_init(prob, alg.initialization, alg, f, fu, u, p; stats, linsolve, maxiters, internalnorm) - abstol, reltol, termination_cache = init_termination_cache( - prob, abstol, reltol, fu, u, termination_condition) + abstol, reltol, termination_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fu, u, termination_condition, Val(:regular)) linsolve_kwargs = merge((; abstol, reltol), linsolve_kwargs) J = initialization_cache(nothing) diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl index ebdaf1fc8..13980c154 100644 --- a/src/core/generalized_first_order.jl +++ b/src/core/generalized_first_order.jl @@ -150,7 +150,7 @@ function SciMLBase.__init( prob::AbstractNonlinearProblem{uType, iip}, alg::GeneralizedFirstOrderAlgorithm, args...; stats = empty_nlstats(), alias_u0 = false, maxiters = 1000, abstol = nothing, reltol = nothing, maxtime = nothing, - termination_condition = nothing, internalnorm = DEFAULT_NORM, + termination_condition = nothing, internalnorm = L2_NORM, linsolve_kwargs = (;), kwargs...) where {uType, iip} timer = get_timer_output() @static_timeit timer "cache construction" begin @@ -161,8 +161,8 @@ function SciMLBase.__init( linsolve = get_linear_solver(alg.descent) - abstol, reltol, termination_cache = init_termination_cache( - prob, abstol, reltol, fu, u, termination_condition) + abstol, reltol, termination_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fu, u, termination_condition, Val(:regular)) linsolve_kwargs = merge((; abstol, reltol), linsolve_kwargs) jac_cache = JacobianCache( diff --git a/src/core/spectral_methods.jl b/src/core/spectral_methods.jl index d141a627b..31e988b70 100644 --- a/src/core/spectral_methods.jl +++ b/src/core/spectral_methods.jl @@ -131,8 +131,8 @@ function SciMLBase.__init(prob::AbstractNonlinearProblem, alg::GeneralizedDFSane linesearch_cache = init(prob, alg.linesearch, fu, u; stats, kwargs...) - abstol, reltol, tc_cache = init_termination_cache( - prob, abstol, reltol, fu, u_cache, termination_condition) + abstol, reltol, tc_cache = NonlinearSolveBase.init_termination_cache( + prob, abstol, reltol, fu, u_cache, termination_condition, Val(:regular)) trace = init_nonlinearsolve_trace(prob, alg, u, fu, nothing, du; kwargs...) if alg.σ_1 === nothing diff --git a/src/default.jl b/src/default.jl index 967b2e0e8..a7cc550d8 100644 --- a/src/default.jl +++ b/src/default.jl @@ -101,7 +101,7 @@ for (probType, pType) in ((:NonlinearProblem, :NLS), (:NonlinearLeastSquaresProb @eval begin function SciMLBase.__init( prob::$probType, alg::$algType{N}, args...; stats = empty_nlstats(), - maxtime = nothing, maxiters = 1000, internalnorm = DEFAULT_NORM, + maxtime = nothing, maxiters = 1000, internalnorm = L2_NORM, alias_u0 = false, verbose = true, kwargs...) where {N} if (alias_u0 && !ismutable(prob.u0)) verbose && @warn "`alias_u0` has been set to `true`, but `u0` is \ @@ -309,7 +309,7 @@ for (probType, pType) in ((:NonlinearProblem, :NLS), (:NonlinearLeastSquaresProb push!(calls, quote resids = tuple($(Tuple(resids)...)) - minfu, idx = __findmin(DEFAULT_NORM, resids) + minfu, idx = __findmin(L2_NORM, resids) end) for i in 1:N diff --git a/src/descent/dogleg.jl b/src/descent/dogleg.jl index 5d0bb1c7c..4c96c98f6 100644 --- a/src/descent/dogleg.jl +++ b/src/descent/dogleg.jl @@ -49,7 +49,7 @@ end function __internal_init(prob::AbstractNonlinearProblem, alg::Dogleg, J, fu, u; pre_inverted::Val{INV} = False, linsolve_kwargs = (;), - abstol = nothing, reltol = nothing, internalnorm::F = DEFAULT_NORM, + abstol = nothing, reltol = nothing, internalnorm::F = L2_NORM, shared::Val{N} = Val(1), kwargs...) where {F, INV, N} newton_cache = __internal_init(prob, alg.newton_descent, J, fu, u; pre_inverted, linsolve_kwargs, abstol, reltol, shared, kwargs...) diff --git a/src/descent/geodesic_acceleration.jl b/src/descent/geodesic_acceleration.jl index 136795057..8e8a305f0 100644 --- a/src/descent/geodesic_acceleration.jl +++ b/src/descent/geodesic_acceleration.jl @@ -87,7 +87,7 @@ end function __internal_init(prob::AbstractNonlinearProblem, alg::GeodesicAcceleration, J, fu, u; shared::Val{N} = Val(1), pre_inverted::Val{INV} = False, linsolve_kwargs = (;), abstol = nothing, reltol = nothing, - internalnorm::F = DEFAULT_NORM, kwargs...) where {INV, N, F} + internalnorm::F = L2_NORM, kwargs...) where {INV, N, F} T = promote_type(eltype(u), eltype(fu)) @bb δu = similar(u) δus = N ≤ 1 ? nothing : map(2:N) do i diff --git a/src/globalization/trust_region.jl b/src/globalization/trust_region.jl index e6e2cba17..8ff6de905 100644 --- a/src/globalization/trust_region.jl +++ b/src/globalization/trust_region.jl @@ -57,7 +57,7 @@ end function __internal_init( prob::AbstractNonlinearProblem, alg::LevenbergMarquardtTrustRegion, f::F, fu, - u, p, args...; stats, internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF} + u, p, args...; stats, internalnorm::IF = L2_NORM, kwargs...) where {F, IF} T = promote_type(eltype(u), eltype(fu)) @bb v = copy(u) @bb u_cache = similar(u) @@ -367,7 +367,7 @@ end function __internal_init( prob::AbstractNonlinearProblem, alg::GenericTrustRegionScheme, f::F, fu, u, - p, args...; stats, internalnorm::IF = DEFAULT_NORM, kwargs...) where {F, IF} + p, args...; stats, internalnorm::IF = L2_NORM, kwargs...) where {F, IF} T = promote_type(eltype(u), eltype(fu)) u0_norm = internalnorm(u) fu_norm = internalnorm(fu) diff --git a/src/internal/approximate_initialization.jl b/src/internal/approximate_initialization.jl index 985234ac4..72e46a97e 100644 --- a/src/internal/approximate_initialization.jl +++ b/src/internal/approximate_initialization.jl @@ -65,14 +65,14 @@ end function __internal_init( prob::AbstractNonlinearProblem, alg::IdentityInitialization, solver, f::F, - fu, u::Number, p; internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN} + fu, u::Number, p; internalnorm::IN = L2_NORM, kwargs...) where {F, IN} α = __initial_alpha(alg.alpha, u, fu, internalnorm) return InitializedApproximateJacobianCache( α, alg.structure, alg, nothing, true, internalnorm) end function __internal_init(prob::AbstractNonlinearProblem, alg::IdentityInitialization, solver, f::F, fu::StaticArray, u::StaticArray, p; - internalnorm::IN = DEFAULT_NORM, kwargs...) where {IN, F} + internalnorm::IN = L2_NORM, kwargs...) where {IN, F} α = __initial_alpha(alg.alpha, u, fu, internalnorm) if alg.structure isa DiagonalStructure @assert length(u)==length(fu) "Diagonal Jacobian Structure must be square!" @@ -91,7 +91,7 @@ function __internal_init(prob::AbstractNonlinearProblem, alg::IdentityInitializa end function __internal_init( prob::AbstractNonlinearProblem, alg::IdentityInitialization, solver, - f::F, fu, u, p; internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN} + f::F, fu, u, p; internalnorm::IN = L2_NORM, kwargs...) where {F, IN} α = __initial_alpha(alg.alpha, u, fu, internalnorm) if alg.structure isa DiagonalStructure @assert length(u)==length(fu) "Diagonal Jacobian Structure must be square!" @@ -147,7 +147,7 @@ end function __internal_init(prob::AbstractNonlinearProblem, alg::TrueJacobianInitialization, solver, f::F, fu, u, p; stats, linsolve = missing, - internalnorm::IN = DEFAULT_NORM, kwargs...) where {F, IN} + internalnorm::IN = L2_NORM, kwargs...) where {F, IN} autodiff = get_concrete_forward_ad( alg.autodiff, prob; check_forward_mode = false, kwargs...) jac_cache = JacobianCache(prob, solver, prob.f, fu, u, p; stats, autodiff, linsolve) diff --git a/src/internal/termination.jl b/src/internal/termination.jl index ef3f7c4c0..7728aea69 100644 --- a/src/internal/termination.jl +++ b/src/internal/termination.jl @@ -1,34 +1,9 @@ -function init_termination_cache(prob::NonlinearProblem, abstol, reltol, du, u, ::Nothing) - return init_termination_cache(prob, abstol, reltol, du, u, - AbsSafeBestTerminationMode(Base.Fix1(maximum, abs); max_stalled_steps = 32)) -end -function init_termination_cache( - prob::NonlinearLeastSquaresProblem, abstol, reltol, du, u, ::Nothing) - return init_termination_cache(prob, abstol, reltol, du, u, - AbsSafeBestTerminationMode(Base.Fix2(norm, 2); max_stalled_steps = 32)) -end - -function init_termination_cache( - prob::Union{NonlinearProblem, NonlinearLeastSquaresProblem}, - abstol, reltol, du, u, tc::AbstractNonlinearTerminationMode) - tc_ = if hasfield(typeof(tc), :internalnorm) && tc.internalnorm === nothing - internalnorm = ifelse( - prob isa NonlinearProblem, Base.Fix1(maximum, abs), Base.Fix2(norm, 2)) - DiffEqBase.set_termination_mode_internalnorm(tc, internalnorm) - else - tc - end - tc_cache = init(du, u, tc_; abstol, reltol, use_deprecated_retcodes = Val(false)) - return DiffEqBase.get_abstol(tc_cache), DiffEqBase.get_reltol(tc_cache), tc_cache -end - function check_and_update!(cache, fu, u, uprev) return check_and_update!(cache.termination_cache, cache, fu, u, uprev) end function check_and_update!(tc_cache, cache, fu, u, uprev) - return check_and_update!( - tc_cache, cache, fu, u, uprev, DiffEqBase.get_termination_mode(tc_cache)) + return check_and_update!(tc_cache, cache, fu, u, uprev, tc_cache.mode) end function check_and_update!(tc_cache, cache, fu, u, uprev, mode) @@ -40,8 +15,7 @@ function check_and_update!(tc_cache, cache, fu, u, uprev, mode) end function update_from_termination_cache!(tc_cache, cache, u = get_u(cache)) - return update_from_termination_cache!( - tc_cache, cache, DiffEqBase.get_termination_mode(tc_cache), u) + return update_from_termination_cache!(tc_cache, cache, tc_cache.mode, u) end function update_from_termination_cache!( diff --git a/src/utils.jl b/src/utils.jl index 6ceb4c9d8..069fde86e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -1,16 +1,8 @@ # Defaults -@inline DEFAULT_NORM(args...) = DiffEqBase.NONLINEARSOLVE_DEFAULT_NORM(args...) @inline DEFAULT_PRECS(W, du, u, p, t, newW, Plprev, Prprev, cachedata) = nothing, nothing -@inline DEFAULT_TOLERANCE(args...) = DiffEqBase._get_tolerance(args...) # Helper Functions -@static if VERSION ≤ v"1.10-" - @inline @generated function __hasfield(::T, ::Val{field}) where {T, field} - return :($(field ∉ fieldnames(T))) - end -else - @inline __hasfield(::T, ::Val{field}) where {T, field} = hasfield(T, field) -end +@inline __hasfield(::T, ::Val{field}) where {T, field} = hasfield(T, field) @generated function __getproperty(s::S, ::Val{X}) where {S, X} hasfield(S, X) && return :(s.$X) @@ -86,12 +78,10 @@ LazyArrays.applied_axes(::typeof(__zero), x) = axes(x) @inline __is_complex(::Type{T}) where {T} = false @inline __findmin_caches(f::F, caches) where {F} = __findmin(f ∘ get_fu, caches) -# FIXME: DEFAULT_NORM makes an Array of NaNs not a NaN (atleast according to `isnan`) +# FIXME: L2_NORM makes an Array of NaNs not a NaN (atleast according to `isnan`) @generated function __findmin(f::F, x) where {F} # JET shows dynamic dispatch if this is not written as a generated function - if F === typeof(DEFAULT_NORM) - return :(return __findmin_impl(Base.Fix1(maximum, abs), x)) - end + F === typeof(L2_NORM) && return :(return __findmin_impl(Base.Fix1(maximum, abs), x)) return :(return __findmin_impl(f, x)) end @inline @views function __findmin_impl(f::F, x) where {F} From 89d76b06e5efa5f0cdfe05ff2167d4bb2368f53d Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 20:06:35 -0400 Subject: [PATCH 83/92] chore: fix QA testing --- lib/BracketingNonlinearSolve/test/qa_tests.jl | 3 ++- src/NonlinearSolve.jl | 6 +++--- src/abstract_types.jl | 3 ++- test/core/rootfind_tests.jl | 17 ++++++++++++----- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/BracketingNonlinearSolve/test/qa_tests.jl b/lib/BracketingNonlinearSolve/test/qa_tests.jl index de27e33a4..c01c493f4 100644 --- a/lib/BracketingNonlinearSolve/test/qa_tests.jl +++ b/lib/BracketingNonlinearSolve/test/qa_tests.jl @@ -10,7 +10,8 @@ end import ForwardDiff using ExplicitImports, BracketingNonlinearSolve - @test check_no_implicit_imports(BracketingNonlinearSolve; skip = (Base, Core)) === nothing + @test check_no_implicit_imports(BracketingNonlinearSolve; skip = (Base, Core)) === + nothing @test check_no_stale_explicit_imports(BracketingNonlinearSolve) === nothing @test check_all_qualified_accesses_via_owners(BracketingNonlinearSolve) === nothing end diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index fd2c0bc3d..5f21936a1 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -13,14 +13,14 @@ using LinearAlgebra: LinearAlgebra, ColumnNorm, Diagonal, I, LowerTriangular, Sy UpperTriangular, axpy!, cond, diag, diagind, dot, issuccess, istril, istriu, lu, mul!, norm, pinv, tril!, triu! using LineSearch: LineSearch, AbstractLineSearchCache, LineSearchesJL, NoLineSearch, - RobustNonMonotoneLineSearch, BackTracking, LiFukushimaLineSearch + RobustNonMonotoneLineSearch, BackTracking, LiFukushimaLineSearch using LinearSolve: LinearSolve, QRFactorization, needs_concrete_A, AbstractFactorization, DefaultAlgorithmChoice, DefaultLinearSolver using MaybeInplace: @bb using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution, nonlinearsolve_∂f_∂p, - nonlinearsolve_∂f_∂u, L2_NORM, AbstractNonlinearTerminationMode, - AbstractSafeNonlinearTerminationMode, + nonlinearsolve_∂f_∂u, L2_NORM, AbsNormTerminationMode, + AbstractNonlinearTerminationMode, AbstractSafeBestNonlinearTerminationMode using Printf: @printf using Preferences: Preferences, @load_preference, @set_preferences! diff --git a/src/abstract_types.jl b/src/abstract_types.jl index 415dc5797..255c5e541 100644 --- a/src/abstract_types.jl +++ b/src/abstract_types.jl @@ -382,7 +382,8 @@ Abstract Type for all Approximate Jacobian Update Rules used in NonlinearSolve.j ```julia __internal_init( prob::AbstractNonlinearProblem, alg::AbstractApproximateJacobianUpdateRule, J, fu, u, - du, args...; internalnorm::F = L2_NORM, kwargs...) where {F} --> AbstractApproximateJacobianUpdateRuleCache{INV} + du, args...; internalnorm::F = L2_NORM, kwargs...) where {F} --> +AbstractApproximateJacobianUpdateRuleCache{INV} ``` """ abstract type AbstractApproximateJacobianUpdateRule{INV} end diff --git a/test/core/rootfind_tests.jl b/test/core/rootfind_tests.jl index e3f92fd26..2d1662570 100644 --- a/test/core/rootfind_tests.jl +++ b/test/core/rootfind_tests.jl @@ -1,8 +1,8 @@ @testsetup module CoreRootfindTesting using Reexport @reexport using BenchmarkTools, LinearSolve, NonlinearSolve, StaticArrays, Random, - LinearAlgebra, ForwardDiff, Zygote, Enzyme, DiffEqBase, - SparseConnectivityTracer + LinearAlgebra, ForwardDiff, Zygote, Enzyme, SparseConnectivityTracer, + NonlinearSolveBase using LineSearches: LineSearches _nameof(x) = applicable(nameof, x) ? nameof(x) : _nameof(typeof(x)) @@ -22,9 +22,16 @@ function newton_fails(u, p) end const TERMINATION_CONDITIONS = [ - NormTerminationMode(), RelTerminationMode(), RelNormTerminationMode(), - AbsTerminationMode(), AbsNormTerminationMode(), RelSafeTerminationMode(), - AbsSafeTerminationMode(), RelSafeBestTerminationMode(), AbsSafeBestTerminationMode()] + NormTerminationMode(Base.Fix1(maximum, abs)), + RelTerminationMode(), + RelNormTerminationMode(Base.Fix1(maximum, abs)), + RelNormSafeTerminationMode(Base.Fix1(maximum, abs)), + RelNormSafeBestTerminationMode(Base.Fix1(maximum, abs)), + AbsTerminationMode(), + AbsNormTerminationMode(Base.Fix1(maximum, abs)), + AbsNormSafeTerminationMode(Base.Fix1(maximum, abs)), + AbsNormSafeBestTerminationMode(Base.Fix1(maximum, abs)) +] function benchmark_nlsolve_oop(f, u0, p = 2.0; solver, kwargs...) prob = NonlinearProblem{false}(f, u0, p) From f5a06cbcd829a4d66ccb765c40669d69b59f3e88 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 20:34:34 -0400 Subject: [PATCH 84/92] fix: dispatch forwarddiff on `__init` and `__solve` --- src/NonlinearSolve.jl | 3 +- src/internal/forward_diff.jl | 58 ++++++++++++++++++++++-------------- 2 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 5f21936a1..625477a88 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -67,7 +67,6 @@ include("descent/damped_newton.jl") include("descent/geodesic_acceleration.jl") include("internal/jacobian.jl") -include("internal/forward_diff.jl") include("internal/linear_solve.jl") include("internal/termination.jl") include("internal/tracing.jl") @@ -93,6 +92,8 @@ include("algorithms/levenberg_marquardt.jl") include("algorithms/trust_region.jl") include("algorithms/extension_algs.jl") +include("internal/forward_diff.jl") # we need to define after the algorithms + include("utils.jl") include("default.jl") diff --git a/src/internal/forward_diff.jl b/src/internal/forward_diff.jl index a4238674e..c2adc70e2 100644 --- a/src/internal/forward_diff.jl +++ b/src/internal/forward_diff.jl @@ -1,14 +1,24 @@ -# XXX: dispatch on `__solve` & `__init` -function SciMLBase.solve( - prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip, - <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}}, - alg::Union{Nothing, AbstractNonlinearAlgorithm}, - args...; - kwargs...) where {T, V, P, iip} - sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) - dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) - return SciMLBase.build_solution( - prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) +const DualNonlinearProblem = NonlinearProblem{<:Union{Number, <:AbstractArray}, iip, + <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}} where {iip, T, V, P} +const DualNonlinearLeastSquaresProblem = NonlinearLeastSquaresProblem{ + <:Union{Number, <:AbstractArray}, iip, + <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}} where {iip, T, V, P} +const DualAbstractNonlinearProblem = Union{ + DualNonlinearProblem, DualNonlinearLeastSquaresProblem} + +for algType in ( + Nothing, AbstractNonlinearSolveAlgorithm, GeneralizedDFSane, + GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm, + LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL, + SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL +) + @eval function SciMLBase.__solve( + prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...) + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) + return SciMLBase.build_solution( + prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) + end end @concrete mutable struct NonlinearSolveForwardDiffCache @@ -32,17 +42,21 @@ function reinit_cache!(cache::NonlinearSolveForwardDiffCache; return cache end -function SciMLBase.init( - prob::NonlinearProblem{<:Union{Number, <:AbstractArray}, iip, - <:Union{<:Dual{T, V, P}, <:AbstractArray{<:Dual{T, V, P}}}}, - alg::Union{Nothing, AbstractNonlinearAlgorithm}, - args...; - kwargs...) where {T, V, P, iip} - p = __value(prob.p) - newprob = NonlinearProblem(prob.f, __value(prob.u0), p; prob.kwargs...) - cache = init(newprob, alg, args...; kwargs...) - return NonlinearSolveForwardDiffCache( - cache, newprob, alg, prob.p, p, ForwardDiff.partials(prob.p)) +for algType in ( + Nothing, AbstractNonlinearSolveAlgorithm, GeneralizedDFSane, + SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm, + GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm, + LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL, + SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL +) + @eval function SciMLBase.__init( + prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...) + p = __value(prob.p) + newprob = NonlinearProblem(prob.f, __value(prob.u0), p; prob.kwargs...) + cache = init(newprob, alg, args...; kwargs...) + return NonlinearSolveForwardDiffCache( + cache, newprob, alg, prob.p, p, ForwardDiff.partials(prob.p)) + end end function SciMLBase.solve!(cache::NonlinearSolveForwardDiffCache) From 3ded2fa26a61b9dde801106e8ddd04763f7a1fb7 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Mon, 21 Oct 2024 22:30:10 -0400 Subject: [PATCH 85/92] feat: forwarddiff support for sundials --- Project.toml | 2 ++ ext/NonlinearSolveSundialsExt.jl | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 ext/NonlinearSolveSundialsExt.jl diff --git a/Project.toml b/Project.toml index 8841eca18..63df3592a 100644 --- a/Project.toml +++ b/Project.toml @@ -44,6 +44,7 @@ NLSolvers = "337daf1e-9722-11e9-073e-8b9effe078ba" NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4" SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412" +Sundials = "c3572dad-4567-51f8-b174-8c6c989267f4" [extensions] NonlinearSolveBandedMatricesExt = "BandedMatrices" @@ -55,6 +56,7 @@ NonlinearSolveNLSolversExt = "NLSolvers" NonlinearSolveNLsolveExt = ["NLsolve", "LineSearches"] NonlinearSolveSIAMFANLEquationsExt = "SIAMFANLEquations" NonlinearSolveSpeedMappingExt = "SpeedMapping" +NonlinearSolveSundialsExt = "Sundials" [compat] ADTypes = "1.9" diff --git a/ext/NonlinearSolveSundialsExt.jl b/ext/NonlinearSolveSundialsExt.jl new file mode 100644 index 000000000..edea3a49b --- /dev/null +++ b/ext/NonlinearSolveSundialsExt.jl @@ -0,0 +1,16 @@ +module NonlinearSolveSundialsExt + +using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve, + nonlinearsolve_dual_solution +using NonlinearSolve: DualNonlinearProblem +using SciMLBase: SciMLBase +using Sundials: KINSOL + +function SciMLBase.__solve(prob::DualNonlinearProblem, alg::KINSOL, args...; kwargs...) + sol, partials = nonlinearsolve_forwarddiff_solve(prob, alg, args...; kwargs...) + dual_soln = nonlinearsolve_dual_solution(sol.u, partials, prob.p) + return SciMLBase.build_solution( + prob, alg, dual_soln, sol.resid; sol.retcode, sol.stats, sol.original) +end + +end From 621c1b4d895b9374f4ba33adcfb8a407127e052f Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 09:08:37 -0400 Subject: [PATCH 86/92] refactor: use reexports --- docs/src/release_notes.md | 9 ++++++--- lib/SimpleNonlinearSolve/Project.toml | 2 ++ lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl | 9 +++------ src/NonlinearSolve.jl | 5 +++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/src/release_notes.md b/docs/src/release_notes.md index 0208b1e66..3aa147e27 100644 --- a/docs/src/release_notes.md +++ b/docs/src/release_notes.md @@ -4,13 +4,16 @@ ### Breaking Changes in `NonlinearSolve.jl` v4 + - See [common breaking changes](@ref common-breaking-changes-v4v2) below. + ### Breaking Changes in `SimpleNonlinearSolve.jl` v2 - - `Auto*` structs are no longer exported. Load `ADTypes` to access them. + - See [common breaking changes](@ref common-breaking-changes-v4v2) below. + +### [Common Breaking Changes](@id common-breaking-changes-v4v2) + - Use of termination conditions from `DiffEqBase` has been removed. Use the termination conditions from `NonlinearSolveBase` instead. - - We no longer export the entire `SciMLBase`. Instead selected functionality relevant to - `SimpleNonlinearSolve` has been exported. - If no autodiff is provided, we now choose from a list of autodiffs based on the packages loaded. For example, if `Enzyme` is loaded, we will default to that. In general, we don't guarantee the exact autodiff selected if `autodiff` is not provided (i.e. diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 82f69c6ad..586341f67 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -19,6 +19,7 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MaybeInplace = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" PrecompileTools = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +Reexport = "189a3867-3050-52da-a836-e630ba90ab69" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" @@ -60,6 +61,7 @@ Pkg = "1.10" PolyesterForwardDiff = "0.1" PrecompileTools = "1.2" Random = "1.10" +Reexport = "1.2" ReverseDiff = "1.15" SciMLBase = "2.50" StaticArrays = "1.9" diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index 6d1187668..f15630e24 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -8,6 +8,7 @@ using LineSearch: LiFukushimaLineSearch using LinearAlgebra: LinearAlgebra, dot using MaybeInplace: @bb, setindex_trait, CannotSetindex, CanSetindex using PrecompileTools: @compile_workload, @setup_workload +using Reexport: @reexport using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, NonlinearFunction, NonlinearProblem, NonlinearLeastSquaresProblem, IntervalNonlinearProblem, ReturnCode, remake using StaticArraysCore: StaticArray, SArray, SVector, MArray @@ -18,7 +19,6 @@ using DifferentiationInterface: DifferentiationInterface using FiniteDiff: FiniteDiff using ForwardDiff: ForwardDiff -using BracketingNonlinearSolve: Alefeld, Bisection, Brent, Falsi, ITP, Ridder using NonlinearSolveBase: NonlinearSolveBase, ImmutableNonlinearProblem, L2_NORM, nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution @@ -138,11 +138,8 @@ function solve_adjoint_internal end end end -export IntervalNonlinearProblem - -export Alefeld, Bisection, Brent, Falsi, ITP, Ridder - -export NonlinearFunction, NonlinearProblem, NonlinearLeastSquaresProblem +# Rexexports +@reexport using ADTypes, SciMLBase, BracketingNonlinearSolve, NonlinearSolveBase export SimpleBroyden, SimpleKlement, SimpleLimitedMemoryBroyden export SimpleDFSane diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 625477a88..e6ca6cb00 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -48,8 +48,6 @@ using SparseArrays: AbstractSparseMatrix, SparseMatrixCSC using SparseMatrixColorings: ConstantColoringAlgorithm, GreedyColoringAlgorithm, LargestFirst -@reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase - const DI = DifferentiationInterface const True = Val(true) @@ -157,6 +155,9 @@ include("default.jl") end end +# Rexexports +@reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase + # Core Algorithms export NewtonRaphson, PseudoTransient, Klement, Broyden, LimitedMemoryBroyden, DFSane export GaussNewton, LevenbergMarquardt, TrustRegion From 8f00979f53e7689d94529a15a48246563ec99c10 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 16:04:25 -0400 Subject: [PATCH 87/92] refactor: centralize autodiff selection --- Project.toml | 2 + docs/src/native/solvers.md | 11 +- docs/src/release_notes.md | 6 +- lib/NonlinearSolveBase/src/autodiff.jl | 32 +++-- lib/SimpleNonlinearSolve/src/utils.jl | 1 - src/NonlinearSolve.jl | 130 +++++++++++---------- src/algorithms/broyden.jl | 59 +++++----- src/algorithms/dfsane.jl | 1 + src/algorithms/gauss_newton.jl | 13 ++- src/algorithms/klement.jl | 36 +++--- src/algorithms/lbroyden.jl | 13 +-- src/algorithms/levenberg_marquardt.jl | 16 ++- src/algorithms/pseudo_transient.jl | 18 +-- src/algorithms/raphson.jl | 15 +-- src/algorithms/trust_region.jl | 23 ++-- src/core/approximate_jacobian.jl | 9 +- src/core/generalized_first_order.jl | 78 +++++++------ src/globalization/trust_region.jl | 14 +-- src/internal/approximate_initialization.jl | 3 +- src/internal/helpers.jl | 59 ---------- src/internal/jacobian.jl | 9 -- 21 files changed, 247 insertions(+), 301 deletions(-) diff --git a/Project.toml b/Project.toml index 63df3592a..d6fec9aa1 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ version = "4.0.0" [deps] ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +CommonSolve = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" ConcreteStructs = "2569d6c7-a4a2-43d3-a901-331e8e4be471" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" @@ -65,6 +66,7 @@ ArrayInterface = "7.16" BandedMatrices = "1.5" BenchmarkTools = "1.4" CUDA = "5.5" +CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" DiffEqBase = "6.158.3" DifferentiationInterface = "0.6.1" diff --git a/docs/src/native/solvers.md b/docs/src/native/solvers.md index aebaee379..5653f1fab 100644 --- a/docs/src/native/solvers.md +++ b/docs/src/native/solvers.md @@ -22,15 +22,16 @@ documentation. preconditioners. For more information on specifying preconditioners for LinearSolve algorithms, consult the [LinearSolve.jl documentation](https://docs.sciml.ai/LinearSolve/stable/). - - `linesearch`: the line search algorithm to use. Defaults to [`NoLineSearch()`](@extref LineSearch.NoLineSearch), - which means that no line search is performed. - - `autodiff`/`jacobian_ad`: etermines the backend used for the Jacobian. Note that this + - `linesearch`: the line search algorithm to use. Defaults to + [`NoLineSearch()`](@extref LineSearch.NoLineSearch), which means that no line search is + performed. + - `autodiff`: etermines the backend used for the Jacobian. Note that this argument is ignored if an analytical Jacobian is passed, as that will be used instead. Defaults to `nothing` which means that a default is selected according to the problem specification! Valid choices are types from ADTypes.jl. - - `forward_ad`/`vjp_autodiff`: similar to `autodiff`, but is used to compute Jacobian + - `vjp_autodiff`: similar to `autodiff`, but is used to compute Jacobian Vector Products. Ignored if the NonlinearFunction contains the `jvp` function. - - `reverse_ad`/`vjp_autodiff`: similar to `autodiff`, but is used to compute Vector + - `vjp_autodiff`: similar to `autodiff`, but is used to compute Vector Jacobian Products. Ignored if the NonlinearFunction contains the `vjp` function. - `concrete_jac`: whether to build a concrete Jacobian. If a Krylov-subspace method is used, then the Jacobian will not be constructed and instead direct Jacobian-Vector diff --git a/docs/src/release_notes.md b/docs/src/release_notes.md index 3aa147e27..1dc3d9433 100644 --- a/docs/src/release_notes.md +++ b/docs/src/release_notes.md @@ -15,9 +15,9 @@ - Use of termination conditions from `DiffEqBase` has been removed. Use the termination conditions from `NonlinearSolveBase` instead. - If no autodiff is provided, we now choose from a list of autodiffs based on the packages - loaded. For example, if `Enzyme` is loaded, we will default to that. In general, we - don't guarantee the exact autodiff selected if `autodiff` is not provided (i.e. - `nothing`). + loaded. For example, if `Enzyme` is loaded, we will default to that (for reverse mode). + In general, we don't guarantee the exact autodiff selected if `autodiff` is not provided + (i.e. `nothing`). ## Dec '23 diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index 83b6aa1f7..2102ae09b 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -3,24 +3,34 @@ # Ordering is important here. We want to select the first one that is compatible with the # problem. -const ReverseADs = ( - ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), - ADTypes.AutoZygote(), - ADTypes.AutoTracker(), - ADTypes.AutoReverseDiff(; compile = true), - ADTypes.AutoReverseDiff(), - ADTypes.AutoFiniteDiff() -) +# XXX: Remove this once Enzyme is properly supported on Julia 1.11+ +@static if VERSION ≥ v"1.11-" + const ReverseADs = ( + ADTypes.AutoZygote(), + ADTypes.AutoTracker(), + ADTypes.AutoReverseDiff(; compile = true), + ADTypes.AutoReverseDiff(), + ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), + ADTypes.AutoFiniteDiff() + ) +else + const ReverseADs = ( + ADTypes.AutoEnzyme(; mode = EnzymeCore.Reverse), + ADTypes.AutoZygote(), + ADTypes.AutoTracker(), + ADTypes.AutoReverseDiff(; compile = true), + ADTypes.AutoReverseDiff(), + ADTypes.AutoFiniteDiff() + ) +end const ForwardADs = ( - ADTypes.AutoEnzyme(; mode = EnzymeCore.Forward), ADTypes.AutoPolyesterForwardDiff(), ADTypes.AutoForwardDiff(), + ADTypes.AutoEnzyme(; mode = EnzymeCore.Forward), ADTypes.AutoFiniteDiff() ) -# TODO: Handle Sparsity - function select_forward_mode_autodiff( prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) diff --git a/lib/SimpleNonlinearSolve/src/utils.jl b/lib/SimpleNonlinearSolve/src/utils.jl index 311762429..bd7368bd7 100644 --- a/lib/SimpleNonlinearSolve/src/utils.jl +++ b/lib/SimpleNonlinearSolve/src/utils.jl @@ -130,7 +130,6 @@ function prepare_jacobian(prob, autodiff, _, x::Number) if SciMLBase.has_jac(prob.f) || SciMLBase.has_vjp(prob.f) || SciMLBase.has_jvp(prob.f) return AnalyticJacobian() end - # return DI.prepare_derivative(prob.f, autodiff, x, Constant(prob.p)) return DINoPreparation() end function prepare_jacobian(prob, autodiff, fx, x) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index e6ca6cb00..a5e71f13b 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -5,6 +5,7 @@ using PrecompileTools: @compile_workload, @setup_workload using ArrayInterface: ArrayInterface, can_setindex, restructure, fast_scalar_indexing, ismutable +using CommonSolve: solve, init, solve! using ConcreteStructs: @concrete using DiffEqBase: DiffEqBase # Needed for `init` / `solve` dispatches using FastClosures: @closure @@ -21,13 +22,18 @@ using NonlinearSolveBase: NonlinearSolveBase, nonlinearsolve_forwarddiff_solve, nonlinearsolve_dual_solution, nonlinearsolve_∂f_∂p, nonlinearsolve_∂f_∂u, L2_NORM, AbsNormTerminationMode, AbstractNonlinearTerminationMode, - AbstractSafeBestNonlinearTerminationMode + AbstractSafeBestNonlinearTerminationMode, + select_forward_mode_autodiff, select_reverse_mode_autodiff, + select_jacobian_autodiff using Printf: @printf using Preferences: Preferences, @load_preference, @set_preferences! using RecursiveArrayTools: recursivecopy! -using SciMLBase: AbstractNonlinearAlgorithm, AbstractNonlinearProblem, _unwrap_val, - isinplace, NLStats +using SciMLBase: SciMLBase, AbstractNonlinearAlgorithm, AbstractNonlinearProblem, + _unwrap_val, isinplace, NLStats, NonlinearFunction, + NonlinearLeastSquaresProblem, NonlinearProblem, ReturnCode, get_du, step!, + set_u!, LinearProblem, IdentityOperator using SciMLOperators: AbstractSciMLOperator +using SimpleNonlinearSolve: SimpleNonlinearSolve using StaticArraysCore: StaticArray, SVector, SArray, MArray, Size, SMatrix using SymbolicIndexingInterface: SymbolicIndexingInterface, ParameterIndexingProxy, symbolic_container, parameter_values, state_values, getu, @@ -95,65 +101,65 @@ include("internal/forward_diff.jl") # we need to define after the algorithms include("utils.jl") include("default.jl") -@setup_workload begin - nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1), - (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1])) - probs_nls = NonlinearProblem[] - for (fn, u0) in nlfuncs - push!(probs_nls, NonlinearProblem(fn, u0, 2.0)) - end - - nls_algs = ( - NewtonRaphson(), - TrustRegion(), - LevenbergMarquardt(), - Broyden(), - Klement(), - nothing - ) - - probs_nlls = NonlinearLeastSquaresProblem[] - nlfuncs = ( - (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), - (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]), - ( - NonlinearFunction{true}( - (du, u, p) -> du[1] = u[1] * u[1] - p, resid_prototype = zeros(1)), - [0.1, 0.0]), - ( - NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p), - resid_prototype = zeros(4)), - [0.1, 0.1] - ) - ) - for (fn, u0) in nlfuncs - push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0)) - end - - nlls_algs = ( - LevenbergMarquardt(), - GaussNewton(), - TrustRegion(), - nothing - ) - - @compile_workload begin - @sync begin - for T in (Float32, Float64), (fn, u0) in nlfuncs - Threads.@spawn NonlinearProblem(fn, T.(u0), T(2)) - end - for (fn, u0) in nlfuncs - Threads.@spawn NonlinearLeastSquaresProblem(fn, u0, 2.0) - end - for prob in probs_nls, alg in nls_algs - Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) - end - for prob in probs_nlls, alg in nlls_algs - Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) - end - end - end -end +# @setup_workload begin +# nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1), +# (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1])) +# probs_nls = NonlinearProblem[] +# for (fn, u0) in nlfuncs +# push!(probs_nls, NonlinearProblem(fn, u0, 2.0)) +# end + +# nls_algs = ( +# NewtonRaphson(), +# TrustRegion(), +# LevenbergMarquardt(), +# Broyden(), +# Klement(), +# nothing +# ) + +# probs_nlls = NonlinearLeastSquaresProblem[] +# nlfuncs = ( +# (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), +# (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]), +# ( +# NonlinearFunction{true}( +# (du, u, p) -> du[1] = u[1] * u[1] - p, resid_prototype = zeros(1)), +# [0.1, 0.0]), +# ( +# NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p), +# resid_prototype = zeros(4)), +# [0.1, 0.1] +# ) +# ) +# for (fn, u0) in nlfuncs +# push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0)) +# end + +# nlls_algs = ( +# LevenbergMarquardt(), +# GaussNewton(), +# TrustRegion(), +# nothing +# ) + +# @compile_workload begin +# @sync begin +# for T in (Float32, Float64), (fn, u0) in nlfuncs +# Threads.@spawn NonlinearProblem(fn, T.(u0), T(2)) +# end +# for (fn, u0) in nlfuncs +# Threads.@spawn NonlinearLeastSquaresProblem(fn, u0, 2.0) +# end +# for prob in probs_nls, alg in nls_algs +# Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) +# end +# for prob in probs_nlls, alg in nlls_algs +# Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) +# end +# end +# end +# end # Rexexports @reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase diff --git a/src/algorithms/broyden.jl b/src/algorithms/broyden.jl index 4b4b3df9d..39962bc6a 100644 --- a/src/algorithms/broyden.jl +++ b/src/algorithms/broyden.jl @@ -1,5 +1,5 @@ """ - Broyden(; max_resets::Int = 100, linesearch = NoLineSearch(), reset_tolerance = nothing, + Broyden(; max_resets::Int = 100, linesearch = nothing, reset_tolerance = nothing, init_jacobian::Val = Val(:identity), autodiff = nothing, alpha = nothing) An implementation of `Broyden`'s Method [broyden1965class](@cite) with resetting and line @@ -29,36 +29,37 @@ search. problem """ function Broyden(; - max_resets = 100, linesearch = NoLineSearch(), reset_tolerance = nothing, - init_jacobian::Val{IJ} = Val(:identity), autodiff = nothing, - alpha = nothing, update_rule::Val{UR} = Val(:good_broyden)) where {IJ, UR} - if IJ === :identity - if UR === :diagonal - initialization = IdentityInitialization(alpha, DiagonalStructure()) - else - initialization = IdentityInitialization(alpha, FullStructure()) - end - elseif IJ === :true_jacobian - initialization = TrueJacobianInitialization(FullStructure(), autodiff) - else - throw(ArgumentError("`init_jacobian` must be one of `:identity` or \ - `:true_jacobian`")) - end + max_resets = 100, linesearch = nothing, reset_tolerance = nothing, + init_jacobian = Val(:identity), autodiff = nothing, alpha = nothing, + update_rule = Val(:good_broyden)) + initialization = broyden_init(init_jacobian, update_rule, autodiff, alpha) + update_rule = broyden_update_rule(update_rule) + return ApproximateJacobianSolveAlgorithm{ + init_jacobian isa Val{:true_jacobian}, :Broyden}(; + linesearch, descent = NewtonDescent(), update_rule, max_resets, initialization, + reinit_rule = NoChangeInStateReset(; reset_tolerance)) +end - update_rule = if UR === :good_broyden - GoodBroydenUpdateRule() - elseif UR === :bad_broyden - BadBroydenUpdateRule() - elseif UR === :diagonal - GoodBroydenUpdateRule() - else - throw(ArgumentError("`update_rule` must be one of `:good_broyden`, `:bad_broyden`, \ - or `:diagonal`")) - end +function broyden_init(::Val{:identity}, ::Val{:diagonal}, autodiff, alpha) + return IdentityInitialization(alpha, DiagonalStructure()) +end +function broyden_init(::Val{:identity}, ::Val, autodiff, alpha) + IdentityInitialization(alpha, FullStructure()) +end +function broyden_init(::Val{:true_jacobian}, ::Val, autodiff, alpha) + return TrueJacobianInitialization(FullStructure(), autodiff) +end +function broyden_init(::Val{IJ}, ::Val{UR}, autodiff, alpha) where {IJ, UR} + error("Unknown combination of `init_jacobian = Val($(Meta.quot(IJ)))` and \ + `update_rule = Val($(Meta.quot(UR)))`. Please choose a valid combination.") +end - return ApproximateJacobianSolveAlgorithm{IJ === :true_jacobian, :Broyden}(; - linesearch, descent = NewtonDescent(), update_rule, max_resets, - initialization, reinit_rule = NoChangeInStateReset(; reset_tolerance)) +broyden_update_rule(::Val{:good_broyden}) = GoodBroydenUpdateRule() +broyden_update_rule(::Val{:bad_broyden}) = BadBroydenUpdateRule() +broyden_update_rule(::Val{:diagonal}) = GoodBroydenUpdateRule() +function broyden_update_rule(::Val{UR}) where {UR} + error("Unknown update rule `update_rule = Val($(Meta.quot(UR)))`. Please choose a \ + valid update rule.") end # Checks for no significant change for `nsteps` diff --git a/src/algorithms/dfsane.jl b/src/algorithms/dfsane.jl index b42544055..1ece5f5da 100644 --- a/src/algorithms/dfsane.jl +++ b/src/algorithms/dfsane.jl @@ -1,3 +1,4 @@ +# XXX: remove kwargs with unicode """ DFSane(; σ_min = 1 // 10^10, σ_max = 1e10, σ_1 = 1, M::Int = 10, γ = 1 // 10^4, τ_min = 1 // 10, τ_max = 1 // 2, n_exp::Int = 2, max_inner_iterations::Int = 100, diff --git a/src/algorithms/gauss_newton.jl b/src/algorithms/gauss_newton.jl index 54b7193fc..0f4ec5cd9 100644 --- a/src/algorithms/gauss_newton.jl +++ b/src/algorithms/gauss_newton.jl @@ -1,14 +1,15 @@ """ - GaussNewton(; concrete_jac = nothing, linsolve = nothing, linesearch = NoLineSearch(), - precs = DEFAULT_PRECS, adkwargs...) + GaussNewton(; concrete_jac = nothing, linsolve = nothing, precs = DEFAULT_PRECS, + linesearch = nothing, vjp_autodiff = nothing, autodiff = nothing, + jvp_autodiff = nothing) An advanced GaussNewton implementation with support for efficient handling of sparse matrices via colored automatic differentiation and preconditioned linear solvers. Designed for large-scale and numerically-difficult nonlinear least squares problems. """ function GaussNewton(; concrete_jac = nothing, linsolve = nothing, precs = DEFAULT_PRECS, - linesearch = NoLineSearch(), vjp_autodiff = nothing, autodiff = nothing) - descent = NewtonDescent(; linsolve, precs) - return GeneralizedFirstOrderAlgorithm(; concrete_jac, name = :GaussNewton, descent, - jacobian_ad = autodiff, reverse_ad = vjp_autodiff, linesearch) + linesearch = nothing, vjp_autodiff = nothing, autodiff = nothing, + jvp_autodiff = nothing) + return GeneralizedFirstOrderAlgorithm{concrete_jac, :GaussNewton}(; linesearch, + descent = NewtonDescent(; linsolve, precs), autodiff, vjp_autodiff, jvp_autodiff) end diff --git a/src/algorithms/klement.jl b/src/algorithms/klement.jl index f5d3edf3d..be94efc9c 100644 --- a/src/algorithms/klement.jl +++ b/src/algorithms/klement.jl @@ -1,5 +1,5 @@ """ - Klement(; max_resets = 100, linsolve = NoLineSearch(), linesearch = nothing, + Klement(; max_resets = 100, linsolve = nothing, linesearch = nothing, precs = DEFAULT_PRECS, alpha = nothing, init_jacobian::Val = Val(:identity), autodiff = nothing) @@ -25,27 +25,31 @@ over this. differentiable problems. """ function Klement(; max_resets::Int = 100, linsolve = nothing, alpha = nothing, - linesearch = NoLineSearch(), precs = DEFAULT_PRECS, - autodiff = nothing, init_jacobian::Val{IJ} = Val(:identity)) where {IJ} - if IJ === :identity - initialization = IdentityInitialization(alpha, DiagonalStructure()) - elseif IJ === :true_jacobian - initialization = TrueJacobianInitialization(FullStructure(), autodiff) - elseif IJ === :true_jacobian_diagonal - initialization = TrueJacobianInitialization(DiagonalStructure(), autodiff) - else - throw(ArgumentError("`init_jacobian` must be one of `:identity`, `:true_jacobian`, \ - or `:true_jacobian_diagonal`")) - end - - CJ = IJ === :true_jacobian || IJ === :true_jacobian_diagonal - + linesearch = nothing, precs = DEFAULT_PRECS, + autodiff = nothing, init_jacobian::Val = Val(:identity)) + initialization = klement_init(init_jacobian, autodiff, alpha) + CJ = init_jacobian isa Val{:true_jacobian} || + init_jacobian isa Val{:true_jacobian_diagonal} return ApproximateJacobianSolveAlgorithm{CJ, :Klement}(; linesearch, descent = NewtonDescent(; linsolve, precs), update_rule = KlementUpdateRule(), reinit_rule = IllConditionedJacobianReset(), max_resets, initialization) end +function klement_init(::Val{:identity}, autodiff, alpha) + return IdentityInitialization(alpha, DiagonalStructure()) +end +function klement_init(::Val{:true_jacobian}, autodiff, alpha) + return TrueJacobianInitialization(FullStructure(), autodiff) +end +function klement_init(::Val{:true_jacobian_diagonal}, autodiff, alpha) + return TrueJacobianInitialization(DiagonalStructure(), autodiff) +end +function klement_init(::Val{IJ}, autodiff, alpha) where {IJ} + error("Unknown `init_jacobian = Val($(Meta.quot(IJ)))`. Please choose a valid \ + `init_jacobian`.") +end + # Essentially checks ill conditioned Jacobian """ IllConditionedJacobianReset() diff --git a/src/algorithms/lbroyden.jl b/src/algorithms/lbroyden.jl index 8fb0db748..89df6ece5 100644 --- a/src/algorithms/lbroyden.jl +++ b/src/algorithms/lbroyden.jl @@ -1,5 +1,5 @@ """ - LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = NoLineSearch(), + LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = nothing, threshold::Val = Val(10), reset_tolerance = nothing, alpha = nothing) An implementation of `LimitedMemoryBroyden` [ziani2008autoadaptative](@cite) with resetting @@ -15,16 +15,13 @@ and line search. - `alpha`: The initial Jacobian inverse is set to be `(αI)⁻¹`. Defaults to `nothing` which implies `α = max(norm(u), 1) / (2 * norm(fu))`. """ -function LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = NoLineSearch(), +function LimitedMemoryBroyden(; max_resets::Int = 3, linesearch = nothing, threshold::Union{Val, Int} = Val(10), reset_tolerance = nothing, alpha = nothing) threshold isa Int && (threshold = Val(threshold)) + initialization = BroydenLowRankInitialization{_unwrap_val(threshold)}(alpha, threshold) return ApproximateJacobianSolveAlgorithm{false, :LimitedMemoryBroyden}(; linesearch, - descent = NewtonDescent(), - update_rule = GoodBroydenUpdateRule(), - max_resets, - initialization = BroydenLowRankInitialization{_unwrap_val(threshold)}( - alpha, threshold), - reinit_rule = NoChangeInStateReset(; reset_tolerance)) + descent = NewtonDescent(), update_rule = GoodBroydenUpdateRule(), + max_resets, initialization, reinit_rule = NoChangeInStateReset(; reset_tolerance)) end """ diff --git a/src/algorithms/levenberg_marquardt.jl b/src/algorithms/levenberg_marquardt.jl index 2b7c08f95..01896bd14 100644 --- a/src/algorithms/levenberg_marquardt.jl +++ b/src/algorithms/levenberg_marquardt.jl @@ -3,7 +3,8 @@ precs = DEFAULT_PRECS, damping_initial::Real = 1.0, α_geodesic::Real = 0.75, damping_increase_factor::Real = 2.0, damping_decrease_factor::Real = 3.0, finite_diff_step_geodesic = 0.1, b_uphill::Real = 1.0, autodiff = nothing, - min_damping_D::Real = 1e-8, disable_geodesic = Val(false)) + min_damping_D::Real = 1e-8, disable_geodesic = Val(false), vjp_autodiff = nothing, + jvp_autodiff = nothing) An advanced Levenberg-Marquardt implementation with the improvements suggested in [transtrum2012improvements](@citet). Designed for large-scale and numerically-difficult @@ -34,20 +35,17 @@ function LevenbergMarquardt(; linsolve = nothing, precs = DEFAULT_PRECS, damping_initial::Real = 1.0, α_geodesic::Real = 0.75, damping_increase_factor::Real = 2.0, damping_decrease_factor::Real = 3.0, finite_diff_step_geodesic = 0.1, - b_uphill::Real = 1.0, autodiff = nothing, - min_damping_D::Real = 1e-8, disable_geodesic = False) - descent = DampedNewtonDescent(; linsolve, - precs, - initial_damping = damping_initial, + b_uphill::Real = 1.0, min_damping_D::Real = 1e-8, disable_geodesic = False, + autodiff = nothing, vjp_autodiff = nothing, jvp_autodiff = nothing) + descent = DampedNewtonDescent(; linsolve, precs, initial_damping = damping_initial, damping_fn = LevenbergMarquardtDampingFunction( damping_increase_factor, damping_decrease_factor, min_damping_D)) if disable_geodesic === False descent = GeodesicAcceleration(descent, finite_diff_step_geodesic, α_geodesic) end trustregion = LevenbergMarquardtTrustRegion(b_uphill) - return GeneralizedFirstOrderAlgorithm(; - concrete_jac = true, name = :LevenbergMarquardt, - trustregion, descent, jacobian_ad = autodiff) + return GeneralizedFirstOrderAlgorithm{true, :LevenbergMarquardt}(; + trustregion, descent, autodiff, vjp_autodiff, jvp_autodiff) end @concrete struct LevenbergMarquardtDampingFunction <: AbstractDampingFunction diff --git a/src/algorithms/pseudo_transient.jl b/src/algorithms/pseudo_transient.jl index 1266d0702..effb904bc 100644 --- a/src/algorithms/pseudo_transient.jl +++ b/src/algorithms/pseudo_transient.jl @@ -1,6 +1,7 @@ """ PseudoTransient(; concrete_jac = nothing, linsolve = nothing, - linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing) + linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing, + jvp_autodiff = nothing, vjp_autodiff = nothing) An implementation of PseudoTransient Method [coffey2003pseudotransient](@cite) that is used to solve steady state problems in an accelerated manner. It uses an adaptive time-stepping @@ -14,13 +15,14 @@ This implementation specifically uses "switched evolution relaxation" - `alpha_initial` : the initial pseudo time step. It defaults to `1e-3`. If it is small, you are going to need more iterations to converge but it can be more stable. """ -function PseudoTransient(; concrete_jac = nothing, linsolve = nothing, - linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing, - alpha_initial = 1e-3) +function PseudoTransient(; + concrete_jac = nothing, linsolve = nothing, linesearch = nothing, + precs = DEFAULT_PRECS, alpha_initial = 1e-3, autodiff = nothing, + jvp_autodiff = nothing, vjp_autodiff = nothing) descent = DampedNewtonDescent(; linsolve, precs, initial_damping = alpha_initial, damping_fn = SwitchedEvolutionRelaxation()) - return GeneralizedFirstOrderAlgorithm(; - concrete_jac, name = :PseudoTransient, linesearch, descent, jacobian_ad = autodiff) + return GeneralizedFirstOrderAlgorithm{concrete_jac, :PseudoTransient}(; + linesearch, descent, autodiff, vjp_autodiff, jvp_autodiff) end """ @@ -42,11 +44,11 @@ Cache for the [`SwitchedEvolutionRelaxation`](@ref) method. internalnorm end -function requires_normal_form_jacobian(cache::Union{ +function requires_normal_form_jacobian(::Union{ SwitchedEvolutionRelaxation, SwitchedEvolutionRelaxationCache}) return false end -function requires_normal_form_rhs(cache::Union{ +function requires_normal_form_rhs(::Union{ SwitchedEvolutionRelaxation, SwitchedEvolutionRelaxationCache}) return false end diff --git a/src/algorithms/raphson.jl b/src/algorithms/raphson.jl index c6847f54c..e5dee4c91 100644 --- a/src/algorithms/raphson.jl +++ b/src/algorithms/raphson.jl @@ -1,14 +1,15 @@ """ - NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = NoLineSearch(), - precs = DEFAULT_PRECS, autodiff = nothing) + NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = missing, + precs = DEFAULT_PRECS, autodiff = nothing, vjp_autodiff = nothing, + jvp_autodiff = nothing) An advanced NewtonRaphson implementation with support for efficient handling of sparse matrices via colored automatic differentiation and preconditioned linear solvers. Designed for large-scale and numerically-difficult nonlinear systems. """ -function NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, - linesearch = NoLineSearch(), precs = DEFAULT_PRECS, autodiff = nothing) - descent = NewtonDescent(; linsolve, precs) - return GeneralizedFirstOrderAlgorithm(; - concrete_jac, name = :NewtonRaphson, linesearch, descent, jacobian_ad = autodiff) +function NewtonRaphson(; concrete_jac = nothing, linsolve = nothing, linesearch = nothing, + precs = DEFAULT_PRECS, autodiff = nothing, vjp_autodiff = nothing, + jvp_autodiff = nothing) + return GeneralizedFirstOrderAlgorithm{concrete_jac, :NewtonRaphson}(; linesearch, + descent = NewtonDescent(; linsolve, precs), autodiff, vjp_autodiff, jvp_autodiff) end diff --git a/src/algorithms/trust_region.jl b/src/algorithms/trust_region.jl index d68e2d9ec..dab6843e8 100644 --- a/src/algorithms/trust_region.jl +++ b/src/algorithms/trust_region.jl @@ -4,7 +4,8 @@ initial_trust_radius::Real = 0 // 1, step_threshold::Real = 1 // 10000, shrink_threshold::Real = 1 // 4, expand_threshold::Real = 3 // 4, shrink_factor::Real = 1 // 4, expand_factor::Real = 2 // 1, - max_shrink_times::Int = 32, vjp_autodiff = nothing, autodiff = nothing) + max_shrink_times::Int = 32, + vjp_autodiff = nothing, autodiff = nothing, jvp_autodiff = nothing) An advanced TrustRegion implementation with support for efficient handling of sparse matrices via colored automatic differentiation and preconditioned linear solvers. Designed @@ -24,22 +25,12 @@ function TrustRegion(; concrete_jac = nothing, linsolve = nothing, precs = DEFAU initial_trust_radius::Real = 0 // 1, step_threshold::Real = 1 // 10000, shrink_threshold::Real = 1 // 4, expand_threshold::Real = 3 // 4, shrink_factor::Real = 1 // 4, expand_factor::Real = 2 // 1, - max_shrink_times::Int = 32, autodiff = nothing, vjp_autodiff = nothing) + max_shrink_times::Int = 32, + autodiff = nothing, vjp_autodiff = nothing, jvp_autodiff = nothing) descent = Dogleg(; linsolve, precs) - if autodiff !== nothing && ADTypes.mode(autodiff) isa ADTypes.ForwardMode - forward_ad = autodiff - else - forward_ad = nothing - end - if isnothing(vjp_autodiff) && - autodiff isa Union{ADTypes.AutoFiniteDiff, ADTypes.AutoFiniteDifferences} - # TODO: why not just ForwardMode? - vjp_autodiff = autodiff - end trustregion = GenericTrustRegionScheme(; method = radius_update_scheme, step_threshold, shrink_threshold, expand_threshold, - shrink_factor, expand_factor, reverse_ad = vjp_autodiff, forward_ad) - return GeneralizedFirstOrderAlgorithm(; - concrete_jac, name = :TrustRegion, trustregion, descent, - jacobian_ad = autodiff, reverse_ad = vjp_autodiff, max_shrink_times) + shrink_factor, expand_factor, initial_trust_radius, max_trust_radius) + return GeneralizedFirstOrderAlgorithm{concrete_jac, :TrustRegion}(; + trustregion, descent, autodiff, vjp_autodiff, jvp_autodiff, max_shrink_times) end diff --git a/src/core/approximate_jacobian.jl b/src/core/approximate_jacobian.jl index 3522a39f4..b8f479a6d 100644 --- a/src/core/approximate_jacobian.jl +++ b/src/core/approximate_jacobian.jl @@ -174,7 +174,10 @@ function SciMLBase.__init( reinit_rule_cache = __internal_init(alg.reinit_rule, J, fu, u, du) - if alg.trustregion !== missing && alg.linesearch !== missing + has_linesearch = alg.linesearch !== missing && alg.linesearch !== nothing + has_trustregion = alg.trustregion !== missing && alg.trustregion !== nothing + + if has_trustregion && has_linesearch error("TrustRegion and LineSearch methods are algorithmically incompatible.") end @@ -182,7 +185,7 @@ function SciMLBase.__init( linesearch_cache = nothing trustregion_cache = nothing - if alg.trustregion !== missing + if has_trustregion supports_trust_region(alg.descent) || error("Trust Region not supported by \ $(alg.descent).") trustregion_cache = __internal_init( @@ -190,7 +193,7 @@ function SciMLBase.__init( GB = :TrustRegion end - if alg.linesearch !== missing + if has_linesearch supports_line_search(alg.descent) || error("Line Search not supported by \ $(alg.descent).") linesearch_cache = init( diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl index 13980c154..7b77847d9 100644 --- a/src/core/generalized_first_order.jl +++ b/src/core/generalized_first_order.jl @@ -1,7 +1,7 @@ """ GeneralizedFirstOrderAlgorithm{concrete_jac, name}(; descent, linesearch = missing, - trustregion = missing, jacobian_ad = nothing, forward_ad = nothing, - reverse_ad = nothing, max_shrink_times::Int = typemax(Int)) + trustregion = missing, autodiff = nothing, vjp_autodiff = nothing, + jvp_autodiff = nothing, max_shrink_times::Int = typemax(Int)) GeneralizedFirstOrderAlgorithm(; concrete_jac = nothing, name::Symbol = :unknown, kwargs...) @@ -11,7 +11,9 @@ common example of this is Newton-Raphson Method. First Order here refers to the order of differentiation, and should not be confused with the order of convergence. -`trustregion` and `linesearch` cannot be specified together. +!!! danger + + `trustregion` and `linesearch` cannot be specified together. ### Keyword Arguments @@ -28,9 +30,10 @@ order of convergence. trustregion descent max_shrink_times::Int - jacobian_ad - forward_ad - reverse_ad + + autodiff + vjp_autodiff + jvp_autodiff end function __show_algorithm(io::IO, alg::GeneralizedFirstOrderAlgorithm, name, indent) @@ -38,9 +41,9 @@ function __show_algorithm(io::IO, alg::GeneralizedFirstOrderAlgorithm, name, ind __is_present(alg.linesearch) && push!(modifiers, "linesearch = $(alg.linesearch)") __is_present(alg.trustregion) && push!(modifiers, "trustregion = $(alg.trustregion)") push!(modifiers, "descent = $(alg.descent)") - __is_present(alg.jacobian_ad) && push!(modifiers, "jacobian_ad = $(alg.jacobian_ad)") - __is_present(alg.forward_ad) && push!(modifiers, "forward_ad = $(alg.forward_ad)") - __is_present(alg.reverse_ad) && push!(modifiers, "reverse_ad = $(alg.reverse_ad)") + __is_present(alg.autodiff) && push!(modifiers, "autodiff = $(alg.autodiff)") + __is_present(alg.vjp_autodiff) && push!(modifiers, "vjp_autodiff = $(alg.vjp_autodiff)") + __is_present(alg.jvp_autodiff) && push!(modifiers, "jvp_autodiff = $(alg.jvp_autodiff)") spacing = " "^indent * " " spacing_last = " "^indent print(io, "$(name)(\n$(spacing)$(join(modifiers, ",\n$(spacing)"))\n$(spacing_last))") @@ -53,22 +56,11 @@ end function GeneralizedFirstOrderAlgorithm{concrete_jac, name}(; descent, linesearch = missing, trustregion = missing, - jacobian_ad = nothing, forward_ad = nothing, reverse_ad = nothing, + autodiff = nothing, jvp_autodiff = nothing, vjp_autodiff = nothing, max_shrink_times::Int = typemax(Int)) where {concrete_jac, name} - forward_ad = ifelse(forward_ad !== nothing, - forward_ad, - ifelse( - jacobian_ad !== nothing && ADTypes.mode(jacobian_ad) isa ADTypes.ForwardMode, - jacobian_ad, nothing)) - reverse_ad = ifelse(reverse_ad !== nothing, - reverse_ad, - ifelse( - jacobian_ad !== nothing && ADTypes.mode(jacobian_ad) isa ADTypes.ReverseMode, - jacobian_ad, nothing)) - return GeneralizedFirstOrderAlgorithm{concrete_jac, name}( linesearch, trustregion, descent, max_shrink_times, - jacobian_ad, forward_ad, reverse_ad) + autodiff, jvp_autodiff, vjp_autodiff) end concrete_jac(::GeneralizedFirstOrderAlgorithm{CJ}) where {CJ} = CJ @@ -152,6 +144,22 @@ function SciMLBase.__init( abstol = nothing, reltol = nothing, maxtime = nothing, termination_condition = nothing, internalnorm = L2_NORM, linsolve_kwargs = (;), kwargs...) where {uType, iip} + autodiff = select_jacobian_autodiff(prob, alg.autodiff) + jvp_autodiff = if alg.jvp_autodiff === nothing && alg.autodiff !== nothing && + (ADTypes.mode(alg.autodiff) isa ADTypes.ForwardMode || + ADTypes.mode(alg.autodiff) isa ADTypes.ForwardOrReverseMode) + select_forward_mode_autodiff(prob, alg.autodiff) + else + select_forward_mode_autodiff(prob, alg.jvp_autodiff) + end + vjp_autodiff = if alg.vjp_autodiff === nothing && alg.autodiff !== nothing && + (ADTypes.mode(alg.autodiff) isa ADTypes.ReverseMode || + ADTypes.mode(alg.autodiff) isa ADTypes.ForwardOrReverseMode) + select_reverse_mode_autodiff(prob, alg.autodiff) + else + select_reverse_mode_autodiff(prob, alg.vjp_autodiff) + end + timer = get_timer_output() @static_timeit timer "cache construction" begin (; f, u0, p) = prob @@ -166,14 +174,16 @@ function SciMLBase.__init( linsolve_kwargs = merge((; abstol, reltol), linsolve_kwargs) jac_cache = JacobianCache( - prob, alg, f, fu, u, p; stats, autodiff = alg.jacobian_ad, linsolve, - jvp_autodiff = alg.forward_ad, vjp_autodiff = alg.reverse_ad) + prob, alg, f, fu, u, p; stats, autodiff, linsolve, jvp_autodiff, vjp_autodiff) J = jac_cache(nothing) descent_cache = __internal_init(prob, alg.descent, J, fu, u; stats, abstol, reltol, internalnorm, linsolve_kwargs, timer) du = get_du(descent_cache) - if alg.trustregion !== missing && alg.linesearch !== missing + has_linesearch = alg.linesearch !== missing && alg.linesearch !== nothing + has_trustregion = alg.trustregion !== missing && alg.trustregion !== nothing + + if has_trustregion && has_linesearch error("TrustRegion and LineSearch methods are algorithmically incompatible.") end @@ -181,28 +191,20 @@ function SciMLBase.__init( linesearch_cache = nothing trustregion_cache = nothing - if alg.trustregion !== missing + if has_trustregion supports_trust_region(alg.descent) || error("Trust Region not supported by \ $(alg.descent).") trustregion_cache = __internal_init( - prob, alg.trustregion, f, fu, u, p; stats, internalnorm, kwargs...) + prob, alg.trustregion, f, fu, u, p; stats, internalnorm, kwargs..., + autodiff, jvp_autodiff, vjp_autodiff) GB = :TrustRegion end - if alg.linesearch !== missing + if has_linesearch supports_line_search(alg.descent) || error("Line Search not supported by \ $(alg.descent).") - linesearch_ad = alg.forward_ad === nothing ? - (alg.reverse_ad === nothing ? alg.jacobian_ad : - alg.reverse_ad) : alg.forward_ad - if linesearch_ad !== nothing && iip && !DI.check_inplace(linesearch_ad) - @warn "$(linesearch_ad) doesn't support in-place problems." - linesearch_ad = nothing - end - linesearch_ad = get_concrete_forward_ad( - linesearch_ad, prob, False; check_forward_mode = false) linesearch_cache = init( - prob, alg.linesearch, fu, u; stats, autodiff = linesearch_ad, kwargs...) + prob, alg.linesearch, fu, u; stats, autodiff = jvp_autodiff, kwargs...) GB = :LineSearch end diff --git a/src/globalization/trust_region.jl b/src/globalization/trust_region.jl index 8ff6de905..248f5307c 100644 --- a/src/globalization/trust_region.jl +++ b/src/globalization/trust_region.jl @@ -198,8 +198,7 @@ const RUS = RadiusUpdateSchemes GenericTrustRegionScheme(; method = RadiusUpdateSchemes.Simple, max_trust_radius = nothing, initial_trust_radius = nothing, step_threshold = nothing, shrink_threshold = nothing, expand_threshold = nothing, - shrink_factor = nothing, expand_factor = nothing, forward_ad = nothing, - reverse_ad = nothing) + shrink_factor = nothing, expand_factor = nothing) Trust Region Method that updates and stores the current trust region radius in `trust_region`. For any of the keyword arguments, if the value is `nothing`, then we use @@ -245,8 +244,6 @@ the value used in the respective paper. expand_threshold = nothing max_trust_radius = nothing initial_trust_radius = nothing - forward_ad = nothing - reverse_ad = nothing end function Base.show(io::IO, alg::GenericTrustRegionScheme) @@ -367,7 +364,8 @@ end function __internal_init( prob::AbstractNonlinearProblem, alg::GenericTrustRegionScheme, f::F, fu, u, - p, args...; stats, internalnorm::IF = L2_NORM, kwargs...) where {F, IF} + p, args...; stats, internalnorm::IF = L2_NORM, vjp_autodiff = nothing, + jvp_autodiff = nothing, kwargs...) where {F, IF} T = promote_type(eltype(u), eltype(fu)) u0_norm = internalnorm(u) fu_norm = internalnorm(fu) @@ -386,13 +384,11 @@ function __internal_init( p1, p2, p3, p4 = __get_parameters(T, alg.method) ϵ = T(1e-8) - reverse_ad = get_concrete_reverse_ad(alg.reverse_ad, prob; check_reverse_mode = true) vjp_operator = alg.method isa RUS.__Yuan || alg.method isa RUS.__Bastin ? - VecJacOperator(prob, fu, u; autodiff = reverse_ad) : nothing + VecJacOperator(prob, fu, u; autodiff = vjp_autodiff) : nothing - forward_ad = get_concrete_forward_ad(alg.forward_ad, prob; check_forward_mode = true) jvp_operator = alg.method isa RUS.__Bastin ? - JacVecOperator(prob, fu, u; autodiff = forward_ad) : nothing + JacVecOperator(prob, fu, u; autodiff = jvp_autodiff) : nothing if alg.method isa RUS.__Yuan Jᵀfu_cache = StatefulJacobianOperator(vjp_operator, u, prob.p) * _vec(fu) diff --git a/src/internal/approximate_initialization.jl b/src/internal/approximate_initialization.jl index 72e46a97e..a8196c367 100644 --- a/src/internal/approximate_initialization.jl +++ b/src/internal/approximate_initialization.jl @@ -148,8 +148,7 @@ end function __internal_init(prob::AbstractNonlinearProblem, alg::TrueJacobianInitialization, solver, f::F, fu, u, p; stats, linsolve = missing, internalnorm::IN = L2_NORM, kwargs...) where {F, IN} - autodiff = get_concrete_forward_ad( - alg.autodiff, prob; check_forward_mode = false, kwargs...) + autodiff = select_jacobian_autodiff(prob, alg.autodiff) jac_cache = JacobianCache(prob, solver, prob.f, fu, u, p; stats, autodiff, linsolve) J = alg.structure(jac_cache(nothing)) return InitializedApproximateJacobianCache( diff --git a/src/internal/helpers.jl b/src/internal/helpers.jl index 5334f11dc..e57518e2f 100644 --- a/src/internal/helpers.jl +++ b/src/internal/helpers.jl @@ -28,65 +28,6 @@ function evaluate_f!!(f::NonlinearFunction{iip}, fu, u, p) where {iip} return f(u, p) end -# AutoDiff Selection Functions -function get_concrete_forward_ad( - autodiff::ADTypes.AbstractADType, prob, sp::Val{test_sparse} = True, - args...; check_forward_mode = true, kwargs...) where {test_sparse} - if !isa(ADTypes.mode(autodiff), ADTypes.ForwardMode) && check_forward_mode - @warn "$(autodiff)::$(typeof(autodiff)) is not a `ForwardMode`. Use with caution." maxlog=1 - end - return autodiff -end -function get_concrete_forward_ad( - autodiff, prob, sp::Val{test_sparse} = True, args...; kwargs...) where {test_sparse} - if test_sparse - (; sparsity, jac_prototype) = prob.f - use_sparse_ad = sparsity !== nothing || jac_prototype !== nothing - else - use_sparse_ad = false - end - ad = if !ForwardDiff.can_dual(eltype(prob.u0)) # Use Finite Differencing - use_sparse_ad ? AutoSparse(AutoFiniteDiff()) : AutoFiniteDiff() - else - use_sparse_ad ? AutoSparse(AutoForwardDiff()) : AutoForwardDiff() - end - return ad -end - -function get_concrete_reverse_ad( - autodiff::ADTypes.AbstractADType, prob, sp::Val{test_sparse} = True, - args...; check_reverse_mode = true, kwargs...) where {test_sparse} - if !isa(ADTypes.mode(autodiff), ADTypes.ReverseMode) && - !isa(autodiff, ADTypes.AutoFiniteDiff) && # User specified finite differencing - check_reverse_mode - @warn "$(autodiff)::$(typeof(autodiff)) is not a `ReverseMode`. Use with caution." maxlog=1 - end - if autodiff isa Union{AutoZygote, AutoSparse{<:AutoZygote}} && isinplace(prob) - @warn "Attempting to use Zygote.jl for inplace problems. Switching to FiniteDiff. \ - Sparsity even if present will be ignored for correctness purposes. Set \ - the reverse ad option to `nothing` to automatically select the best option \ - and exploit sparsity." - return AutoFiniteDiff() # colorvec confusion will occur if we use FiniteDiff - else - return autodiff - end -end -function get_concrete_reverse_ad( - autodiff, prob, sp::Val{test_sparse} = True, args...; kwargs...) where {test_sparse} - if test_sparse - (; sparsity, jac_prototype) = prob.f - use_sparse_ad = sparsity !== nothing || jac_prototype !== nothing - else - use_sparse_ad = false - end - ad = if isinplace(prob) || !DI.check_available(AutoZygote()) # Use Finite Differencing - use_sparse_ad ? AutoSparse(AutoFiniteDiff()) : AutoFiniteDiff() - else - use_sparse_ad ? AutoSparse(AutoZygote()) : AutoZygote() - end - return ad -end - # Callbacks """ callback_into_cache!(cache, internalcache, args...) diff --git a/src/internal/jacobian.jl b/src/internal/jacobian.jl index 358bca792..70cc7021e 100644 --- a/src/internal/jacobian.jl +++ b/src/internal/jacobian.jl @@ -56,9 +56,6 @@ function JacobianCache(prob, alg, f::F, fu_, u, p; stats, autodiff = nothing, @bb fu = similar(fu_) - autodiff = get_concrete_forward_ad( - autodiff, prob, Val(false); check_forward_mode = false) - if !has_analytic_jac && needs_jac autodiff = construct_concrete_adtype(f, autodiff) di_extras = if iip @@ -71,10 +68,6 @@ function JacobianCache(prob, alg, f::F, fu_, u, p; stats, autodiff = nothing, end J = if !needs_jac - jvp_autodiff = get_concrete_forward_ad( - jvp_autodiff, prob, Val(false); check_forward_mode = true) - vjp_autodiff = get_concrete_reverse_ad( - vjp_autodiff, prob, Val(false); check_reverse_mode = false) JacobianOperator(prob, fu, u; jvp_autodiff, vjp_autodiff) else if f.jac_prototype === nothing @@ -109,8 +102,6 @@ function JacobianCache(prob, alg, f::F, ::Number, u::Number, p; stats, if SciMLBase.has_jac(f) || SciMLBase.has_vjp(f) || SciMLBase.has_jvp(f) return JacobianCache{false}(u, f, fu, u, p, stats, autodiff, nothing) end - autodiff = get_dense_ad(get_concrete_forward_ad( - autodiff, prob; check_forward_mode = false)) di_extras = DI.prepare_derivative(f, get_dense_ad(autodiff), u, Constant(prob.p)) return JacobianCache{false}(u, f, fu, u, p, stats, autodiff, di_extras) end From 2f36adeb9f0a8601b3aa36e894861347916175ef Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 16:26:47 -0400 Subject: [PATCH 88/92] fix: `:misc` testing --- src/NonlinearSolve.jl | 4 ++-- src/internal/forward_diff.jl | 6 ++++-- src/internal/helpers.jl | 6 ++++-- src/internal/linear_solve.jl | 15 +++++---------- test/misc/aliasing_tests.jl | 4 ++-- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index a5e71f13b..560006413 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -96,11 +96,11 @@ include("algorithms/levenberg_marquardt.jl") include("algorithms/trust_region.jl") include("algorithms/extension_algs.jl") -include("internal/forward_diff.jl") # we need to define after the algorithms - include("utils.jl") include("default.jl") +include("internal/forward_diff.jl") # we need to define after the algorithms + # @setup_workload begin # nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1), # (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1])) diff --git a/src/internal/forward_diff.jl b/src/internal/forward_diff.jl index c2adc70e2..1259480b8 100644 --- a/src/internal/forward_diff.jl +++ b/src/internal/forward_diff.jl @@ -10,7 +10,8 @@ for algType in ( Nothing, AbstractNonlinearSolveAlgorithm, GeneralizedDFSane, GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm, LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL, - SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL + SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL, + NonlinearSolvePolyAlgorithm{:NLLS, <:Any}, NonlinearSolvePolyAlgorithm{:NLS, <:Any} ) @eval function SciMLBase.__solve( prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...) @@ -47,7 +48,8 @@ for algType in ( SimpleNonlinearSolve.AbstractSimpleNonlinearSolveAlgorithm, GeneralizedFirstOrderAlgorithm, ApproximateJacobianSolveAlgorithm, LeastSquaresOptimJL, FastLevenbergMarquardtJL, CMINPACK, NLsolveJL, NLSolversJL, - SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL + SpeedMappingJL, FixedPointAccelerationJL, SIAMFANLEquationsJL, + NonlinearSolvePolyAlgorithm{:NLLS, <:Any}, NonlinearSolvePolyAlgorithm{:NLS, <:Any} ) @eval function SciMLBase.__init( prob::DualNonlinearProblem, alg::$(algType), args...; kwargs...) diff --git a/src/internal/helpers.jl b/src/internal/helpers.jl index e57518e2f..30e4596bd 100644 --- a/src/internal/helpers.jl +++ b/src/internal/helpers.jl @@ -109,9 +109,11 @@ function __construct_extension_f(prob::AbstractNonlinearProblem; alias_u0::Bool end function __construct_extension_jac(prob, alg, u0, fu; can_handle_oop::Val = False, - can_handle_scalar::Val = False, kwargs...) + can_handle_scalar::Val = False, autodiff = nothing, kwargs...) + autodiff = select_jacobian_autodiff(prob, autodiff) + Jₚ = JacobianCache( - prob, alg, prob.f, fu, u0, prob.p; stats = empty_nlstats(), kwargs...) + prob, alg, prob.f, fu, u0, prob.p; stats = empty_nlstats(), autodiff, kwargs...) 𝓙 = (can_handle_scalar === False && prob.u0 isa Number) ? @closure(u->[Jₚ(u[1])]) : Jₚ diff --git a/src/internal/linear_solve.jl b/src/internal/linear_solve.jl index a0c1ba664..707790ff3 100644 --- a/src/internal/linear_solve.jl +++ b/src/internal/linear_solve.jl @@ -149,8 +149,7 @@ function (cache::LinearSolverCache)(; if linres.retcode === ReturnCode.Failure structured_mat = ArrayInterface.isstructured(cache.lincache.A) is_gpuarray = ArrayInterface.device(cache.lincache.A) isa ArrayInterface.GPU - if !(cache.linsolve isa QRFactorization{ColumnNorm}) && - !is_gpuarray && + if !(cache.linsolve isa QRFactorization{ColumnNorm}) && !is_gpuarray && !structured_mat if verbose @warn "Potential Rank Deficient Matrix Detected. Attempting to solve using \ @@ -177,15 +176,11 @@ function (cache::LinearSolverCache)(; return LinearSolveResult(; u = linres.u) elseif !(cache.linsolve isa QRFactorization{ColumnNorm}) if verbose - if structured_mat + if structured_mat || is_gpuarray + mat_desc = structured_mat ? "Structured" : "GPU" @warn "Potential Rank Deficient Matrix Detected. But Matrix is \ - Structured. Currently, we don't attempt to solve Rank Deficient \ - Structured Matrices. Please open an issue at \ - https://github.com/SciML/NonlinearSolve.jl" - elseif is_gpuarray - @warn "Potential Rank Deficient Matrix Detected. But Matrix is on GPU. \ - Currently, we don't attempt to solve Rank Deficient GPU \ - Matrices. Please open an issue at \ + $(mat_desc). Currently, we don't attempt to solve Rank Deficient \ + $(mat_desc) Matrices. Please open an issue at \ https://github.com/SciML/NonlinearSolve.jl" end end diff --git a/test/misc/aliasing_tests.jl b/test/misc/aliasing_tests.jl index 653490dfa..78b8ec798 100644 --- a/test/misc/aliasing_tests.jl +++ b/test/misc/aliasing_tests.jl @@ -9,7 +9,7 @@ # If aliasing is not handled properly this will diverge sol = solve(prob; abstol = 1e-6, alias_u0 = true, - termination_condition = AbsNormTerminationMode()) + termination_condition = AbsNormTerminationMode(Base.Fix1(maximum, abs))) @test sol.u === prob.u0 @test SciMLBase.successful_retcode(sol.retcode) @@ -17,7 +17,7 @@ prob = remake(prob; u0 = copy(u0)) cache = init(prob; abstol = 1e-6, alias_u0 = true, - termination_condition = AbsNormTerminationMode()) + termination_condition = AbsNormTerminationMode(Base.Fix1(maximum, abs))) sol = solve!(cache) @test sol.u === prob.u0 From 5073358491850141ca5490a86af131c86ccdbd25 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 16:43:46 -0400 Subject: [PATCH 89/92] fix: `:wrappers` testing --- ext/NonlinearSolveMINPACKExt.jl | 3 ++- ext/NonlinearSolveNLsolveExt.jl | 3 ++- ext/NonlinearSolveSIAMFANLEquationsExt.jl | 3 ++- src/algorithms/pseudo_transient.jl | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ext/NonlinearSolveMINPACKExt.jl b/ext/NonlinearSolveMINPACKExt.jl index 3761a817f..88adf5753 100644 --- a/ext/NonlinearSolveMINPACKExt.jl +++ b/ext/NonlinearSolveMINPACKExt.jl @@ -28,7 +28,8 @@ function SciMLBase.__solve( original = MINPACK.fsolve( f!, u0, m; tol, show_trace, tracing, method, iterations = maxiters) else - _jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; alg.autodiff) + autodiff = alg.autodiff === missing ? nothing : alg.autodiff + _jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; autodiff) jac! = @closure (J, u) -> (_jac!(J, u); Cint(0)) original = MINPACK.fsolve( f!, jac!, u0, m; tol, show_trace, tracing, method, iterations = maxiters) diff --git a/ext/NonlinearSolveNLsolveExt.jl b/ext/NonlinearSolveNLsolveExt.jl index 301ed7137..73d98c062 100644 --- a/ext/NonlinearSolveNLsolveExt.jl +++ b/ext/NonlinearSolveNLsolveExt.jl @@ -18,7 +18,8 @@ function SciMLBase.__solve( if prob.f.jac === nothing && alg.autodiff isa Symbol df = OnceDifferentiable(f!, u0, resid; alg.autodiff) else - jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; alg.autodiff) + autodiff = alg.autodiff isa Symbol ? nothing : alg.autodiff + jac! = NonlinearSolve.__construct_extension_jac(prob, alg, u0, resid; autodiff) if prob.f.jac_prototype === nothing J = similar( u0, promote_type(eltype(u0), eltype(resid)), length(u0), length(resid)) diff --git a/ext/NonlinearSolveSIAMFANLEquationsExt.jl b/ext/NonlinearSolveSIAMFANLEquationsExt.jl index 65154b63a..2468064fb 100644 --- a/ext/NonlinearSolveSIAMFANLEquationsExt.jl +++ b/ext/NonlinearSolveSIAMFANLEquationsExt.jl @@ -91,10 +91,11 @@ function SciMLBase.__solve(prob::NonlinearProblem, alg::SIAMFANLEquationsJL, arg f, u, m, zeros(T, N, 2 * m + 4); atol, rtol, maxit = maxiters, beta) end else + autodiff = alg.autodiff === missing ? nothing : alg.autodiff FPS = prob.f.jac_prototype !== nothing ? zero(prob.f.jac_prototype) : __zeros_like(u, N, N) jac = NonlinearSolve.__construct_extension_jac( - prob, alg, u, resid; alg.autodiff) + prob, alg, u, resid; autodiff) AJ! = @closure (J, u, x) -> jac(J, x) if method == :newton sol = nsol(f, u, FS, FPS, AJ!; sham = 1, atol, diff --git a/src/algorithms/pseudo_transient.jl b/src/algorithms/pseudo_transient.jl index effb904bc..1e6b94763 100644 --- a/src/algorithms/pseudo_transient.jl +++ b/src/algorithms/pseudo_transient.jl @@ -17,7 +17,7 @@ This implementation specifically uses "switched evolution relaxation" """ function PseudoTransient(; concrete_jac = nothing, linsolve = nothing, linesearch = nothing, - precs = DEFAULT_PRECS, alpha_initial = 1e-3, autodiff = nothing, + precs = DEFAULT_PRECS, alpha_initial = 1e-3, autodiff = nothing, jvp_autodiff = nothing, vjp_autodiff = nothing) descent = DampedNewtonDescent(; linsolve, precs, initial_damping = alpha_initial, damping_fn = SwitchedEvolutionRelaxation()) From f3b317dd42381062d50570065c9b2070ec3ba9a4 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 16:59:45 -0400 Subject: [PATCH 90/92] fix: mode warning printing --- lib/NonlinearSolveBase/src/autodiff.jl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index 2102ae09b..9dbad5472 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -33,7 +33,8 @@ const ForwardADs = ( function select_forward_mode_autodiff( prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) - if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) + if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) && + !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) @warn "The chosen AD backend $(ad) is not a forward mode AD. Use with caution." end if incompatible_backend_and_problem(prob, ad) @@ -58,7 +59,8 @@ end function select_reverse_mode_autodiff( prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) - if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ReverseMode) + if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ReverseMode) && + !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) if !is_finite_differences_backend(ad) @warn "The chosen AD backend $(ad) is not a reverse mode AD. Use with caution." else From 1b77877b8530128c1b73cd6faf1befbb0aec96d9 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 17:09:06 -0400 Subject: [PATCH 91/92] fix: minor cleanups --- lib/NonlinearSolveBase/src/autodiff.jl | 13 +++++-------- .../src/SimpleNonlinearSolve.jl | 1 + src/core/generalized_first_order.jl | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/NonlinearSolveBase/src/autodiff.jl b/lib/NonlinearSolveBase/src/autodiff.jl index 9dbad5472..395580924 100644 --- a/lib/NonlinearSolveBase/src/autodiff.jl +++ b/lib/NonlinearSolveBase/src/autodiff.jl @@ -34,7 +34,8 @@ const ForwardADs = ( function select_forward_mode_autodiff( prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ForwardMode) && - !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) + !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) && + !is_finite_differences_backend(ad) @warn "The chosen AD backend $(ad) is not a forward mode AD. Use with caution." end if incompatible_backend_and_problem(prob, ad) @@ -60,13 +61,9 @@ end function select_reverse_mode_autodiff( prob::AbstractNonlinearProblem, ad::AbstractADType; warn_check_mode::Bool = true) if warn_check_mode && !(ADTypes.mode(ad) isa ADTypes.ReverseMode) && - !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) - if !is_finite_differences_backend(ad) - @warn "The chosen AD backend $(ad) is not a reverse mode AD. Use with caution." - else - @warn "The chosen AD backend $(ad) is a finite differences backend. This might \ - be slow and inaccurate. Use with caution." - end + !(ADTypes.mode(ad) isa ADTypes.ForwardOrReverseMode) && + !is_finite_differences_backend(ad) + @warn "The chosen AD backend $(ad) is not a reverse mode AD. Use with caution." end if incompatible_backend_and_problem(prob, ad) adₙ = select_reverse_mode_autodiff(prob, nothing; warn_check_mode) diff --git a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl index f15630e24..cee8f8dd3 100644 --- a/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl +++ b/lib/SimpleNonlinearSolve/src/SimpleNonlinearSolve.jl @@ -1,6 +1,7 @@ module SimpleNonlinearSolve using Accessors: @reset +using BracketingNonlinearSolve: BracketingNonlinearSolve using CommonSolve: CommonSolve, solve, init, solve! using ConcreteStructs: @concrete using FastClosures: @closure diff --git a/src/core/generalized_first_order.jl b/src/core/generalized_first_order.jl index 7b77847d9..d5ee3eeab 100644 --- a/src/core/generalized_first_order.jl +++ b/src/core/generalized_first_order.jl @@ -60,7 +60,7 @@ function GeneralizedFirstOrderAlgorithm{concrete_jac, name}(; max_shrink_times::Int = typemax(Int)) where {concrete_jac, name} return GeneralizedFirstOrderAlgorithm{concrete_jac, name}( linesearch, trustregion, descent, max_shrink_times, - autodiff, jvp_autodiff, vjp_autodiff) + autodiff, vjp_autodiff, jvp_autodiff) end concrete_jac(::GeneralizedFirstOrderAlgorithm{CJ}) where {CJ} = CJ From 8f5ecf3a5d31679d7b6b5ae632d2d46f44ade262 Mon Sep 17 00:00:00 2001 From: Avik Pal Date: Tue, 22 Oct 2024 18:16:46 -0400 Subject: [PATCH 92/92] docs: update documentation --- .github/workflows/Documentation.yml | 2 +- .github/workflows/Downgrade.yml | 35 +++++- Project.toml | 8 +- docs/Project.toml | 9 +- docs/make.jl | 8 +- docs/pages.jl | 4 +- docs/src/basics/diagnostics_api.md | 20 ++-- docs/src/basics/solve.md | 2 +- docs/src/basics/termination_condition.md | 22 ++-- docs/src/devdocs/algorithm_helpers.md | 3 - docs/src/native/bracketingnonlinearsolve.md | 21 ++++ docs/src/native/simplenonlinearsolve.md | 16 +-- docs/src/tutorials/modelingtoolkit.md | 28 ++--- lib/BracketingNonlinearSolve/README.md | 23 ++++ lib/NonlinearSolveBase/Project.toml | 2 +- lib/NonlinearSolveBase/src/public.jl | 2 +- lib/SciMLJacobianOperators/Project.toml | 4 +- lib/SimpleNonlinearSolve/Project.toml | 4 +- lib/SimpleNonlinearSolve/README.md | 23 ++++ lib/SimpleNonlinearSolve/src/lbroyden.jl | 4 +- src/NonlinearSolve.jl | 118 ++++++++++---------- test/gpu/core_tests.jl | 7 +- 22 files changed, 225 insertions(+), 140 deletions(-) create mode 100644 docs/src/native/bracketingnonlinearsolve.md create mode 100644 lib/BracketingNonlinearSolve/README.md create mode 100644 lib/SimpleNonlinearSolve/README.md diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml index 84f38404f..9dc416799 100644 --- a/.github/workflows/Documentation.yml +++ b/.github/workflows/Documentation.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1' + version: '1.10' - name: Install dependencies run: | import Pkg diff --git a/.github/workflows/Downgrade.yml b/.github/workflows/Downgrade.yml index 83001be29..172457df8 100644 --- a/.github/workflows/Downgrade.yml +++ b/.github/workflows/Downgrade.yml @@ -14,15 +14,42 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - version: ["1.10"] + version: + - "1.10" + group: + - Core + - Downstream + - Misc + - Wrappers steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} - uses: julia-actions/julia-downgrade-compat@v1 + - name: "Install Dependencies and Run Tests" + run: | + import Pkg + Pkg.Registry.update() + # Install packages present in subdirectories + dev_pks = Pkg.PackageSpec[] + for path in ("lib/SciMLJacobianOperators", "lib/BracketingNonlinearSolve", "lib/NonlinearSolveBase", "lib/SimpleNonlinearSolve") + push!(dev_pks, Pkg.PackageSpec(; path)) + end + Pkg.develop(dev_pks) + Pkg.instantiate() + Pkg.test(; coverage=true) + shell: julia --color=yes --code-coverage=user --depwarn=yes --project=. {0} + env: + GROUP: ${{ matrix.group }} + - uses: julia-actions/julia-processcoverage@v1 with: - skip: Pkg,TOML - - uses: julia-actions/julia-buildpkg@v1 - - uses: julia-actions/julia-runtest@v1 + directories: src,ext,lib/SciMLJacobianOperators/src + - uses: codecov/codecov-action@v4 + with: + file: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + fail_ci_if_error: true diff --git a/Project.toml b/Project.toml index d6fec9aa1..03e6519dd 100644 --- a/Project.toml +++ b/Project.toml @@ -68,9 +68,9 @@ BenchmarkTools = "1.4" CUDA = "5.5" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" -DiffEqBase = "6.158.3" -DifferentiationInterface = "0.6.1" -Enzyme = "0.13.2" +DiffEqBase = "6.155.3" +DifferentiationInterface = "0.6.16" +Enzyme = "0.13.11" ExplicitImports = "1.5" FastClosures = "0.3.2" FastLevenbergMarquardt = "0.1" @@ -108,7 +108,7 @@ SciMLOperators = "0.3.10" SimpleNonlinearSolve = "2" SparseArrays = "1.10" SparseConnectivityTracer = "0.6.5" -SparseMatrixColorings = "0.4.2" +SparseMatrixColorings = "0.4.5" SpeedMapping = "0.3" StableRNGs = "1" StaticArrays = "1.9" diff --git a/docs/Project.toml b/docs/Project.toml index d01ae32a1..4e72b257d 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,6 +3,7 @@ ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" AlgebraicMultigrid = "2169fc97-5a83-5252-b627-83903c6c433c" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +BracketingNonlinearSolve = "70df07ce-3d50-431d-a3e7-ca6ddb60ac1e" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DifferentiationInterface = "a0c0ee7d-e4b9-4e03-894e-1c5f64a51d63" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" @@ -11,8 +12,8 @@ DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656" IncompleteLU = "40713840-3770-5561-ab4c-a76e7d0d7895" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" LinearSolve = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" -ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" NonlinearSolve = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" +NonlinearSolveBase = "be0214bd-f91f-a760-ac4e-3421ce2b2da0" OrdinaryDiffEqTsit5 = "b1df2697-797e-41e3-8120-5422d3b24e4a" Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" @@ -29,16 +30,16 @@ ADTypes = "1.9.0" AlgebraicMultigrid = "0.5, 0.6" ArrayInterface = "6, 7" BenchmarkTools = "1" -DiffEqBase = "6.136" -DifferentiationInterface = "0.6.1" +BracketingNonlinearSolve = "1" +DifferentiationInterface = "0.6.16" Documenter = "1" DocumenterCitations = "1" DocumenterInterLinks = "1.0.0" IncompleteLU = "0.2" InteractiveUtils = "<0.0.1, 1" LinearSolve = "2" -ModelingToolkit = "9" NonlinearSolve = "4" +NonlinearSolveBase = "1" OrdinaryDiffEqTsit5 = "1.1.0" Plots = "1" Random = "1.10" diff --git a/docs/make.jl b/docs/make.jl index eec25f4f7..bdc2b7fe9 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,7 +1,8 @@ using Documenter, DocumenterCitations, DocumenterInterLinks using NonlinearSolve, SimpleNonlinearSolve, Sundials, SteadyStateDiffEq, SciMLBase, - DiffEqBase + BracketingNonlinearSolve, NonlinearSolveBase using SciMLJacobianOperators +import DiffEqBase cp(joinpath(@__DIR__, "Manifest.toml"), joinpath(@__DIR__, "src/assets/Manifest.toml"), force = true) @@ -20,8 +21,9 @@ interlinks = InterLinks( makedocs(; sitename = "NonlinearSolve.jl", authors = "Chris Rackauckas", - modules = [NonlinearSolve, SimpleNonlinearSolve, SteadyStateDiffEq, - Sundials, DiffEqBase, SciMLBase, SciMLJacobianOperators], + modules = [NonlinearSolve, SimpleNonlinearSolve, SteadyStateDiffEq, DiffEqBase, + Sundials, NonlinearSolveBase, SciMLBase, SciMLJacobianOperators, + BracketingNonlinearSolve], clean = true, doctest = false, linkcheck = true, diff --git a/docs/pages.jl b/docs/pages.jl index cfb2754c0..8da01762a 100644 --- a/docs/pages.jl +++ b/docs/pages.jl @@ -1,6 +1,7 @@ # Put in a separate page so it can be used by SciMLDocs.jl -pages = ["index.md", +pages = [ + "index.md", "Getting Started with Nonlinear Rootfinding in Julia" => "tutorials/getting_started.md", "Tutorials" => Any[ "tutorials/code_optimization.md", @@ -31,6 +32,7 @@ pages = ["index.md", "Native Functionalities" => Any[ "native/solvers.md", "native/simplenonlinearsolve.md", + "native/bracketingnonlinearsolve.md", "native/steadystatediffeq.md", "native/descent.md", "native/globalization.md", diff --git a/docs/src/basics/diagnostics_api.md b/docs/src/basics/diagnostics_api.md index c8b207544..795348bd6 100644 --- a/docs/src/basics/diagnostics_api.md +++ b/docs/src/basics/diagnostics_api.md @@ -27,20 +27,16 @@ Note that you will have to restart Julia to disable the timer outputs once enabl ## Example Usage ```@example diagnostics_example -using ModelingToolkit, NonlinearSolve +using NonlinearSolve -@variables x y z -@parameters σ ρ β +function nlfunc(resid, u0, p) + resid[1] = u0[1] * u0[1] - p + resid[2] = u0[2] * u0[2] - p + resid[3] = u0[3] * u0[3] - p + nothing +end -# Define a nonlinear system -eqs = [0 ~ σ * (y - x), 0 ~ x * (ρ - z) - y, 0 ~ x * y - β * z] -@mtkbuild ns = NonlinearSystem(eqs, [x, y, z], [σ, ρ, β]) - -u0 = [x => 1.0, y => 0.0, z => 0.0] - -ps = [σ => 10.0 ρ => 26.0 β => 8 / 3] - -prob = NonlinearProblem(ns, u0, ps) +prob = NonlinearProblem(nlfunc, [1.0, 3.0, 5.0], 2.0) solve(prob) ``` diff --git a/docs/src/basics/solve.md b/docs/src/basics/solve.md index f7c238966..3bc7df056 100644 --- a/docs/src/basics/solve.md +++ b/docs/src/basics/solve.md @@ -1,7 +1,7 @@ # [Common Solver Options (Solve Keyword Arguments)](@id solver_options) ```@docs -solve(prob::SciMLBase.NonlinearProblem, args...; kwargs...) +solve(::NonlinearProblem, args...; kwargs...) ``` ## General Controls diff --git a/docs/src/basics/termination_condition.md b/docs/src/basics/termination_condition.md index 9a7c718ec..4ea98b4a5 100644 --- a/docs/src/basics/termination_condition.md +++ b/docs/src/basics/termination_condition.md @@ -23,17 +23,23 @@ terminated = cache(du, u, uprev) ### Absolute Tolerance ```@docs -AbsTerminationMode -AbsNormTerminationMode -AbsNormSafeTerminationMode -AbsNormSafeBestTerminationMode +NonlinearSolveBase.AbsTerminationMode +NonlinearSolveBase.AbsNormTerminationMode +NonlinearSolveBase.AbsNormSafeTerminationMode +NonlinearSolveBase.AbsNormSafeBestTerminationMode ``` ### Relative Tolerance ```@docs -RelTerminationMode -RelNormTerminationMode -RelNormSafeTerminationMode -RelNormSafeBestTerminationMode +NonlinearSolveBase.RelTerminationMode +NonlinearSolveBase.RelNormTerminationMode +NonlinearSolveBase.RelNormSafeTerminationMode +NonlinearSolveBase.RelNormSafeBestTerminationMode +``` + +### Both Tolerances + +```@docs +NonlinearSolveBase.NormTerminationMode ``` diff --git a/docs/src/devdocs/algorithm_helpers.md b/docs/src/devdocs/algorithm_helpers.md index 7b0f91a9f..c945b6003 100644 --- a/docs/src/devdocs/algorithm_helpers.md +++ b/docs/src/devdocs/algorithm_helpers.md @@ -60,9 +60,6 @@ NonlinearSolve.GenericTrustRegionScheme ## Miscellaneous ```@docs -SimpleNonlinearSolve.__nextfloat_tdir -SimpleNonlinearSolve.__prevfloat_tdir -SimpleNonlinearSolve.__max_tdir NonlinearSolve.callback_into_cache! NonlinearSolve.concrete_jac ``` diff --git a/docs/src/native/bracketingnonlinearsolve.md b/docs/src/native/bracketingnonlinearsolve.md new file mode 100644 index 000000000..2201378e1 --- /dev/null +++ b/docs/src/native/bracketingnonlinearsolve.md @@ -0,0 +1,21 @@ +# BracketingNonlinearSolve.jl + +These methods can be used independently of the rest of NonlinearSolve.jl + +```@index +Pages = ["bracketingnonlinearsolve.md"] +``` + +## Interval Methods + +These methods are suited for interval (scalar) root-finding problems, +i.e. [`IntervalNonlinearProblem`](@ref). + +```@docs +ITP +Alefeld +Bisection +Falsi +Ridder +Brent +``` diff --git a/docs/src/native/simplenonlinearsolve.md b/docs/src/native/simplenonlinearsolve.md index 0ff386898..777a81ef8 100644 --- a/docs/src/native/simplenonlinearsolve.md +++ b/docs/src/native/simplenonlinearsolve.md @@ -6,20 +6,6 @@ These methods can be used independently of the rest of NonlinearSolve.jl Pages = ["simplenonlinearsolve.md"] ``` -## Interval Methods - -These methods are suited for interval (scalar) root-finding problems, -i.e. `IntervalNonlinearProblem`. - -```@docs -ITP -Alefeld -Bisection -Falsi -Ridder -Brent -``` - ## General Methods These methods are suited for any general nonlinear root-finding problem, i.e. @@ -54,6 +40,6 @@ Squares problems. [^1]: Needs [`StaticArrays.jl`](https://github.com/JuliaArrays/StaticArrays.jl) to be installed and loaded for the non-allocating version. [^2]: This method is non-allocating if the termination condition is set to either `nothing` - (default) or [`AbsNormTerminationMode`](@ref). + (default) or [`NonlinearSolveBase.AbsNormTerminationMode`](@ref). [^3]: Only the defaults are guaranteed to work inside kernels. We try to provide warnings if the used version is not non-allocating. diff --git a/docs/src/tutorials/modelingtoolkit.md b/docs/src/tutorials/modelingtoolkit.md index 87b0a4405..e2016dde0 100644 --- a/docs/src/tutorials/modelingtoolkit.md +++ b/docs/src/tutorials/modelingtoolkit.md @@ -5,7 +5,7 @@ modeling system for the Julia SciML ecosystem. It adds a high-level interactive for the numerical solvers which can make it easy to symbolically modify and generate equations to be solved. The basic form of using ModelingToolkit looks as follows: -```@example mtk +```julia using ModelingToolkit, NonlinearSolve @variables x y z @@ -30,20 +30,20 @@ sol = solve(prob, NewtonRaphson()) As a symbolic system, ModelingToolkit can be used to represent the equations and derive new forms. For example, let's look at the equations: -```@example mtk +```julia equations(ns) ``` We can ask it what the Jacobian of our system is via `calculate_jacobian`: -```@example mtk +```julia calculate_jacobian(ns) ``` We can tell MTK to generate a computable form of this analytical Jacobian via `jac = true` to help the solver use efficient forms: -```@example mtk +```julia prob = NonlinearProblem(ns, u0, ps, jac = true) sol = solve(prob, NewtonRaphson()) ``` @@ -54,7 +54,7 @@ One of the major reasons for using ModelingToolkit is to allow structural simpli the systems. It's very easy to write down a mathematical model that, in theory, could be solved more simply. Let's take a look at a quick system: -```@example mtk +```julia @variables u1 u2 u3 u4 u5 eqs = [0 ~ u1 - sin(u5), 0 ~ u2 - cos(u1), 0 ~ u3 - hypot(u1, u2), 0 ~ u4 - hypot(u2, u3), 0 ~ u5 - hypot(u4, u1)] @@ -63,23 +63,23 @@ eqs = [0 ~ u1 - sin(u5), 0 ~ u2 - cos(u1), 0 ~ u3 - hypot(u1, u2), If we run structural simplification, we receive the following form: -```@example mtk +```julia sys = structural_simplify(sys) ``` -```@example mtk +```julia equations(sys) ``` How did it do this? Let's look at the `observed` to see the relationships that it found: -```@example mtk +```julia observed(sys) ``` Using ModelingToolkit, we can build and solve the simplified system: -```@example mtk +```julia u0 = [u5 .=> 1.0] prob = NonlinearProblem(sys, u0) sol = solve(prob, NewtonRaphson()) @@ -87,23 +87,23 @@ sol = solve(prob, NewtonRaphson()) We can then use symbolic indexing to retrieve any variable: -```@example mtk +```julia sol[u1] ``` -```@example mtk +```julia sol[u2] ``` -```@example mtk +```julia sol[u3] ``` -```@example mtk +```julia sol[u4] ``` -```@example mtk +```julia sol[u5] ``` diff --git a/lib/BracketingNonlinearSolve/README.md b/lib/BracketingNonlinearSolve/README.md new file mode 100644 index 000000000..85839107e --- /dev/null +++ b/lib/BracketingNonlinearSolve/README.md @@ -0,0 +1,23 @@ +# BracketingNonlinearSolve.jl + +Fast implementations of interval root finding algorithms in Julia that satisfy the SciML +common interface. BracketingNonlinearSolve.jl focuses on low-dependency implementations of +very fast methods for very small and simple problems. For the full set of solvers, see +[NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which +BracketingNonlinearSolve.jl is just one solver set. + +For information on using the package, +[see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the +[in-development documentation](https://docs.sciml.ai/NonlinearSolve/dev/) for the version of +the documentation which contains the unreleased features. + +## High Level Examples + +```julia +using BracketingNonlinearSolve + +f(u, p) = u .* u .- 2.0 +u0 = (1.0, 2.0) # brackets +probB = IntervalNonlinearProblem(f, u0) +sol = solve(probB, ITP()) +``` diff --git a/lib/NonlinearSolveBase/Project.toml b/lib/NonlinearSolveBase/Project.toml index 162ce2742..70c0f97d4 100644 --- a/lib/NonlinearSolveBase/Project.toml +++ b/lib/NonlinearSolveBase/Project.toml @@ -37,7 +37,7 @@ CommonSolve = "0.2.4" Compat = "4.15" ConcreteStructs = "0.2.3" DiffEqBase = "6.149" -DifferentiationInterface = "0.6.1" +DifferentiationInterface = "0.6.16" EnzymeCore = "0.8" ExplicitImports = "1.10.1" FastClosures = "0.3" diff --git a/lib/NonlinearSolveBase/src/public.jl b/lib/NonlinearSolveBase/src/public.jl index eceea6d75..b39aa26d2 100644 --- a/lib/NonlinearSolveBase/src/public.jl +++ b/lib/NonlinearSolveBase/src/public.jl @@ -81,7 +81,7 @@ for norm_type in (:RelNorm, :AbsNorm), safety in (:Safe, :SafeBest) supertype_name = Symbol(:Abstract, safety, :NonlinearTerminationMode) doctring = safety == :Safe ? - "Essentially [`$(norm_type)NormTerminationMode`](@ref) + terminate if there \ + "Essentially [`$(norm_type)TerminationMode`](@ref) + terminate if there \ has been no improvement for the last `patience_steps` + terminate if the \ solution blows up (diverges)." : "Essentially [`$(norm_type)SafeTerminationMode`](@ref), but caches the best\ diff --git a/lib/SciMLJacobianOperators/Project.toml b/lib/SciMLJacobianOperators/Project.toml index 73ee5a55d..c889eb199 100644 --- a/lib/SciMLJacobianOperators/Project.toml +++ b/lib/SciMLJacobianOperators/Project.toml @@ -18,8 +18,8 @@ ADTypes = "1.8.1" Aqua = "0.8.7" ConcreteStructs = "0.2.3" ConstructionBase = "1.5" -DifferentiationInterface = "0.6.1" -Enzyme = "0.12, 0.13" +DifferentiationInterface = "0.6.16" +Enzyme = "0.13.11" ExplicitImports = "1.9.0" FastClosures = "0.3.2" FiniteDiff = "2.24.0" diff --git a/lib/SimpleNonlinearSolve/Project.toml b/lib/SimpleNonlinearSolve/Project.toml index 586341f67..cfda24544 100644 --- a/lib/SimpleNonlinearSolve/Project.toml +++ b/lib/SimpleNonlinearSolve/Project.toml @@ -45,8 +45,8 @@ ChainRulesCore = "1.24" CommonSolve = "0.2.4" ConcreteStructs = "0.2.3" DiffEqBase = "6.149" -DifferentiationInterface = "0.6.1" -Enzyme = "0.13" +DifferentiationInterface = "0.6.16" +Enzyme = "0.13.11" ExplicitImports = "1.9" FastClosures = "0.3.2" FiniteDiff = "2.24.0" diff --git a/lib/SimpleNonlinearSolve/README.md b/lib/SimpleNonlinearSolve/README.md new file mode 100644 index 000000000..562a5a4ae --- /dev/null +++ b/lib/SimpleNonlinearSolve/README.md @@ -0,0 +1,23 @@ +# SimpleNonlinearSolve.jl + +Fast implementations of root finding algorithms in Julia that satisfy the SciML common interface. +SimpleNonlinearSolve.jl focuses on low-dependency implementations of very fast methods for +very small and simple problems. For the full set of solvers, see +[NonlinearSolve.jl](https://github.com/SciML/NonlinearSolve.jl), of which +SimpleNonlinearSolve.jl is just one solver set. + +For information on using the package, +[see the stable documentation](https://docs.sciml.ai/NonlinearSolve/stable/). Use the +[in-development documentation](https://docs.sciml.ai/NonlinearSolve/dev/) for the version of +the documentation which contains the unreleased features. + +## High Level Examples + +```julia +using SimpleNonlinearSolve, StaticArrays + +f(u, p) = u .* u .- 2 +u0 = @SVector[1.0, 1.0] +probN = NonlinearProblem{false}(f, u0) +solver = solve(probN, SimpleNewtonRaphson(), abstol = 1e-9) +``` diff --git a/lib/SimpleNonlinearSolve/src/lbroyden.jl b/lib/SimpleNonlinearSolve/src/lbroyden.jl index 3a17e0936..d2bd6ef83 100644 --- a/lib/SimpleNonlinearSolve/src/lbroyden.jl +++ b/lib/SimpleNonlinearSolve/src/lbroyden.jl @@ -44,8 +44,8 @@ function SciMLBase.__solve( end @warn "Specifying `termination_condition = $(termination_condition)` for \ `SimpleLimitedMemoryBroyden` with `SArray` is not non-allocating. Use \ - either `termination_condition = AbsNormTerminationMode()` or \ - `termination_condition = nothing`." maxlog=1 + either `termination_condition = AbsNormTerminationMode(Base.Fix2(norm, Inf))` \ + or `termination_condition = nothing`." maxlog=1 end return internal_generic_solve(prob, alg, args...; termination_condition, kwargs...) end diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl index 560006413..60e2f0663 100644 --- a/src/NonlinearSolve.jl +++ b/src/NonlinearSolve.jl @@ -101,65 +101,65 @@ include("default.jl") include("internal/forward_diff.jl") # we need to define after the algorithms -# @setup_workload begin -# nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1), -# (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1])) -# probs_nls = NonlinearProblem[] -# for (fn, u0) in nlfuncs -# push!(probs_nls, NonlinearProblem(fn, u0, 2.0)) -# end - -# nls_algs = ( -# NewtonRaphson(), -# TrustRegion(), -# LevenbergMarquardt(), -# Broyden(), -# Klement(), -# nothing -# ) - -# probs_nlls = NonlinearLeastSquaresProblem[] -# nlfuncs = ( -# (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), -# (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]), -# ( -# NonlinearFunction{true}( -# (du, u, p) -> du[1] = u[1] * u[1] - p, resid_prototype = zeros(1)), -# [0.1, 0.0]), -# ( -# NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p), -# resid_prototype = zeros(4)), -# [0.1, 0.1] -# ) -# ) -# for (fn, u0) in nlfuncs -# push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0)) -# end - -# nlls_algs = ( -# LevenbergMarquardt(), -# GaussNewton(), -# TrustRegion(), -# nothing -# ) - -# @compile_workload begin -# @sync begin -# for T in (Float32, Float64), (fn, u0) in nlfuncs -# Threads.@spawn NonlinearProblem(fn, T.(u0), T(2)) -# end -# for (fn, u0) in nlfuncs -# Threads.@spawn NonlinearLeastSquaresProblem(fn, u0, 2.0) -# end -# for prob in probs_nls, alg in nls_algs -# Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) -# end -# for prob in probs_nlls, alg in nlls_algs -# Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) -# end -# end -# end -# end +@setup_workload begin + nlfuncs = ((NonlinearFunction{false}((u, p) -> u .* u .- p), 0.1), + (NonlinearFunction{true}((du, u, p) -> du .= u .* u .- p), [0.1])) + probs_nls = NonlinearProblem[] + for (fn, u0) in nlfuncs + push!(probs_nls, NonlinearProblem(fn, u0, 2.0)) + end + + nls_algs = ( + NewtonRaphson(), + TrustRegion(), + LevenbergMarquardt(), + Broyden(), + Klement(), + nothing + ) + + probs_nlls = NonlinearLeastSquaresProblem[] + nlfuncs = ( + (NonlinearFunction{false}((u, p) -> (u .^ 2 .- p)[1:1]), [0.1, 0.0]), + (NonlinearFunction{false}((u, p) -> vcat(u .* u .- p, u .* u .- p)), [0.1, 0.1]), + ( + NonlinearFunction{true}( + (du, u, p) -> du[1] = u[1] * u[1] - p, resid_prototype = zeros(1)), + [0.1, 0.0]), + ( + NonlinearFunction{true}((du, u, p) -> du .= vcat(u .* u .- p, u .* u .- p), + resid_prototype = zeros(4)), + [0.1, 0.1] + ) + ) + for (fn, u0) in nlfuncs + push!(probs_nlls, NonlinearLeastSquaresProblem(fn, u0, 2.0)) + end + + nlls_algs = ( + LevenbergMarquardt(), + GaussNewton(), + TrustRegion(), + nothing + ) + + @compile_workload begin + @sync begin + for T in (Float32, Float64), (fn, u0) in nlfuncs + Threads.@spawn NonlinearProblem(fn, T.(u0), T(2)) + end + for (fn, u0) in nlfuncs + Threads.@spawn NonlinearLeastSquaresProblem(fn, u0, 2.0) + end + for prob in probs_nls, alg in nls_algs + Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) + end + for prob in probs_nlls, alg in nlls_algs + Threads.@spawn solve(prob, alg; abstol = 1e-2, verbose = false) + end + end + end +end # Rexexports @reexport using SciMLBase, SimpleNonlinearSolve, NonlinearSolveBase diff --git a/test/gpu/core_tests.jl b/test/gpu/core_tests.jl index 75087aa30..91d6178a4 100644 --- a/test/gpu/core_tests.jl +++ b/test/gpu/core_tests.jl @@ -15,7 +15,8 @@ SOLVERS = ( NewtonRaphson(), LevenbergMarquardt(; linsolve = QRFactorization()), - LevenbergMarquardt(; linsolve = KrylovJL_GMRES()), + # XXX: Fails currently + # LevenbergMarquardt(; linsolve = KrylovJL_GMRES()), PseudoTransient(), Klement(), Broyden(; linesearch = LiFukushimaLineSearch()), @@ -27,7 +28,7 @@ ) @testset "[IIP] GPU Solvers" begin - for alg in SOLVERS + @testset "$(nameof(typeof(alg)))" for alg in SOLVERS @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5) end end @@ -37,7 +38,7 @@ prob = NonlinearProblem{false}(linear_f, u0) @testset "[OOP] GPU Solvers" begin - for alg in SOLVERS + @testset "$(nameof(typeof(alg)))" for alg in SOLVERS @test_nowarn sol = solve(prob, alg; abstol = 1.0f-5, reltol = 1.0f-5) end end