More debugging, use local wfact

.buildkite/Manifest.toml

@@ -2,7 +2,7 @@
 
 julia_version = "1.10.6"
 manifest_format = "2.0"
-project_hash = "8c1158513d32dc08d4dad3376a95c8915662a919"
+project_hash = "fc780a0dc833e209a503e8cdbd13bf068bf6b381"
 
 [[deps.ADTypes]]
 git-tree-sha1 = "eea5d80188827b35333801ef97a40c2ed653b081"

.buildkite/Project.toml

@@ -20,6 +20,7 @@ Krylov = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7"
 LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 LinearOperators = "5c8ed15e-5a4c-59e4-a42b-c7e8811fb125"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 ODEConvergenceTester = "42a5c2e1-f365-4540-8ca5-3684de3ecd95"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"

docs/src/dev/compute_convergence.jl

@@ -111,15 +111,19 @@ function predicted_convergence_order(algorithm_name::AbstractAlgorithmName, ode_
     return 0
 end
 
+import Logging
 function export_convergence_results(alg_name, test_problem, num_steps; kwargs...)
+    @info "=========================== Exporting convergence for $(test_problem.test_name)..."
     out_dict = Dict()
     (; test_name) = test_problem
     out_dict[string(test_name)] = Dict()
     out_dict[string(test_name)][string(alg_name)] = Dict()
     out_dict[string(test_name)]["args"] = (alg_name, test_problem, num_steps)
     out_dict[string(test_name)]["kwargs"] = kwargs
     compute_convergence!(out_dict, alg_name, test_problem, num_steps; kwargs...)
-    JLD2.save_object("convergence_$(alg_name)_$(test_problem.test_name).jld2", out_dict)
+    Logging.with_logger(Logging.NullLogger()) do
+        JLD2.save_object("convergence_$(alg_name)_$(test_problem.test_name).jld2", out_dict)
+    end
 end
 
 
@@ -279,8 +283,10 @@ function compute_convergence!(
         mkpath(out_path)
         fname(i) = replace("$(key1)_$(key2)_step_$(i).png", " " => "_", "(" => "", ")" => "")
         for (i, u) in enumerate(plot2_values.u)
-            f = joinpath(out_path, fname(i))
-            Plots.png(Plots.plot(u.u), joinpath(out_path, fname(i)))
+            if !all(isnan, u.u)
+                f = joinpath(out_path, fname(i))
+                Plots.png(Plots.plot(u.u), joinpath(out_path, fname(i)))
+            end
         end
         # error("NaN found in plot2_values in problem $(test_name)")
     end

test/problems.jl

@@ -510,15 +510,24 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
         dss_tendency && Spaces.weighted_dss!(tendency.u)
         _FT = Spaces.undertype(axes(state.u))
         set_boundaries!(tendency.u, _FT(0))
-        e1 = extrema(divgradu)
-        e2 = extrema(src_term)
-        e3 = extrema(φ_sin_sin_term)
-        @show e1, e2, e3, exp_term
+        if t ≤ t_end*0.01
+            e0 = extrema(state.u)
+            e1 = extrema(divgradu)
+            @show e0, e1, count(isnan, parent(state.u)), length(parent(state.u))
+        end
         return nothing
     end
 
     function dss!(state, _, t)
+        if t ≤ t_end*0.01
+            e0_dss = extrema(state.u)
+            @show e0_dss, count(isnan, parent(state.u)), length(parent(state.u))
+        end
         dss_tendency || Spaces.weighted_dss!(state.u)
+        if t ≤ t_end*0.01
+            e1_dss = extrema(state.u)
+            @show e1_dss, count(isnan, parent(state.u)), length(parent(state.u))
+        end
     end
 
     function analytic_sol(t)
@@ -537,7 +546,7 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
     T_imp! = SciMLBase.ODEFunction(
         (Yₜ, u, _, t) -> nothing;
         jac_prototype = FieldMatrixWithSolver(jacobian, init_state),
-        Wfact = Wfact!,
+        Wfact = (W, Y, p, dtγ, t) -> nothing,
         tgrad = (∂Y∂t, Y, p, t) -> (∂Y∂t .= 0),
     )