include/pops/coupling/system/system_coupler.hpp Source FileΒΆ

adc_cpp: include/pops/coupling/system/system_coupler.hpp Source File
adc_cpp 0.3.0
Model-free C++23 core for coupled hyperbolic-elliptic systems on adaptive (AMR) meshes, with MPI and GPU (Kokkos) backends
system_coupler.hpp
Go to the documentation of this file.
1#pragma once
2
5#include <pops/coupling/base/aux_fill.hpp> // detail::derive_aux_bc + detail::fill_bz_box (shared)
22
23#include <algorithm>
24#include <functional>
25#include <type_traits>
26#include <utility>
27
39
40namespace pops {
41
42namespace detail {
43template <class>
44inline constexpr bool always_false_v = false;
45
46template <class Block>
48 Block& block;
50
51 ScopedBlockState(Block& b, MultiFab& stage_state) : block(b), old_state(b.state) {
52 block.state = &stage_state;
53 }
54
55 // RULE OF FIVE (C.21): scope-guard with a side effect in the dtor (restores block.state). Copy/move
56 // BY DEFAULT -> double restoration or restoration from a dead copy. Never copied nor moved
57 // (always a block-scoped local variable): delete the four operations.
62
64};
65} // namespace detail
66
67// === ASSEMBLER: fields (system Poisson + aux) + block residual. No stepping. ======
71template <CoupledSystemLike System, class RhsAssembler, class Elliptic = GeometricMG>
73 static_assert(EllipticSolver<Elliptic>, "the elliptic backend must model EllipticSolver");
74
75 public:
76 // bz: out-of-plane magnetic field B_z(x, y) supplied by the user (constant or field),
77 // shared by ALL blocks. The SHARED aux channel is allocated at the MAXIMUM width requested
78 // by the blocks (aux_comps): a block reading B_z (n_aux=4) sees it, a base block (3)
79 // ignores the component. Without an extra-field block the width stays 3 -> allocation and numerics
80 // strictly bit-identical to history.
81 SystemAssembler(System system, const Geometry& geom, const BoxArray& ba, const BCRec& bcPhi,
82 RhsAssembler rhs_assembler, std::function<bool(Real, Real)> active = {},
83 std::function<Real(Real, Real)> bz = {})
84 : system_(std::move(system)),
85 rhs_assembler_(std::move(rhs_assembler)),
86 geom_(geom),
87 ba_(ba),
88 dm_(ba.size(), n_ranks()),
89 bcPhi_(bcPhi),
90 aux_bc_(detail::derive_aux_bc(bcPhi)),
91 mg_(geom, ba, bcPhi, std::move(active)),
92 aux_ncomp_(system_aux_comps(system_)),
93 aux_(ba, dm_, aux_ncomp_, 1),
94 bz_(std::move(bz)) {
95 fill_bz(); // populates B_z (no-op if no block requests it or if bz is empty)
96 }
97
98 System& system() { return system_; }
99 const System& system() const { return system_; }
100 MultiFab& phi() { return mg_.phi(); }
101 MultiFab& aux() { return aux_; }
102 const MultiFab& aux() const { return aux_; }
103 const Geometry& geom() const { return geom_; }
104 const BoxArray& ba() const { return ba_; }
105 const DistributionMapping& dm() const { return dm_; }
106
110 rhs_assembler_(system_, mg_.rhs());
111 mg_.solve();
112 derive_aux();
113 }
114
118 template <class Limiter, class NumericalFlux, class Block>
119 void block_residual(Block& block, MultiFab& state, MultiFab& R, bool recompute_aux) {
120 if (recompute_aux) {
121 detail::ScopedBlockState<Block> swap(block, state);
122 solve_fields();
123 }
124 fill_ghosts(state, geom_.domain, block.bc);
125 assemble_rhs<Limiter, NumericalFlux>(block.model, state, aux_, geom_, R);
126 }
127
128 private:
129 void derive_aux() {
130 fill_ghosts(mg_.phi(), geom_.domain, bcPhi_);
131 const Real cx = Real(1) / (2 * geom_.dx());
132 const Real cy = Real(1) / (2 * geom_.dy());
133 field_postprocess(mg_.phi(), aux_, cx, cy,
134 FieldPostProcess{FieldPostProcess::GradSign::Plus, true});
135 fill_ghosts(aux_, geom_.domain, aux_bc_);
136 }
137
138 // Width of the SHARED aux channel: maximum of aux_comps<Model> over all blocks (at least
139 // kAuxBaseComps). The shared channel must be at least as wide as the most demanding block
140 // so that load_aux<aux_comps<Model>> never reads out of bounds; a less demanding block
141 // simply ignores the extra components.
142 static int system_aux_comps(const System& sys) {
143 int w = kAuxBaseComps;
144 sys.for_each_block([&](const auto& b) {
145 using Model = std::decay_t<decltype(b.model)>;
146 const int c = aux_comps<Model>();
147 if (c > w)
148 w = c;
149 });
150 return w;
151 }
152
153 // Populates the aux B_z component (index kAuxBaseComps) of the shared channel from bz_(x, y), a
154 // single time (static B_z). No-op if no block declares B_z (width 3) or if bz_ is empty:
155 // RUNTIME guard on aux_ncomp_ (the width is only known at construction). Halos are then
156 // maintained by derive_aux (aux_bc_); field_postprocess only writes phi/grad (comp 0..2).
157 void fill_bz() {
158 if (!bz_ || aux_ncomp_ <= kAuxBaseComps)
159 return;
160 for (int li = 0; li < aux_.local_size(); ++li)
161 detail::fill_bz_box(aux_.fab(li), aux_.box(li), geom_, bz_); // valid box
162 fill_ghosts(aux_, geom_.domain, aux_bc_); // B_z halos before the 1st solve
163 }
164
165 System system_;
166 RhsAssembler rhs_assembler_;
167 Geometry geom_;
168 BoxArray ba_;
169 DistributionMapping dm_;
170 BCRec bcPhi_, aux_bc_;
171 Elliptic mg_;
172 int aux_ncomp_; // width of the shared aux channel (max over blocks); init before aux_
173 MultiFab aux_;
174 std::function<Real(Real, Real)> bz_; // external B_z(x, y) (empty if not supplied)
175};
176
177// === DRIVER: advances the system. Owns an Assembler, delegates the fields to it. =========
181template <CoupledSystemLike System, class RhsAssembler, class Elliptic = GeometricMG>
183 public:
186 SystemDriver(System system, const Geometry& geom, const BoxArray& ba, const BCRec& bcPhi,
187 RhsAssembler rhs_assembler, std::function<bool(Real, Real)> active = {},
188 std::function<Real(Real, Real)> bz = {})
189 : asm_(std::move(system), geom, ba, bcPhi, std::move(rhs_assembler), std::move(active),
190 std::move(bz)) {}
191
192 // Accessors delegated to the assembler (compat with the old SystemCoupler API).
193 System& system() { return asm_.system(); }
194 const System& system() const { return asm_.system(); }
195 MultiFab& phi() { return asm_.phi(); }
196 const MultiFab& aux() const { return asm_.aux(); }
197 void solve_fields() { asm_.solve_fields(); }
199
202 template <class ImplicitAdvance>
203 void step(Real dt, ImplicitAdvance&& implicit_advance) {
204 ImplicitAdvance& advance_implicit = implicit_advance;
205 // macro_step_: a block of cadence `stride` only advances 1 macro-step out of stride
206 // (then by an effective step stride*dt). stride=1 -> every step (history).
207 advance_subcycled(asm_.system(), dt, macro_step_, [&](auto& block, Real h, int s, int n) {
208 advance_block_dispatch(block, h, s, n, advance_implicit);
209 });
210 ++macro_step_;
211 }
212
216 template <class ImplicitAdvance>
217 Real step_adaptive(Real cfl, ImplicitAdvance&& implicit_advance) {
218 ImplicitAdvance& advance_implicit = implicit_advance;
219 asm_.solve_fields(); // aux up to date for the wave speeds
220 const Real h = std::min(asm_.geom().dx(), asm_.geom().dy());
221 const Real wmax = system_max_wave_speed();
222 const Real macro_dt = cfl * h / std::max(wmax, Real(1e-30));
223 asm_.system().for_each_block([&](auto& block) {
224 using Block = std::decay_t<decltype(block)>;
225 if constexpr (block_time_treatment_v<Block> != TimeTreatment::Prescribed) {
226 const Real w_s = max_wave_speed_mf(block.model, block.U(), asm_.aux());
227 const int stride = (w_s <= Real(0)) ? 1 : std::max(1, static_cast<int>(wmax / w_s));
228 if (macro_step_ % stride == 0) {
229 constexpr int n = block_substeps_v<Block>;
230 const Real hh = (macro_dt * static_cast<Real>(stride)) / static_cast<Real>(n);
231 for (int s = 0; s < n; ++s)
232 advance_block_dispatch(block, hh, s, n, advance_implicit);
233 }
234 }
235 });
236 ++macro_step_;
237 return macro_dt;
238 }
240 return step_adaptive(cfl, [](auto&, auto& block, Real, int, int) {
241 using Block = std::decay_t<decltype(block)>;
242 static_assert(detail::always_false_v<Block>,
243 "SystemDriver::step_adaptive(cfl) cannot advance an "
244 "implicit/IMEX block without a callback");
245 });
246 }
247
248 // Convenience overload for a fully explicit system.
249 void step(Real dt) {
250 step(dt, [](auto&, auto& block, Real, int, int) {
251 using Block = std::decay_t<decltype(block)>;
252 static_assert(detail::always_false_v<Block>,
253 "SystemDriver::step(dt) cannot advance an "
254 "implicit/IMEX block without a callback");
255 });
256 }
257
261 asm_.solve_fields();
262 const Real h = std::min(asm_.geom().dx(), asm_.geom().dy());
263 return cfl * h / std::max(system_max_wave_speed(), Real(1e-30));
264 }
265 template <class ImplicitAdvance>
266 Real step_cfl(Real cfl, ImplicitAdvance&& implicit_advance) {
267 const Real dt = cfl_dt(cfl);
268 step(dt, std::forward<ImplicitAdvance>(implicit_advance));
269 return dt;
270 }
272 const Real dt = cfl_dt(cfl);
273 step(dt);
274 return dt;
275 }
276
279 template <class CoupledSource>
280 void coupled_source_step(CoupledSource&& src, Real dt) {
282 "coupled_source_step expects a CoupledSource: "
283 "apply(system, aux, dt)");
284 asm_.solve_fields();
285 src.apply(asm_.system(), asm_.aux(), dt);
286 }
287
288 private:
289 // Largest wave speed over ALL species (aux assumed up to date). Fixes the CFL step.
290 Real system_max_wave_speed() {
291 Real wmax = 0;
292 asm_.system().for_each_block([&](auto& block) {
293 wmax = std::max(wmax, max_wave_speed_mf(block.model, block.U(), asm_.aux()));
294 });
295 return wmax;
296 }
297
298 // Dispatch of a (sub-)step for ONE block, per its treatment. Shared by step (compile-time
299 // cadence) and step_adaptive (runtime CFL cadence): no duplication.
300 // Explicit: advance via TimeStepper. Implicit/IMEX: re-solve the fields, (IMEX)
301 // explicit transport, then implicit source via the callback.
302 template <class Block, class ImplicitAdvance>
303 void advance_block_dispatch(Block& block, Real h, int s, int n,
304 ImplicitAdvance& advance_implicit) {
305 constexpr TimeTreatment treatment = block_time_treatment_v<Block>;
306 if constexpr (treatment == TimeTreatment::Explicit) {
307 advance_explicit_block(block, h);
308 } else if constexpr (treatment == TimeTreatment::Implicit || treatment == TimeTreatment::IMEX) {
309 asm_.solve_fields();
310 if constexpr (treatment == TimeTreatment::IMEX)
311 explicit_transport(block, h);
312 advance_implicit(*this, block, h, s, n);
313 }
314 }
315
316 // Explicit advance of a block: DELEGATES the scheme to a TimeStepper object (core SSPRK2/3
317 // or user integrator), passing it the assembler's residual evaluator.
318 template <class Block>
319 void advance_explicit_block(Block& block, Real dt) {
320 using Time = TimePolicyTraits<typename Block::Time>;
321 using Method = typename Time::Method;
322 using Limiter = typename Block::Spatial::Limiter;
323 using NumericalFlux = typename Block::Spatial::NumericalFlux;
324 static_assert(Time::treatment == TimeTreatment::Explicit,
325 "advance_explicit_block expects an explicit block");
326
327 auto rhs_eval = [&](MultiFab& stage, MultiFab& R) {
328 asm_.template block_residual<Limiter, NumericalFlux>(block, stage, R,
329 /*recompute_aux=*/true);
330 };
331 if constexpr (std::is_same_v<Method, SSPRK3>)
332 SSPRK3Step{}.take_step(rhs_eval, block.U(), dt);
333 else if constexpr (std::is_same_v<Method, SSPRK2>)
334 SSPRK2Step{}.take_step(rhs_eval, block.U(), dt);
335 else if constexpr (TimeStepper<Method>)
336 Method{}.take_step(rhs_eval, block.U(), dt);
337 else
338 static_assert(detail::always_false_v<Method>,
339 "explicit Method must be SSPRK2, SSPRK3, or a TimeStepper "
340 "(object with take_step(rhs_eval, U, dt)) supplied by the user");
341 }
342
343 // EXPLICIT half-step of an IMEX block: transport only (-div F, source-free), forward Euler.
344 // The stiff source is handled separately in implicit form by the callback. aux assumed up to date.
345 template <class Block>
346 void explicit_transport(Block& block, Real dt) {
347 using Model = typename Block::Model;
348 using Limiter = typename Block::Spatial::Limiter;
349 using NumericalFlux = typename Block::Spatial::NumericalFlux;
350 const SourceFreeModel<Model> sf{block.model};
351 MultiFab R(asm_.ba(), asm_.dm(), Model::n_vars, 0);
352 fill_ghosts(block.U(), asm_.geom().domain, block.bc);
353 assemble_rhs<Limiter, NumericalFlux>(sf, block.U(), asm_.aux(), asm_.geom(), R);
354 saxpy(block.U(), dt, R);
355 }
356
357 SystemAssembler<System, RhsAssembler, Elliptic> asm_;
358 int macro_step_ = 0; // macro-step counter (per-block stride cadence)
359};
360
361// Compat / historical naming: SystemCoupler == the Driver (which advances). We keep the alias
362// SystemCoupler (tests, MultiSpeciesSolver facade) AND SystemDriver (the "advances" name).
363template <CoupledSystemLike System, class RhsAssembler, class Elliptic = GeometricMG>
365
366// Friendly builder for the SystemCoupler alias. CTAD written directly on an alias template
367// (`SystemCoupler sim(...)`) is accepted by GCC but REJECTED by clang -- alias-template argument
368// deduction (P1814) is not implemented the same way, so `SystemCoupler sim(...)` fails to compile
369// under clang ("alias template requires template arguments"). This factory deduces the parameters
370// through the underlying class template (CTAD on SystemDriver, which every compiler supports) and
371// forwards to its constructor. Use `auto sim = make_system_coupler(system, geom, ba, bc, rhs);`.
372template <class... Args>
373auto make_system_coupler(Args&&... args) {
374 return SystemDriver(std::forward<Args>(args)...);
375}
376
377} // namespace pops
Helpers shared by the three couplers (single-block Coupler, SystemAssembler, AmrSystemCoupler) for th...
BoxArray: the set of boxes tiling a level (disjoint, covering).
Ordered list of boxes tiling a level.
Definition box_array.hpp:22
Owning MPI rank of each box, indexed by GLOBAL box index (parallel to a BoxArray).
Definition distribution_mapping.hpp:19
Field distributed over a level: decomposition (BoxArray) + distribution (DistributionMapping) + ncomp...
Definition multifab.hpp:33
Fab2D & fab(int li)
Local fab at index li (0 <= li < local_size()), for writing.
Definition multifab.hpp:67
const Box2D & box(int li) const
VALID box of local fab li.
Definition multifab.hpp:71
int local_size() const
Number of fabs OWNED by this rank (bound on local indices).
Definition multifab.hpp:65
ASSEMBLES the fields (system Poisson + shared aux) and a block residual evaluator.
Definition system_coupler.hpp:72
MultiFab & aux()
Definition system_coupler.hpp:101
const System & system() const
Definition system_coupler.hpp:99
const MultiFab & aux() const
Definition system_coupler.hpp:102
void block_residual(Block &block, MultiFab &state, MultiFab &R, bool recompute_aux)
Residual R = -div F + S of a block at a stage (with field re-solve if recompute_aux).
Definition system_coupler.hpp:119
const DistributionMapping & dm() const
Definition system_coupler.hpp:105
void solve_fields()
Solves the system RHS (Sum_s q_s n_s), the Poisson, then derives aux = (phi, grad phi).
Definition system_coupler.hpp:109
const Geometry & geom() const
Definition system_coupler.hpp:103
MultiFab & phi()
Definition system_coupler.hpp:100
const BoxArray & ba() const
Definition system_coupler.hpp:104
SystemAssembler(System system, const Geometry &geom, const BoxArray &ba, const BCRec &bcPhi, RhsAssembler rhs_assembler, std::function< bool(Real, Real)> active={}, std::function< Real(Real, Real)> bz={})
Definition system_coupler.hpp:81
System & system()
Definition system_coupler.hpp:98
ADVANCES the system: carries the schedule (per-species subcycling, adaptive multirate,...
Definition system_coupler.hpp:182
void step(Real dt)
Definition system_coupler.hpp:249
const System & system() const
Definition system_coupler.hpp:194
const MultiFab & aux() const
Definition system_coupler.hpp:196
Real step_adaptive(Real cfl, ImplicitAdvance &&implicit_advance)
FULLY ADAPTIVE multirate: macro-step fixed by the fastest species (CFL cfl), stride of each species d...
Definition system_coupler.hpp:217
SystemDriver(System system, const Geometry &geom, const BoxArray &ba, const BCRec &bcPhi, RhsAssembler rhs_assembler, std::function< bool(Real, Real)> active={}, std::function< Real(Real, Real)> bz={})
Builds the driver (which builds the underlying assembler).
Definition system_coupler.hpp:186
SystemAssembler< System, RhsAssembler, Elliptic > & assembler()
Definition system_coupler.hpp:198
void coupled_source_step(CoupledSource &&src, Real dt)
Applies an inter-species COUPLING source (forward-Euler splitting): refreshes phi (aux) then calls sr...
Definition system_coupler.hpp:280
void step(Real dt, ImplicitAdvance &&implicit_advance)
Advances the blocks by a macro-step dt per their TimePolicy: explicit ones via TimeStepper,...
Definition system_coupler.hpp:203
void solve_fields()
Definition system_coupler.hpp:197
System & system()
Definition system_coupler.hpp:193
Real step_cfl(Real cfl)
Definition system_coupler.hpp:271
Real step_cfl(Real cfl, ImplicitAdvance &&implicit_advance)
Definition system_coupler.hpp:266
Real step_adaptive(Real cfl)
Definition system_coupler.hpp:239
MultiFab & phi()
Definition system_coupler.hpp:195
Real cfl_dt(Real cfl)
Macro-step chosen by multi-species CFL: dt = cfl * min(dx, dy) / w_max (w_max = largest wave speed ov...
Definition system_coupler.hpp:260
Coupled multi-species system, composed at runtime from generic bricks.
Definition system.hpp:89
Parallel seam: minimal MPI abstraction (rank/size + collectives) with serial fallback.
Concept: C is a valid coupling source for System if System is a CoupledSystem and if C exposes apply(...
Definition coupled_source.hpp:24
Definition elliptic_solver.hpp:30
CoupledSourceFor / NoCoupledSource: contract of an inter-species COUPLING source.
CoupledSystem: heterogeneous collection of equation blocks (multi-species, multi-scheme).
DistributionMapping: maps each box (by global index) to its owning MPI rank.
DESCRIPTIVE types of the elliptic stage: EllipticProblem (problem definition) and FieldPostProcess (f...
Elliptic (Poisson) RIGHT-HAND-SIDE assemblers: single-model and N-species.
EllipticSolver concept: common contract for elliptic solvers at the MultiFab level (solve D phi = f),...
GeometricMG: in-house geometric multigrid (V-cycle) for the elliptic operator, Gauss-Seidel smoother ...
Geometry: index-space (Box2D) <-> Cartesian physical-space mapping; PolarGeometry: SIBLING for a glob...
Implicit / IMEX block step as a named CONTRACT.
MultiFab arithmetic (saxpy, lincomb, norm_inf, dot) over VALID cells.
MultiFab: a field DISTRIBUTED over a level (equivalent of AMReX's MultiFab).
constexpr bool always_false_v
Definition system_coupler.hpp:44
void fill_bz_box(Fab2D &f, const Box2D &box, const Geometry &g, const Bz &bz)
Writes B_z(x, y) at component kAuxBaseComps on box box of fab f, sampling bz at the cell centers of g...
Definition aux_fill.hpp:42
Definition amr_hierarchy.hpp:29
auto make_system_coupler(Args &&... args)
Definition system_coupler.hpp:373
void advance_subcycled(System &system, Real dt, int macro_step, AdvanceBlock &&advance_block)
Definition scheduler.hpp:45
constexpr int kAuxBaseComps
Definition state.hpp:147
double Real
Definition types.hpp:30
int n_ranks()
Definition comm.hpp:139
TimeTreatment
Definition time_integrator.hpp:29
void saxpy(MultiFab &y, Real a, const MultiFab &x)
y <- y + a x over ALL components of the valid cells. Identical layouts required.
Definition mf_arith.hpp:102
void field_postprocess(const MultiFab &phi, MultiFab &out, Real cx, Real cy, FieldPostProcess spec)
Definition elliptic_problem.hpp:104
void fill_ghosts(MultiFab &mf, const Box2D &domain, const BCRec &bc)
COMPLETE ghost filling: fill_boundary (interior + periodic, periodicity deduced from bc) THEN fill_ph...
Definition physical_bc.hpp:227
Real max_wave_speed_mf(const Model &model, const MultiFab &U, const MultiFab &aux)
max_wave_speed_mf: global max of the wave speed over the whole MultiFab (CFL).
Definition wave_speed.hpp:66
PHYSICAL boundary conditions at the domain edge (BCType, BCRec, fill_physical_bc, fill_ghosts).
Minimal scheduler for coupled systems: advance_subcycled reads each block's time policy (traits block...
Cartesian spatial operator: assembles R(U, aux) = -div F + S over the cells of a level.
Boundary conditions for the FOUR faces of the domain (type + associated Dirichlet value).
Definition physical_bc.hpp:29
Definition elliptic_problem.hpp:73
Cartesian geometry of a level: index domain + physical bounds [xlo, xhi] x [ylo, yhi].
Definition geometry.hpp:20
POPS_HD Real dy() const
Grid spacing in y (= (yhi - ylo) / domain.ny()). POPS_HD.
Definition geometry.hpp:33
POPS_HD Real dx() const
Grid spacing in x (= (xhi - xlo) / domain.nx()). POPS_HD.
Definition geometry.hpp:31
Box2D domain
Definition geometry.hpp:21
Definition system_coupler.hpp:47
Block & block
Definition system_coupler.hpp:48
ScopedBlockState & operator=(ScopedBlockState &&)=delete
MultiFab * old_state
Definition system_coupler.hpp:49
ScopedBlockState(const ScopedBlockState &)=delete
ScopedBlockState & operator=(const ScopedBlockState &)=delete
~ScopedBlockState()
Definition system_coupler.hpp:63
ScopedBlockState(Block &b, MultiFab &stage_state)
Definition system_coupler.hpp:51
ScopedBlockState(ScopedBlockState &&)=delete
Time integrators as first-class OBJECTS with a take_step method: TimeStepper concept,...
Base scalar types and the POPS_HD macro (host+device portability).