PoPS/doxygen/multifab_8hpp_source.html

#pragma once


#include <pops/core/foundation/types.hpp>

#include <pops/mesh/index/box2d.hpp>

#include <pops/mesh/layout/box_array.hpp>

#include <pops/mesh/layout/distribution_mapping.hpp>

#include <pops/mesh/storage/fab2d.hpp>

#include <pops/mesh/execution/for_each.hpp>       // device_fence, sync_host, sync_device

#include <pops/mesh/boundary/halo_schedule.hpp>  // memoized fill_boundary schedule (ADC-260)

#include <pops/parallel/comm.hpp>


#include <memory>

#include <utility>

#include <vector>


namespace pops {


class MultiFab {

 public:

  MultiFab() = default;


  MultiFab(BoxArray ba, DistributionMapping dm, int ncomp, int ngrow)

      : ba_(std::move(ba)),

        dm_(std::move(dm)),

        ncomp_(ncomp),

        ngrow_(ngrow),

        local_index_(ba_.size(), -1) {

    const int me = my_rank();

    for (int i = 0; i < ba_.size(); ++i) {

      if (dm_[i] == me) {

        local_index_[i] = static_cast<int>(fabs_.size());

        global_of_local_.push_back(i);

        fabs_.emplace_back(ba_[i], ncomp_, ngrow_);

      }

    }

  }


  const BoxArray& box_array() const { return ba_; }

  const DistributionMapping& dmap() const { return dm_; }

  int ncomp() const { return ncomp_; }

  int n_grow() const { return ngrow_; }


  int local_size() const { return static_cast<int>(fabs_.size()); }

  Fab2D& fab(int li) { return fabs_[li]; }

  const Fab2D& fab(int li) const { return fabs_[li]; }

  const Box2D& box(int li) const { return fabs_[li].box(); }

  int global_index(int li) const { return global_of_local_[li]; }

  int local_index_of(int global) const { return local_index_[global]; }


  void sync_host() { pops::sync_host(); }

  void sync_device() { pops::sync_device(); }


  void set_val(Real v) {

    sync_host();  // a kernel may have written these fabs; make the host residence

                  // valid before the host fill (otherwise a host/kernel write

                  // race). Under unified memory = a device_fence().

    for (auto& f : fabs_)

      f.set_val(v);

  }


  HaloScheduleCache& halo_cache() const {

    if (!halo_cache_)

      halo_cache_ = std::make_shared<HaloScheduleCache>();

    return *halo_cache_;

  }


 private:

  BoxArray ba_{};

  DistributionMapping dm_{};

  int ncomp_{1};

  int ngrow_{0};

  std::vector<Fab2D> fabs_{};           // locally owned fabs

  std::vector<int> local_index_{};      // global box -> local index (-1 otherwise)

  std::vector<int> global_of_local_{};  // local index -> global box

  // Memoized fill_boundary schedule (ADC-260). mutable: caching is logically const; lazily built.

  mutable std::shared_ptr<HaloScheduleCache> halo_cache_{};

};


inline Real sum(const MultiFab& mf, int comp = 0) {

  Real s = 0;

  for (int li = 0; li < mf.local_size(); ++li) {

    const ConstArray4 a = mf.fab(li).const_array();

    s += for_each_cell_reduce_sum(mf.box(li),

                                  [a, comp] POPS_HD(int i, int j) { return a(i, j, comp); });

  }

  return static_cast<Real>(all_reduce_sum(static_cast<double>(s)));

}


}  // namespace pops

box2d.hpp
Box2D: the integer index space of a 2D cell-centered Cartesian grid.

box_array.hpp
BoxArray: the set of boxes tiling a level (disjoint, covering).

pops::BoxArray
Ordered list of boxes tiling a level.
Definition box_array.hpp:22

pops::BoxArray::size
int size() const
Number of boxes in the tiling.
Definition box_array.hpp:42

pops::DistributionMapping
Owning MPI rank of each box, indexed by GLOBAL box index (parallel to a BoxArray).
Definition distribution_mapping.hpp:19

pops::Fab2D
Single-grid data on a Box2D: VALID box + ng ghost layers, ncomp components, component-slow layout.
Definition fab2d.hpp:59

pops::Fab2D::const_array
ConstArray4 const_array() const
READ handle (POD device-copyable) over this Fab. Valid as long as the Fab lives.
Definition fab2d.hpp:96

pops::HaloScheduleCache
Small per-MultiFab cache of halo schedules, one entry per distinct (Periodicity, domain).
Definition halo_schedule.hpp:57

pops::MultiFab
Field distributed over a level: decomposition (BoxArray) + distribution (DistributionMapping) + ncomp...
Definition multifab.hpp:33

pops::MultiFab::MultiFab
MultiFab()=default

pops::MultiFab::fab
Fab2D & fab(int li)
Local fab at index li (0 <= li < local_size()), for writing.
Definition multifab.hpp:67

pops::MultiFab::local_index_of
int local_index_of(int global) const
LOCAL index of the global box global, or -1 if it is not owned by this rank.
Definition multifab.hpp:75

pops::MultiFab::global_index
int global_index(int li) const
GLOBAL index (in box_array) of local fab li.
Definition multifab.hpp:73

pops::MultiFab::ncomp
int ncomp() const
Number of components.
Definition multifab.hpp:60

pops::MultiFab::halo_cache
HaloScheduleCache & halo_cache() const
Internal (ADC-260): memoized halo-exchange schedule used by fill_boundary.
Definition multifab.hpp:100

pops::MultiFab::sync_host
void sync_host()
Makes the HOST residence valid (before a host access: operator(), loop, set_val).
Definition multifab.hpp:79

pops::MultiFab::sync_device
void sync_device()
Marks a DEVICE residence (before a kernel). No-op under unified memory.
Definition multifab.hpp:81

pops::MultiFab::fab
const Fab2D & fab(int li) const
Local fab at index li, for reading.
Definition multifab.hpp:69

pops::MultiFab::dmap
const DistributionMapping & dmap() const
GLOBAL distribution (owner rank per box).
Definition multifab.hpp:58

pops::MultiFab::n_grow
int n_grow() const
Number of ghost layers.
Definition multifab.hpp:62

pops::MultiFab::box_array
const BoxArray & box_array() const
GLOBAL decomposition of the level (all boxes, all ranks).
Definition multifab.hpp:56

pops::MultiFab::box
const Box2D & box(int li) const
VALID box of local fab li.
Definition multifab.hpp:71

pops::MultiFab::MultiFab
MultiFab(BoxArray ba, DistributionMapping dm, int ncomp, int ngrow)
Builds the field: allocates one Fab2D (ncomp components, ngrow ghosts) for EACH box that this rank ow...
Definition multifab.hpp:39

pops::MultiFab::set_val
void set_val(Real v)
Fills all cells (valid + ghosts) of every local fab with v.
Definition multifab.hpp:85

pops::MultiFab::local_size
int local_size() const
Number of fabs OWNED by this rank (bound on local indices).
Definition multifab.hpp:65

comm.hpp
Parallel seam: minimal MPI abstraction (rank/size + collectives) with serial fallback.

distribution_mapping.hpp
DistributionMapping: maps each box (by global index) to its owning MPI rank.

fab2d.hpp
Fab2D: single-grid data on a Box2D (in-house equivalent of AMReX's FArrayBox); Array4 / ConstArray4: ...

for_each.hpp
for_each_cell and reductions: the parallelism SEAM over the cells of a Box2D; sync_host / sync_device...

halo_schedule.hpp
HaloSchedule: memoized intra-level halo-exchange plan for fill_boundary (ADC-260).

pops
Definition amr_hierarchy.hpp:29

pops::for_each_cell_reduce_sum
Real for_each_cell_reduce_sum(const Box2D &b, F f)
SUM reduction of f(i, j) over box b.
Definition for_each.hpp:199

pops::Real
double Real
Definition types.hpp:30

pops::all_reduce_sum
double all_reduce_sum(double x)
Definition comm.hpp:143

pops::sync_host
void sync_host()
Makes the HOST residency valid before a host access (read/write from the host).
Definition for_each.hpp:125

pops::my_rank
int my_rank()
Definition comm.hpp:136

pops::sum
Real sum(const MultiFab &mf, int comp=0)
Sum of the VALID cells of component comp, reduced over ALL ranks (all_reduce).
Definition multifab.hpp:121

pops::sync_device
void sync_device()
Marks a DEVICE residency (upcoming kernel).
Definition for_each.hpp:132

pops::Box2D
2D integer index space, cell-centered.
Definition box2d.hpp:37

pops::ConstArray4
READ-only handle (const counterpart of Array4): same layout and same contract (POD device-copyable,...
Definition fab2d.hpp:44

types.hpp
Base scalar types and the POPS_HD macro (host+device portability).

POPS_HD
#define POPS_HD
Definition types.hpp:25