include/pops/parallel/comm.hpp Source File

adc_cpp 0.3.0 Model-free C++23 core for coupled hyperbolic-elliptic systems on adaptive (AMR) meshes, with MPI and GPU (Kokkos) backends

• include
• pops
• parallel

comm.hpp

Go to the documentation of this file.

1#pragma once

2

21

22#ifdef POPS_HAS_MPI

23#include <mpi.h>

24#endif

25

26namespace pops {

27

28#ifdef POPS_HAS_MPI

29

30inline bool comm_active() {

31 int inited = 0, fin = 0;

32 MPI_Initialized(&inited);

33 MPI_Finalized(&fin);

34 return inited && !fin;

35}

36

37inline void comm_init(int* argc = nullptr, char*** argv = nullptr) {

38 int inited = 0;

39 MPI_Initialized(&inited);

40 if (!inited)

41 MPI_Init(argc, argv);

42}

43

44inline void comm_finalize() {

45 int fin = 0;

46 MPI_Finalized(&fin);

47 if (!fin)

48 MPI_Finalize();

49}

50

51inline int my_rank() {

52 if (!comm_active())

53 return 0;

54 int r = 0;

55 MPI_Comm_rank(MPI_COMM_WORLD, &r);

56 return r;

57}

58

59inline int n_ranks() {

60 if (!comm_active())

61 return 1;

62 int s = 1;

63 MPI_Comm_size(MPI_COMM_WORLD, &s);

64 return s;

65}

66

67inline void barrier() {

68 if (comm_active())

69 MPI_Barrier(MPI_COMM_WORLD);

70}

71

72inline double all_reduce_sum(double x) {

73 if (!comm_active())

74 return x;

75 double r = x;

76 MPI_Allreduce(&x, &r, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);

77 return r;

78}

79

80inline double all_reduce_max(double x) {

81 if (!comm_active())

82 return x;

83 double r = x;

84 MPI_Allreduce(&x, &r, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);

85 return r;

86}

87

88// Global min (counterpart of all_reduce_max). Brick for GLOBAL time step bounds

89// (System::add_dt_bound): the host callback is evaluated PER RANK, the global min guarantees a dt

90// IDENTICAL on all ranks (otherwise the collectives of the step -- Krylov, fill_boundary --

91// would diverge -> deadlock). In serial: identity.

92inline double all_reduce_min(double x) {

93 if (!comm_active())

94 return x;

95 double r = x;

96 MPI_Allreduce(&x, &r, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD);

97 return r;

98}

99

100// Element-by-element sum of an array, in place, on all ranks. Base brick of the

101// distributed multi-patch AMR reflux: each rank fills the contributions of

102// its local patches (0 elsewhere), all-reduce -> each rank has the complete register.

103inline void all_reduce_sum_inplace(double* buf, int n) {

104 if (!comm_active() || n <= 0)

105 return;

106 MPI_Allreduce(MPI_IN_PLACE, buf, n, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD);

107}

108

109inline long all_reduce_sum(long x) {

110 if (!comm_active())

111 return x;

112 long r = x;

113 MPI_Allreduce(&x, &r, 1, MPI_LONG, MPI_SUM, MPI_COMM_WORLD);

114 return r;

115}

116

117// Element-by-element logical OR of a marker array (0/1), in place, on all ranks.

118// Brick of the DISTRIBUTED-COARSE AMR regrid: each rank tags ONLY its local coarse

119// boxes (tag_cells iterating over local_size()), so nobody sees the complete tag grid;

120// the global OR gathers the tags on each rank before the Berger-Rigoutsos clustering, which then

121// produces IDENTICAL fine patches everywhere (otherwise the fine BoxArray would differ per rank -> MPI

122// desynchronized). See the note in tag_box.hpp ("the distributed tags will be gathered before clustering").

123inline void all_reduce_or_inplace(char* buf, int n) {

124 if (!comm_active() || n <= 0)

125 return;

126 MPI_Allreduce(MPI_IN_PLACE, buf, n, MPI_CHAR, MPI_BOR, MPI_COMM_WORLD);

127}

128

129#else // ----- serial -----

130

131inline bool comm_active() {

132 return false;

133}

134inline void comm_init(int* = nullptr, char*** = nullptr) {}

135inline void comm_finalize() {}

136inline int my_rank() {

137 return 0;

138}

139inline int n_ranks() {

140 return 1;

141}

142inline void barrier() {}

143inline double all_reduce_sum(double x) {

144 return x;

145}

146inline double all_reduce_max(double x) {

147 return x;

148}

149inline double all_reduce_min(double x) {

150 return x;

151}

152inline long all_reduce_sum(long x) {

153 return x;

154}

155inline void all_reduce_sum_inplace(double*, int) {} // serial: identity

156inline void all_reduce_or_inplace(char*, int) {} // serial: identity

157

158#endif

159

160} // namespace pops

pops

Definition amr_hierarchy.hpp:29

pops::comm_init

void comm_init(int *=nullptr, char ***=nullptr)

Definition comm.hpp:134

pops::n_ranks

int n_ranks()

Definition comm.hpp:139

pops::all_reduce_or_inplace

void all_reduce_or_inplace(char *, int)

Definition comm.hpp:156

pops::all_reduce_sum

double all_reduce_sum(double x)

Definition comm.hpp:143

pops::comm_finalize

void comm_finalize()

Definition comm.hpp:135

pops::all_reduce_sum_inplace

void all_reduce_sum_inplace(double *, int)

Definition comm.hpp:155

pops::barrier

void barrier()

Definition comm.hpp:142

pops::comm_active

bool comm_active()

Definition comm.hpp:131

pops::my_rank

int my_rank()

Definition comm.hpp:136

pops::all_reduce_max

double all_reduce_max(double x)

Definition comm.hpp:146

pops::all_reduce_min

double all_reduce_min(double x)

Definition comm.hpp:149

---

Generated by  1.9.8