diff --git a/runtime/read_from_file.hpp b/runtime/read_from_file.hpp
index ddbc65e517cac5e13bbc56548cd8dbc10771d63f..3376b132125b12c8e1f9b0d524d045ef760a7b8b 100644
--- a/runtime/read_from_file.hpp
+++ b/runtime/read_from_file.hpp
@@ -14,8 +14,6 @@ size_t read_particle_data(PairsSimulation *ps, const char *filename, double *gri
std::string line;
size_t n = 0;
int read_grid_data = 0;
- // TODO: store this from PairsSimulation class
- const int num_dims = 3;
if(in_file.is_open()) {
while(std::getline(in_file, line)) {
@@ -25,6 +23,8 @@ size_t read_particle_data(PairsSimulation *ps, const char *filename, double *gri
while(std::getline(line_stream, in0, ',')) {
if(!read_grid_data) {
+ // TODO: store this from PairsSimulation class
+ const int num_dims = 3;
PAIRS_ASSERT(i < num_dims * 2);
grid_buffer[i] = std::stod(in0);
} else {
diff --git a/src/pairs/sim/comm.py b/src/pairs/sim/comm.py
index af9137814181e34eafcf2940af3f8f07a4672277..fc88339b03969e5e6e2678921b239db7dd0ca19f 100644
--- a/src/pairs/sim/comm.py
+++ b/src/pairs/sim/comm.py
@@ -35,35 +35,36 @@ class Comm:
@pairs_inline
def synchronize(self):
prop_list = [self.sim.property(p) for p in ['position']]
- for d in range(self.sim.ndims()):
- PackGhostParticles(self, d, prop_list)
- CommunicateData(self, d, prop_list)
- UnpackGhostParticles(self, d, prop_list)
+ for step in range(self.dom_part.number_of_steps()):
+ PackGhostParticles(self, step, prop_list)
+ CommunicateData(self, step, prop_list)
+ UnpackGhostParticles(self, step, prop_list)
@pairs_inline
def borders(self):
prop_list = [self.sim.property(p) for p in ['mass', 'position']]
self.nsend_all.set(0)
- for d in range(self.sim.ndims()):
- DetermineGhostParticles(self, d, self.sim.cell_spacing())
- CommunicateSizes(self, d)
- SetCommunicationOffsets(self, d)
- PackGhostParticles(self, d, prop_list)
- CommunicateData(self, d, prop_list)
- UnpackGhostParticles(self, d, prop_list)
+ for step in range(self.dom_part.number_of_steps()):
+ DetermineGhostParticles(self, step, self.sim.cell_spacing())
+ CommunicateSizes(self, step)
+ SetCommunicationOffsets(self, step)
+ PackGhostParticles(self, step, prop_list)
+ CommunicateData(self, step, prop_list)
+ UnpackGhostParticles(self, step, prop_list)
@pairs_inline
def exchange(self):
prop_list = [self.sim.property(p) for p in ['mass', 'position', 'velocity']]
- for d in range(self.sim.ndims()):
- DetermineGhostParticles(self, d, 0.0)
- PackGhostParticles(self, d, prop_list)
+ for step in range(self.dom_part.number_of_steps()):
+ self.nsend_all.set(0)
+ DetermineGhostParticles(self, step, 0.0)
+ PackGhostParticles(self, step, prop_list)
RemoveExchangedParticles_part1(self)
RemoveExchangedParticles_part2(self, prop_list)
- CommunicateSizes(self, d)
- CommunicateData(self, d, prop_list)
- ChangeSizeAfterExchange(self)
- UnpackGhostParticles(self, d, prop_list)
+ CommunicateSizes(self, step)
+ CommunicateData(self, step, prop_list)
+ ChangeSizeAfterExchange(self, step)
+ UnpackGhostParticles(self, step, prop_list)
class CommunicateSizes(Lowerable):
@@ -135,9 +136,13 @@ class SetCommunicationOffsets(Lowerable):
recv_offsets = self.comm.recv_offsets
self.sim.module_name(f"set_communication_offsets{self.step}")
+ isend = 0
+ irecv = 0
for j in self.comm.dom_part.step_indexes(self.step):
- send_offsets[j].set(send_offsets[j - 1] + nsend[j - 1] if j > 0 else 0)
- recv_offsets[j].set(recv_offsets[j - 1] + nrecv[j - 1] if j > 0 else 0)
+ send_offsets[j].set(isend)
+ recv_offsets[j].set(irecv)
+ isend += nsend[j]
+ irecv += nrecv[j]
class PackGhostParticles(Lowerable):
@@ -159,7 +164,9 @@ class PackGhostParticles(Lowerable):
elems_per_particle = self.get_elems_per_particle()
self.sim.module_name(f"pack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list]))
- for i in For(self.sim, 0, sum([self.comm.nsend[j] for j in self.comm.dom_part.step_indexes(self.step)])):
+ step_indexes = self.comm.dom_part.step_indexes(self.step)
+ start = self.comm.send_offsets[step_indexes[0]]
+ for i in For(self.sim, start, start + sum([self.comm.nsend[j] for j in step_indexes])):
p_offset = 0
m = send_map[i]
buffer_index = i * elems_per_particle
@@ -198,7 +205,9 @@ class UnpackGhostParticles(Lowerable):
elems_per_particle = self.get_elems_per_particle()
self.sim.module_name(f"unpack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list]))
- for i in For(self.sim, 0, sum([self.comm.nrecv[j] for j in self.comm.dom_part.step_indexes(self.step)])):
+ step_indexes = self.comm.dom_part.step_indexes(self.step)
+ start = self.comm.recv_offsets[step_indexes[0]]
+ for i in For(self.sim, start, start + sum([self.comm.nrecv[j] for j in step_indexes])):
p_offset = 0
buffer_index = i * elems_per_particle
for p in self.prop_list:
@@ -225,8 +234,8 @@ class RemoveExchangedParticles_part1(Lowerable):
self.sim.module_name("remove_exchanged_particles_pt1")
send_pos = self.sim.add_temp_var(self.sim.nparticles)
for i in For(self.sim, 0, self.comm.nsend_all):
- for is_local in Branch(self.sim, self.comm.send_map[i] < self.sim.nlocal - self.comm.nsend_all):
- if is_local:
+ for need_copy in Branch(self.sim, self.comm.send_map[i] < self.sim.nlocal - self.comm.nsend_all):
+ if need_copy:
for _ in While(self.sim, BinOp.cmp(self.comm.exchg_flag[send_pos], 1)):
send_pos.set(send_pos - 1)
@@ -263,14 +272,15 @@ class RemoveExchangedParticles_part2(Lowerable):
class ChangeSizeAfterExchange(Lowerable):
- def __init__(self, comm):
+ def __init__(self, comm, step):
super().__init__(comm.sim)
self.comm = comm
+ self.step = step
self.sim.add_statement(self)
@pairs_host_block
def lower(self):
sim = self.sim
- sim.module_name("change_size_after_exchange")
+ sim.module_name(f"change_size_after_exchange{self.step}")
sim.check_resize(self.sim.particle_capacity, self.sim.nlocal)
- self.sim.nlocal.set(sim.nlocal + self.comm.nrecv)
+ self.sim.nlocal.set(sim.nlocal + sum([self.comm.nrecv[j] for j in self.comm.dom_part.step_indexes(self.step)]))