Skip to content
Snippets Groups Projects
Commit ce8d5537 authored by Rafael Ravedutti's avatar Rafael Ravedutti
Browse files

First fixes for waLBerla communication

parent ff496c99
No related branches found
No related tags found
No related merge requests found
#include <map>
#include <mpi.h> #include <mpi.h>
#include <vector> #include <vector>
//--- //---
...@@ -21,6 +22,8 @@ ...@@ -21,6 +22,8 @@
namespace pairs { namespace pairs {
void BlockForest::updateNeighborhood() { void BlockForest::updateNeighborhood() {
std::map<int, std::vector<walberla::math::AABB>> neighborhood;
std::map<int, std::vector<walberla::BlockID>> blocks_pushed;
auto me = walberla::mpi::MPIManager::instance()->rank(); auto me = walberla::mpi::MPIManager::instance()->rank();
this->nranks = 0; this->nranks = 0;
this->total_aabbs = 0; this->total_aabbs = 0;
......
#include <map>
//---
#include <blockforest/BlockForest.h> #include <blockforest/BlockForest.h>
#include <blockforest/Initialization.h> #include <blockforest/Initialization.h>
#include <blockforest/loadbalancing/DynamicCurve.h> #include <blockforest/loadbalancing/DynamicCurve.h>
...@@ -27,8 +25,6 @@ private: ...@@ -27,8 +25,6 @@ private:
PairsSimulation *ps; PairsSimulation *ps;
std::shared_ptr<walberla::BlockForest> forest; std::shared_ptr<walberla::BlockForest> forest;
std::shared_ptr<walberla::blockforest::InfoCollection> info; std::shared_ptr<walberla::blockforest::InfoCollection> info;
std::map<int, std::vector<walberla::math::AABB>> neighborhood;
std::map<int, std::vector<walberla::BlockID>> blocks_pushed;
std::vector<int> ranks; std::vector<int> ranks;
std::vector<int> naabbs; std::vector<int> naabbs;
std::vector<int> aabb_offsets; std::vector<int> aabb_offsets;
......
...@@ -93,7 +93,7 @@ class Comm: ...@@ -93,7 +93,7 @@ class Comm:
CommunicateData(self, step, prop_list) CommunicateData(self, step, prop_list)
UnpackGhostParticles(self, step, prop_list) UnpackGhostParticles(self, step, prop_list)
step_nrecv = sum([self.nrecv[j] for j in self.dom_part.step_indexes(step)]) step_nrecv = self.dom_part.reduce_sum_step_indexes(step, self.nrecv)
Assign(self.sim, self.sim.nghost, self.sim.nghost + step_nrecv) Assign(self.sim, self.sim.nghost, self.sim.nghost + step_nrecv)
@pairs_inline @pairs_inline
...@@ -306,9 +306,9 @@ class PackGhostParticles(Lowerable): ...@@ -306,9 +306,9 @@ class PackGhostParticles(Lowerable):
send_mult = self.comm.send_mult send_mult = self.comm.send_mult
self.sim.module_name(f"pack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list])) self.sim.module_name(f"pack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list]))
step_indexes = self.comm.dom_part.step_indexes(self.step) start = self.comm.send_offsets[self.comm.dom_part.first_step_index(self.step)]
start = self.comm.send_offsets[step_indexes[0]] end = ScalarOp.inline(start + self.comm.dom_part.reduce_sum_step_indexes(self.step, self.comm.nsend))
for i in For(self.sim, start, ScalarOp.inline(start + sum([self.comm.nsend[j] for j in step_indexes]))): for i in For(self.sim, start, end):
p_offset = 0 p_offset = 0
m = send_map[i] m = send_map[i]
for p in self.prop_list: for p in self.prop_list:
...@@ -347,9 +347,9 @@ class UnpackGhostParticles(Lowerable): ...@@ -347,9 +347,9 @@ class UnpackGhostParticles(Lowerable):
recv_buffer.set_stride(1, self.get_elems_per_particle()) recv_buffer.set_stride(1, self.get_elems_per_particle())
self.sim.module_name(f"unpack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list])) self.sim.module_name(f"unpack_ghost_particles{self.step}_" + "_".join([str(p.id()) for p in self.prop_list]))
step_indexes = self.comm.dom_part.step_indexes(self.step) start = self.comm.recv_offsets[self.comm.dom_part.first_step_index(self.step)]
start = self.comm.recv_offsets[step_indexes[0]] end = ScalarOp.inline(start + self.comm.dom_part.reduce_sum_step_indexes(self.step, self.comm.nrecv))
for i in For(self.sim, start, ScalarOp.inline(start + sum([self.comm.nrecv[j] for j in step_indexes]))): for i in For(self.sim, start, end):
p_offset = 0 p_offset = 0
for p in self.prop_list: for p in self.prop_list:
if not Types.is_scalar(p.type()): if not Types.is_scalar(p.type()):
...@@ -422,9 +422,7 @@ class UnpackAllGhostParticles(Lowerable): ...@@ -422,9 +422,7 @@ class UnpackAllGhostParticles(Lowerable):
recv_buffer.set_stride(1, self.get_elems_per_particle()) recv_buffer.set_stride(1, self.get_elems_per_particle())
self.sim.module_name(f"unpack_all_ghost_particles" + "_".join([str(p.id()) for p in self.prop_list])) self.sim.module_name(f"unpack_all_ghost_particles" + "_".join([str(p.id()) for p in self.prop_list]))
nrecv_size = sum([len(dom_part.step_indexes(s)) for s in range(dom_part.number_of_steps())]) nrecv_all = self.comm.dom_part.reduce_sum_all_steps(self.comm.nrecv)
nrecv_all = sum([self.comm.nrecv[j] for j in range(nrecv_size)])
for i in For(self.sim, 0, nrecv_all): for i in For(self.sim, 0, nrecv_all):
p_offset = 0 p_offset = 0
for p in self.prop_list: for p in self.prop_list:
...@@ -516,7 +514,8 @@ class ChangeSizeAfterExchange(Lowerable): ...@@ -516,7 +514,8 @@ class ChangeSizeAfterExchange(Lowerable):
def lower(self): def lower(self):
self.sim.module_name(f"change_size_after_exchange{self.step}") self.sim.module_name(f"change_size_after_exchange{self.step}")
self.sim.check_resize(self.sim.particle_capacity, self.sim.nlocal) self.sim.check_resize(self.sim.particle_capacity, self.sim.nlocal)
Assign(self.sim, self.sim.nlocal, self.sim.nlocal + sum([self.comm.nrecv[j] for j in self.comm.dom_part.step_indexes(self.step)])) nrecv = self.comm.dom_part.reduce_sum_step_indexes(self.step, self.comm.nrecv)
Assign(self.sim, self.sim.nlocal, self.sim.nlocal + nrecv)
class PackContactHistoryData(Lowerable): class PackContactHistoryData(Lowerable):
...@@ -611,7 +610,6 @@ class UnpackContactHistoryData(Lowerable): ...@@ -611,7 +610,6 @@ class UnpackContactHistoryData(Lowerable):
contact_used = self.sim._contact_history.contact_used contact_used = self.sim._contact_history.contact_used
self.sim.module_name(f"unpack_contact_history{self.step}") self.sim.module_name(f"unpack_contact_history{self.step}")
step_indexes = self.comm.dom_part.step_indexes(self.step)
nelems_per_contact = sum([Types.number_of_elements(self.sim, cp.type()) \ nelems_per_contact = sum([Types.number_of_elements(self.sim, cp.type()) \
for cp in self.sim.contact_properties]) + 1 for cp in self.sim.contact_properties]) + 1
......
...@@ -29,6 +29,16 @@ class DimensionRanges: ...@@ -29,6 +29,16 @@ class DimensionRanges:
def step_indexes(self, step): def step_indexes(self, step):
return [step * 2 + 0, step * 2 + 1] return [step * 2 + 0, step * 2 + 1]
def first_step_index(self, step):
return self.step_indexes(step)[0]
def reduce_sum_all_steps(self, array):
total_size = sum([len(self.step_indexes(s)) for s in range(self.number_of_steps())])
return sum([array[i] for i in range(total_size)])
def reduce_sum_step_indexes(self, step, array):
return sum([array[i] for i in self.step_indexes(step)])
def initialize(self): def initialize(self):
grid_array = [(self.sim.grid.min(d), self.sim.grid.max(d)) for d in range(self.sim.ndims())] grid_array = [(self.sim.grid.min(d), self.sim.grid.max(d)) for d in range(self.sim.ndims())]
Call_Void(self.sim, "pairs->initDomain", [param for delim in grid_array for param in delim]) Call_Void(self.sim, "pairs->initDomain", [param for delim in grid_array for param in delim])
...@@ -96,7 +106,20 @@ class BlockForest: ...@@ -96,7 +106,20 @@ class BlockForest:
return 1 return 1
def step_indexes(self, step): def step_indexes(self, step):
return [step] yield from For(self.sim, 0, self.nranks)
def first_step_index(self, step):
return 0
def reduce_sum_all_steps(self, array):
return self.reduce_sum_step_indexes(0, array)
def reduce_sum_step_indexes(self, step, array):
nsend_sum = self.sim.add_temp_var(0)
for i in For(self.sim, 0, self.nranks):
Assign(self.sim, nsend_sum, nsend_sum + array[i])
return nsend_sum
def initialize(self): def initialize(self):
grid_array = [(self.sim.grid.min(d), self.sim.grid.max(d)) for d in range(self.sim.ndims())] grid_array = [(self.sim.grid.min(d), self.sim.grid.max(d)) for d in range(self.sim.ndims())]
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment