From 90b7454216d3e26c225f7862f21ff1990b7611b4 Mon Sep 17 00:00:00 2001
From: Rafael Ravedutti <rafaelravedutti@gmail.com>
Date: Sat, 24 Sep 2022 02:57:37 +0200
Subject: [PATCH] First fixes for communication code
Signed-off-by: Rafael Ravedutti <rafaelravedutti@gmail.com>
---
runtime/domain/regular_6d_stencil.hpp | 6 +++---
src/pairs/analysis/devices.py | 10 ++++++++++
src/pairs/analysis/modules.py | 10 ++++++++++
src/pairs/sim/comm.py | 8 +++++---
4 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/runtime/domain/regular_6d_stencil.hpp b/runtime/domain/regular_6d_stencil.hpp
index 2891232..e65058d 100644
--- a/runtime/domain/regular_6d_stencil.hpp
+++ b/runtime/domain/regular_6d_stencil.hpp
@@ -56,12 +56,12 @@ public:
MPI_Comm cartesian;
int myloc[ndims];
int periods[ndims];
- int rank_length[ndims];
+ real_t rank_length[ndims];
int reorder = 0;
for(int d = 0; d < ndims; d++) {
periods[d] = 1;
- rank_length[d] = (this->grid_max[d] - this->grid_min[d]) / this->nranks[d];
+ rank_length[d] = (this->grid_max[d] - this->grid_min[d]) / (real_t)this->nranks[d];
}
MPI_Cart_create(MPI_COMM_WORLD, ndims, this->nranks, periods, reorder, &cartesian);
@@ -70,7 +70,7 @@ public:
MPI_Cart_shift(cartesian, d, 1, &(this->prev[d]), &(this->next[d]));
this->pbc_prev[d] = (myloc[d] == 0) ? 1 : 0;
this->pbc_next[d] = (myloc[d] == this->nranks[d]) ? -1 : 0;
- this->subdom_min[d] = this->grid_min[d] + rank_length[d] * myloc[d];
+ this->subdom_min[d] = this->grid_min[d] + rank_length[d] * (real_t)myloc[d];
this->subdom_max[d] = this->subdom_min[d] + rank_length[d];
}
diff --git a/src/pairs/analysis/devices.py b/src/pairs/analysis/devices.py
index 7aa0905..3746ac3 100644
--- a/src/pairs/analysis/devices.py
+++ b/src/pairs/analysis/devices.py
@@ -32,6 +32,16 @@ class FetchKernelReferences(Visitor):
self.writing = False
self.visit(ast_node.sources())
+ def visit_AtomicAdd(self, ast_node):
+ self.writing = True
+ self.visit(ast_node.elem)
+ self.writing = False
+ self.visit(ast_node.value)
+
+ if ast_node.resize is not None:
+ self.visit(ast_node.resize)
+ self.visit(ast_node.capacity)
+
def visit_Kernel(self, ast_node):
kernel_id = ast_node.kernel_id
self.kernel_decls[kernel_id] = []
diff --git a/src/pairs/analysis/modules.py b/src/pairs/analysis/modules.py
index 5e33c8e..c4a2c7a 100644
--- a/src/pairs/analysis/modules.py
+++ b/src/pairs/analysis/modules.py
@@ -23,6 +23,16 @@ class FetchModulesReferences(Visitor):
self.writing = False
self.visit(ast_node.sources())
+ def visit_AtomicAdd(self, ast_node):
+ self.writing = True
+ self.visit(ast_node.elem)
+ self.writing = False
+ self.visit(ast_node.value)
+
+ if ast_node.resize is not None:
+ self.visit(ast_node.resize)
+ self.visit(ast_node.capacity)
+
def visit_Module(self, ast_node):
self.module_stack.append(ast_node)
self.visit_children(ast_node)
diff --git a/src/pairs/sim/comm.py b/src/pairs/sim/comm.py
index 8fd6c49..1ef393d 100644
--- a/src/pairs/sim/comm.py
+++ b/src/pairs/sim/comm.py
@@ -16,8 +16,8 @@ class Comm:
self.sim = sim
self.dom_part = dom_part
self.nsend_all = sim.add_var('nsend_all', Types.Int32)
- self.send_capacity = sim.add_var('send_capacity', Types.Int32, 100)
- self.recv_capacity = sim.add_var('recv_capacity', Types.Int32, 100)
+ self.send_capacity = sim.add_var('send_capacity', Types.Int32, 10000)
+ self.recv_capacity = sim.add_var('recv_capacity', Types.Int32, 10000)
self.elem_capacity = sim.add_var('elem_capacity', Types.Int32, 10)
self.neigh_capacity = sim.add_var('neigh_capacity', Types.Int32, 6)
self.nsend = sim.add_array('nsend', [self.neigh_capacity], Types.Int32)
@@ -45,6 +45,7 @@ class Comm:
def borders(self):
prop_list = [self.sim.property(p) for p in ['mass', 'position']]
self.nsend_all.set(0)
+ self.sim.nghost.set(0)
for step in range(self.dom_part.number_of_steps()):
DetermineGhostParticles(self, step, self.sim.cell_spacing())
CommunicateSizes(self, step)
@@ -52,6 +53,7 @@ class Comm:
PackGhostParticles(self, step, prop_list)
CommunicateData(self, step, prop_list)
UnpackGhostParticles(self, step, prop_list)
+ self.sim.nghost.add(sum([self.nrecv[j] for j in self.dom_part.step_indexes(step)]))
@pairs_inline
def exchange(self):
@@ -257,7 +259,7 @@ class RemoveExchangedParticles_part2(Lowerable):
@pairs_device_block
def lower(self):
self.sim.module_name("remove_exchanged_particles_pt2")
- for i in ParticleFor(self.sim):
+ for i in For(self.sim, 0, self.comm.nsend_all):
src = self.comm.exchg_copy_to[i]
for _ in Filter(self.sim, src > 0):
dst = self.comm.send_map[i]
--
GitLab