First fixes for communication code

Signed-off-by: Rafael Ravedutti <rafaelravedutti@gmail.com>

runtime/domain/regular_6d_stencil.hpp

@@ -56,12 +56,12 @@ public:
         MPI_Comm cartesian;
         int myloc[ndims];
         int periods[ndims];
-        int rank_length[ndims];
+        real_t rank_length[ndims];
         int reorder = 0;
 
         for(int d = 0; d < ndims; d++) {
             periods[d] = 1;
-            rank_length[d] = (this->grid_max[d] - this->grid_min[d]) / this->nranks[d];
+            rank_length[d] = (this->grid_max[d] - this->grid_min[d]) / (real_t)this->nranks[d];
         }
 
         MPI_Cart_create(MPI_COMM_WORLD, ndims, this->nranks, periods, reorder, &cartesian);
@@ -70,7 +70,7 @@ public:
             MPI_Cart_shift(cartesian, d, 1, &(this->prev[d]), &(this->next[d]));
             this->pbc_prev[d] = (myloc[d] == 0) ? 1 : 0;
             this->pbc_next[d] = (myloc[d] == this->nranks[d]) ? -1 : 0;
-            this->subdom_min[d] = this->grid_min[d] + rank_length[d] * myloc[d];
+            this->subdom_min[d] = this->grid_min[d] + rank_length[d] * (real_t)myloc[d];
             this->subdom_max[d] = this->subdom_min[d] + rank_length[d];
         }
 

src/pairs/analysis/devices.py

@@ -32,6 +32,16 @@ class FetchKernelReferences(Visitor):
         self.writing = False
         self.visit(ast_node.sources())
 
+    def visit_AtomicAdd(self, ast_node):
+        self.writing = True
+        self.visit(ast_node.elem)
+        self.writing = False
+        self.visit(ast_node.value)
+
+        if ast_node.resize is not None:
+            self.visit(ast_node.resize)
+            self.visit(ast_node.capacity)
+
     def visit_Kernel(self, ast_node):
         kernel_id = ast_node.kernel_id
         self.kernel_decls[kernel_id] = []

src/pairs/analysis/modules.py

@@ -23,6 +23,16 @@ class FetchModulesReferences(Visitor):
         self.writing = False
         self.visit(ast_node.sources())
 
+    def visit_AtomicAdd(self, ast_node):
+        self.writing = True
+        self.visit(ast_node.elem)
+        self.writing = False
+        self.visit(ast_node.value)
+
+        if ast_node.resize is not None:
+            self.visit(ast_node.resize)
+            self.visit(ast_node.capacity)
+
     def visit_Module(self, ast_node):
         self.module_stack.append(ast_node)
         self.visit_children(ast_node)

src/pairs/sim/comm.py

@@ -16,8 +16,8 @@ class Comm:
         self.sim = sim
         self.dom_part = dom_part
         self.nsend_all      = sim.add_var('nsend_all', Types.Int32)
-        self.send_capacity  = sim.add_var('send_capacity', Types.Int32, 100)
-        self.recv_capacity  = sim.add_var('recv_capacity', Types.Int32, 100)
+        self.send_capacity  = sim.add_var('send_capacity', Types.Int32, 10000)
+        self.recv_capacity  = sim.add_var('recv_capacity', Types.Int32, 10000)
         self.elem_capacity  = sim.add_var('elem_capacity', Types.Int32, 10)
         self.neigh_capacity = sim.add_var('neigh_capacity', Types.Int32, 6)
         self.nsend          = sim.add_array('nsend', [self.neigh_capacity], Types.Int32)
@@ -45,6 +45,7 @@ class Comm:
     def borders(self):
         prop_list = [self.sim.property(p) for p in ['mass', 'position']]
         self.nsend_all.set(0)
+        self.sim.nghost.set(0)
         for step in range(self.dom_part.number_of_steps()):
             DetermineGhostParticles(self, step, self.sim.cell_spacing())
             CommunicateSizes(self, step)
@@ -52,6 +53,7 @@ class Comm:
             PackGhostParticles(self, step, prop_list)
             CommunicateData(self, step, prop_list)
             UnpackGhostParticles(self, step, prop_list)
+            self.sim.nghost.add(sum([self.nrecv[j] for j in self.dom_part.step_indexes(step)]))
 
     @pairs_inline
     def exchange(self):
@@ -257,7 +259,7 @@ class RemoveExchangedParticles_part2(Lowerable):
     @pairs_device_block
     def lower(self):
         self.sim.module_name("remove_exchanged_particles_pt2")
-        for i in ParticleFor(self.sim):
+        for i in For(self.sim, 0, self.comm.nsend_all):
             src = self.comm.exchg_copy_to[i]
             for _ in Filter(self.sim, src > 0):
                 dst = self.comm.send_map[i]