Adjustments to GPU/MPI version

Signed-off-by: Rafael Ravedutti <rafaelravedutti@gmail.com>

runtime/pairs.cpp

@@ -40,6 +40,12 @@ Array &PairsSimulation::getArrayByName(std::string name) {
     return *a;
 }
 
+Array &PairsSimulation::getArrayByHostPointer(const void *h_ptr) {
+    auto a = std::find_if(arrays.begin(), arrays.end(), [h_ptr](Array a) { return a.getHostPointer() == h_ptr; });
+    PAIRS_ASSERT(a != std::end(arrays));
+    return *a;
+}
+
 void PairsSimulation::addProperty(Property prop) {
     int id = prop.getId();
     auto p = std::find_if(properties.begin(), properties.end(), [id](Property p) { return p.getId() == id; });
@@ -100,6 +106,14 @@ void PairsSimulation::copyPropertyToHost(Property &prop) {
 }
 
 void PairsSimulation::communicateSizes(int dim, const int *send_sizes, int *recv_sizes) {
+    auto nsend_id = getArrayByHostPointer(send_sizes).getId();
+    auto nrecv_id = getArrayByHostPointer(recv_sizes).getId();
+
+    copyArrayToHost(nsend_id);
+    array_flags->setHostFlag(nsend_id);
+    array_flags->clearDeviceFlag(nsend_id);
+    array_flags->setHostFlag(nrecv_id);
+    array_flags->clearDeviceFlag(nrecv_id);
     this->getDomainPartitioner()->communicateSizes(dim, send_sizes, recv_sizes);
     PAIRS_DEBUG("send_sizes=[%d, %d], recv_sizes=[%d, %d]\n", send_sizes[dim * 2 + 0], send_sizes[dim * 2 + 1], recv_sizes[dim * 2 + 0], recv_sizes[dim * 2 + 1]);
 }
@@ -109,6 +123,9 @@ void PairsSimulation::communicateData(
     const real_t *send_buf, const int *send_offsets, const int *nsend,
     real_t *recv_buf, const int *recv_offsets, const int *nrecv) {
 
+    auto recv_buf_id = getArrayByHostPointer(recv_buf).getId();
+    array_flags->setHostFlag(recv_buf_id);
+    array_flags->clearDeviceFlag(recv_buf_id);
     this->getDomainPartitioner()->communicateData(dim, elem_size, send_buf, send_offsets, nsend, recv_buf, recv_offsets, nrecv);
 
     /*

runtime/pairs.hpp

@@ -57,6 +57,7 @@ public:
 
     Array &getArray(array_t id);
     Array &getArrayByName(std::string name);
+    Array &getArrayByHostPointer(const void *h_ptr);
 
     template<typename T_ptr> void addProperty(
         property_t id, std::string name, T_ptr **h_ptr, std::nullptr_t, PropertyType type, layout_t layout, size_t sx, size_t sy = 1);

src/pairs/ir/module.py

@@ -65,13 +65,15 @@ class Module(ASTNode):
         return self._host_references
 
     def properties_to_synchronize(self):
-        return {p for p in self._properties if self._properties[p][0] == 'r'}
+        #return {p for p in self._properties if self._properties[p][0] == 'r'}
+        return {p for p in self._properties}
 
     def write_properties(self):
         return {p for p in self._properties if 'w' in self._properties[p]}
 
     def arrays_to_synchronize(self):
-        return {a for a in self._arrays if a.sync() and self._arrays[a][0] == 'r'}
+        #return {a for a in self._arrays if a.sync() and self._arrays[a][0] == 'r'}
+        return {a for a in self._arrays if a.sync()}
 
     def write_arrays(self):
         return {a for a in self._arrays if a.sync() and 'w' in self._arrays[a]}

src/pairs/sim/comm.py

@@ -115,6 +115,7 @@ class DetermineGhostParticles(Lowerable):
         self.spacing = spacing
         self.sim.add_statement(self)
 
+    #@pairs_host_block
     @pairs_device_block
     def lower(self):
         nsend_all = self.comm.nsend_all
@@ -189,6 +190,7 @@ class PackGhostParticles(Lowerable):
     def get_elems_per_particle(self):
         return sum([self.sim.ndims() if p.type() == Types.Vector else 1 for p in self.prop_list])
 
+    #@pairs_host_block
     @pairs_device_block
     def lower(self):
         send_buffer = self.comm.send_buffer
@@ -230,6 +232,7 @@ class UnpackGhostParticles(Lowerable):
     def get_elems_per_particle(self):
         return sum([self.sim.ndims() if p.type() == Types.Vector else 1 for p in self.prop_list])
 
+    #@pairs_host_block
     @pairs_device_block
     def lower(self):
         nlocal = self.sim.nlocal
@@ -285,6 +288,7 @@ class RemoveExchangedParticles_part2(Lowerable):
         self.prop_list = prop_list
         self.sim.add_statement(self)
 
+    #@pairs_host_block
     @pairs_device_block
     def lower(self):
         self.sim.module_name("remove_exchanged_particles_pt2")