From 810e9fa88009559c695e59b3666802aef162c9b4 Mon Sep 17 00:00:00 2001
From: Rafael Ravedutti <rafaelravedutti@gmail.com>
Date: Wed, 28 Sep 2022 01:38:14 +0200
Subject: [PATCH] Refactoring and properly fill send and recv buffers
Signed-off-by: Rafael Ravedutti <rafaelravedutti@gmail.com>
---
runtime/comm.hpp | 31 ---------------------------
runtime/domain/regular_6d_stencil.hpp | 17 +++++++++------
runtime/pairs.hpp | 19 +++++++++++++++-
src/pairs/code_gen/cgen.py | 12 ++++++++---
src/pairs/sim/comm.py | 4 ++--
src/pairs/sim/simulation.py | 4 ++--
6 files changed, 42 insertions(+), 45 deletions(-)
delete mode 100644 runtime/comm.hpp
diff --git a/runtime/comm.hpp b/runtime/comm.hpp
deleted file mode 100644
index 29c45a4..0000000
--- a/runtime/comm.hpp
+++ /dev/null
@@ -1,31 +0,0 @@
-#include "pairs.hpp"
-
-#pragma once
-
-namespace pairs {
-
-template<int ndims>
-void initDomain(PairsSimulation<ndims> *pairs, int *argc, char ***argv, real_t xmin, real_t xmax, real_t ymin, real_t ymax, real_t zmin, real_t zmax) {
- pairs->initDomain(argc, argv, xmin, xmax, ymin, ymax, zmin, zmax);
-}
-
-template<int ndims>
-void communicateSizes(PairsSimulation<ndims> *pairs, int dim, const int *send_sizes, int *recv_sizes) {
- pairs->getDomainPartitioner()->communicateSizes(dim, send_sizes, recv_sizes);
-}
-
-template<int ndims>
-void communicateData(
- PairsSimulation<ndims> *pairs, int dim, int elem_size,
- const real_t *send_buf, const int *send_offsets, const int *nsend,
- real_t *recv_buf, const int *recv_offsets, const int *nrecv) {
-
- pairs->getDomainPartitioner()->communicateData(dim, elem_size, send_buf, send_offsets, nsend, recv_buf, recv_offsets, nrecv);
-}
-
-template<int ndims>
-void fillCommunicationArrays(PairsSimulation<ndims> *pairs, int neighbor_ranks[], int pbc[], real_t subdom[]) {
- pairs->getDomainPartitioner()->fillArrays(neighbor_ranks, pbc, subdom);
-}
-
-}
diff --git a/runtime/domain/regular_6d_stencil.hpp b/runtime/domain/regular_6d_stencil.hpp
index e65058d..6414e4c 100644
--- a/runtime/domain/regular_6d_stencil.hpp
+++ b/runtime/domain/regular_6d_stencil.hpp
@@ -121,21 +121,26 @@ public:
const real_t *send_buf, const int *send_offsets, const int *nsend,
real_t *recv_buf, const int *recv_offsets, const int *nrecv) {
+ const real_t *send_prev = &send_buf[send_offsets[dim * 2 + 0] * elem_size];
+ const real_t *send_next = &send_buf[send_offsets[dim * 2 + 1] * elem_size];
+ real_t *recv_prev = &recv_buf[recv_offsets[dim * 2 + 0] * elem_size];
+ real_t *recv_next = &recv_buf[recv_offsets[dim * 2 + 1] * elem_size];
+
if(prev[dim] != rank) {
- MPI_Send(&send_buf[send_offsets[dim * 2 + 0]], nsend[dim * 2 + 0] * elem_size, MPI_DOUBLE, prev[dim], 0, MPI_COMM_WORLD);
- MPI_Recv(&recv_buf[recv_offsets[dim * 2 + 0]], nrecv[dim * 2 + 0] * elem_size, MPI_DOUBLE, next[dim], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ MPI_Send(send_prev, nsend[dim * 2 + 0] * elem_size, MPI_DOUBLE, prev[dim], 0, MPI_COMM_WORLD);
+ MPI_Recv(recv_prev, nrecv[dim * 2 + 0] * elem_size, MPI_DOUBLE, next[dim], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
} else {
for(int i = 0; i < nsend[dim * 2 + 0] * elem_size; i++) {
- recv_buf[recv_offsets[dim * 2 + 0] + i] = send_buf[send_offsets[dim * 2 + 0] + i];
+ recv_prev[i] = send_prev[i];
}
}
if(next[dim] != rank) {
- MPI_Send(&send_buf[send_offsets[dim * 2 + 1]], nsend[dim * 2 + 1] * elem_size, MPI_DOUBLE, next[dim], 0, MPI_COMM_WORLD);
- MPI_Recv(&recv_buf[recv_offsets[dim * 2 + 1]], nrecv[dim * 2 + 1] * elem_size, MPI_DOUBLE, prev[dim], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+ MPI_Send(send_next, nsend[dim * 2 + 1] * elem_size, MPI_DOUBLE, next[dim], 0, MPI_COMM_WORLD);
+ MPI_Recv(recv_next, nrecv[dim * 2 + 1] * elem_size, MPI_DOUBLE, prev[dim], 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
} else {
for(int i = 0; i < nsend[dim * 2 + 1] * elem_size; i++) {
- recv_buf[recv_offsets[dim * 2 + 1] + i] = send_buf[send_offsets[dim * 2 + 1] + i];
+ recv_next[i] = send_next[i];
}
}
}
diff --git a/runtime/pairs.hpp b/runtime/pairs.hpp
index 91ca0ae..690e707 100644
--- a/runtime/pairs.hpp
+++ b/runtime/pairs.hpp
@@ -25,7 +25,7 @@
# define PAIRS_EXCEPTION(a)
#endif
-#define PAIRS_ERROR(...) fprintf(stderr, __VA_ARGS__)
+#define PAIRS_ERROR(...) fprintf(stderr, __VA_ARGS__)
namespace pairs {
@@ -515,6 +515,23 @@ public:
pairs::copy_to_host(prop.getDevicePointer(), prop.getHostPointer(), prop.getTotalSize());
}
}
+
+ void communicateSizes(int dim, const int *send_sizes, int *recv_sizes) {
+ this->getDomainPartitioner()->communicateSizes(dim, send_sizes, recv_sizes);
+ PAIRS_DEBUG("send_sizes=[%d, %d], recv_sizes=[%d, %d]\n", send_sizes[dim * 2 + 0], recv_sizes[dim * 2 + 1], recv_sizes[dim * 2 + 0], recv_sizes[dim * 2 + 1]);
+ }
+
+ void communicateData(
+ int dim, int elem_size,
+ const real_t *send_buf, const int *send_offsets, const int *nsend,
+ real_t *recv_buf, const int *recv_offsets, const int *nrecv) {
+
+ this->getDomainPartitioner()->communicateData(dim, elem_size, send_buf, send_offsets, nsend, recv_buf, recv_offsets, nrecv);
+ }
+
+ void fillCommunicationArrays(int neighbor_ranks[], int pbc[], real_t subdom[]) {
+ this->getDomainPartitioner()->fillArrays(neighbor_ranks, pbc, subdom);
+ }
};
}
diff --git a/src/pairs/code_gen/cgen.py b/src/pairs/code_gen/cgen.py
index 34e6709..bd41f7a 100644
--- a/src/pairs/code_gen/cgen.py
+++ b/src/pairs/code_gen/cgen.py
@@ -55,7 +55,6 @@ class CGen:
self.print("#include <stdlib.h>")
self.print("//---")
self.print("#include \"runtime/pairs.hpp\"")
- self.print("#include \"runtime/comm.hpp\"")
self.print("#include \"runtime/read_from_file.hpp\"")
self.print("#include \"runtime/vtk.hpp\"")
@@ -531,8 +530,15 @@ class CGen:
return f"e{ast_node.id()}"
if isinstance(ast_node, Call):
- extra_params = [] if ast_node.name() != "pairs::initDomain" else ["&argc", "&argv"]
- params = ", ".join(["pairs"] + extra_params + [str(self.generate_expression(p)) for p in ast_node.parameters()])
+ extra_params = []
+
+ if ast_node.name().startswith("pairs::"):
+ extra_params += ["pairs"]
+
+ if ast_node.name() == "pairs->initDomain":
+ extra_params += ["&argc", "&argv"]
+
+ params = ", ".join(extra_params + [str(self.generate_expression(p)) for p in ast_node.parameters()])
return f"{ast_node.name()}({params})"
if isinstance(ast_node, Cast):
diff --git a/src/pairs/sim/comm.py b/src/pairs/sim/comm.py
index 4b29317..b95000c 100644
--- a/src/pairs/sim/comm.py
+++ b/src/pairs/sim/comm.py
@@ -79,7 +79,7 @@ class CommunicateSizes(Lowerable):
@pairs_inline
def lower(self):
- Call_Void(self.sim, "pairs::communicateSizes", [self.step, self.comm.nsend, self.comm.nrecv])
+ Call_Void(self.sim, "pairs->communicateSizes", [self.step, self.comm.nsend, self.comm.nrecv])
class CommunicateData(Lowerable):
@@ -93,7 +93,7 @@ class CommunicateData(Lowerable):
@pairs_inline
def lower(self):
elem_size = sum([self.sim.ndims() if p.type() == Types.Vector else 1 for p in self.prop_list])
- Call_Void(self.sim, "pairs::communicateData", [self.step, elem_size,
+ Call_Void(self.sim, "pairs->communicateData", [self.step, elem_size,
self.comm.send_buffer, self.comm.send_offsets, self.comm.nsend,
self.comm.recv_buffer, self.comm.recv_offsets, self.comm.nrecv])
diff --git a/src/pairs/sim/simulation.py b/src/pairs/sim/simulation.py
index dad95ed..2e73b7b 100644
--- a/src/pairs/sim/simulation.py
+++ b/src/pairs/sim/simulation.py
@@ -245,8 +245,8 @@ class Simulation:
self.capture_statements(False)
grid_array = [[self.grid.min(d), self.grid.max(d)] for d in range(self.ndims())]
self.setups.add_statement([
- Call_Void(self, "pairs::initDomain", [param for delim in grid_array for param in delim]),
- Call_Void(self, "pairs::fillCommunicationArrays", [dom_part.neighbor_ranks, dom_part.pbc, dom_part.subdom])
+ Call_Void(self, "pairs->initDomain", [param for delim in grid_array for param in delim]),
+ Call_Void(self, "pairs->fillCommunicationArrays", [dom_part.neighbor_ranks, dom_part.pbc, dom_part.subdom])
])
self.capture_statements() # TODO: check if this is actually required
--
GitLab