From a766964957f1d7ac9e1963fe963fb8b929059085 Mon Sep 17 00:00:00 2001
From: Behzad Safaei <iwia103h@a0229.nhr.fau.de>
Date: Wed, 19 Mar 2025 20:00:23 +0100
Subject: [PATCH] Fix issue with array reallocations for load balancing
---
src/pairs/sim/comm.py | 20 +++++++------
src/pairs/sim/domain_partitioning.py | 43 ++++++++++++++++------------
2 files changed, 36 insertions(+), 27 deletions(-)
diff --git a/src/pairs/sim/comm.py b/src/pairs/sim/comm.py
index 30ca8e9..10f78d4 100644
--- a/src/pairs/sim/comm.py
+++ b/src/pairs/sim/comm.py
@@ -36,10 +36,12 @@ class Comm:
self.recv_buffer = sim.add_array('recv_buffer', [self.recv_capacity, self.elem_capacity], Types.Real, arr_sync=False)
self.recv_map = sim.add_array('recv_map', [self.recv_capacity], Types.Int32)
self.recv_mult = sim.add_array('recv_mult', [self.recv_capacity, sim.ndims()], Types.Int32)
- self.nsend_contact = sim.add_array('nsend_contact', [dom_part.nranks_capacity], Types.Int32)
- self.nrecv_contact = sim.add_array('nrecv_contact', [dom_part.nranks_capacity], Types.Int32)
- self.contact_soffsets = sim.add_array('contact_soffsets', [dom_part.nranks_capacity], Types.Int32)
- self.contact_roffsets = sim.add_array('contact_roffsets', [dom_part.nranks_capacity], Types.Int32)
+
+ if self.sim._use_contact_history:
+ self.nsend_contact = sim.add_array('nsend_contact', [dom_part.nranks_capacity], Types.Int32)
+ self.nrecv_contact = sim.add_array('nrecv_contact', [dom_part.nranks_capacity], Types.Int32)
+ self.contact_soffsets = sim.add_array('contact_soffsets', [dom_part.nranks_capacity], Types.Int32)
+ self.contact_roffsets = sim.add_array('contact_roffsets', [dom_part.nranks_capacity], Types.Int32)
if self.sim.properties.reduction_props():
self.nsend_reverse = sim.add_array('nsend_reverse', [dom_part.nranks_capacity], Types.Int32)
@@ -138,10 +140,12 @@ class Exchange(Lowerable):
Assign(self.comm.sim, self.comm.nrecv[j], 0)
Assign(self.comm.sim, self.comm.send_offsets[j], 0)
Assign(self.comm.sim, self.comm.recv_offsets[j], 0)
- Assign(self.comm.sim, self.comm.nsend_contact[j], 0)
- Assign(self.comm.sim, self.comm.nrecv_contact[j], 0)
- Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
- Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
+
+ if self.sim._use_contact_history:
+ Assign(self.comm.sim, self.comm.nsend_contact[j], 0)
+ Assign(self.comm.sim, self.comm.nrecv_contact[j], 0)
+ Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
+ Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
if self.sim._target.is_gpu():
CopyArray(self.comm.sim, self.comm.nsend, Contexts.Device, Actions.Ignore)
diff --git a/src/pairs/sim/domain_partitioning.py b/src/pairs/sim/domain_partitioning.py
index fc5b87b..0229e19 100644
--- a/src/pairs/sim/domain_partitioning.py
+++ b/src/pairs/sim/domain_partitioning.py
@@ -199,26 +199,31 @@ class BlockForest:
Assign(self.sim, self.nranks, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborRanks", []))
Assign(self.sim, self.ntotal_aabbs, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborAABBs", []))
- for _ in Filter(self.sim, self.nranks_capacity < self.nranks):
- Assign(self.sim, self.nranks_capacity, self.nranks + 10)
- self.ranks.realloc()
- self.naabbs.realloc()
- self.aabb_offsets.realloc()
-
- for _ in Filter(self.sim, self.aabb_capacity < self.ntotal_aabbs):
- Assign(self.sim, self.aabb_capacity, self.ntotal_aabbs + 20)
- self.aabbs.realloc()
+ for _ in Filter(self.sim, ScalarOp.neq(self.nranks, 0)):
+ for _ in Filter(self.sim, self.nranks_capacity < self.nranks):
+ Assign(self.sim, self.nranks_capacity, self.nranks + 10)
+ for arr in self.nranks_capacity.bonded_arrays():
+ arr.realloc()
+
+ for _ in Filter(self.sim, self.aabb_capacity < self.ntotal_aabbs):
+ Assign(self.sim, self.aabb_capacity, self.ntotal_aabbs + 20)
+ for arr in self.aabb_capacity.bonded_arrays():
+ arr.realloc()
+
+ if self.sim._target.is_gpu():
+ CopyArray(self.sim, self.ranks, Contexts.Host, Actions.WriteOnly, self.nranks)
+ CopyArray(self.sim, self.naabbs, Contexts.Host, Actions.WriteOnly, self.nranks)
+ CopyArray(self.sim, self.aabb_offsets, Contexts.Host, Actions.WriteOnly, self.nranks)
+ CopyArray(self.sim, self.aabbs, Contexts.Host, Actions.WriteOnly, self.ntotal_aabbs * 6)
+
+ Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['ranks', self.ranks, self.nranks])
+ Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['naabbs', self.naabbs, self.nranks])
+ Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabb_offsets', self.aabb_offsets, self.nranks])
+ Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabbs', self.aabbs, self.ntotal_aabbs * 6])
- CopyArray(self.sim, self.ranks, Contexts.Host, Actions.WriteOnly, self.nranks)
- CopyArray(self.sim, self.naabbs, Contexts.Host, Actions.WriteOnly, self.nranks)
- CopyArray(self.sim, self.aabb_offsets, Contexts.Host, Actions.WriteOnly, self.nranks)
- CopyArray(self.sim, self.aabbs, Contexts.Host, Actions.WriteOnly, self.ntotal_aabbs * 6)
- CopyArray(self.sim, self.subdom, Contexts.Host, Actions.WriteOnly)
-
- Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['ranks', self.ranks, self.nranks])
- Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['naabbs', self.naabbs, self.nranks])
- Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabb_offsets', self.aabb_offsets, self.nranks])
- Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabbs', self.aabbs, self.ntotal_aabbs * 6])
+ if self.sim._target.is_gpu():
+ CopyArray(self.sim, self.subdom, Contexts.Host, Actions.WriteOnly)
+
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['subdom', self.subdom, self.sim.ndims() * 2])
if isinstance(self.sim.grid, MutableGrid):
--
GitLab