Skip to content
Snippets Groups Projects
Commit a7669649 authored by Behzad Safaei's avatar Behzad Safaei
Browse files

Fix issue with array reallocations for load balancing

parent a4a623cf
Branches
No related tags found
No related merge requests found
...@@ -36,10 +36,12 @@ class Comm: ...@@ -36,10 +36,12 @@ class Comm:
self.recv_buffer = sim.add_array('recv_buffer', [self.recv_capacity, self.elem_capacity], Types.Real, arr_sync=False) self.recv_buffer = sim.add_array('recv_buffer', [self.recv_capacity, self.elem_capacity], Types.Real, arr_sync=False)
self.recv_map = sim.add_array('recv_map', [self.recv_capacity], Types.Int32) self.recv_map = sim.add_array('recv_map', [self.recv_capacity], Types.Int32)
self.recv_mult = sim.add_array('recv_mult', [self.recv_capacity, sim.ndims()], Types.Int32) self.recv_mult = sim.add_array('recv_mult', [self.recv_capacity, sim.ndims()], Types.Int32)
self.nsend_contact = sim.add_array('nsend_contact', [dom_part.nranks_capacity], Types.Int32)
self.nrecv_contact = sim.add_array('nrecv_contact', [dom_part.nranks_capacity], Types.Int32) if self.sim._use_contact_history:
self.contact_soffsets = sim.add_array('contact_soffsets', [dom_part.nranks_capacity], Types.Int32) self.nsend_contact = sim.add_array('nsend_contact', [dom_part.nranks_capacity], Types.Int32)
self.contact_roffsets = sim.add_array('contact_roffsets', [dom_part.nranks_capacity], Types.Int32) self.nrecv_contact = sim.add_array('nrecv_contact', [dom_part.nranks_capacity], Types.Int32)
self.contact_soffsets = sim.add_array('contact_soffsets', [dom_part.nranks_capacity], Types.Int32)
self.contact_roffsets = sim.add_array('contact_roffsets', [dom_part.nranks_capacity], Types.Int32)
if self.sim.properties.reduction_props(): if self.sim.properties.reduction_props():
self.nsend_reverse = sim.add_array('nsend_reverse', [dom_part.nranks_capacity], Types.Int32) self.nsend_reverse = sim.add_array('nsend_reverse', [dom_part.nranks_capacity], Types.Int32)
...@@ -138,10 +140,12 @@ class Exchange(Lowerable): ...@@ -138,10 +140,12 @@ class Exchange(Lowerable):
Assign(self.comm.sim, self.comm.nrecv[j], 0) Assign(self.comm.sim, self.comm.nrecv[j], 0)
Assign(self.comm.sim, self.comm.send_offsets[j], 0) Assign(self.comm.sim, self.comm.send_offsets[j], 0)
Assign(self.comm.sim, self.comm.recv_offsets[j], 0) Assign(self.comm.sim, self.comm.recv_offsets[j], 0)
Assign(self.comm.sim, self.comm.nsend_contact[j], 0)
Assign(self.comm.sim, self.comm.nrecv_contact[j], 0) if self.sim._use_contact_history:
Assign(self.comm.sim, self.comm.contact_soffsets[j], 0) Assign(self.comm.sim, self.comm.nsend_contact[j], 0)
Assign(self.comm.sim, self.comm.contact_soffsets[j], 0) Assign(self.comm.sim, self.comm.nrecv_contact[j], 0)
Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
Assign(self.comm.sim, self.comm.contact_soffsets[j], 0)
if self.sim._target.is_gpu(): if self.sim._target.is_gpu():
CopyArray(self.comm.sim, self.comm.nsend, Contexts.Device, Actions.Ignore) CopyArray(self.comm.sim, self.comm.nsend, Contexts.Device, Actions.Ignore)
......
...@@ -199,26 +199,31 @@ class BlockForest: ...@@ -199,26 +199,31 @@ class BlockForest:
Assign(self.sim, self.nranks, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborRanks", [])) Assign(self.sim, self.nranks, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborRanks", []))
Assign(self.sim, self.ntotal_aabbs, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborAABBs", [])) Assign(self.sim, self.ntotal_aabbs, Call_Int(self.sim, "pairs_runtime->getNumberOfNeighborAABBs", []))
for _ in Filter(self.sim, self.nranks_capacity < self.nranks): for _ in Filter(self.sim, ScalarOp.neq(self.nranks, 0)):
Assign(self.sim, self.nranks_capacity, self.nranks + 10) for _ in Filter(self.sim, self.nranks_capacity < self.nranks):
self.ranks.realloc() Assign(self.sim, self.nranks_capacity, self.nranks + 10)
self.naabbs.realloc() for arr in self.nranks_capacity.bonded_arrays():
self.aabb_offsets.realloc() arr.realloc()
for _ in Filter(self.sim, self.aabb_capacity < self.ntotal_aabbs): for _ in Filter(self.sim, self.aabb_capacity < self.ntotal_aabbs):
Assign(self.sim, self.aabb_capacity, self.ntotal_aabbs + 20) Assign(self.sim, self.aabb_capacity, self.ntotal_aabbs + 20)
self.aabbs.realloc() for arr in self.aabb_capacity.bonded_arrays():
arr.realloc()
if self.sim._target.is_gpu():
CopyArray(self.sim, self.ranks, Contexts.Host, Actions.WriteOnly, self.nranks)
CopyArray(self.sim, self.naabbs, Contexts.Host, Actions.WriteOnly, self.nranks)
CopyArray(self.sim, self.aabb_offsets, Contexts.Host, Actions.WriteOnly, self.nranks)
CopyArray(self.sim, self.aabbs, Contexts.Host, Actions.WriteOnly, self.ntotal_aabbs * 6)
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['ranks', self.ranks, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['naabbs', self.naabbs, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabb_offsets', self.aabb_offsets, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabbs', self.aabbs, self.ntotal_aabbs * 6])
CopyArray(self.sim, self.ranks, Contexts.Host, Actions.WriteOnly, self.nranks) if self.sim._target.is_gpu():
CopyArray(self.sim, self.naabbs, Contexts.Host, Actions.WriteOnly, self.nranks) CopyArray(self.sim, self.subdom, Contexts.Host, Actions.WriteOnly)
CopyArray(self.sim, self.aabb_offsets, Contexts.Host, Actions.WriteOnly, self.nranks)
CopyArray(self.sim, self.aabbs, Contexts.Host, Actions.WriteOnly, self.ntotal_aabbs * 6)
CopyArray(self.sim, self.subdom, Contexts.Host, Actions.WriteOnly)
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['ranks', self.ranks, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['naabbs', self.naabbs, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabb_offsets', self.aabb_offsets, self.nranks])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['aabbs', self.aabbs, self.ntotal_aabbs * 6])
Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['subdom', self.subdom, self.sim.ndims() * 2]) Call_Void(self.sim, "pairs_runtime->copyRuntimeArray", ['subdom', self.subdom, self.sim.ndims() * 2])
if isinstance(self.sim.grid, MutableGrid): if isinstance(self.sim.grid, MutableGrid):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment