diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4a44474058c2ce51ba80bdd8651fc7d3ba8624a5..ae96208da8541f44fc3d5cd468a07694cf2f003d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,6 @@
###############################################################################
## ##
-## Genral settings ##
+## General settings ##
## ##
###############################################################################
@@ -970,6 +970,71 @@ msvc-14.1_MpiOnly:
- triggers
+###############################################################################
+## ##
+## macOS Builds ##
+## ##
+###############################################################################
+
+
+.mac_build_template: &mac_build_definition
+ script:
+ - export NUM_CORES=$(system_profiler SPHardwareDataType | grep 'Total Number of Cores' | awk '{print $5}')
+ - export MAX_BUILD_CORES=$(( $(system_profiler SPHardwareDataType | grep 'Memory' | awk '{print $2}') / 4 ))
+ - "[[ $MAX_BUILD_CORES -lt $NUM_CORES ]] && export NUM_BUILD_CORES=$MAX_BUILD_CORES || export NUM_BUILD_CORES=$NUM_CORES"
+ - c++ --version
+ - cmake --version
+ - mpirun --version
+ - mkdir build
+ - cd build
+ - cmake .. -DWALBERLA_BUILD_TESTS=ON -DWALBERLA_BUILD_BENCHMARKS=ON -DWALBERLA_BUILD_TUTORIALS=ON -DWALBERLA_BUILD_TOOLS=ON -DWALBERLA_BUILD_WITH_MPI=$WALBERLA_BUILD_WITH_MPI -DWALBERLA_BUILD_WITH_PYTHON=$WALBERLA_BUILD_WITH_PYTHON -DWALBERLA_BUILD_WITH_OPENMP=$WALBERLA_BUILD_WITH_OPENMP -DWALBERLA_BUILD_WITH_CUDA=$WALBERLA_BUILD_WITH_CUDA -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DWARNING_ERROR=ON
+ - cmake . -LAH
+ - make -j $NUM_BUILD_CORES -l $NUM_CORES
+ - ctest -LE "$CTEST_EXCLUDE_LABELS|cuda" -C $CMAKE_BUILD_TYPE --output-on-failure -j $NUM_CORES
+ tags:
+ - mac
+
+mac_Serial_Dbg:
+ <<: *mac_build_definition
+ variables:
+ CMAKE_BUILD_TYPE: "DebugOptimized"
+ CTEST_EXCLUDE_LABELS: "longrun"
+ WALBERLA_BUILD_WITH_MPI: "OFF"
+ WALBERLA_BUILD_WITH_OPENMP: "OFF"
+ WALBERLA_BUILD_WITH_PYTHON: "ON"
+ WALBERLA_BUILD_WITH_CUDA: "ON"
+
+mac_Serial:
+ <<: *mac_build_definition
+ variables:
+ CMAKE_BUILD_TYPE: "Release"
+ CTEST_EXCLUDE_LABELS: "longrun"
+ WALBERLA_BUILD_WITH_MPI: "OFF"
+ WALBERLA_BUILD_WITH_OPENMP: "OFF"
+ WALBERLA_BUILD_WITH_PYTHON: "ON"
+ WALBERLA_BUILD_WITH_CUDA: "ON"
+
+mac_MpiOnly_Dbg:
+ <<: *mac_build_definition
+ variables:
+ CMAKE_BUILD_TYPE: "DebugOptimized"
+ CTEST_EXCLUDE_LABELS: "longrun"
+ WALBERLA_BUILD_WITH_MPI: "ON"
+ WALBERLA_BUILD_WITH_OPENMP: "OFF"
+ WALBERLA_BUILD_WITH_PYTHON: "ON"
+ WALBERLA_BUILD_WITH_CUDA: "ON"
+
+mac_MpiOnly:
+ <<: *mac_build_definition
+ variables:
+ CMAKE_BUILD_TYPE: "Release"
+ CTEST_EXCLUDE_LABELS: "longrun"
+ WALBERLA_BUILD_WITH_MPI: "ON"
+ WALBERLA_BUILD_WITH_OPENMP: "OFF"
+ WALBERLA_BUILD_WITH_PYTHON: "ON"
+ WALBERLA_BUILD_WITH_CUDA: "ON"
+
+
###############################################################################
## ##
## Deploy jobs ##
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6ad1c28c18ca6f2b44b4f436bc654ab63d25b2c8..85125dcaf83ccf81a7ef108eb38f92b4b3ee89ed 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -360,17 +360,6 @@ if( NOT WARNING_DEPRECATED)
endif()
endif()
-# Treat warnings as errors
-if ( WARNING_ERROR )
- if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
- add_flag ( CMAKE_CXX_FLAGS "-pedantic-errors -Werror" )
- elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
- add_flag ( CMAKE_CXX_FLAGS "/WX" )
- elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
- add_flag ( CMAKE_CXX_FLAGS "-h error_on_warning" )
- endif()
-endif ( )
-
if ( WALBERLA_CXX_COMPILER_IS_CLANG )
add_flag ( CMAKE_CXX_FLAGS "-Wall -Wconversion -Wshadow -Wno-c++11-extensions -Qunused-arguments" )
@@ -784,18 +773,16 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
if ( WIN32 )
message ( STATUS "Enter Workaround Routine for Windows and OpenMPI: PRESS CONFIGURE ONE MORE TIME!" )
string ( REGEX REPLACE "(.*)/bin/.*" "\\1" MPI_PATH ${MPI_CXX_COMPILER} )
- find_path ( MPI_INCLUDE_PATH mpi.h
+ find_path ( MPI_C_INCLUDE_PATH mpi.h
HINTS ${MPI_PATH}
PATH_SUFFIXES include Inc)
- set ( MPI_CXX_INCLUDE_PATH ${MPI_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
- set ( MPI_C_INCLUDE_PATH ${MPI_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
+ set ( MPI_CXX_INCLUDE_PATH ${MPI_C_INCLUDE_PATH} CACHE FILEPATH "" FORCE )
set ( MPI_CXX_LIBRARIES "MPI_CXX_LIBRARIES-NOTFOUND" CACHE FILEPATH "Cleared" FORCE )
find_library ( MPI_CXX_LIBRARIES
NAMES mpi++ mpicxx cxx mpi_cxx libmpi++ libmpicxx libcxx libmpi_cxx
HINTS ${MPI_PATH}
PATH_SUFFIXES lib )
- set ( MPI_LIBRARY "MPI_CXX_LIBRARIES" CACHE FILEPATH "" FORCE )
if ( NOT MPI_CXX_LIBRARIES STREQUAL "MPI_CXX_LIBRARIES-NOTFOUND" )
set ( MPI_CXX_FOUND ON FORCE )
@@ -806,7 +793,6 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
NAMES mpi mpich mpich2 msmpi libmpi libmpich libmpich2 libmsmpi
HINTS ${MPI_PATH}
PATH_SUFFIXES lib )
- set ( MPI_EXTRA_LIBRARY "MPI_C_LIBRARIES" CACHE FILEPATH "" FORCE )
if ( NOT MPI_C_LIBRARIES STREQUAL "MPI_C_LIBRARIES-NOTFOUND" )
set ( MPI_C_FOUND ON FORCE )
@@ -823,23 +809,14 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
endif ( )
if ( MPI_FOUND )
- if ( MPI_CXX_FOUND )
- include_directories ( SYSTEM ${MPI_CXX_INCLUDE_PATH} ${MPI_C_INCLUDE_PATH} )
- foreach( LIB ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES} )
- if ( LIB )
- list ( APPEND SERVICE_LIBS ${LIB} )
- endif ( )
- endforeach ( )
- add_flag ( CMAKE_CXX_FLAGS "${MPI_CXX_COMPILE_FLAGS}" )
- add_flag ( CMAKE_C_FLAGS "${MPI_C_COMPILE_FLAGS}" )
- else ( ) # For older CMake versions
- include_directories ( SYSTEM ${MPI_INCLUDE_PATH} )
- list ( APPEND SERVICE_LIBS ${MPI_LIBRARY} )
- if ( MPI_EXTRA_LIBRARY )
- list ( APPEND SERVICE_LIBS ${MPI_EXTRA_LIBRARY} )
+ include_directories ( SYSTEM ${MPI_CXX_INCLUDE_PATH} ${MPI_C_INCLUDE_PATH} )
+ foreach( LIB ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES} )
+ if ( LIB )
+ list ( APPEND SERVICE_LIBS ${LIB} )
endif ( )
- add_flag ( CMAKE_C_FLAGS "${MPI_COMPILE_FLAGS}" )
- endif ( )
+ endforeach ( )
+ add_flag ( CMAKE_CXX_FLAGS "${MPI_CXX_COMPILE_FLAGS}" )
+ add_flag ( CMAKE_C_FLAGS "${MPI_C_COMPILE_FLAGS}" )
add_flag ( CMAKE_MODULE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
add_flag ( CMAKE_EXE_LINKER_FLAGS "${MPI_CXX_LINK_FLAGS}" )
@@ -847,7 +824,7 @@ if ( WALBERLA_BUILD_WITH_MPI AND NOT WALBERLA_CXX_COMPILER_IS_MPI_WRAPPER )
# When using Intel MPI, mpi.h has to be included before including the standard library
# therefore we use the -include flag to enforce this.
- if ( MPI_INCLUDE_PATH MATCHES "intel" )
+ if ( MPI_C_INCLUDE_PATH MATCHES "intel" )
message (STATUS "Activating IntelMPI include workaround for mpi.h" )
add_flag ( CMAKE_CXX_FLAGS "-include mpi.h" )
add_flag ( CMAKE_C_FLAGS "-include mpi.h" )
@@ -1233,7 +1210,23 @@ if ( WALBERLA_BUILD_WITH_LTO )
endif( )
endif ( )
+
############################################################################################################################
+##
+## Some more compiler flags that need to happen after any try_compile (e.g. inside FindMPI)
+##
+############################################################################################################################
+
+# Treat warnings as errors
+if ( WARNING_ERROR )
+ if( WALBERLA_CXX_COMPILER_IS_GNU OR WALBERLA_CXX_COMPILER_IS_INTEL OR WALBERLA_CXX_COMPILER_IS_CLANG )
+ add_flag ( CMAKE_CXX_FLAGS "-pedantic-errors -Werror" )
+ elseif( WALBERLA_CXX_COMPILER_IS_MSVC )
+ add_flag ( CMAKE_CXX_FLAGS "/WX" )
+ elseif ( WALBERLA_CXX_COMPILER_IS_CRAY )
+ add_flag ( CMAKE_CXX_FLAGS "-h error_on_warning" )
+ endif()
+endif ( )
############################################################################################################################
##
diff --git a/cmake/waLBerlaHelperFunctions.cmake b/cmake/waLBerlaHelperFunctions.cmake
index face2f95f2fdb187cefc274fe314bfc20875b5d0..4d1dafe0e5ce02f35d55151550750fcf3dcdcef2 100644
--- a/cmake/waLBerlaHelperFunctions.cmake
+++ b/cmake/waLBerlaHelperFunctions.cmake
@@ -156,6 +156,11 @@ function ( waLBerla_export )
# Export link paths
set ( WALBERLA_LINK_DIRS ${LINK_DIRS} CACHE INTERNAL "waLBerla link directories" )
+
+ set( WALBERLA_CXX_STANDARD ${CMAKE_CXX_STANDARD} CACHE INTERNAL "CXX standard")
+ set( WALBERLA_CXX_STANDARD_REQUIRED ${CMAKE_CXX_STANDARD_REQUIRED} CACHE INTERNAL "CXX Standard Required")
+ set( WALBERLA_CXX_EXTENSIONS ${CMAKE_CXX_EXTENSIONS} CACHE INTERNAL "CXX Extensions")
+
endfunction( waLBerla_export)
#######################################################################################################################
@@ -210,7 +215,11 @@ function ( waLBerla_import )
endif()
set( SERVICE_LIBS ${WALBERLA_SERVICE_LIBS} PARENT_SCOPE )
-
+
+ set( CMAKE_CXX_STANDARD ${WALBERLA_CXX_STANDARD} PARENT_SCOPE)
+ set( CMAKE_CXX_STANDARD_REQUIRED ${WALBERLA_STANDARD_REQUIRED} PARENT_SCOPE)
+ set( CMAKE_CXX_EXTENSIONS ${WALBERLA_EXTENSIONS} PARENT_SCOPE)
+
link_directories( ${WALBERLA_LINK_DIRS} )
endfunction( waLBerla_import)
#######################################################################################################################
diff --git a/src/blockforest/loadbalancing/DynamicParMetis.cpp b/src/blockforest/loadbalancing/DynamicParMetis.cpp
index a612d47759d6e5425c9b699ef290cd1237387c7b..0ec01197de2e98535cf4e0dc225ef3453518e718 100644
--- a/src/blockforest/loadbalancing/DynamicParMetis.cpp
+++ b/src/blockforest/loadbalancing/DynamicParMetis.cpp
@@ -30,20 +30,28 @@
#include "core/mpi/MPIManager.h"
#include "core/mpi/Gather.h"
#include "core/mpi/Gatherv.h"
+#include "core/mpi/Reduce.h"
#include "core/timing/Timer.h"
#include <boost/algorithm/string/case_conv.hpp>
#include <boost/algorithm/string/trim.hpp>
+#include <array>
+#include <vector>
+
namespace walberla {
namespace blockforest {
-std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phantomForest, MPI_Comm comm )
+std::pair<uint_t, uint_t> getBlockSequenceRange( uint_t numLocalBlocks, MPI_Comm comm )
{
const uint_t rank = uint_c(mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, MPIManager::instance()->rank()));
-
- uint_t numLocalBlocks = phantomForest.getNumberOfBlocks();
+ WALBERLA_DEBUG_SECTION()
+ {
+ int rankRaw;
+ MPI_Comm_rank(comm, &rankRaw);
+ WALBERLA_ASSERT_EQUAL(rank, rankRaw);
+ }
size_t sequenceStartOnProcess = 0;
MPI_Exscan( &numLocalBlocks, &sequenceStartOnProcess, 1, MPITrait<uint_t>::type(), MPI_SUM, comm );
@@ -53,14 +61,13 @@ std::pair<uint_t, uint_t> getBlockSequenceRange( const PhantomBlockForest & phan
return std::make_pair( sequenceStartOnProcess, sequenceStartOnProcess + numLocalBlocks );
}
-std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest & phantomForest, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm )
+std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const PhantomBlockForest& phantomForest, const std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess, const std::pair<uint_t, uint_t> & blockSequenceRange, MPI_Comm comm )
{
std::map< blockforest::BlockID, uint_t > mapping;
- const auto & blockMap = phantomForest.getBlockMap();
uint_t sequenceId = blockSequenceRange.first;
- for( auto it = blockMap.begin(); it != blockMap.end(); ++it )
- mapping.insert( std::make_pair( it->first, sequenceId++ ) );
+ for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it )
+ mapping.insert( std::make_pair( it->first->getId(), sequenceId++ ) );
WALBERLA_ASSERT_EQUAL( sequenceId, blockSequenceRange.second );
const std::vector<uint_t> neighborProcesses = phantomForest.getNeighboringProcesses();
@@ -68,7 +75,11 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
mpi::BufferSystem bs( comm );
for( auto it = neighborProcesses.begin(); it != neighborProcesses.end(); ++it )
- bs.sendBuffer( mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it)) ) << mapping;
+ {
+ auto destRank = mpi::translateRank(mpi::MPIManager::instance()->comm(), comm, int_c(*it));
+ if (destRank != -1)
+ bs.sendBuffer( destRank ) << mapping;
+ }
bs.setReceiverInfoFromSendBufferState( false, true );
@@ -101,6 +112,8 @@ T * ptr( std::vector<T> & v )
typedef uint_t idx_t;
+
+
bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *, uint_t > > & targetProcess,
std::set< uint_t > & processesToRecvFrom,
const PhantomBlockForest & phantomForest,
@@ -111,11 +124,11 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
globalTimer.start();
//create new communicator which excludes processes which do not have blocks
- MPI_Comm subComm;
+ MPI_Comm subComm = MPI_COMM_NULL;
MPI_Group allGroup, subGroup;
MPI_Comm_group( MPIManager::instance()->comm(), &allGroup );
std::vector<int> ranks;
- if (phantomForest.getNumberOfBlocks() > 0)
+ if (targetProcess.size() > 0)
ranks.push_back( MPIManager::instance()->rank() );
ranks = mpi::allGatherv( ranks, MPIManager::instance()->comm() );
auto numSubProcesses = ranks.size();
@@ -123,22 +136,51 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
MPI_Group_incl(allGroup, int_c(ranks.size()), &ranks[0], &subGroup);
MPI_Comm_create( MPIManager::instance()->comm(), subGroup, &subComm);
+ if ( targetProcess.size() != 0)
+ {
+ int subRank;
+ int subSize;
+ MPI_Comm_rank(subComm, &subRank);
+ MPI_Comm_size(subComm, &subSize);
+ WALBERLA_CHECK_GREATER_EQUAL(subRank, 0);
+ WALBERLA_CHECK_LESS(subRank, subSize);
+ } else
+ {
+ int subRank;
+ MPI_Comm_rank(subComm, &subRank);
+ WALBERLA_CHECK_EQUAL(subRank, MPI_UNDEFINED);
+ }
+
int64_t edgecut = 0;
- std::vector<int64_t> part( phantomForest.getNumberOfBlocks(), int64_c( MPIManager::instance()->rank() ) );
+ WALBERLA_CHECK_EQUAL( phantomForest.getNumberOfBlocks(), targetProcess.size() );
+ std::vector<int64_t> part( targetProcess.size(), int64_c( MPIManager::instance()->rank() ) );
if (subComm != MPI_COMM_NULL)
{
- const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( phantomForest, subComm );
- const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, blockSequenceRange, subComm );
+ int subRank;
+ int subSize;
+ MPI_Comm_rank(subComm, &subRank);
+ MPI_Comm_size(subComm, &subSize);
+
+ WALBERLA_CHECK_UNEQUAL(targetProcess.size(), 0);
+ const std::pair<uint_t, uint_t> blockSequenceRange = getBlockSequenceRange( targetProcess.size(), subComm );
+ const std::map< blockforest::BlockID, uint_t > mapping = getBlockIdToSequenceMapping( phantomForest, targetProcess, blockSequenceRange, subComm ); //blockid to vertex id
std::vector<int64_t> vtxdist = mpi::allGather( int64_c( blockSequenceRange.second ), subComm );
vtxdist.insert( vtxdist.begin(), uint_t( 0 ) );
+ WALBERLA_CHECK_EQUAL( vtxdist.size(), subSize + 1 );
+ for (size_t i = 1; i < vtxdist.size(); ++i)
+ {
+ WALBERLA_ASSERT_LESS( vtxdist[i-1], vtxdist[i] );
+ }
std::vector<int64_t> adjncy, xadj, vsize, vwgt, adjwgt;
std::vector<double> xyz;
- for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it )
+ uint_t blockIndex = 0;
+ for( auto it = targetProcess.begin(); it != targetProcess.end(); ++it, ++blockIndex )
{
+ WALBERLA_CHECK_EQUAL(blockIndex, mapping.find(it->first->getId())->second - blockSequenceRange.first);
xadj.push_back( int64_c( adjncy.size() ) );
const PhantomBlock & block = *( it->first );
auto bi = block.getData< DynamicParMetisBlockInfo >();
@@ -150,9 +192,12 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{
auto mapIt = mapping.find( nit->getId() );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
+ WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
+ WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) );
auto edgeWeightIt = bi.getEdgeWeights().find( nit->getId() );
- adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 0 ) : edgeWeightIt->second );
+ //WALBERLA_CHECK_UNEQUAL( edgeWeightIt->second, 0 ); // perhaps WARNING
+ adjwgt.push_back( edgeWeightIt == bi.getEdgeWeights().end() ? int64_t( 1 ) : edgeWeightIt->second );
}
break;
case PARMETIS_EDGES_FROM_EDGE_WEIGHTS:
@@ -160,10 +205,15 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
{
auto mapIt = mapping.find( edgeIt->first );
WALBERLA_ASSERT_UNEQUAL( mapIt, mapping.end(), "BlockId of neighbor is not contained in sequence mapping!" );
+ WALBERLA_CHECK_GREATER_EQUAL( mapIt->second, 0 );
+ WALBERLA_CHECK_LESS( mapIt->second, vtxdist.back() );
adjncy.push_back( int64_c( mapIt->second ) );
+ //WALBERLA_CHECK_UNEQUAL( edgeIt->second, 0 ); // perhaps WARNING
adjwgt.push_back( edgeIt->second );
}
+ break;
}
+ WALBERLA_CHECK_UNEQUAL( bi.getVertexWeight(), 0 );
vwgt.push_back( bi.getVertexWeight() );
vsize.push_back( bi.getVertexSize() );
xyz.push_back( bi.getVertexCoords()[0] );
@@ -172,45 +222,58 @@ bool DynamicParMetis::operator()( std::vector< std::pair< const PhantomBlock *,
}
xadj.push_back( int64_c( adjncy.size() ) );
- WALBERLA_ASSERT_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) );
- WALBERLA_ASSERT_EQUAL( xadj.size(), phantomForest.getNumberOfBlocks() + 1 );
- WALBERLA_ASSERT_EQUAL( vwgt.size(), phantomForest.getNumberOfBlocks() );
- WALBERLA_ASSERT_EQUAL( vsize.size(), phantomForest.getNumberOfBlocks() );
- WALBERLA_ASSERT_EQUAL( adjncy.size(), adjwgt.size() );
-
- int64_t wgtflag = weightsToUse_;
- int64_t numflag = 0; // C-style ordering
- int64_t ncon = 1; // Number of constraints
- int64_t ndims = 3; // Number of dimensions
- double ubvec[] = { real_t( 1.05 ) }; // imbalance tolerance
- int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
- double ipc2redist = real_t( 1000000.0 ); // compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
- MPI_Comm comm = subComm; //MPIManager::instance()->comm();
+ WALBERLA_CHECK_EQUAL( vtxdist.size(), numSubProcesses + uint_t( 1 ) );
+ WALBERLA_CHECK_EQUAL( xadj.size(), targetProcess.size() + 1 );
+ WALBERLA_CHECK_EQUAL( xadj.front(), 0);
+ WALBERLA_CHECK_EQUAL( xadj.back(), adjncy.size() );
+ for (size_t i = 1; i < xadj.size(); ++i)
+ {
+ WALBERLA_ASSERT_LESS( xadj[i-1], xadj[i] );
+ }
+ WALBERLA_CHECK_EQUAL( vwgt.size(), targetProcess.size() );
+ WALBERLA_CHECK_EQUAL( vsize.size(), targetProcess.size() );
+ WALBERLA_CHECK_EQUAL( xyz.size(), targetProcess.size() * 3 );
+ WALBERLA_CHECK_EQUAL( adjncy.size(), adjwgt.size() );
+ WALBERLA_CHECK_EQUAL( adjwgt.size(), xadj.back() );
+
+ int64_t wgtflag = weightsToUse_;
+ int64_t numflag = 0; // C-style ordering
+ int64_t ncon = 1; // Number of constraints
+ int64_t ndims = 3; // Number of dimensions
+ std::vector<double> ubvec( uint_c(ncon), double_c( 1.05 ) ); // imbalance tolerance
+ int64_t nparts = int64_c( MPIManager::instance()->numProcesses() ); // number of subdomains
+ double ipc2redist = double_c(ipc2redist_);
+ MPI_Comm comm = subComm; //MPIManager::instance()->comm();
std::vector<double> tpwgts( uint_c(nparts * ncon), 1.0 / double_c( nparts ) ); // vertex weight fraction that is stored in a subdomain
- int64_t options[] = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
+ std::vector<int64_t> options = { int64_t( 1 ), int64_t( 0 ), int64_t( 23 ), int64_t( 1 ) };
int metisResult = core::METIS_OK;
switch( algorithm_ )
{
+ case PARMETIS_PART_GEOM:
+ parmetisTimer.start();
+ metisResult = core::ParMETIS_V3_PartGeom( ptr( vtxdist ), &ndims, ptr( xyz ), ptr( part ), &comm );
+ parmetisTimer.end();
+ break;
case PARMETIS_PART_GEOM_KWAY:
parmetisTimer.start();
- metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
+ metisResult = core::ParMETIS_V3_PartGeomKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ndims, ptr( xyz ), &ncon, &nparts, ptr( tpwgts ), ptr( ubvec ), ptr( options ), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_PART_KWAY:
parmetisTimer.start();
- metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
+ metisResult = core::ParMETIS_V3_PartKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_ADAPTIVE_REPART:
parmetisTimer.start();
- metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, &ipc2redist, options, &edgecut, ptr( part ), &comm );
+ metisResult = core::ParMETIS_V3_AdaptiveRepart( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( vsize ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), &ipc2redist, ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
case PARMETIS_REFINE_KWAY:
parmetisTimer.start();
- metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ubvec, options, &edgecut, ptr( part ), &comm );
+ metisResult = core::ParMETIS_V3_RefineKway( ptr( vtxdist ), ptr( xadj ), ptr( adjncy ), ptr( vwgt ), ptr( adjwgt ), &wgtflag, &numflag, &ncon, &nparts, ptr( tpwgts ), ptr(ubvec), ptr(options), &edgecut, ptr( part ), &comm );
parmetisTimer.end();
break;
}
@@ -265,6 +328,8 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s )
if( s == "PART_GEOM_KWAY" )
return PARMETIS_PART_GEOM_KWAY;
+ else if( s == "PART_GEOM" )
+ return PARMETIS_PART_GEOM;
else if( s == "PART_KWAY" )
return PARMETIS_PART_KWAY;
else if( s == "PART_ADAPTIVE_REPART" )
@@ -272,7 +337,7 @@ DynamicParMetis::Algorithm DynamicParMetis::stringToAlgorithm( std::string s )
else if( s == "REFINE_KWAY" )
return PARMETIS_REFINE_KWAY;
else
- WALBERLA_ABORT( "Illegal ParMetis algorithm specified! Valid choices are: \"PART_GEOM_KWAY\", \"PART_KWAY\", \"PART_ADAPTIVE_REPART\", or \"REFINE_KWAY\"." );
+ WALBERLA_ABORT( "Illegal ParMetis algorithm specified (" << s << ")! Valid choices are: \"PART_GEOM_KWAY\", \"PART_KWAY\", \"PART_ADAPTIVE_REPART\", or \"REFINE_KWAY\"." );
}
@@ -290,7 +355,7 @@ DynamicParMetis::WeightsToUse DynamicParMetis::stringToWeightsToUse( std::string
else if( s == "BOTH_WEIGHTS" )
return PARMETIS_BOTH_WEIGHTS;
else
- WALBERLA_ABORT( "Illegal ParMetis weights usage specified! Valid choices are: \"NO_WEIGHTS\", \"EDGE_WEIGHTS\", \"VERTEX_WEIGHTS\", or \"BOTH_WEIGHTS\"." );
+ WALBERLA_ABORT( "Illegal ParMetis weights usage specified (" << s << ")! Valid choices are: \"NO_WEIGHTS\", \"EDGE_WEIGHTS\", \"VERTEX_WEIGHTS\", or \"BOTH_WEIGHTS\"." );
}
@@ -304,7 +369,56 @@ DynamicParMetis::EdgeSource DynamicParMetis::stringToEdgeSource( std::string s )
else if( s == "EDGES_FROM_EDGE_WEIGHTS" )
return PARMETIS_EDGES_FROM_EDGE_WEIGHTS;
else
- WALBERLA_ABORT( "Illegal ParMetis weights usage specified! Valid choices are: \"EDGES_FROM_FOREST\" or \"EDGES_FROM_EDGE_WEIGHTS\"" );
+ WALBERLA_ABORT( "Illegal ParMetis weights usage specified (" << s << ")! Valid choices are: \"EDGES_FROM_FOREST\" or \"EDGES_FROM_EDGE_WEIGHTS\"" );
+}
+
+
+std::string DynamicParMetis::algorithmToString( ) const
+{
+ switch (algorithm_)
+ {
+ case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM_KWAY:
+ return "PART_GEOM_KWAY";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_PART_GEOM:
+ return "PART_GEOM";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_PART_KWAY:
+ return "PART_KWAY";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_ADAPTIVE_REPART:
+ return "PART_ADAPTIVE_REPART";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_REFINE_KWAY:
+ return "PARMETIS_REFINE_KWAY";
+ }
+ return "Unknown";
+}
+
+
+std::string DynamicParMetis::weightsToUseToString( ) const
+{
+ switch (weightsToUse_)
+ {
+ case walberla::blockforest::DynamicParMetis::PARMETIS_NO_WEIGHTS:
+ return "NO_WEIGHTS";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_EDGE_WEIGHTS:
+ return "EDGE_WEIGHTS";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_VERTEX_WEIGHTS:
+ return "VERTEX_WEIGHTS";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_BOTH_WEIGHTS:
+ return "BOTH_WEIGHTS";
+ }
+ return "Unknown";
+}
+
+
+std::string DynamicParMetis::edgeSourceToString( ) const
+{
+ switch (edgeSource_)
+ {
+ case walberla::blockforest::DynamicParMetis::PARMETIS_EDGES_FROM_FOREST:
+ return "EDGES_FROM_FOREST";
+ case walberla::blockforest::DynamicParMetis::PARMETIS_EDGES_FROM_EDGE_WEIGHTS:
+ return "EDGES_FROM_EDGE_WEIGHTS";
+ }
+ return "Unknown";
}
diff --git a/src/blockforest/loadbalancing/DynamicParMetis.h b/src/blockforest/loadbalancing/DynamicParMetis.h
index 927ca25698c5257a667674535d9838191634c1a6..d1b3acd1f877c755279f8a4b3f0d2a2586549cd7 100644
--- a/src/blockforest/loadbalancing/DynamicParMetis.h
+++ b/src/blockforest/loadbalancing/DynamicParMetis.h
@@ -23,10 +23,12 @@
#include "blockforest/PhantomBlockForest.h"
+#include "core/debug/Debug.h"
#include "core/DataTypes.h"
#include "core/math/Vector3.h"
#include "core/mpi/MPIWrapper.h"
+#include <cmath>
#include <map>
namespace walberla {
@@ -40,7 +42,7 @@ std::map< blockforest::BlockID, uint_t > getBlockIdToSequenceMapping( const Phan
class DynamicParMetis
{
public:
- enum Algorithm { PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
+ enum Algorithm { PARMETIS_PART_GEOM, PARMETIS_PART_GEOM_KWAY, PARMETIS_PART_KWAY, PARMETIS_ADAPTIVE_REPART, PARMETIS_REFINE_KWAY };
enum WeightsToUse { PARMETIS_NO_WEIGHTS = 0, PARMETIS_EDGE_WEIGHTS = 1, PARMETIS_VERTEX_WEIGHTS = 2, PARMETIS_BOTH_WEIGHTS = 3 };
enum EdgeSource { PARMETIS_EDGES_FROM_FOREST, PARMETIS_EDGES_FROM_EDGE_WEIGHTS };
@@ -54,6 +56,10 @@ public:
const PhantomBlockForest & phantomForest,
const uint_t iteration ) const;
+
+ void setipc2redist(double val) {ipc2redist_ = val;}
+ double getipc2redist() const {return ipc2redist_;}
+
bool edgeWeightsUsed() const { return ( weightsToUse_ == PARMETIS_EDGE_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexWeightsUsed() const { return ( weightsToUse_ == PARMETIS_VERTEX_WEIGHTS ) || ( weightsToUse_ == PARMETIS_BOTH_WEIGHTS ); }
bool vertexSizeUsed() const { return algorithm_ == PARMETIS_ADAPTIVE_REPART; }
@@ -62,10 +68,16 @@ public:
static WeightsToUse stringToWeightsToUse( std::string s );
static EdgeSource stringToEdgeSource( std::string s );
+ std::string algorithmToString() const;
+ std::string weightsToUseToString() const;
+ std::string edgeSourceToString() const;
+
protected:
Algorithm algorithm_;
WeightsToUse weightsToUse_;
EdgeSource edgeSource_;
+
+ double ipc2redist_ = real_t( 1000.0 ); ///< compute repartitioning with low edge cut (set lower (down to 0.000001) to get minimal repartitioning )
};
class DynamicParMetisBlockInfo
@@ -90,7 +102,7 @@ public:
vsize_t getVertexSize() const { return vertexSize_; }
void setVertexSize( const vsize_t size ) { vertexSize_ = size; }
- const Vector3<double> & getVertexCoords() const { return vertexCoords_; }
+ const Vector3<double> & getVertexCoords() const { WALBERLA_ASSERT( !std::isnan(vertexCoords_[0]) && !std::isnan(vertexCoords_[1]) && !std::isnan(vertexCoords_[2]) ); return vertexCoords_; }
void setVertexCoords( const Vector3<double> & p ) { vertexCoords_ = p; }
void setEdgeWeight( const blockforest::BlockID & blockId, const weight_t edgeWeight ) { edgeWeights_[blockId] = edgeWeight; }
@@ -107,7 +119,7 @@ private:
weight_t vertexWeight_; /// Defines the weight of a block
vsize_t vertexSize_; /// Defines the cost of rebalancing a block
- Vector3<double> vertexCoords_; /// Defines where the block is located in space. Needed by some ParMetis algorithms
+ Vector3<double> vertexCoords_ = Vector3<double>(std::numeric_limits<double>::signaling_NaN()); /// Defines where the block is located in space. Needed by some ParMetis algorithms
std::map< blockforest::BlockID, weight_t > edgeWeights_; /// Defines the cost of communication with other blocks
};
diff --git a/src/core/mpi/MPIHelper.cpp b/src/core/mpi/MPIHelper.cpp
index 8822c198d4ba19d6f76005c8a34aa6d374245970..94f239a90e20e36fe35d7193976206b0e1add64e 100644
--- a/src/core/mpi/MPIHelper.cpp
+++ b/src/core/mpi/MPIHelper.cpp
@@ -20,28 +20,67 @@
#include "MPIHelper.h"
+#include <core/debug/CheckFunctions.h>
+
namespace walberla {
namespace mpi {
+//!
+//! \brief This functions maps the rank in one communicator to the rank in another communicator.
+//! \param srcComm source communicator
+//! \param destComm destination communicator
+//! \param srcRank rank in the source communicator
+//! \return rank in the destination communicator or -1 if not available
+//!
int translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const int srcRank)
{
+ if (srcComm == destComm)
+ {
+ return srcRank;
+ }
+
int destRank = -1;
MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, 1, const_cast<int*>(&srcRank), destGroup, &destRank);
+ int size;
+ MPI_Comm_size(destComm, &size);
+ if (destRank == MPI_UNDEFINED) destRank = -1;
+ WALBERLA_CHECK_GREATER_EQUAL(destRank, -1);
+ WALBERLA_CHECK_LESS(destRank, size);
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
}
+//!
+//! \brief This functions converts a array of ranks in one communicator to an array of ranks in another communicator.
+//! \param srcComm source communicator
+//! \param destComm destination communicator
+//! \param srcRank source ranks
+//! \return converted ranks, -1 if not available
+//!
std::vector<int> translateRank(const MPI_Comm srcComm, const MPI_Comm destComm, const std::vector<int>& srcRank)
{
+ if (srcComm == destComm)
+ {
+ return srcRank;
+ }
+
std::vector<int> destRank(srcRank.size(), -1);
MPI_Group srcGroup, destGroup;
MPI_Comm_group(srcComm, &srcGroup);
MPI_Comm_group(destComm, &destGroup);
MPI_Group_translate_ranks(srcGroup, int_c(srcRank.size()), const_cast<int*>(&srcRank[0]), destGroup, &destRank[0]);
+ int size;
+ MPI_Comm_size(destComm, &size);
+ for (auto& dstRnk : destRank)
+ {
+ if (dstRnk == MPI_UNDEFINED) dstRnk = -1;
+ WALBERLA_CHECK_GREATER_EQUAL(dstRnk, -1);
+ WALBERLA_CHECK_LESS(dstRnk, size);
+ }
MPI_Group_free(&srcGroup);
MPI_Group_free(&destGroup);
return destRank;
diff --git a/src/pe/amr/weight_assignment/MetisAssignmentFunctor.h b/src/pe/amr/weight_assignment/MetisAssignmentFunctor.h
index bed28616694dbdbff30914cba828d0ed22c6d6fe..720c80a2e363302f6b60a582c7628000cada10ea 100644
--- a/src/pe/amr/weight_assignment/MetisAssignmentFunctor.h
+++ b/src/pe/amr/weight_assignment/MetisAssignmentFunctor.h
@@ -35,26 +35,38 @@ public:
typedef blockforest::DynamicParMetisBlockInfo PhantomBlockWeight;
typedef blockforest::DynamicParMetisBlockInfoPackUnpack PhantomBlockWeightPackUnpackFunctor;
- MetisAssignmentFunctor( const shared_ptr<InfoCollection>& ic ) : ic_( ic )
- {}
+ MetisAssignmentFunctor( shared_ptr<InfoCollection>& ic, const real_t baseWeight = real_t(10.0) ) : ic_(ic), baseWeight_(baseWeight) {}
void operator()( std::vector< std::pair< const PhantomBlock *, walberla::any > > & blockData, const PhantomBlockForest & )
{
for( auto it = blockData.begin(); it != blockData.end(); ++it )
{
- const uint_t& weight = ic_->find( it->first->getId() )->second.numberOfLocalBodies;
- blockforest::DynamicParMetisBlockInfo info( int64_c(weight) );
- info.setVertexSize(int64_c( weight ));
+ const double weight = double_c( ic_->find( it->first->getId() )->second.numberOfLocalBodies ) + baseWeight_;
+ //const double commWeight = double_c( edgeWeightFactor * (double_c(ic_->find( it->first->getId() )->second.numberOfShadowBodies) + baseWeight_)) + 1;
+ blockforest::DynamicParMetisBlockInfo info( 0 );
+ info.setVertexWeight( int64_c(weight) );
+ info.setVertexSize( int64_c( weight ) );
+ info.setVertexCoords( it->first->getAABB().center() );
for( uint_t nb = uint_t(0); nb < it->first->getNeighborhoodSize(); ++nb )
{
- info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(weight) );
+ info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(edgeWeight_) );
}
it->second = info;
}
}
+ inline void setBaseWeight( const double weight) { baseWeight_ = weight;}
+ inline double getBaseWeight() const { return baseWeight_; }
+
+ inline void setEdgeWeight( const double weight) { edgeWeight_ = weight;}
+ inline double getEdgeWeight() const { return edgeWeight_; }
+
private:
shared_ptr< InfoCollection > ic_;
+
+ ///Base weight due to allocated data structures. A weight of zero for blocks is dangerous as empty blocks might accumulate on one process!
+ double baseWeight_ = 10.0;
+ double edgeWeight_ = 1.0;
};
}
diff --git a/src/pe/cr/DEM.h b/src/pe/cr/DEM.h
index 9d0887e774253fcfbb7c76e4ded484a30abbd843..dcc19af292c646addf2f0eac477d3c87e5a4218f 100644
--- a/src/pe/cr/DEM.h
+++ b/src/pe/cr/DEM.h
@@ -79,7 +79,7 @@ private:
size_t numberOfContactsTreated_;
};
-class DEM : public DEMSolver<IntegrateImplictEuler, ResolveContactSpringDashpotHaffWerner>
+class DEM : public DEMSolver<IntegrateImplicitEuler, ResolveContactSpringDashpotHaffWerner>
{
public:
DEM( const shared_ptr<BodyStorage>& globalBodyStorage
@@ -88,8 +88,8 @@ public:
, domain_decomposition::BlockDataID ccdID
, domain_decomposition::BlockDataID fcdID
, WcTimingTree* tt = NULL)
- : DEMSolver<IntegrateImplictEuler, ResolveContactSpringDashpotHaffWerner>(
- IntegrateImplictEuler(), ResolveContactSpringDashpotHaffWerner(),
+ : DEMSolver<IntegrateImplicitEuler, ResolveContactSpringDashpotHaffWerner>(
+ IntegrateImplicitEuler(), ResolveContactSpringDashpotHaffWerner(),
globalBodyStorage, blockStorage, storageID, ccdID, fcdID, tt )
{
}
diff --git a/src/pe/cr/DEM.impl.h b/src/pe/cr/DEM.impl.h
index c64669ba210b48ca9f8101d7c09883c10405cc73..eb3a6b4e325616444808d21cf8d26db9fe9cc364 100644
--- a/src/pe/cr/DEM.impl.h
+++ b/src/pe/cr/DEM.impl.h
@@ -128,7 +128,7 @@ void DEMSolver<Integrator,ContactResolver>::timestep( real_t dt )
WALBERLA_ASSERT( bodyIt->checkInvariants(), "Invalid body state detected" );
WALBERLA_ASSERT( !bodyIt->hasSuperBody(), "Invalid superordinate body detected" );
- // Moving the capsule according to the acting forces (don't move a sleeping capsule)
+ // Moving the body according to the acting forces (don't move a sleeping body)
if( bodyIt->isAwake() && !bodyIt->hasInfiniteMass() )
{
integrate_( *bodyIt, dt, *this );
@@ -138,7 +138,7 @@ void DEMSolver<Integrator,ContactResolver>::timestep( real_t dt )
bodyIt->resetForceAndTorque();
// Checking the state of the rigid body
- WALBERLA_ASSERT( bodyIt->checkInvariants(), "Invalid capsule state detected" );
+ WALBERLA_ASSERT( bodyIt->checkInvariants(), "Invalid body state detected" );
// Resetting the acting forces
bodyIt->resetForceAndTorque();
diff --git a/src/pe/cr/Integrators.h b/src/pe/cr/Integrators.h
index 30b56e50a9e308917872d03156447d9e3b520157..1ea365f6661f542c73fde4ad31255cb5b18c2097 100644
--- a/src/pe/cr/Integrators.h
+++ b/src/pe/cr/Integrators.h
@@ -30,7 +30,7 @@ namespace pe {
namespace cr {
//*************************************************************************************************
-/*!\brief Integrate the trajectory of one body using implict Euler.
+/*!\brief Integrate the trajectory of one body using implicit Euler.
*
* \param id Body ID.
* \param dt Time step size.
@@ -40,7 +40,7 @@ namespace cr {
* The implicit Euler algorithm, also known as backward Euler, is used. It is a first-order
* integrator that does conserves energy (i.e. it is symplectic.)
*/
-class IntegrateImplictEuler {
+class IntegrateImplicitEuler {
public:
void operator()( BodyID id, real_t dt, ICR & solver ) const
{
@@ -72,7 +72,7 @@ public:
// Setting the axis-aligned bounding box
id->calcBoundingBox();
- // Calculating the current motion of the capsule
+ // Calculating the current motion of the body
id->calcMotion();
}
};
@@ -88,7 +88,7 @@ public:
* The explicit Euler algorithm, also known as forward Euler, is used. It is a first-order
* integrator that does not conserve energy (i.e. it is not symplectic.)
*/
-class IntegrateExplictEuler {
+class IntegrateExplicitEuler {
public:
void operator()( BodyID id, real_t dt, ICR & solver ) const
{
@@ -120,7 +120,7 @@ public:
// Setting the axis-aligned bounding box
id->calcBoundingBox();
- // Calculating the current motion of the capsule
+ // Calculating the current motion of the body
id->calcMotion();
}
};
diff --git a/src/pe/cr/PlainIntegrator.h b/src/pe/cr/PlainIntegrator.h
index 271847f5d3d9b601452553d08d4a85bae8a4eb71..92abbbfb3746b59f08927fb9222b4b72bd00045f 100644
--- a/src/pe/cr/PlainIntegrator.h
+++ b/src/pe/cr/PlainIntegrator.h
@@ -61,14 +61,14 @@ private:
WcTimingTree* tt_;
};
-class PlainIntegrator : public PlainIntegratorSolver<IntegrateImplictEuler>
+class PlainIntegrator : public PlainIntegratorSolver<IntegrateImplicitEuler>
{
public:
PlainIntegrator( const shared_ptr<BodyStorage>& globalBodyStorage
, const shared_ptr<BlockStorage>& blockStorage
, domain_decomposition::BlockDataID storageID
, WcTimingTree* tt = NULL)
- : PlainIntegratorSolver<IntegrateImplictEuler>( IntegrateImplictEuler(), globalBodyStorage, blockStorage,
+ : PlainIntegratorSolver<IntegrateImplicitEuler>( IntegrateImplicitEuler(), globalBodyStorage, blockStorage,
storageID, tt )
{
}
diff --git a/src/pe/cr/PlainIntegrator.impl.h b/src/pe/cr/PlainIntegrator.impl.h
index 7b9c4ce45101e774ed936da68e81caab088f1f95..b96a6e2e0e46cac743062832c3ee4e8fc19f906b 100644
--- a/src/pe/cr/PlainIntegrator.impl.h
+++ b/src/pe/cr/PlainIntegrator.impl.h
@@ -65,14 +65,14 @@ void PlainIntegratorSolver<Integrator>::timestep( const real_t dt )
BodyStorage& localStorage = (*storage)[0];
for (auto bd = localStorage.begin(); bd != localStorage.end(); ++bd){
// Checking the state of the body
- WALBERLA_ASSERT( bd->checkInvariants(), "Invalid capsule state detected" );
+ WALBERLA_ASSERT( bd->checkInvariants(), "Invalid body state detected" );
WALBERLA_ASSERT( !bd->hasSuperBody(), "Invalid superordinate body detected" );
// Resetting the contact node and removing all attached contacts
// bd->resetNode();
bd->clearContacts();
- // Moving the capsule according to the acting forces (don't move a sleeping capsule)
+ // Moving the body according to the acting forces (don't move a sleeping body)
if( bd->isAwake() && !bd->hasInfiniteMass() ) {
integrate_( *bd, dt, *this );
}
@@ -80,8 +80,8 @@ void PlainIntegratorSolver<Integrator>::timestep( const real_t dt )
// Resetting the acting forces
bd->resetForceAndTorque();
- // Checking the state of the capsule
- WALBERLA_ASSERT( bd->checkInvariants(), "Invalid capsule state detected" );
+ // Checking the state of the body
+ WALBERLA_ASSERT( bd->checkInvariants(), "Invalid body state detected" );
}
}
if (tt_!=NULL) tt_->stop("Integrate Bodies");
diff --git a/src/pe/rigidbody/CylindricalBoundary.cpp b/src/pe/rigidbody/CylindricalBoundary.cpp
index 6b99c946384f11f2ecc5cd9f3d88a4e65956622d..66f639e49015a66ab41caf851690e678149c507b 100644
--- a/src/pe/rigidbody/CylindricalBoundary.cpp
+++ b/src/pe/rigidbody/CylindricalBoundary.cpp
@@ -56,7 +56,7 @@ namespace pe {
CylindricalBoundary::CylindricalBoundary( id_t sid, id_t uid, const Vec3& gpos, const real_t radius,
MaterialID material )
: GeomPrimitive( getStaticTypeID(), sid, uid, material ) // Initializing the base object
- , radius_(radius) // Radius of the cylinder // Length of the capsule
+ , radius_(radius) // Radius of the cylinder
{
//boundaries are always considered locally and have infinite mass
setGlobal( true );
@@ -66,11 +66,11 @@ CylindricalBoundary::CylindricalBoundary( id_t sid, id_t uid, const Vec3& gpos,
// Checking the radius
// Since the constructor is never directly called but only used in a small number
- // of functions that already check the capsule arguments, only asserts are used here to
+ // of functions that already check the cylinder arguments, only asserts are used here to
// double check the arguments.
- WALBERLA_ASSERT_GREATER( radius, real_t(0), "Invalid capsule radius" );
+ WALBERLA_ASSERT_GREATER( radius, real_t(0), "Invalid cylinder radius" );
- // Initializing the instantiated capsule
+ // Initializing the instantiated cylinder
gpos_ = gpos;
q_ = Quat(); // Setting the orientation
R_ = q_.toRotationMatrix(); // Setting the rotation matrix
diff --git a/src/pe/rigidbody/CylindricalBoundary.h b/src/pe/rigidbody/CylindricalBoundary.h
index 4c095c1a20bae85959c756a856500ec7b4332b93..5d9b4c9d5da23922f0858500e065b25ca884a6f0 100644
--- a/src/pe/rigidbody/CylindricalBoundary.h
+++ b/src/pe/rigidbody/CylindricalBoundary.h
@@ -175,7 +175,7 @@ inline id_t CylindricalBoundary::getStaticTypeID()
//=================================================================================================
//*************************************************************************************************
-/*!\name Capsule operators */
+/*!\name Cylinder operators */
//@{
std::ostream& operator<<( std::ostream& os, const CylindricalBoundary& cb );
std::ostream& operator<<( std::ostream& os, CylindricalBoundaryID cb );
diff --git a/src/python_coupling/Manager.cpp b/src/python_coupling/Manager.cpp
index 264ba49b6eaa356cac7579415bfea13c5bb01d24..9ccf4a597e06b5a80f0346702484112ecdbd41f1 100644
--- a/src/python_coupling/Manager.cpp
+++ b/src/python_coupling/Manager.cpp
@@ -75,11 +75,9 @@ void Manager::addPath( const std::string & path )
{
WALBERLA_ASSERT( initialized_ );
- object main_module = import("__main__");
- dict globals = extract<dict>( main_module.attr( "__dict__" ) );
- exec( "import sys", globals );
- std::string pathCommand = std::string ( "sys.path.append( \"") + path + "\" ) ";
- exec( str(pathCommand), globals );
+ object sys = import("sys");
+ list sys_path = extract<list>( sys.attr("path") );
+ sys_path.append(path);
}
void Manager::triggerInitialization()
@@ -116,10 +114,24 @@ void Manager::triggerInitialization()
}
catch ( boost::python::error_already_set & ) {
- PyErr_Print();
+ PyObject *type_ptr = NULL, *value_ptr = NULL, *traceback_ptr = NULL;
+ PyErr_Fetch(&type_ptr, &value_ptr, &traceback_ptr);
+
+ if( type_ptr )
+ {
+ extract<std::string> type_str(( str( handle<>( type_ptr ) ) ));
+ if( type_str.check() )
+ WALBERLA_LOG_DEVEL( type_str() );
+ }
+ if(value_ptr)
+ {
+ extract<std::string> value_str(( str( handle<>( value_ptr ) ) ));
+ if ( value_str.check() )
+ WALBERLA_LOG_DEVEL( value_str() );
+ }
+
WALBERLA_ABORT( "Error while initializing Python" );
}
-
}
@@ -159,4 +171,3 @@ int someSymbolSoThatLinkerDoesNotComplain=0;
-
diff --git a/src/waLBerlaDefinitions.in.h b/src/waLBerlaDefinitions.in.h
index 26b6627685ab451ae693fb32f8af4dfe0080d8aa..563ec863a1c6694b69dab1efd3b5cca2253aaac7 100644
--- a/src/waLBerlaDefinitions.in.h
+++ b/src/waLBerlaDefinitions.in.h
@@ -31,6 +31,8 @@
#cmakedefine WALBERLA_BUILD_WITH_CUDA
+#cmakedefine WALBERLA_BUILD_WITH_CODEGEN
+
#cmakedefine WALBERLA_BUFFER_DEBUG
#cmakedefine WALBERLA_THREAD_SAFE_LOGGING
diff --git a/tests/pe/CMakeLists.txt b/tests/pe/CMakeLists.txt
index 64a077d081657246870a0bbac6583cc58bb8acb0..ce10e244c19ea13befd35797f6ce1d7e87991499 100644
--- a/tests/pe/CMakeLists.txt
+++ b/tests/pe/CMakeLists.txt
@@ -69,6 +69,11 @@ waLBerla_execute_test( NAME PE_OVERLAP )
waLBerla_compile_test( NAME PE_PARALLELEQUIVALENCE FILES ParallelEquivalence.cpp DEPENDS core blockforest )
waLBerla_execute_test( NAME PE_PARALLELEQUIVALENCE PROCESSES 4 )
+if( WALBERLA_BUILD_WITH_PARMETIS )
+ waLBerla_compile_test( NAME PE_PARMETIS FILES ParMetis.cpp DEPENDS core blockforest )
+ waLBerla_execute_test( NAME PE_PARMETIS PROCESSES 64 )
+endif()
+
waLBerla_compile_test( NAME PE_PARSEMESSAGE FILES ParseMessage.cpp DEPENDS core )
waLBerla_execute_test( NAME PE_PARSEMESSAGE )
diff --git a/tests/pe/ParMetis.cpp b/tests/pe/ParMetis.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e35a8cb2336015a2f6aa3f925a1c4a74c9ed263b
--- /dev/null
+++ b/tests/pe/ParMetis.cpp
@@ -0,0 +1,144 @@
+//======================================================================================================================
+//
+// This file is part of waLBerla. waLBerla is free software: you can
+// redistribute it and/or modify it under the terms of the GNU General Public
+// License as published by the Free Software Foundation, either version 3 of
+// the License, or (at your option) any later version.
+//
+// waLBerla is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+// for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with waLBerla (see COPYING.txt). If not, see <http://www.gnu.org/licenses/>.
+//
+//! \file ParMetis.cpp
+//! \author Sebastian Eibl <sebastian.eibl@fau.de>
+//
+//======================================================================================================================
+
+#include "pe/utility/CreateWorld.h"
+
+#include <blockforest/loadbalancing/DynamicParMetis.h>
+#include <core/Environment.h>
+#include <core/logging/Logging.h>
+#include <core/math/Sample.h>
+
+using namespace walberla;
+using namespace walberla::pe;
+
+class ReGrid
+{
+public:
+ void operator()( std::vector< std::pair< const Block *, uint_t > > & minTargetLevels,
+ std::vector< const Block * > &, const BlockForest & /*forest*/ )
+ {
+ std::for_each( minTargetLevels.begin(),
+ minTargetLevels.end(),
+ [](auto& pair){pair.second = pair.first->getLevel() + 1;} );
+ }
+
+};
+
+class MetisAssignmentFunctor
+{
+public:
+
+ typedef blockforest::DynamicParMetisBlockInfo PhantomBlockWeight;
+ typedef blockforest::DynamicParMetisBlockInfoPackUnpack PhantomBlockWeightPackUnpackFunctor;
+
+ void operator()( std::vector< std::pair< const PhantomBlock *, walberla::any > > & blockData, const PhantomBlockForest & )
+ {
+ for( auto it = blockData.begin(); it != blockData.end(); ++it )
+ {
+ const auto& corner = it->first->getAABB().maxCorner();
+ const int weight = int_c( corner[0] + corner[1] + corner[2] );
+ blockforest::DynamicParMetisBlockInfo info(0);
+ info.setVertexWeight( weight );
+ info.setVertexSize( weight );
+ info.setVertexCoords( it->first->getAABB().center() );
+ for( uint_t nb = uint_t(0); nb < it->first->getNeighborhoodSize(); ++nb )
+ {
+ info.setEdgeWeight(it->first->getNeighborId(nb), int64_c(weight) );
+ }
+ it->second = info;
+ }
+ }
+};
+
+int parmetisTest(const std::string& algorithm,
+ const std::string& weightsToUse,
+ const std::string& edgeSource)
+{
+ walberla::MPIManager::instance()->resetMPI();
+ walberla::MPIManager::instance()->useWorldComm();
+
+ WALBERLA_LOG_INFO_ON_ROOT("****** " << algorithm << " | " << weightsToUse << " | " << edgeSource);
+
+ // create forest
+ shared_ptr< BlockForest > forest = createBlockForest( math::AABB(0,0,0,40,40,40),
+ Vector3<uint_t>(4, 4, 4),
+ Vector3<bool>(false, false, false),
+ 64,
+ 0);
+ if (!forest)
+ {
+ WALBERLA_LOG_INFO_ON_ROOT( "No BlockForest created ... exiting!");
+ return EXIT_SUCCESS;
+ }
+
+ forest->setRefreshMinTargetLevelDeterminationFunction( ReGrid() );
+
+ auto assFunc = MetisAssignmentFunctor();
+ forest->setRefreshPhantomBlockDataAssignmentFunction( assFunc );
+ forest->setRefreshPhantomBlockDataPackFunction( MetisAssignmentFunctor::PhantomBlockWeightPackUnpackFunctor() );
+ forest->setRefreshPhantomBlockDataUnpackFunction( MetisAssignmentFunctor::PhantomBlockWeightPackUnpackFunctor() );
+
+ auto alg = blockforest::DynamicParMetis::stringToAlgorithm( algorithm );
+ auto vWeight = blockforest::DynamicParMetis::stringToWeightsToUse( weightsToUse );
+ auto eWeight = blockforest::DynamicParMetis::stringToEdgeSource( edgeSource );
+
+ auto prepFunc = blockforest::DynamicParMetis( alg, vWeight, eWeight );
+ forest->setRefreshPhantomBlockMigrationPreparationFunction( prepFunc );
+
+ forest->refresh();
+
+ math::Sample numBlocks;
+ numBlocks.castToRealAndInsert( forest->size() );
+ numBlocks.mpiGatherRoot();
+ WALBERLA_LOG_INFO_ON_ROOT("#blocks: " << numBlocks.format() );
+
+ int weight = 0;
+ for (const auto& block : *forest)
+ {
+ const auto& corner = block.getAABB().maxCorner();
+ weight += int_c( corner[0] + corner[1] + corner[2] );
+ }
+ math::Sample weightSample;
+ weightSample.castToRealAndInsert( weight );
+ weightSample.mpiGatherRoot();
+ WALBERLA_LOG_INFO_ON_ROOT("#weights: " << weightSample.format() );
+
+ return EXIT_SUCCESS;
+}
+
+int main( int argc, char ** argv )
+{
+ walberla::debug::enterTestMode();
+ walberla::MPIManager::instance()->initializeMPI( &argc, &argv );
+ walberla::MPIManager::instance()->useWorldComm();
+
+ std::vector<std::string> algs = {"PART_GEOM", "PART_GEOM_KWAY", "PART_KWAY", "PART_ADAPTIVE_REPART", "REFINE_KWAY"};
+ std::vector<std::string> wtu = {"NO_WEIGHTS", "EDGE_WEIGHTS", "VERTEX_WEIGHTS", "BOTH_WEIGHTS"};
+ std::vector<std::string> es = {"EDGES_FROM_FOREST", "EDGES_FROM_EDGE_WEIGHTS"};
+
+ for (const auto& a : algs)
+ for (const auto& w : wtu)
+ for (const auto& e : es)
+ {
+ parmetisTest(a,w,e);
+ }
+
+ return EXIT_SUCCESS;
+}
diff --git a/tests/pe_coupling/momentum_exchange_method/SettlingSphereMEMDynamicRefinement.cpp b/tests/pe_coupling/momentum_exchange_method/SettlingSphereMEMDynamicRefinement.cpp
index 44b17eb70ee97b509d096e0b09133d97bc40efc8..0e44a2e57f402bee733f69a1d9ad9b00656056cc 100644
--- a/tests/pe_coupling/momentum_exchange_method/SettlingSphereMEMDynamicRefinement.cpp
+++ b/tests/pe_coupling/momentum_exchange_method/SettlingSphereMEMDynamicRefinement.cpp
@@ -869,9 +869,18 @@ int main( int argc, char **argv )
{
auto & forest = blocks->getBlockForest();
pe::createWithNeighborhood(forest, bodyStorageID, *peInfoCollection);
- pe::clearSynchronization( blockforest, bodyStorageID);
+ uint_t stampBefore = blocks->getBlockForest().getModificationStamp();
blocks->refresh();
+ uint_t stampAfter = blocks->getBlockForest().getModificationStamp();
+
+ if(stampBefore == stampAfter)
+ {
+ // nothing has changed
+ continue;
+ }
+
+ pe::clearSynchronization( blockforest, bodyStorageID);
for( uint_t syncStep = 0; syncStep < uint_c(diameter / real_c(minBlockSizeInCells)) + 1; ++syncStep)
syncCall();