diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000000000000000000000000000000..a42fb023f2ab03da6df441e4871a05ef9e4843b7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/walberla"] + path = external/walberla + url = https://i10git.cs.fau.de/walberla/walberla.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 6d5bae8986940a742550776d0dbe228a238e0581..963313c7f7a5821bdec123e658a07aa6233b28a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,7 @@ set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -DDEBUG") option(USE_MPI "USE_MPI" ON) +option(USE_WALBERLA "Enable waLBerla support for using BlockForest domain partitioning and dynamic load balancing" OFF) option(COMPILE_CUDA "COMPILE_CUDA" OFF) set(USER_SOURCE_FILES "" CACHE STRING "List of source files to compile (semicolon-separated)") @@ -96,14 +97,8 @@ endif() #================================================================================ # waLBerla ====================================================================== #================================================================================ -set(WALBERLA_DIR ${WALBERLA_DIR} CACHE PATH "Path to waLBerla source directory (required only when using BlockForest partitioning).") - -if(WALBERLA_DIR) - if(EXISTS "${WALBERLA_DIR}") - target_compile_definitions(${PAIRS_TARGET} PUBLIC USE_WALBERLA) - else() - message(FATAL_ERROR "Invalid WALBERLA_DIR: '${WALBERLA_DIR}' does not exist.") - endif() +if(USE_WALBERLA) + target_compile_definitions(${PAIRS_TARGET} PUBLIC USE_WALBERLA) set(RUNTIME_WALBERLA_FILES runtime/domain/block_forest.cpp @@ -115,13 +110,14 @@ if(WALBERLA_DIR) else() list(APPEND RUNTIME_WALBERLA_FILES runtime/boundary_weights.cpp) endif() - + target_sources(${PAIRS_TARGET} PRIVATE ${RUNTIME_WALBERLA_FILES}) - ## Linking walberla modules - set(PAIRS_WALBERLA_DEPENDENCIES walberla::blockforest walberla::core walberla::pe) - find_package(waLBerla REQUIRED) - set(WALBERLA_LINK_LIBRARIES_KEYWORD PUBLIC) + add_subdirectory(external/walberla EXCLUDE_FROM_ALL) + waLBerla_import() + + # Recent issue from the waLBerla side: warnings about empty CUDA_ARCHITECTURES for walberla_gpu + set(PAIRS_WALBERLA_DEPENDENCIES walberla::blockforest walberla::core) target_link_libraries(${PAIRS_TARGET} PUBLIC ${PAIRS_WALBERLA_DEPENDENCIES}) ## TODO: PAIRS_LINK_DIRS and PAIRS_LINK_LIBRARIES for walberla modules *AND* their dependencies diff --git a/cmake/FindwaLBerla.cmake b/cmake/FindwaLBerla.cmake deleted file mode 100644 index 8f87e88a03902f1c1af3900a3d4d38b921996682..0000000000000000000000000000000000000000 --- a/cmake/FindwaLBerla.cmake +++ /dev/null @@ -1,13 +0,0 @@ -if ( WALBERLA_DIR ) - # WALBERLA_DIR has to point to the waLBerla source directory - # this command builds waLBerla (again) in the current build directory in the subfolder "walberla" (second argument) - add_subdirectory( ${WALBERLA_DIR} walberla EXCLUDE_FROM_ALL) - - waLBerla_import() - # Adds the 'src' and 'tests' directory of current app - list( APPEND WALBERLA_MODULE_DIRS "${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/tests" ) - list( REMOVE_DUPLICATES WALBERLA_MODULE_DIRS ) - set ( WALBERLA_MODULE_DIRS ${WALBERLA_MODULE_DIRS} CACHE INTERNAL "All folders that contain modules or tests" ) -else() - message( FATAL_ERROR "waLBerla not found - Use 'cmake -DWALBERLA_DIR=path_to_waLBerla_sources pathToApplicationSources' " ) -endif() diff --git a/runtime/domain/block_forest.cpp b/runtime/domain/block_forest.cpp index 7bdac59664b5c343246a95b8c55cf2bf67c5209f..77c082b709787e96ea3fb4da923f06ef060f4ddc 100644 --- a/runtime/domain/block_forest.cpp +++ b/runtime/domain/block_forest.cpp @@ -255,13 +255,12 @@ void BlockForest::initialize(int *argc, char ***argv) { rank = mpiManager->rank(); auto block_config = balance_workload ? walberla::Vector3<int>(1, 1, 1) : getBlockConfig(); - auto procs = mpiManager->numProcesses(); - auto ref_level = balance_workload ? getInitialRefinementLevel(procs) : 0; + auto ref_level = balance_workload ? getInitialRefinementLevel(world_size) : 0; // PBC's are forced to true here and sperately handled when determining ghosts walberla::Vector3<bool> pbc(true, true, true); walberla::math::AABB domain(grid_min[0], grid_min[1], grid_min[2], grid_max[0], grid_max[1], grid_max[2]); - forest = walberla::blockforest::createBlockForest(domain, block_config, pbc, procs, ref_level); + forest = walberla::blockforest::createBlockForest(domain, block_config, pbc, world_size, ref_level); this->info = make_shared<walberla::blockforest::InfoCollection>();