From ae82a7fd5e90afab08c8e4a7b57e5e090f335478 Mon Sep 17 00:00:00 2001
From: Behzad Safaei <iwia103h@alex2.nhr.fau.de>
Date: Wed, 4 Dec 2024 11:50:26 +0100
Subject: [PATCH] Add support for integrating pairs into other projects

---
 CMakeLists.txt                                | 236 +++++++++++-------
 .../FindwaLBerla.cmake                        |   0
 cmake/pairs-config.cmake.in                   |   6 +
 examples/dem_sd.py                            |   6 +-
 runtime/devices/cuda.cu                       |   2 +-
 src/pairs/code_gen/cgen.py                    |   7 +-
 6 files changed, 162 insertions(+), 95 deletions(-)
 rename FindwaLBerla.cmake => cmake/FindwaLBerla.cmake (100%)
 create mode 100644 cmake/pairs-config.cmake.in

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3f0b768..ed0f398 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,15 +1,6 @@
-cmake_minimum_required(VERSION 3.8.2 FATAL_ERROR)
-# cmake_policy(SET CMP0074 NEW)
+cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
 project(pairs CXX)
 
-set(TESTCASE ${TESTCASE} CACHE STRING "Select the testcase from the following: md, dem")
-
-if(NOT TESTCASE)
-    set(TESTCASE md CACHE STRING "Select the testcase from the following: md, dem" FORCE)
-endif()
-
-set(TARGET_BIN ${TESTCASE})
-
 # Set default build type if none is specified
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build (Debug, Release, etc.)" FORCE)
@@ -18,25 +9,47 @@ endif()
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 set(CMAKE_CXX_FLAGS_DEBUG "-g -O0 -DDEBUG")
 
-string(TOLOWER "${TESTCASE}" TESTCASE)
-message(STATUS "Selected testcase: ${TESTCASE}")
-
-option(USE_WALBERLA "USE_WALBERLA" ON)
+option(USE_WALBERLA "Enable waLBerla support for using BlockForest partitioning" ON)
 option(USE_MPI "USE_MPI" ON)
 option(COMPILE_CUDA "COMPILE_CUDA" ON)
 option(ENABLE_GPU_DIRECT "ENABLE_GPU_DIRECT" ON)
-option(GENERATE_WHOLE_PROGRAM "GENERATE_WHOLE_PROGRAM" OFF)
+option(GENERATE_WHOLE_PROGRAM "Generate the whole program (i.e. including the 'main' function). No additional source files are needed." OFF)
+option(BUILD_APP "Build a stand-alone app which uses the P4IRS modular interface. Provide your source files with -DUSER_SOURCE_FILES" OFF)
 
-if(USE_WALBERLA)
-    SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_WALBERLA_LOAD_BALANCING -DUSE_WALBERLA")
-    set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}")
-    find_package(waLBerla REQUIRED)
-    waLBerla_import()
+if(GENERATE_WHOLE_PROGRAM AND BUILD_APP)
+    message(FATAL_ERROR "You must choose either GENERATE_WHOLE_PROGRAM or BUILD_APP or neither.\n
+        Choose neither if you only want to use the generated libraries in your project (in a seperate build system).")
+endif()
+
+set(PYTHON_SCRIPT ${PYTHON_SCRIPT} CACHE PATH "The python script triggering code generation")
+if(NOT EXISTS ${PYTHON_SCRIPT})
+    message(FATAL_ERROR "PYTHON_SCRIPT doesn't exist! Specify it with -DPYTHON_SCRIPT=/path/to/script.py")
+endif()
+get_filename_component(PYTHON_SCRIPT_NAME ${PYTHON_SCRIPT} NAME_WE)
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
+
+#================================================================================
+# Setup directories =============================================================
+#================================================================================
+file(COPY runtime DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+file(COPY data DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
+
+set(OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/output)
+if(EXISTS ${OUTPUT_DIR})
+    file(REMOVE_RECURSE ${OUTPUT_DIR})
 endif()
+file(MAKE_DIRECTORY ${OUTPUT_DIR})
 
-set(GEN_HEADER ${CMAKE_CURRENT_BINARY_DIR}/runtime/interfaces/last_generated.hpp)
-set(GEN_HEADER_DIR ${CMAKE_CURRENT_BINARY_DIR}/runtime/interfaces)
+#================================================================================
+# Generated header (internally used by runtime files) ===========================
+#================================================================================
+set(GEN_INTERFACE_HEADER ${CMAKE_CURRENT_BINARY_DIR}/runtime/interfaces/last_generated.hpp)
+set(GEN_INTERFACE_DIR ${CMAKE_CURRENT_BINARY_DIR}/runtime/interfaces)
 
+#================================================================================
+# RUNTIME_COMMON_FILES ==========================================================
+#================================================================================
 set(RUNTIME_COMMON_FILES
     runtime/pairs.cpp
     runtime/copper_fcc_lattice.cpp
@@ -48,55 +61,96 @@ set(RUNTIME_COMMON_FILES
     runtime/timing.cpp
     runtime/vtk.cpp
     runtime/domain/regular_6d_stencil.cpp)
-    
-set(RUNTIME_WALBERLA_FILES
-    # runtime/boundary_weights.cpp  # avoid compiling this for now. TODO: generate the host and device functions
-    runtime/domain/block_forest.cpp)
 
-set(PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE} CACHE STRING "Python executable not set.")
+#================================================================================
+# PAIRS_TARGET ==================================================================
+#================================================================================
+set(PAIRS_TARGET "pairs")
 
-if(NOT GENERATE_WHOLE_PROGRAM)
-    file(
-        COPY ${CMAKE_CURRENT_SOURCE_DIR}/examples/main.cpp
-        DESTINATION ${CMAKE_BINARY_DIR})
+# PAIRS dependencies 
+set(PAIRS_LINK_LIBRARIES)
+set(PAIRS_LINK_DIRS ${CMAKE_CURRENT_BINARY_DIR})
+set(PAIRS_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR})
 
-    set(EXEC_FILES ${CMAKE_BINARY_DIR}/main.cpp ${RUNTIME_COMMON_FILES})
+# The target can either be an executable or a static library
+if(GENERATE_WHOLE_PROGRAM OR BUILD_APP)
+    add_executable(${PAIRS_TARGET} ${RUNTIME_COMMON_FILES})
 else()
-    set(EXEC_FILES ${RUNTIME_COMMON_FILES})
+    add_library(${PAIRS_TARGET} STATIC ${RUNTIME_COMMON_FILES})
+    list(APPEND PAIRS_LINK_LIBRARIES ${PAIRS_TARGET})
 endif()
 
-if(USE_WALBERLA)
-    set(EXEC_FILES ${EXEC_FILES} ${RUNTIME_WALBERLA_FILES})
-endif()
+set_target_properties(${PAIRS_TARGET} PROPERTIES 
+    CXX_STANDARD_REQUIRED ON
+    CXX_STANDARD 17
+)
+
+#================================================================================
+# USER_SOURCE_FILES =============================================================
+#================================================================================
+if(BUILD_APP)
+    set(USER_SOURCE_FILES "" CACHE STRING "List of source files to compile (semicolon-separated)")
+    if(NOT USER_SOURCE_FILES)
+        message(FATAL_ERROR "BUILD_APP is ON. You have to specify source files like this:\n
+            -DUSER_SOURCE_FILES=src/main.cpp;src/helper.cpp")
+    endif()
 
-if(NOT PYTHON_EXECUTABLE)
-    set(PYTHON_EXECUTABLE python3)
+    foreach(file ${USER_SOURCE_FILES})
+        if(NOT EXISTS ${file})
+            message(FATAL_ERROR "File '${file}' does not exist!")
+        endif()
+    endforeach()
+    target_sources(${PAIRS_TARGET} PRIVATE ${USER_SOURCE_FILES})
 endif()
 
-file(COPY runtime DESTINATION ${CMAKE_BINARY_DIR})
-file(COPY data DESTINATION ${CMAKE_BINARY_DIR})
+#================================================================================
+# waLBerla ======================================================================
+#================================================================================
+if(USE_WALBERLA)
+    set(RUNTIME_WALBERLA_FILES
+        runtime/domain/block_forest.cpp
+        # runtime/boundary_weights.cpp  # avoid compiling this for now. TODO: generate the host and device functions
+    )
+    target_sources(${PAIRS_TARGET} PRIVATE ${RUNTIME_WALBERLA_FILES})
+    target_compile_definitions(${PAIRS_TARGET} PUBLIC USE_WALBERLA)
+
+    ## Linking walberla modules
+    set(PAIRS_WALBERLA_DEPENDENCIES blockforest core pe)
+    find_package(waLBerla REQUIRED)
+    set(WALBERLA_LINK_LIBRARIES_KEYWORD PUBLIC)
+    target_link_modules(${PAIRS_TARGET} ${PAIRS_WALBERLA_DEPENDENCIES})     # This is a walberla helper function
 
-set(OUTPUT_DIR ${CMAKE_BINARY_DIR}/output)
-if(EXISTS ${OUTPUT_DIR})
-    file(REMOVE_RECURSE ${OUTPUT_DIR})
+    ## TODO: PAIRS_LINK_DIRS and PAIRS_LINK_LIBRARIES for walberla modules *AND* their dependencies
+    ## This implemention only works if the consumer of the library is itself a walberla app (made within the build system of walberla)
+    list(APPEND PAIRS_LINK_LIBRARIES ${PAIRS_WALBERLA_DEPENDENCIES})
 endif()
 
-file(MAKE_DIRECTORY ${OUTPUT_DIR})
+#================================================================================
+# Install pairs python package ==================================================
+#================================================================================
+set(PYTHON_EXECUTABLE ${PYTHON_EXECUTABLE} CACHE STRING "Python executable")
+
+if(NOT PYTHON_EXECUTABLE)
+    set(PYTHON_EXECUTABLE python3)
+endif()
 
 execute_process(
     COMMAND ${PYTHON_EXECUTABLE} setup.py build
+    OUTPUT_QUIET
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
 execute_process(
     COMMAND ${PYTHON_EXECUTABLE} setup.py install --user
+    OUTPUT_QUIET
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 
-# CUDA compilation
+#================================================================================
+# CUDA ==========================================================================
+#================================================================================
 if(COMPILE_CUDA)
-    cmake_policy(SET CMP0074 NEW)
     find_package(CUDA REQUIRED)
     enable_language(CUDA)
-    set(GEN_SOURCES "${CMAKE_BINARY_DIR}/${TESTCASE}.cu")
+    set(GEN_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/${PYTHON_SCRIPT_NAME}.cu")
     set(CUDA_ARCH ${CUDA_ARCH} CACHE STRING "CUDA_ARCH environment variable must be set.")
     set(TARGET_ARG "gpu")
 
@@ -105,7 +159,6 @@ if(COMPILE_CUDA)
         set(CUDA_ARCH 80)
     endif()
 
-    set(CMAKE_CUDA_ARCHITECTURES ${CUDA_ARCH})
     set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -rdc=true")
 
     if(CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -114,70 +167,69 @@ if(COMPILE_CUDA)
         set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O3")
     endif()
 
-    if(USE_WALBERLA)
-        waLBerla_add_executable(
-            NAME ${TARGET_BIN}
-            FILES ${EXEC_FILES} ${GEN_SOURCES}
-            DEPENDS blockforest core pe)
-    else()
-        add_executable(${TARGET_BIN} ${GEN_SOURCES} ${EXEC_FILES})
-    endif()
-
     if(ENABLE_GPU_DIRECT)
-        target_compile_definitions(${TARGET_BIN} PRIVATE ENABLE_CUDA_AWARE_MPI)
+        target_compile_definitions(${PAIRS_TARGET} PRIVATE ENABLE_CUDA_AWARE_MPI)
     endif()
 
-    add_library(runtime STATIC runtime/devices/cuda.cu)
-    target_compile_features(runtime PUBLIC cxx_std_11)
-    set_target_properties(runtime PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-    set_target_properties(runtime PROPERTIES CUDA_ARCHITECTURES ${CUDA_ARCH})
-    target_compile_definitions(runtime PRIVATE PAIRS_TARGET_CUDA)
-    target_include_directories(runtime PRIVATE ${CUDA_INCLUDE_DIRS})
+    target_sources(${PAIRS_TARGET} PRIVATE runtime/devices/cuda.cu)
+    target_compile_definitions(${PAIRS_TARGET} PUBLIC PAIRS_TARGET_CUDA)
+    target_include_directories(${PAIRS_TARGET} PUBLIC ${CUDA_INCLUDE_DIRS})
+    list(APPEND PAIRS_INCLUDE_DIRS ${CUDA_INCLUDE_DIRS})
 
-    set_target_properties(${TARGET_BIN} PROPERTIES CUDA_ARCHITECTURES ${CUDA_ARCH})
-    target_compile_definitions(${TARGET_BIN} PRIVATE PAIRS_TARGET_CUDA)
-    target_include_directories(${TARGET_BIN} PRIVATE ${CUDA_INCLUDE_DIRS} ${GEN_HEADER_DIR})
+    set_target_properties(${PAIRS_TARGET} PROPERTIES 
+        CUDA_RESOLVE_DEVICE_SYMBOLS ON
+        CUDA_STANDARD 17
+        CUDA_SEPARABLE_COMPILATION ON
+        CUDA_ARCHITECTURES ${CUDA_ARCH})
 
-# CPU compilation
+    target_link_libraries(${PAIRS_TARGET} PUBLIC ${CUDA_LIBRARIES})
+    list(APPEND PAIRS_LINK_LIBRARIES ${CUDA_LIBRARIES})
 else()
-    set(GEN_SOURCES "${CMAKE_BINARY_DIR}/${TESTCASE}.cpp")
+    set(GEN_SOURCES "${CMAKE_CURRENT_BINARY_DIR}/${PYTHON_SCRIPT_NAME}.cpp")
     set(TARGET_ARG "cpu")
-
-    if(USE_WALBERLA)
-        waLBerla_add_executable(
-            NAME ${TARGET_BIN}
-            FILES ${EXEC_FILES} ${GEN_SOURCES}
-            DEPENDS blockforest core pe)
-    else()
-        add_executable(${TARGET_BIN} ${GEN_SOURCES} ${EXEC_FILES})
-    endif()
-
-    add_library(runtime STATIC runtime/devices/dummy.cpp)
-    target_include_directories(${TARGET_BIN} PRIVATE ${GEN_HEADER_DIR})
+    target_sources(${PAIRS_TARGET} PRIVATE runtime/devices/dummy.cpp)
 endif()
 
-target_link_libraries(${TARGET_BIN} runtime)
-
+#================================================================================
+# Generate code and add them to PAIRS_TARGET ====================================
+#================================================================================
 add_custom_command(
-    OUTPUT ${GEN_SOURCES} ${GEN_HEADER}
-    COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/examples/${TESTCASE}.py ${TARGET_ARG}
+    OUTPUT ${GEN_SOURCES} ${GEN_INTERFACE_HEADER}
+    COMMAND ${PYTHON_EXECUTABLE} ${PYTHON_SCRIPT} ${TARGET_ARG}
     COMMENT "Generating code with P4IRS"
-    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/examples/${TESTCASE}.py)
+    DEPENDS ${PYTHON_SCRIPT})
 
-add_custom_target(generated_code DEPENDS ${GEN_SOURCES} ${GEN_HEADER})
-add_dependencies(${TARGET_BIN} generated_code)
+add_custom_target(generated_code DEPENDS ${GEN_SOURCES} ${GEN_INTERFACE_HEADER})
+add_dependencies(${PAIRS_TARGET} generated_code)
 
-target_link_libraries(${TARGET_BIN} ${CMAKE_EXE_LINKER_FLAGS})
-set_target_properties(${TARGET_BIN} PROPERTIES CXX_STANDARD_REQUIRED ON)
-set_target_properties(${TARGET_BIN} PROPERTIES CXX_STANDARD 17)
+target_sources(${PAIRS_TARGET} PRIVATE ${GEN_SOURCES})
+target_include_directories(${PAIRS_TARGET} PRIVATE 
+    ${GEN_INTERFACE_DIR}            # Interface header USED INTERNALLY by pairs is located here
+    ${CMAKE_CURRENT_BINARY_DIR}     # Generated source and header FOR USER is located here
+)    
 
+#================================================================================
+# MPI ===========================================================================
+#================================================================================
 if(USE_MPI)
     find_package(MPI REQUIRED)
     include_directories(SYSTEM ${MPI_INCLUDE_PATH})
-    target_link_libraries(${TARGET_BIN} ${MPI_LIBRARIES})
+    target_link_libraries(${PAIRS_TARGET} PUBLIC ${MPI_LIBRARIES})
+    list(APPEND PAIRS_LINK_LIBRARIES "${MPI_LIBRARIES}")
+    list(APPEND PAIRS_INCLUDE_DIRS "${MPI_INCLUDE_PATH}")
 endif()
 
+#================================================================================
+# LIKWID ========================================================================
+#================================================================================
 if(LIKWID_DIR)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DLIKWID_PERFMON -pthread -L${LIKWID_DIR}/lib -llikwid ")
     include_directories(${LIKWID_DIR}/include)
+    list(APPEND PAIRS_INCLUDE_DIRS "${LIKWID_DIR}/include")
 endif()
+
+#================================================================================
+# config file ===================================================================
+#================================================================================
+configure_file ( "${CMAKE_SOURCE_DIR}/cmake/pairs-config.cmake.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/pairs-config.cmake")
diff --git a/FindwaLBerla.cmake b/cmake/FindwaLBerla.cmake
similarity index 100%
rename from FindwaLBerla.cmake
rename to cmake/FindwaLBerla.cmake
diff --git a/cmake/pairs-config.cmake.in b/cmake/pairs-config.cmake.in
new file mode 100644
index 0000000..fdf10ef
--- /dev/null
+++ b/cmake/pairs-config.cmake.in
@@ -0,0 +1,6 @@
+set ( pairs_SOURCE_DIR @pairs_SOURCE_DIR@ )
+set ( pairs_BINARY_DIR @pairs_BINARY_DIR@ )
+
+set ( PAIRS_LINK_LIBRARIES @PAIRS_LINK_LIBRARIES@ )
+set ( PAIRS_LINK_DIRS @PAIRS_LINK_DIRS@ )
+set ( PAIRS_INCLUDE_DIRS @PAIRS_INCLUDE_DIRS@ )
diff --git a/examples/dem_sd.py b/examples/dem_sd.py
index ac56506..96e1c05 100644
--- a/examples/dem_sd.py
+++ b/examples/dem_sd.py
@@ -1,6 +1,7 @@
 import math
 import pairs
 import sys
+import os
 
 def update_mass_and_inertia(i):
     rotation_matrix[i] = diagonal_matrix(1.0)
@@ -93,8 +94,11 @@ lnDryResCoeff = math.log(restitutionCoefficient)
 frictionStatic = 0.0
 frictionDynamic = frictionCoefficient
 
+file_name = os.path.basename(__file__)
+file_name_without_extension = os.path.splitext(file_name)[0]
+
 psim = pairs.simulation(
-    "dem_sd",
+    file_name_without_extension,
     [pairs.sphere(), pairs.halfspace()],
     timesteps=timeSteps,
     double_prec=True,
diff --git a/runtime/devices/cuda.cu b/runtime/devices/cuda.cu
index 7ee69c1..52e8825 100644
--- a/runtime/devices/cuda.cu
+++ b/runtime/devices/cuda.cu
@@ -2,7 +2,7 @@
 #include <iostream>
 #include <cstring>
 #include "../pairs_common.hpp"
-// #include "device.hpp"
+#include "device.hpp"
 
 #define CUDA_ASSERT(a) { pairs::cuda_assert((a), __FILE__, __LINE__); }
 
diff --git a/src/pairs/code_gen/cgen.py b/src/pairs/code_gen/cgen.py
index 800d047..18d4771 100644
--- a/src/pairs/code_gen/cgen.py
+++ b/src/pairs/code_gen/cgen.py
@@ -140,6 +140,10 @@ class CGen:
     def generate_preamble(self):
         self.print(f"#define APPLICATION_REFERENCE \"{self.ref}\"")
 
+        # TODO: Either do this, or add USE_WALBERLA to you compile definitions
+        if self.sim.partitioner()==DomainPartitioners.BlockForest:
+            self.print("#define USE_WALBERLA")
+
         if self.target.is_gpu():
             self.print("#define PAIRS_TARGET_CUDA")
 
@@ -501,7 +505,8 @@ class CGen:
         self.print.add_indent(-4)
         self.print("};")
 
-        self.generate_host_pairs_accessor_class()
+        if self.sim.partitioner()==DomainPartitioners.BlockForest:
+            self.generate_host_pairs_accessor_class()
         
         self.print.end()
         self.generate_full_object_names = False
-- 
GitLab