[skip-ci] toward improved venv management

CMakeLists.txt

@@ -8,6 +8,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED)
 
 list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake )
 
-include( PrepareSFG )
+include( WalberlaCodegen )
 
 add_subdirectory( lib )

cmake/ManageCodegenVenv.py

+from __future__ import annotations
+
+from typing import Generator
+import sys
+import subprocess
+import json
+import shutil
+from contextlib import contextmanager
+from dataclasses import dataclass, asdict
+from argparse import ArgumentParser
+from pathlib import Path
+
+
+@dataclass
+class VenvState:
+    venv_dir: str | None = None
+    main_requirements_file: str | None = None
+    initialized: bool = False
+
+    @staticmethod
+    @contextmanager
+    def lock(statefile: Path) -> Generator[VenvState, None, None]:
+        statefile_bak = statefile.with_suffix(".json.bak")
+        if statefile.exists():
+            statefile.replace(statefile_bak)
+
+            with statefile_bak.open("r") as f:
+                state_dict = json.load(f)
+
+            state = VenvState(**state_dict)
+        else:
+            state = VenvState()
+
+        yield state
+
+        #   If the consumer raises an error, execution terminates here
+
+        state_dict = asdict(state)
+        with statefile_bak.open("w") as f:
+            json.dump(state_dict, f)
+        statefile_bak.replace(statefile)
+
+
+def action_initialize(args):
+    statefile = Path(args.statefile)
+
+    with VenvState.lock(statefile) as state:
+        if not state.initialized:
+            p_venv_dir = Path(args.venv_dir).resolve()
+            if p_venv_dir.exists():
+                shutil.rmtree(p_venv_dir)
+            reqs_file = Path(args.requirements_file).resolve()
+
+            state.venv_dir = str(p_venv_dir)
+            state.main_requirements_file = str(reqs_file)
+
+            base_py = Path(sys.executable)
+
+            #   Create the virtual environment
+            venv_args = [base_py, "-m", "venv", state.venv_dir]
+            subprocess.run(venv_args).check_returncode()
+
+            #   Install base requirements
+            venv_py = Path(state.venv_dir).absolute() / "bin" / "python"
+            install_args = [venv_py, "-m", "pip", "install", "-r", state.main_requirements_file]
+            subprocess.run(install_args).check_returncode()
+
+            state.initialized = True
+
+
+def main():
+    parser = ArgumentParser("ManageCodegenVenv")
+    parser.add_argument(
+        "-s",
+        "--statefile",
+        required=True,
+        dest="statefile",
+        help="Path to the environment statefile",
+    )
+
+    subparsers = parser.add_subparsers(required=True)
+
+    parser_initialize = subparsers.add_parser("init")
+    parser_initialize.add_argument(
+        "venv_dir",
+        type=str,
+        help="Location of the virtual environment in the filesystem",
+    )
+    parser_initialize.add_argument(
+        "requirements_file",
+        type=str,
+        help="Location of the virtual environment in the filesystem",
+    )
+    parser_initialize.set_defaults(func=action_initialize)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

cmake/PrepareSFG.cmake → cmake/WalberlaCodegen.cmake

@@ -9,10 +9,22 @@ set(
 if( WALBERLA_CODEGEN_PRIVATE_VENV )
     set(WALBERLA_CODEGEN_VENV_PATH ${CMAKE_CURRENT_BINARY_DIR}/codegen-venv CACHE PATH "Location of the virtual environment used for code generation")
     set(_venv_python_exe ${WALBERLA_CODEGEN_VENV_PATH}/bin/python)
+    set(
+        _WALBERLA_CODEGEN_VENV_MANAGER
+        ${sfg_walberla_SOURCE_DIR}/cmake/ManageCodegenVenv.py
+        CACHE INTERNAL
+        "venv manager filepath - for internal use only"
+    )
+    set(
+        _WALBERLA_CODEGEN_VENV_STATEFILE
+        ${CMAKE_CURRENT_BINARY_DIR}/walberla-venv-state.json
+        CACHE INTERNAL
+        "venv statefile - for internal use only"
+    )
 
     set(
         WALBERLA_CODEGEN_VENV_REQUIREMENTS
-        ${sfg_walberla_SOURCE_DIR}/codegen-requirements.txt
+        ${sfg_walberla_SOURCE_DIR}/cmake/codegen-requirements.txt.in
         CACHE PATH
         "Location of the primary requirements file for the codegen virtual environment"
     )
@@ -20,32 +32,32 @@ if( WALBERLA_CODEGEN_PRIVATE_VENV )
 
     find_package( Python COMPONENTS Interpreter REQUIRED )
 
-    if(NOT _sfg_private_venv_done)
-        message( STATUS "Setting up Python virtual environment at ${WALBERLA_CODEGEN_VENV_PATH}" )
-
-        #   Create the venv and register its interpreter with pystencils-sfg
-        if(NOT EXISTS ${WALBERLA_CODEGEN_VENV_PATH})
-            execute_process(
-                COMMAND ${Python_EXECUTABLE} -m venv ${WALBERLA_CODEGEN_VENV_PATH}
-            )
-        endif()
-
-        message( STATUS "Installing required Python packages..." )
-
-        execute_process(
-            COMMAND ${_venv_python_exe} -m pip install -r $CACHE{WALBERLA_CODEGEN_VENV_REQUIREMENTS}
-            OUTPUT_QUIET
-        )
-
-        execute_process(
-            COMMAND ${_venv_python_exe} -m pip install -e ${sfg_walberla_SOURCE_DIR}
-            OUTPUT_QUIET
-        )
-
-        set( _sfg_private_venv_done TRUE CACHE BOOL "" )
-        set( _wlb_codegen_python_init ${_venv_python_exe} )
-        mark_as_advanced(_sfg_private_venv_done)
+    set(
+        _requirements_file
+        ${CMAKE_CURRENT_BINARY_DIR}/codegen-requirements.txt
+    )
+
+    configure_file(
+        ${WALBERLA_CODEGEN_VENV_REQUIREMENTS}
+        ${_requirements_file}
+    )
+
+    execute_process(
+        COMMAND
+            ${Python_EXECUTABLE}
+            ${_WALBERLA_CODEGEN_VENV_MANAGER}
+            -s ${_WALBERLA_CODEGEN_VENV_STATEFILE}
+            init
+            ${WALBERLA_CODEGEN_VENV_PATH}
+            ${_requirements_file}
+        RESULT_VARIABLE _lastResult
+    )
+
+    if( ${_lastResult} )
+        message( FATAL_ERROR "Codegen virtual environment setup failed" )
     endif()
+
+    set( _wlb_codegen_python_init ${_venv_python_exe} )
 else()
     #   Use the external Python environment, but check if all packages are installed
     find_package( Python COMPONENTS Interpreter REQUIRED )

codegen-requirements.txt → cmake/codegen-requirements.txt.in

-# pystencils 2.0 Development Branch
+
 git+https://i10git.cs.fau.de/pycodegen/pystencils.git@v2.0-dev
 
-# lbmpy: feature branch for pystencils-2.0 compatibility
 git+https://i10git.cs.fau.de/pycodegen/lbmpy.git@fhennig/pystencils2.0-compat
 
-# pystencils-sfg: master
 git+https://i10git.cs.fau.de/pycodegen/pystencils-sfg.git
+
+-e ${sfg_walberla_SOURCE_DIR}