From 9f61b1aedea9b0846adb4e134859e880fedb8341 Mon Sep 17 00:00:00 2001 From: Michael Zikeli <michael.zikeli@fau.de> Date: Mon, 10 Feb 2025 09:10:36 +0100 Subject: [PATCH] [no main fork] stop tracking the files that are outsourced to git walberla-config repository. --- .vscode/tasks.json | 71 ----- CMakeUserPresets.json | 142 --------- .../investigateBenchmarkingTimerOutput.ipynb | 277 ------------------ .../investigateBenchmarkingTimerOutput.py | 82 ------ requirements.txt | 50 ---- 5 files changed, 622 deletions(-) delete mode 100644 .vscode/tasks.json delete mode 100644 CMakeUserPresets.json delete mode 100644 post_processing/investigateBenchmarkingTimerOutput.ipynb delete mode 100644 post_processing/investigateBenchmarkingTimerOutput.py delete mode 100644 requirements.txt diff --git a/.vscode/tasks.json b/.vscode/tasks.json deleted file mode 100644 index 54098e63f..000000000 --- a/.vscode/tasks.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "version": "2.0.0", - "tasks": [ - { - "type": "cmake", - "label": "CMake: build debug UniformGridCPU_d3q19_aa_srt", - "command": "build", - "targets": [ - "UniformGridCPU_d3q19_aa_srt" - ], - "preset": "build-debug-UniformGridCPU_d3q19_aa_srt", - "group": "build", - "problemMatcher": [], - "detail": "Build the UniformGridCPU_d3q19_aa_srt target in debug mode" - }, - { - "label": "Run debug UniformGridCPU_d3q19_aa_srt", - "type": "shell", - "command": "mpirun", - "args": [ - "-np", - "1", - "./UniformGridCPU_d3q19_aa_srt", - "simulation_setup/benchmark_configs_RDM.py" - ], - "options": { - "cwd": "${workspaceFolder}/build/benchmark-cpu-debug/apps/benchmarks/UniformGridCPU", - "env": { - "BENCHMARK": "17" - } - }, - "dependsOn": "CMake: build UniformGridCPU_d3q19_aa_srt", - "group": "test", - "problemMatcher": [], - "detail": "Run the UniformGridCPU_d3q19_aa_srt target in debug mode" - }, - { - "type": "cmake", - "label": "CMake: build debug UniformGridGPU_d3q19_aa_srt", - "command": "build", - "targets": [ - "UniformGridGPU_d3q19_aa_srt" - ], - "preset": "build-debug-UniformGridGPU_d3q19_aa_srt", - "group": "build", - "problemMatcher": [], - "detail": "Build the UniformGridGPU_d3q19_aa_srt target in debug mode" - }, - { - "label": "Run debug UniformGridGPU_d3q19_aa_srt", - "type": "shell", - "command": "mpirun", - "args": [ - "-np", - "1", - "./UniformGridGPU_d3q19_aa_srt", - "simulation_setup/benchmark_configs_RDM.py" - ], - "options": { - "cwd": "${workspaceFolder}/build/benchmark-cuda-debug/apps/benchmarks/UniformGridGPU", - "env": { - "BENCHMARK": "17" - } - }, - "dependsOn": "CMake: build debug UniformGridGPU_d3q19_aa_srt", - "group": "test", - "problemMatcher": [], - "detail": "Run the UniformGridGPU_d3q19_aa_srt target in debug mode" - } - ] -} \ No newline at end of file diff --git a/CMakeUserPresets.json b/CMakeUserPresets.json deleted file mode 100644 index 96593b615..000000000 --- a/CMakeUserPresets.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "version": 6, - "cmakeMinimumRequired": { - "major": 3, - "minor": 23, - "patch": 0 - }, - "configurePresets": [ - { - "name": "benchmark-cuda", - "hidden": false, - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/benchmark-cuda", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "DebugOptimized", - "Python_ROOT_DIR": "${sourceDir}/.venv", - "WALBERLA_BUILD_WITH_PYTHON": true, - "WALBERLA_BUILD_WITH_CODEGEN": true, - "WALBERLA_BUILD_WITH_CUDA": true, - "CMAKE_CUDA_ARCHITECTURES": "75", - "WALBERLA_OPTIMIZE_FOR_LOCALHOST": false, - "WALBERLA_BUILD_BENCHMARKS": true, - "WALBERLA_BUILD_TOOLS": true, - "WALBERLA_BUILD_SHOWCASES": false, - "WALBERLA_BUILD_TESTS": false, - "WALBERLA_BUILD_TUTORIALS": false, - "WALBERLA_BUILD_DOC": false - }, - "environment": { - "CC": "gcc", - "CXX": "g++" - } - }, - { - "name": "benchmark-cuda-debug", - "hidden": false, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build/benchmark-cuda-debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "Python_ROOT_DIR": "${sourceDir}/.venv", - "WALBERLA_BUILD_WITH_PYTHON": true, - "WALBERLA_BUILD_WITH_CODEGEN": true, - "WALBERLA_BUILD_WITH_CUDA": true, - "CMAKE_CUDA_ARCHITECTURES": "75", - "WALBERLA_OPTIMIZE_FOR_LOCALHOST": false, - "WALBERLA_BUILD_BENCHMARKS": true, - "WALBERLA_BUILD_TOOLS": true, - "WALBERLA_BUILD_SHOWCASES": false, - "WALBERLA_BUILD_TESTS": false, - "WALBERLA_BUILD_TUTORIALS": false, - "WALBERLA_BUILD_DOC": false - }, - "environment": { - "CC": "clang", - "CXX": "clang++" - } - }, - { - "name": "benchmark-cpu", - "hidden": false, - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/benchmark-cpu", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "DebugOptimized", - "Python_ROOT_DIR": "${sourceDir}/.venv", - "WALBERLA_BUILD_WITH_PYTHON": true, - "WALBERLA_BUILD_WITH_CODEGEN": true, - "WALBERLA_BUILD_WITH_CUDA": false, - "WALBERLA_OPTIMIZE_FOR_LOCALHOST": false, - "WALBERLA_BUILD_BENCHMARKS": true, - "WALBERLA_BUILD_TOOLS": true, - "WALBERLA_BUILD_SHOWCASES": false, - "WALBERLA_BUILD_TESTS": false, - "WALBERLA_BUILD_TUTORIALS": false, - "WALBERLA_BUILD_DOC": false - }, - "environment": { - "CC": "gcc", - "CXX": "g++" - } - }, - { - "name": "benchmark-cpu-debug", - "hidden": false, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build/benchmark-cpu-debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "Python_ROOT_DIR": "${sourceDir}/.venv", - "WALBERLA_BUILD_WITH_PYTHON": true, - "WALBERLA_BUILD_WITH_CODEGEN": true, - "WALBERLA_BUILD_WITH_CUDA": false, - "WALBERLA_OPTIMIZE_FOR_LOCALHOST": false, - "WALBERLA_BUILD_BENCHMARKS": true, - "WALBERLA_BUILD_TOOLS": true, - "WALBERLA_BUILD_SHOWCASES": false, - "WALBERLA_BUILD_TESTS": false, - "WALBERLA_BUILD_TUTORIALS": false, - "WALBERLA_BUILD_DOC": false - }, - "environment": { - "CC": "clang", - "CXX": "clang++" - } - }, - { - "name": "doxygen", - "generator": "Unix Makefiles", - "binaryDir": "${sourceDir}/build/doxygen", - "cacheVariables": { - "WALBERLA_BUILD_WITH_MPI": true, - "WALBERLA_BUILD_WITH_OPENMP": true, - "CMAKE_BUILD_TYPE": "Debug", - "WALBERLA_BUILD_DOC": true, - "WALBERLA_BUILD_TOOLS": true, - "WALBERLA_BUILD_BENCHMARKS": false, - "WALBERLA_BUILD_SHOWCASES": false, - "WALBERLA_BUILD_TESTS": false, - "WALBERLA_BUILD_TUTORIALS": false - } - } - ], - "buildPresets": [ - { - "name": "build-debug-UniformGridCPU_d3q19_aa_srt", - "hidden": false, - "configurePreset": "benchmark-cpu-debug", - "targets": [ - "UniformGridCPU_d3q19_aa_srt" - ] - }, - { - "name": "build-debug-UniformGridGPU_d3q19_aa_srt", - "hidden": false, - "configurePreset": "benchmark-cuda-debug", - "targets": [ - "UniformGridGPU_d3q19_aa_srt" - ] - } - ] -} diff --git a/post_processing/investigateBenchmarkingTimerOutput.ipynb b/post_processing/investigateBenchmarkingTimerOutput.ipynb deleted file mode 100644 index 90e587ff1..000000000 --- a/post_processing/investigateBenchmarkingTimerOutput.ipynb +++ /dev/null @@ -1,277 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "3a718024", - "metadata": {}, - "outputs": [], - "source": [ - "import sqlite3\n", - "import pandas as pd\n", - "import os\n", - "import matplotlib.pyplot as plt\n", - "import warnings" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4c4cbb74", - "metadata": {}, - "outputs": [], - "source": [ - "# Define colors\n", - "color1 = \"#1F3B66\"\n", - "color2 = \"#C14C30\"\n", - "color3 = \"#4AA7A9\"\n", - "color4 = \"#EBAE65\"\n", - "color5 = \"#EBAE65\"\n", - "color6 = \"#36b700\"\n", - "color7 = \"#9d2c00\"\n", - "color8 = \"#c8c8c8\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "22b800b2", - "metadata": {}, - "outputs": [], - "source": [ - "# Database configuration\n", - "GPUs_per_node = 4 # MareNostrum5 ACC has four GPUs per node\n", - "databasename = \"/local/ab04unyc/mywalberla/build/benchmark-debug/apps/benchmarks/UniformGridCPU/cpu_benchmark.sqlite3\"\n", - "\n", - "# Connect to the database\n", - "conn = sqlite3.connect(databasename)\n", - "cursor = conn.cursor()\n", - "cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n", - "all_tables = cursor.fetchall()\n", - "\n", - "# Extract table names\n", - "all_table_names = [str(t[0]) for t in all_tables]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "67b37628-be47-419c-a053-ea0159d8f813", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['runs']" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Debug output\n", - "all_table_names" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "db839b84-0fa6-44f4-9d4a-046dd65b80d7", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Index(['id', 'timestamp', 'omega', 'warmupSteps', 'outerIterations',\n", - " 'timeStepStrategy', 'timesteps', 'initShearFlow', 'vtkWriteFrequency',\n", - " 'remainingTimeLoggerFrequency', 'cartesianSetup', 'numProcesses',\n", - " 'numThreads', 'numCores', 'numberOfCells', 'numberOfFluidCells',\n", - " 'mlups', 'mlupsPerCore', 'mlupsPerProcess', 'stencil',\n", - " 'streamingPattern', 'collisionSetup', 'vectorised', 'nontemporal',\n", - " 'cse_global', 'cse_pdfs', 'Timer_BoundaryConditions_Total',\n", - " 'Timer_BoundaryConditions_Average', 'Timer_BoundaryConditions_Count',\n", - " 'Timer_BoundaryConditions_Min', 'Timer_BoundaryConditions_Max',\n", - " 'Timer_BoundaryConditions_Variance', 'Timer_LBMStreamCollide_Total',\n", - " 'Timer_LBMStreamCollide_Average', 'Timer_LBMStreamCollide_Count',\n", - " 'Timer_LBMStreamCollide_Min', 'Timer_LBMStreamCollide_Max',\n", - " 'Timer_LBMStreamCollide_Variance', 'Timer_VTKOutput_Total',\n", - " 'Timer_VTKOutput_Average', 'Timer_VTKOutput_Count',\n", - " 'Timer_VTKOutput_Min', 'Timer_VTKOutput_Max',\n", - " 'Timer_VTKOutput_Variance', 'Timer_communication_Total',\n", - " 'Timer_communication_Average', 'Timer_communication_Count',\n", - " 'Timer_communication_Min', 'Timer_communication_Max',\n", - " 'Timer_communication_Variance', 'Timer_remainingtimelogger_Total',\n", - " 'Timer_remainingtimelogger_Average', 'Timer_remainingtimelogger_Count',\n", - " 'Timer_remainingtimelogger_Min', 'Timer_remainingtimelogger_Max',\n", - " 'Timer_remainingtimelogger_Variance', 'executable', 'compile_flags',\n", - " 'walberla_version', 'build_machine', 'innerOuterSplit_0',\n", - " 'innerOuterSplit_1', 'innerOuterSplit_2', 'blocks_0', 'blocks_1',\n", - " 'blocks_2', 'cellsPerBlock_0', 'cellsPerBlock_1', 'cellsPerBlock_2',\n", - " 'periodic_0', 'periodic_1', 'periodic_2'],\n", - " dtype='object')" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Load data from all tables\n", - "for table_name in all_table_names:\n", - " df = pd.read_sql(f\"SELECT * FROM {table_name}\", conn)\n", - " if 'full_df' in globals():\n", - " full_df = pd.concat([full_df, df], ignore_index=True)\n", - " else:\n", - " full_df = df.copy()\n", - "\n", - "# Debug output\n", - "df.columns" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "466adb87", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "<div>\n", - "<style scoped>\n", - " .dataframe tbody tr th:only-of-type {\n", - " vertical-align: middle;\n", - " }\n", - "\n", - " .dataframe tbody tr th {\n", - " vertical-align: top;\n", - " }\n", - "\n", - " .dataframe thead th {\n", - " text-align: right;\n", - " }\n", - "</style>\n", - "<table border=\"1\" class=\"dataframe\">\n", - " <thead>\n", - " <tr style=\"text-align: right;\">\n", - " <th></th>\n", - " <th>Min</th>\n", - " <th>Average</th>\n", - " <th>Total</th>\n", - " <th>Count</th>\n", - " <th>Variance</th>\n", - " <th>Max</th>\n", - " </tr>\n", - " </thead>\n", - " <tbody>\n", - " <tr>\n", - " <th>VTKOutput</th>\n", - " <td>0.0</td>\n", - " <td>0.010657</td>\n", - " <td>2.142152</td>\n", - " <td>201</td>\n", - " <td>0.004458</td>\n", - " <td>0.458243</td>\n", - " </tr>\n", - " <tr>\n", - " <th>remainingtimelogger</th>\n", - " <td>0.000001</td>\n", - " <td>0.000001</td>\n", - " <td>0.00027</td>\n", - " <td>201</td>\n", - " <td>0.0</td>\n", - " <td>0.000015</td>\n", - " </tr>\n", - " <tr>\n", - " <th>communication</th>\n", - " <td>0.000006</td>\n", - " <td>0.000008</td>\n", - " <td>0.001645</td>\n", - " <td>201</td>\n", - " <td>0.0</td>\n", - " <td>0.000031</td>\n", - " </tr>\n", - " <tr>\n", - " <th>LBMStreamCollide</th>\n", - " <td>0.027194</td>\n", - " <td>0.030216</td>\n", - " <td>6.073384</td>\n", - " <td>201</td>\n", - " <td>0.000005</td>\n", - " <td>0.044656</td>\n", - " </tr>\n", - " <tr>\n", - " <th>BoundaryConditions</th>\n", - " <td>0.001965</td>\n", - " <td>0.002244</td>\n", - " <td>0.451072</td>\n", - " <td>201</td>\n", - " <td>0.0</td>\n", - " <td>0.003642</td>\n", - " </tr>\n", - " </tbody>\n", - "</table>\n", - "</div>" - ], - "text/plain": [ - " Min Average Total Count Variance Max\n", - "VTKOutput 0.0 0.010657 2.142152 201 0.004458 0.458243\n", - "remainingtimelogger 0.000001 0.000001 0.00027 201 0.0 0.000015\n", - "communication 0.000006 0.000008 0.001645 201 0.0 0.000031\n", - "LBMStreamCollide 0.027194 0.030216 6.073384 201 0.000005 0.044656\n", - "BoundaryConditions 0.001965 0.002244 0.451072 201 0.0 0.003642" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Extract unique timer names and properties\n", - "timer_columns = df.filter(like='Timer')\n", - "timer_names = list(set(col.split('_')[1] for col in timer_columns.columns))\n", - "properties = list(set(col.split('_')[2] for col in timer_columns.columns))\n", - "\n", - "# Create a new dataframe with properties as rows and timer names as columns\n", - "grouped_timers = pd.DataFrame(columns=properties, index=timer_names)\n", - "\n", - "# Fill the new dataframe with the corresponding values\n", - "for timer in timer_names:\n", - " for prop in properties:\n", - " timer_name = f'Timer_{timer}_{prop}'\n", - " try:\n", - " grouped_timers.at[timer, prop] = timer_columns[timer_name].iloc[0]\n", - " except KeyError:\n", - " warnings.warn(f\"{timer_name} not found in timer_columns\")\n", - "\n", - "# Display the grouped timers\n", - "grouped_timers" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/post_processing/investigateBenchmarkingTimerOutput.py b/post_processing/investigateBenchmarkingTimerOutput.py deleted file mode 100644 index ae562f886..000000000 --- a/post_processing/investigateBenchmarkingTimerOutput.py +++ /dev/null @@ -1,82 +0,0 @@ -# %% -import sqlite3 -import pandas as pd -import warnings -import argparse - -def main(databasename: str, verbose: bool) -> None: - """ - Main function to process the benchmarking timer output from the SQLite database. - - Args: - databasename (str): Path to the SQLite database file. - verbose (bool): Flag to enable verbose output. - """ - # Database configuration - GPUs_per_node = 4 # MareNostrum5 ACC has four GPUs per node - - # Connect to the database - conn = sqlite3.connect(databasename) - cursor = conn.cursor() - cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - all_tables = cursor.fetchall() - if verbose: - print(f"Number of tables in {all_tables}: {len(all_tables)}") - - # Extract table names - all_table_names = [str(t[0]) for t in all_tables] - - # Debug output - print("Avalable table names in the database:", all_table_names) - - # Load data from all tables - for table_name in all_table_names: - df = pd.read_sql(f"SELECT * FROM {table_name}", conn) - if 'full_df' in globals(): - full_df = pd.concat([full_df, df], ignore_index=True) - else: - full_df = df.copy() - - # Debug output - print(f"Number of columns in the database: {len(full_df.columns)}") - if verbose: - print("Available columns in the database:", sorted(full_df.columns)) - - # Extract unique timer names and properties - timer_columns = full_df.filter(like='Timer') - timer_names = sorted(list(set(col.split('_')[1] for col in timer_columns.columns))) - properties = sorted(list(set(col.split('_')[2] for col in timer_columns.columns))) - - if verbose: - print("List of timer names:", timer_names) - print("List of properties:", properties) - - print(f"{len(full_df)} available measurements in the database:") - for _, row in full_df.iterrows(): - print(f'\nResults from {row["Identifier"]} at {row["timestamp"]} with {row["timeStepStrategy"]} on {row["blocks_0"]*row["blocks_1"]*row["blocks_2"]} nodes and a domain size of {row["cellsPerBlock_0"]}x{row["cellsPerBlock_1"]}x{row["cellsPerBlock_2"]}') - - # Create a new dataframe with properties as rows and timer names as columns - grouped_timers = pd.DataFrame(columns=properties, index=timer_names) - - # Fill the new dataframe with the corresponding values - for timer in timer_names: - for prop in properties: - timer_name = f'Timer_{timer}_{prop}' - try: - grouped_timers.at[timer, prop] = row[timer_name] - except KeyError: - warnings.warn(f"{timer_name} not found in timer_columns") - - # Display the grouped timers - if verbose: - print(f"in: {databasename.split('/')[-1]}") - print(grouped_timers) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Process benchmarking timer output from an SQLite database.") - parser.add_argument("databasename", type=str, help="Path to the SQLite database file") - parser.add_argument("-v", "--verbose", action="store_true", help="Enable verbose output") - - args = parser.parse_args() - - main(args.databasename, args.verbose) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1a9493d7d..000000000 --- a/requirements.txt +++ /dev/null @@ -1,50 +0,0 @@ -appdirs==1.4.4 -appnope==0.1.4 -asttokens==3.0.0 -comm==0.2.2 -contourpy==1.3.1 -cycler==0.12.1 -debugpy==1.8.12 -decorator==5.1.1 -executing==2.2.0 -fonttools==4.55.8 -ipykernel==6.29.5 -ipython==8.32.0 -jedi==0.19.2 -Jinja2==3.1.5 -joblib==1.4.2 -jupyter_client==8.6.3 -jupyter_core==5.7.2 -kiwisolver==1.4.8 -lbmpy==1.3.7 -MarkupSafe==3.0.2 -matplotlib==3.10.0 -matplotlib-inline==0.1.7 -mpmath==1.3.0 -nest-asyncio==1.6.0 -numpy==2.2.2 -packaging==24.2 -pandas==2.2.3 -parso==0.8.4 -pexpect==4.9.0 -pillow==11.1.0 -platformdirs==4.3.6 -prompt_toolkit==3.0.50 -psutil==6.1.1 -ptyprocess==0.7.0 -pure_eval==0.2.3 -Pygments==2.19.1 -pyparsing==3.2.1 -pystencils==1.3.7 -python-dateutil==2.9.0.post0 -pytz==2025.1 -PyYAML==6.0.2 -pyzmq==26.2.1 -six==1.17.0 -stack-data==0.6.3 -sympy==1.12.1 -tornado==6.4.2 -traitlets==5.14.3 -typing_extensions==4.12.2 -tzdata==2025.1 -wcwidth==0.2.13 -- GitLab