From ee33a95990f09ab7706b40a2f360582a211c1033 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 15 Jan 2024 14:20:24 +0100
Subject: [PATCH] Refactor packaging, part I
---
.gitlab-ci.yml | 6 +-
MANIFEST.in | 7 +-
pyproject.toml | 100 +++++++++++
pystencils/boundaries/createindexlist.py | 156 ++++++++++++------
.../boundaries/createindexlistcython.pyx | 5 +-
quicktest.py | 22 +++
setup.cfg | 11 --
setup.py | 135 +--------------
8 files changed, 239 insertions(+), 203 deletions(-)
create mode 100644 pyproject.toml
create mode 100644 quicktest.py
delete mode 100644 setup.cfg
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f1ac3470f..3680e33aa 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -109,7 +109,7 @@ ubuntu:
before_script:
- apt-get -y remove python3-sympy
- ln -s /usr/include/locale.h /usr/include/xlocale.h
- - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
+ - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' pyproject.toml | sed 's/>/=/g'`
# - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
script:
- export NUM_CORES=$(nproc --all)
@@ -200,7 +200,7 @@ minimal-conda:
- $ENABLE_NIGHTLY_BUILDS
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
script:
- - python setup.py quicktest
+ - python quicktest.py
tags:
- docker
- cuda
@@ -214,7 +214,7 @@ minimal-sympy-master:
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
script:
- python -m pip install --upgrade git+https://github.com/sympy/sympy.git
- - python setup.py quicktest
+ - python quicktest.py
allow_failure: true
tags:
- docker
diff --git a/MANIFEST.in b/MANIFEST.in
index 3d3c47855..db0bf6352 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,8 +1,3 @@
-include README.md
-include COPYING.txt
include AUTHORS.txt
include CONTRIBUTING.md
-CHANGELOG.md
-global-include *.pyx
-include versioneer.py
-include pystencils/_version.py
+include CHANGELOG.md
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 000000000..f815ff304
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,100 @@
+[project]
+name = "pystencils"
+description = "Speeding up stencil computations on CPUs and GPUs"
+dynamic = ["version"]
+readme = "README.md"
+authors = [
+ { name = "Martin Bauer" },
+ { name = "Jan Hönig " },
+ { name = "Markus Holzer" },
+ { name = "Frederik Hennig" },
+ { email = "cs10-codegen@fau.de" },
+]
+license = { file = "COPYING.txt" }
+requires-python = ">=3.10"
+dependencies = ["sympy>=1.6,<=1.11.1", "numpy>=1.8.0", "appdirs", "joblib"]
+classifiers = [
+ "Development Status :: 4 - Beta",
+ "Framework :: Jupyter",
+ "Topic :: Software Development :: Code Generators",
+ "Topic :: Scientific/Engineering :: Physics",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Science/Research",
+ "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+]
+
+[project.urls]
+"Bug Tracker" = "https://i10git.cs.fau.de/pycodegen/pystencils/-/issues"
+"Documentation" = "https://pycodegen.pages.i10git.cs.fau.de/pystencils/"
+"Source Code" = "https://i10git.cs.fau.de/pycodegen/pystencils"
+
+[project.optional-dependencies]
+gpu = ['cupy']
+alltrafos = ['islpy', 'py-cpuinfo']
+bench_db = ['blitzdb', 'pymongo', 'pandas']
+interactive = [
+ 'matplotlib',
+ 'ipy_table',
+ 'imageio',
+ 'jupyter',
+ 'pyevtk',
+ 'rich',
+ 'graphviz',
+]
+use_cython = [
+ 'Cython'
+]
+doc = [
+ 'sphinx',
+ 'sphinx_rtd_theme',
+ 'nbsphinx',
+ 'sphinxcontrib-bibtex',
+ 'sphinx_autodoc_typehints',
+ 'pandoc',
+]
+tests = [
+ 'pytest',
+ 'pytest-cov',
+ 'pytest-html',
+ 'ansi2html',
+ 'pytest-xdist',
+ 'flake8',
+ 'nbformat',
+ 'nbconvert',
+ 'ipython',
+ 'randomgen>=1.18',
+]
+
+[build-system]
+requires = [
+ "setuptools>=69",
+ "versioneer>=0.29",
+ "tomli; python_version < '3.11'",
+ # 'Cython'
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.package-data]
+pystencils = [
+ "include/*.h",
+ "backends/cuda_known_functions.txt",
+ "backends/opencl1.1_known_functions.txt",
+ "boundaries/createindexlistcython.c",
+ "boundaries/createindexlistcython.pyx",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["pystencils", "pystencils.*"]
+namespaces = false
+
+[tool.versioneer]
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+VCS = "git"
+style = "pep440"
+versionfile_source = "pystencils/_version.py"
+versionfile_build = "pystencils/_version.py"
+tag_prefix = "release/"
+parentdir_prefix = "pystencils-"
diff --git a/pystencils/boundaries/createindexlist.py b/pystencils/boundaries/createindexlist.py
index 8619a31d6..462d3f329 100644
--- a/pystencils/boundaries/createindexlist.py
+++ b/pystencils/boundaries/createindexlist.py
@@ -2,26 +2,22 @@ import warnings
import numpy as np
+
try:
- # Try to import right away - assume compiled code is available
- # compile with: python setup.py build_ext --inplace --use-cython
- from pystencils.boundaries.createindexlistcython import create_boundary_neighbor_index_list_2d, \
- create_boundary_neighbor_index_list_3d, create_boundary_cell_index_list_2d, create_boundary_cell_index_list_3d
+ import pyximport
+ pyximport.install(language_level=3)
cython_funcs_available = True
except ImportError:
- try:
- # If not, try development mode and import via pyximport
- import pyximport
-
- pyximport.install(language_level=3)
- cython_funcs_available = True
- except ImportError:
- cython_funcs_available = False
- if cython_funcs_available:
- from pystencils.boundaries.createindexlistcython import create_boundary_neighbor_index_list_2d, \
- create_boundary_neighbor_index_list_3d, create_boundary_cell_index_list_2d, \
- create_boundary_cell_index_list_3d
+ cython_funcs_available = False
+
+if cython_funcs_available:
+ from pystencils.boundaries.createindexlistcython import (
+ create_boundary_neighbor_index_list_2d,
+ create_boundary_neighbor_index_list_3d,
+ create_boundary_cell_index_list_2d,
+ create_boundary_cell_index_list_3d,
+ )
boundary_index_array_coordinate_names = ["x", "y", "z"]
direction_member_name = "dir"
@@ -30,40 +26,59 @@ default_index_array_dtype = np.int32
def numpy_data_type_for_boundary_object(boundary_object, dim):
coordinate_names = boundary_index_array_coordinate_names[:dim]
- return np.dtype([(name, default_index_array_dtype) for name in coordinate_names]
- + [(direction_member_name, default_index_array_dtype)]
- + [(i[0], i[1].numpy_dtype) for i in boundary_object.additional_data], align=True)
-
-
-def _create_index_list_python(flag_field_arr, boundary_mask,
- fluid_mask, stencil, single_link, inner_or_boundary=False, nr_of_ghost_layers=None):
-
+ return np.dtype(
+ [(name, default_index_array_dtype) for name in coordinate_names]
+ + [(direction_member_name, default_index_array_dtype)]
+ + [(i[0], i[1].numpy_dtype) for i in boundary_object.additional_data],
+ align=True,
+ )
+
+
+def _create_index_list_python(
+ flag_field_arr,
+ boundary_mask,
+ fluid_mask,
+ stencil,
+ single_link,
+ inner_or_boundary=False,
+ nr_of_ghost_layers=None,
+):
if inner_or_boundary and nr_of_ghost_layers is None:
- raise ValueError("If inner_or_boundary is set True the number of ghost layers "
- "around the inner domain has to be specified")
+ raise ValueError(
+ "If inner_or_boundary is set True the number of ghost layers "
+ "around the inner domain has to be specified"
+ )
if nr_of_ghost_layers is None:
nr_of_ghost_layers = 0
- coordinate_names = boundary_index_array_coordinate_names[:len(flag_field_arr.shape)]
- index_arr_dtype = np.dtype([(name, default_index_array_dtype) for name in coordinate_names]
- + [(direction_member_name, default_index_array_dtype)])
+ coordinate_names = boundary_index_array_coordinate_names[
+ : len(flag_field_arr.shape)
+ ]
+ index_arr_dtype = np.dtype(
+ [(name, default_index_array_dtype) for name in coordinate_names]
+ + [(direction_member_name, default_index_array_dtype)]
+ )
# boundary cells are extracted via np.where. To ensure continous memory access in the compute kernel these cells
# have to be sorted.
boundary_cells = np.transpose(np.nonzero(flag_field_arr == boundary_mask))
for i in range(len(flag_field_arr.shape)):
- boundary_cells = boundary_cells[boundary_cells[:, i].argsort(kind='mergesort')]
+ boundary_cells = boundary_cells[boundary_cells[:, i].argsort(kind="mergesort")]
# First a set is created to save all fluid cells which are near boundary
fluid_cells = set()
for cell in boundary_cells:
cell = tuple(cell)
for dir_idx, direction in enumerate(stencil):
- neighbor_cell = tuple([cell_i + dir_i for cell_i, dir_i in zip(cell, direction)])
+ neighbor_cell = tuple(
+ [cell_i + dir_i for cell_i, dir_i in zip(cell, direction)]
+ )
# prevent out ouf bounds access. If boundary cell is at the border, some stencil directions would be out.
- if any(not 0 + nr_of_ghost_layers <= e < upper - nr_of_ghost_layers
- for e, upper in zip(neighbor_cell, flag_field_arr.shape)):
+ if any(
+ not 0 + nr_of_ghost_layers <= e < upper - nr_of_ghost_layers
+ for e, upper in zip(neighbor_cell, flag_field_arr.shape)
+ ):
continue
if flag_field_arr[neighbor_cell] & fluid_mask:
fluid_cells.add(neighbor_cell)
@@ -83,9 +98,14 @@ def _create_index_list_python(flag_field_arr, boundary_mask,
cell = tuple(cell)
sum_cells = np.zeros(len(cell))
for dir_idx, direction in enumerate(stencil):
- neighbor_cell = tuple([cell_i + dir_i for cell_i, dir_i in zip(cell, direction)])
+ neighbor_cell = tuple(
+ [cell_i + dir_i for cell_i, dir_i in zip(cell, direction)]
+ )
# prevent out ouf bounds access. If boundary cell is at the border, some stencil directions would be out.
- if any(not 0 <= e < upper for e, upper in zip(neighbor_cell, flag_field_arr.shape)):
+ if any(
+ not 0 <= e < upper
+ for e, upper in zip(neighbor_cell, flag_field_arr.shape)
+ ):
continue
if flag_field_arr[neighbor_cell] & checkmask:
if single_link:
@@ -101,8 +121,15 @@ def _create_index_list_python(flag_field_arr, boundary_mask,
return np.array(result, dtype=index_arr_dtype)
-def create_boundary_index_list(flag_field, stencil, boundary_mask, fluid_mask,
- nr_of_ghost_layers=1, inner_or_boundary=True, single_link=False):
+def create_boundary_index_list(
+ flag_field,
+ stencil,
+ boundary_mask,
+ fluid_mask,
+ nr_of_ghost_layers=1,
+ inner_or_boundary=True,
+ single_link=False,
+):
"""Creates a numpy array storing links (connections) between domain cells and boundary cells.
Args:
@@ -119,11 +146,20 @@ def create_boundary_index_list(flag_field, stencil, boundary_mask, fluid_mask,
"""
dim = len(flag_field.shape)
coordinate_names = boundary_index_array_coordinate_names[:dim]
- index_arr_dtype = np.dtype([(name, default_index_array_dtype) for name in coordinate_names]
- + [(direction_member_name, default_index_array_dtype)])
+ index_arr_dtype = np.dtype(
+ [(name, default_index_array_dtype) for name in coordinate_names]
+ + [(direction_member_name, default_index_array_dtype)]
+ )
stencil = np.array(stencil, dtype=default_index_array_dtype)
- args = (flag_field, nr_of_ghost_layers, boundary_mask, fluid_mask, stencil, single_link)
+ args = (
+ flag_field,
+ nr_of_ghost_layers,
+ boundary_mask,
+ fluid_mask,
+ stencil,
+ single_link,
+ )
args_no_gl = (flag_field, boundary_mask, fluid_mask, stencil, single_link)
if cython_funcs_available:
@@ -142,22 +178,42 @@ def create_boundary_index_list(flag_field, stencil, boundary_mask, fluid_mask,
return np.array(idx_list, dtype=index_arr_dtype)
else:
if flag_field.size > 1e6:
- warnings.warn("Boundary setup may take very long! Consider installing cython to speed it up")
- return _create_index_list_python(*args_no_gl, inner_or_boundary=inner_or_boundary,
- nr_of_ghost_layers=nr_of_ghost_layers)
-
-
-def create_boundary_index_array(flag_field, stencil, boundary_mask, fluid_mask, boundary_object,
- nr_of_ghost_layers=1, inner_or_boundary=True, single_link=False):
- idx_array = create_boundary_index_list(flag_field, stencil, boundary_mask, fluid_mask,
- nr_of_ghost_layers, inner_or_boundary, single_link)
+ warnings.warn(
+ "Boundary setup may take very long! Consider installing cython to speed it up"
+ )
+ return _create_index_list_python(
+ *args_no_gl,
+ inner_or_boundary=inner_or_boundary,
+ nr_of_ghost_layers=nr_of_ghost_layers,
+ )
+
+
+def create_boundary_index_array(
+ flag_field,
+ stencil,
+ boundary_mask,
+ fluid_mask,
+ boundary_object,
+ nr_of_ghost_layers=1,
+ inner_or_boundary=True,
+ single_link=False,
+):
+ idx_array = create_boundary_index_list(
+ flag_field,
+ stencil,
+ boundary_mask,
+ fluid_mask,
+ nr_of_ghost_layers,
+ inner_or_boundary,
+ single_link,
+ )
dim = len(flag_field.shape)
if boundary_object.additional_data:
coordinate_names = boundary_index_array_coordinate_names[:dim]
index_arr_dtype = numpy_data_type_for_boundary_object(boundary_object, dim)
extended_idx_field = np.empty(len(idx_array), dtype=index_arr_dtype)
- for prop in coordinate_names + ['dir']:
+ for prop in coordinate_names + ["dir"]:
extended_idx_field[prop] = idx_array[prop]
idx_array = extended_idx_field
diff --git a/pystencils/boundaries/createindexlistcython.pyx b/pystencils/boundaries/createindexlistcython.pyx
index bd30fc1ba..36f57431b 100644
--- a/pystencils/boundaries/createindexlistcython.pyx
+++ b/pystencils/boundaries/createindexlistcython.pyx
@@ -1,7 +1,4 @@
-# distutils: language=c
-# Workaround for cython bug
-# see https://stackoverflow.com/questions/8024805/cython-compiled-c-extension-importerror-dynamic-module-does-not-define-init-fu
-WORKAROUND = "Something"
+# cython: language_level=3str
import cython
diff --git a/quicktest.py b/quicktest.py
new file mode 100644
index 000000000..04b71cb49
--- /dev/null
+++ b/quicktest.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+from contextlib import redirect_stdout
+import io
+from pystencils_tests.test_quicktests import (
+ test_basic_kernel,
+ test_basic_blocking_staggered,
+ test_basic_vectorization,
+)
+
+quick_tests = [
+ test_basic_kernel,
+ test_basic_blocking_staggered,
+ test_basic_vectorization,
+]
+
+if __name__ == "__main__":
+ print("Running pystencils quicktests")
+ for qt in quick_tests:
+ print(f" -> {qt.__name__}")
+ with redirect_stdout(io.StringIO()):
+ qt()
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 60288a36a..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,11 +0,0 @@
-# See the docstring in versioneer.py for instructions. Note that you must
-# re-run 'versioneer.py setup' after changing this section, and commit the
-# resulting files.
-
-[versioneer]
-VCS = git
-style = pep440
-versionfile_source = pystencils/_version.py
-versionfile_build = pystencils/_version.py
-tag_prefix = release/
-parentdir_prefix = pystencils-
diff --git a/setup.py b/setup.py
index 31392a747..8c99d7c3c 100644
--- a/setup.py
+++ b/setup.py
@@ -1,136 +1,13 @@
-import distutils
-import io
import os
-from contextlib import redirect_stdout
-from importlib import import_module
-import setuptools
+from setuptools import Extension, setup
import versioneer
-try:
- import cython # noqa
-
- USE_CYTHON = True
-except ImportError:
- USE_CYTHON = False
-
-quick_tests = [
- 'test_quicktests.test_basic_kernel',
- 'test_quicktests.test_basic_blocking_staggered',
- 'test_quicktests.test_basic_vectorization',
-]
-
-
-class SimpleTestRunner(distutils.cmd.Command):
- """A custom command to run selected tests"""
-
- description = 'run some quick tests'
- user_options = []
-
- @staticmethod
- def _run_tests_in_module(test):
- """Short test runner function - to work also if py.test is not installed."""
- test = f'pystencils_tests.{test}'
- mod, function_name = test.rsplit('.', 1)
- if isinstance(mod, str):
- mod = import_module(mod)
-
- func = getattr(mod, function_name)
- print(f" -> {function_name} in {mod.__name__}")
- with redirect_stdout(io.StringIO()):
- func()
-
- def initialize_options(self):
- pass
-
- def finalize_options(self):
- pass
-
- def run(self):
- """Run command."""
- for test in quick_tests:
- self._run_tests_in_module(test)
-
-
-def readme():
- with open('README.md') as f:
- return f.read()
-
-
-def cython_extensions(*extensions):
- from distutils.extension import Extension
- if USE_CYTHON:
- ext = '.pyx'
- result = [Extension(e, [os.path.join(*e.split(".")) + ext]) for e in extensions]
- from Cython.Build import cythonize
- result = cythonize(result, language_level=3)
- return result
- elif all([os.path.exists(os.path.join(*e.split(".")) + '.c') for e in extensions]):
- ext = '.c'
- result = [Extension(e, [os.path.join(*e.split(".")) + ext]) for e in extensions]
- return result
- else:
- return None
-
-
def get_cmdclass():
- cmdclass = {"quicktest": SimpleTestRunner}
- cmdclass.update(versioneer.get_cmdclass())
- return cmdclass
-
-
-setuptools.setup(name='pystencils',
- description='Speeding up stencil computations on CPUs and GPUs',
- version=versioneer.get_version(),
- long_description=readme(),
- long_description_content_type="text/markdown",
- author='Martin Bauer, Jan Hönig, Markus Holzer',
- license='AGPLv3',
- author_email='cs10-codegen@fau.de',
- url='https://i10git.cs.fau.de/pycodegen/pystencils/',
- packages=['pystencils'] + ['pystencils.' + s for s in setuptools.find_packages('pystencils')],
- install_requires=['sympy>=1.6,<=1.11.1', 'numpy>=1.8.0', 'appdirs', 'joblib'],
- package_data={'pystencils': ['include/*.h',
- 'backends/cuda_known_functions.txt',
- 'backends/opencl1.1_known_functions.txt',
- 'boundaries/createindexlistcython.c',
- 'boundaries/createindexlistcython.pyx']},
- ext_modules=cython_extensions("pystencils.boundaries.createindexlistcython"),
- classifiers=[
- 'Development Status :: 4 - Beta',
- 'Framework :: Jupyter',
- 'Topic :: Software Development :: Code Generators',
- 'Topic :: Scientific/Engineering :: Physics',
- 'Intended Audience :: Developers',
- 'Intended Audience :: Science/Research',
- 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
- ],
- project_urls={
- "Bug Tracker": "https://i10git.cs.fau.de/pycodegen/pystencils/-/issues",
- "Documentation": "https://pycodegen.pages.i10git.cs.fau.de/pystencils/",
- "Source Code": "https://i10git.cs.fau.de/pycodegen/pystencils",
- },
- extras_require={
- 'gpu': ['cupy'],
- 'alltrafos': ['islpy', 'py-cpuinfo'],
- 'bench_db': ['blitzdb', 'pymongo', 'pandas'],
- 'interactive': ['matplotlib', 'ipy_table', 'imageio', 'jupyter', 'pyevtk', 'rich', 'graphviz'],
- 'doc': ['sphinx', 'sphinx_rtd_theme', 'nbsphinx',
- 'sphinxcontrib-bibtex', 'sphinx_autodoc_typehints', 'pandoc'],
- 'use_cython': ['Cython']
- },
- tests_require=['pytest',
- 'pytest-cov',
- 'pytest-html',
- 'ansi2html',
- 'pytest-xdist',
- 'flake8',
- 'nbformat',
- 'nbconvert',
- 'ipython',
- 'randomgen>=1.18'],
+ return versioneer.get_cmdclass()
- python_requires=">=3.8",
- cmdclass=get_cmdclass()
- )
+setup(
+ version=versioneer.get_version(),
+ cmdclass=get_cmdclass(),
+)
--
GitLab