Merge branch 'master' into backend-rework

.flake8

 [flake8]
 max-line-length=120
-exclude=pystencils/jupyter.py,
-        pystencils/plot.py
-        pystencils/session.py
+exclude=src/pystencils/jupyter.py,
+        src/pystencils/plot.py
+        src/pystencils/session.py
 ignore = W293 W503 W291 C901 E741

.gitignore

@@ -15,12 +15,12 @@ _build
 _local_tmp
 RELEASE-VERSION
 test-report
-pystencils/boundaries/createindexlistcython.c
-pystencils/boundaries/createindexlistcython.*.so
-pystencils_tests/tmp
-pystencils_tests/var
-pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
-pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
+src/pystencils/boundaries/createindexlistcython.c
+src/pystencils/boundaries/createindexlistcython.*.so
+tests/tmp
+tests/var
+tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
+tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
 report.xml
 coverage_report/
 

.gitlab-ci.yml

@@ -13,16 +13,17 @@ tests-and-coverage:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
+  before_script:
+    - pip install -e .
   script:
-    - pip install sympy --upgrade
     - env
     - pip list
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
     - mkdir public
-    - py.test -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
-    - python3 -m coverage xml
+    - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
+    - python -m coverage xml
   tags:
     - docker
     - cuda11
@@ -45,8 +46,10 @@ tests-and-coverage-with-longrun:
   when: manual
   allow_failure: true
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
-  script:
+  before_script:
     - pip install sympy --upgrade
+    - pip install -e .
+  script:
     - env
     - pip list
     - export NUM_CORES=$(nproc --all)
@@ -66,9 +69,12 @@ latest-python:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
+  before_script:
+    - pip install -e .
   script:
     - env
     - pip list
+    - pip install -e .
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
@@ -107,18 +113,16 @@ ubuntu:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
   before_script:
-    - apt-get -y remove python3-sympy
     - ln -s /usr/include/locale.h /usr/include/xlocale.h
-    - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
-    # - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
+    - pip3 install -e .
   script:
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
     - sed -i 's/--doctest-modules //g' pytest.ini
     - env
-    - pip3 list
-    - pytest-3 -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
+    - pip list
+    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
   tags:
     - docker
     - cuda11
@@ -134,6 +138,8 @@ ubuntu:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   before_script: &multiarch_before_script
+    # - pip3 install -v .
+    - export PYTHONPATH=src
     - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"
     - sed -i '/^fail_under.*/d' pytest.ini
   script:
@@ -143,7 +149,7 @@ ubuntu:
     - sed -i 's/--doctest-modules //g' pytest.ini
     - env
     - pip3 list
-    - pytest-3 -v -n $NUM_CORES --junitxml=report.xml pystencils_tests/test_*vec*.py pystencils_tests/test_random.py pystencils_tests/test_half_precision.py
+    - python3 -m pytest -v -n $NUM_CORES --junitxml=report.xml tests/test_*vec*.py tests/test_random.py tests/test_half_precision.py
   tags:
     - docker
     - AVX
@@ -199,8 +205,10 @@ minimal-conda:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
+  before_script:
+    - pip install -e .
   script:
-    - python setup.py quicktest
+    - python quicktest.py
   tags:
     - docker
     - cuda
@@ -212,9 +220,11 @@ minimal-sympy-master:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
+  before_script:
+    - pip install -e .
   script:
     - python -m pip install --upgrade git+https://github.com/sympy/sympy.git
-    - python setup.py quicktest
+    - python quicktest.py
   allow_failure: true
   tags:
     - docker
@@ -275,7 +285,7 @@ flake8-lint:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
   script:
-    - flake8 pystencils
+    - flake8 src/pystencils
   tags:
     - docker
 
@@ -283,8 +293,9 @@ flake8-lint:
 build-documentation:
   stage: test
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
+  before_script:
+    - pip install -e .
   script:
-    - export PYTHONPATH=`pwd`
     - mkdir html_doc
     - sphinx-build -b html doc  html_doc
     - sphinx-build -W -b html doc  html_doc

MANIFEST.in

-include README.md
-include COPYING.txt
 include AUTHORS.txt
 include CONTRIBUTING.md
-CHANGELOG.md
-global-include *.pyx
-include versioneer.py
-include pystencils/_version.py
+include CHANGELOG.md

conftest.py

@@ -39,48 +39,48 @@ def add_path_to_ignore(path):
 
 
 collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py"),
-                  os.path.join(SCRIPT_FOLDER, "pystencils", "opencl", "opencl.autoinit")]
-add_path_to_ignore('pystencils_tests/benchmark')
+                  os.path.join(SCRIPT_FOLDER, "src", "pystencils", "opencl", "opencl.autoinit")]
+add_path_to_ignore('tests/benchmark')
 add_path_to_ignore('_local_tmp')
 
 
 try:
     import cupy
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_gpu.py")]
-    add_path_to_ignore('pystencils/gpu')
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_gpu.py")]
+    add_path_to_ignore('src/pystencils/gpu')
 
 try:
     import waLBerla
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_aligned_array.py"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_datahandling_parallel.py"),
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_aligned_array.py"),
+                       os.path.join(SCRIPT_FOLDER, "tests/test_datahandling_parallel.py"),
                        os.path.join(SCRIPT_FOLDER, "doc/notebooks/03_tutorial_datahandling.ipynb"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/parallel_datahandling.py"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_small_block_benchmark.ipynb")]
+                       os.path.join(SCRIPT_FOLDER, "src/pystencils/datahandling/parallel_datahandling.py"),
+                       os.path.join(SCRIPT_FOLDER, "tests/test_small_block_benchmark.ipynb")]
 
 try:
     import blitzdb
 except ImportError:
-    add_path_to_ignore('pystencils/runhelper')
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_parameterstudy.py")]
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_json_serializer.py")]
+    add_path_to_ignore('src/pystencils/runhelper')
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_parameterstudy.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_json_serializer.py")]
 
 try:
     import islpy
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/integer_set_analysis.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/integer_set_analysis.py")]
 
 try:
     import graphviz
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/backends/dot.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/backends/dot.py")]
     collect_ignore += [os.path.join(SCRIPT_FOLDER, "doc/notebooks/01_tutorial_getting_started.ipynb")]
 
 try:
     import pyevtk
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/vtk.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/datahandling/vtk.py")]
 
 collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
 

pyproject.toml

+[project]
+name = "pystencils"
+description = "Speeding up stencil computations on CPUs and GPUs"
+dynamic = ["version"]
+readme = "README.md"
+authors = [
+    { name = "Martin Bauer" },
+    { name = "Jan Hönig " },
+    { name = "Markus Holzer" },
+    { name = "Frederik Hennig" },
+    { email = "cs10-codegen@fau.de" },
+]
+license = { file = "COPYING.txt" }
+requires-python = ">=3.10"
+dependencies = ["sympy>=1.6,<=1.11.1", "numpy>=1.8.0", "appdirs", "joblib", "pyyaml"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Framework :: Jupyter",
+    "Topic :: Software Development :: Code Generators",
+    "Topic :: Scientific/Engineering :: Physics",
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
+]
+
+[project.urls]
+"Bug Tracker" = "https://i10git.cs.fau.de/pycodegen/pystencils/-/issues"
+"Documentation" = "https://pycodegen.pages.i10git.cs.fau.de/pystencils/"
+"Source Code" = "https://i10git.cs.fau.de/pycodegen/pystencils"
+
+[project.optional-dependencies]
+gpu = ['cupy']
+alltrafos = ['islpy', 'py-cpuinfo']
+bench_db = ['blitzdb', 'pymongo', 'pandas']
+interactive = [
+    'matplotlib',
+    'ipy_table',
+    'imageio',
+    'jupyter',
+    'pyevtk',
+    'rich',
+    'graphviz',
+]
+use_cython = [
+    'Cython'
+]
+doc = [
+    'sphinx',
+    'sphinx_rtd_theme',
+    'nbsphinx',
+    'sphinxcontrib-bibtex',
+    'sphinx_autodoc_typehints',
+    'pandoc',
+]
+tests = [
+    'pytest',
+    'pytest-cov',
+    'pytest-html',
+    'ansi2html',
+    'pytest-xdist',
+    'flake8',
+    'nbformat',
+    'nbconvert',
+    'ipython',
+    'matplotlib',
+    'py-cpuinfo',
+    'randomgen>=1.18',
+]
+
+[build-system]
+requires = [
+    "setuptools>=61",
+    "versioneer>=0.29",
+    "tomli; python_version < '3.11'",
+    # 'Cython'
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.package-data]
+pystencils = [
+    "include/*.h",
+    "boundaries/createindexlistcython.pyx"
+]
+
+[tool.setuptools.packages.find]
+where = ["src"]
+include = ["pystencils", "pystencils.*"]
+namespaces = false
+
+[tool.versioneer]
+# See the docstring in versioneer.py for instructions. Note that you must
+# re-run 'versioneer.py setup' after changing this section, and commit the
+# resulting files.
+VCS = "git"
+style = "pep440"
+versionfile_source = "src/pystencils/_version.py"
+versionfile_build = "pystencils/_version.py"
+tag_prefix = "release/"
+parentdir_prefix = "pystencils-"

pytest.ini

 [pytest]
+testpaths = src tests doc/notebooks
+pythonpath = src
 python_files = test_*.py *_test.py scenario_*.py
 norecursedirs = *.egg-info .git .cache .ipynb_checkpoints htmlcov
 addopts = --doctest-modules --durations=20  --cov-config pytest.ini
@@ -17,20 +19,21 @@ filterwarnings =
 
 [run]
 branch = True
-source = pystencils
-         pystencils_tests
+source = src/pystencils
+         tests
 
 omit = doc/*
-       pystencils_tests/*
+       tests/*
        setup.py
+       quicktest.py
        conftest.py
        versioneer.py
-       pystencils/jupytersetup.py
-       pystencils/cpu/msvc_detection.py
-       pystencils/sympy_gmpy_bug_workaround.py
-       pystencils/cache.py
-       pystencils/pacxx/benchmark.py
-       pystencils/_version.py
+       src/pystencils/jupytersetup.py
+       src/pystencils/cpu/msvc_detection.py
+       src/pystencils/sympy_gmpy_bug_workaround.py
+       src/pystencils/cache.py
+       src/pystencils/pacxx/benchmark.py
+       src/pystencils/_version.py
        venv/
 
 [report]

quicktest.py

+#!/usr/bin/env python3
+
+from contextlib import redirect_stdout
+import io
+from tests.test_quicktests import (
+    test_basic_kernel,
+    test_basic_blocking_staggered,
+    test_basic_vectorization,
+)
+
+quick_tests = [
+    test_basic_kernel,
+    test_basic_blocking_staggered,
+    test_basic_vectorization,
+]
+
+if __name__ == "__main__":
+    print("Running pystencils quicktests")
+    for qt in quick_tests:
+        print(f"   -> {qt.__name__}")
+        with redirect_stdout(io.StringIO()):
+            qt()

setup.cfg

-# See the docstring in versioneer.py for instructions. Note that you must
-# re-run 'versioneer.py setup' after changing this section, and commit the
-# resulting files.
-
-[versioneer]
-VCS = git
-style = pep440
-versionfile_source = pystencils/_version.py
-versionfile_build = pystencils/_version.py
-tag_prefix = release/
-parentdir_prefix = pystencils-

setup.py

-import distutils
-import io
-import os
-from contextlib import redirect_stdout
-from importlib import import_module
+from setuptools import setup, __version__ as setuptools_version
 
-import setuptools
+if int(setuptools_version.split('.')[0]) < 61:
+    raise Exception(
+        "[ERROR] pystencils requires at least setuptools version 61 to install.\n"
+        "If this error occurs during an installation via pip, it is likely that there is a conflict between "
+        "versions of setuptools installed by pip and the system package manager. "
+        "In this case, it is recommended to install pystencils into a virtual environment instead."
+    )
 
 import versioneer
 
-try:
-    import cython  # noqa
-
-    USE_CYTHON = True
-except ImportError:
-    USE_CYTHON = False
-
-quick_tests = [
-    'test_quicktests.test_basic_kernel',
-    'test_quicktests.test_basic_blocking_staggered',
-    'test_quicktests.test_basic_vectorization',
-]
-
-
-class SimpleTestRunner(distutils.cmd.Command):
-    """A custom command to run selected tests"""
-
-    description = 'run some quick tests'
-    user_options = []
-
-    @staticmethod
-    def _run_tests_in_module(test):
-        """Short test runner function - to work also if py.test is not installed."""
-        test = f'pystencils_tests.{test}'
-        mod, function_name = test.rsplit('.', 1)
-        if isinstance(mod, str):
-            mod = import_module(mod)
-
-        func = getattr(mod, function_name)
-        print(f"   -> {function_name} in {mod.__name__}")
-        with redirect_stdout(io.StringIO()):
-            func()
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        """Run command."""
-        for test in quick_tests:
-            self._run_tests_in_module(test)
-
-
-def readme():
-    with open('README.md') as f:
-        return f.read()
-
-
-def cython_extensions(*extensions):
-    from distutils.extension import Extension
-    if USE_CYTHON:
-        ext = '.pyx'
-        result = [Extension(e, [os.path.join(*e.split(".")) + ext]) for e in extensions]
-        from Cython.Build import cythonize
-        result = cythonize(result, language_level=3)
-        return result
-    elif all([os.path.exists(os.path.join(*e.split(".")) + '.c') for e in extensions]):
-        ext = '.c'
-        result = [Extension(e, [os.path.join(*e.split(".")) + ext]) for e in extensions]
-        return result
-    else:
-        return None
-
 
 def get_cmdclass():
-    cmdclass = {"quicktest": SimpleTestRunner}
-    cmdclass.update(versioneer.get_cmdclass())
-    return cmdclass
-
+    return versioneer.get_cmdclass()
 
-setuptools.setup(name='pystencils',
-                 description='Speeding up stencil computations on CPUs and GPUs',
-                 version=versioneer.get_version(),
-                 long_description=readme(),
-                 long_description_content_type="text/markdown",
-                 author='Martin Bauer, Jan Hönig, Markus Holzer',
-                 license='AGPLv3',
-                 author_email='cs10-codegen@fau.de',
-                 url='https://i10git.cs.fau.de/pycodegen/pystencils/',
-                 packages=['pystencils'] + ['pystencils.' + s for s in setuptools.find_packages('pystencils')],
-                 install_requires=['sympy>=1.6,<=1.11.1', 'numpy>=1.8.0', 'pymbolic>=2022.2', 'appdirs', 'joblib'],
-                 package_data={'pystencils': ['include/*.h',
-                                              'backends/cuda_known_functions.txt',
-                                              'backends/opencl1.1_known_functions.txt',
-                                              'boundaries/createindexlistcython.c',
-                                              'boundaries/createindexlistcython.pyx']},
-                 ext_modules=cython_extensions("pystencils.boundaries.createindexlistcython"),
-                 classifiers=[
-                     'Development Status :: 4 - Beta',
-                     'Framework :: Jupyter',
-                     'Topic :: Software Development :: Code Generators',
-                     'Topic :: Scientific/Engineering :: Physics',
-                     'Intended Audience :: Developers',
-                     'Intended Audience :: Science/Research',
-                     'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)',
-                 ],
-                 project_urls={
-                     "Bug Tracker": "https://i10git.cs.fau.de/pycodegen/pystencils/-/issues",
-                     "Documentation": "https://pycodegen.pages.i10git.cs.fau.de/pystencils/",
-                     "Source Code": "https://i10git.cs.fau.de/pycodegen/pystencils",
-                 },
-                 extras_require={
-                     'gpu': ['cupy'],
-                     'alltrafos': ['islpy', 'py-cpuinfo'],
-                     'bench_db': ['blitzdb', 'pymongo', 'pandas'],
-                     'interactive': ['matplotlib', 'ipy_table', 'imageio', 'jupyter', 'pyevtk', 'rich', 'graphviz'],
-                     'doc': ['sphinx', 'sphinx_rtd_theme', 'nbsphinx',
-                             'sphinxcontrib-bibtex', 'sphinx_autodoc_typehints', 'pandoc'],
-                     'use_cython': ['Cython']
-                 },
-                 tests_require=['pytest',
-                                'pytest-cov',
-                                'pytest-html',
-                                'ansi2html',
-                                'pytest-xdist',
-                                'flake8',
-                                'nbformat',
-                                'nbconvert',
-                                'ipython',
-                                'randomgen>=1.18'],
 
-                 python_requires=">=3.10",
-                 cmdclass=get_cmdclass()
-                 )
+setup(
+    version=versioneer.get_version(),
+    cmdclass=get_cmdclass(),
+)