Merge branch 'refactor_packaging' into 'master'

Refactor source tree layout

See merge request !362

.flake8

 [flake8]
 max-line-length=120
-exclude=pystencils/jupyter.py,
-        pystencils/plot.py
-        pystencils/session.py
+exclude=src/pystencils/jupyter.py,
+        src/pystencils/plot.py
+        src/pystencils/session.py
 ignore = W293 W503 W291 C901 E741

.gitignore

@@ -5,7 +5,7 @@ __pycache__
 *.vti
 /build
 /dist
-/*.egg-info
+*.egg-info
 .cache
 _build
 /html_doc
@@ -15,12 +15,12 @@ _build
 _local_tmp
 RELEASE-VERSION
 test-report
-pystencils/boundaries/createindexlistcython.c
-pystencils/boundaries/createindexlistcython.*.so
-pystencils_tests/tmp
-pystencils_tests/var
-pystencils_tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
-pystencils_tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
+src/pystencils/boundaries/createindexlistcython.c
+src/pystencils/boundaries/createindexlistcython.*.so
+tests/tmp
+tests/var
+tests/kerncraft_inputs/.2d-5pt.c_kerncraft/
+tests/kerncraft_inputs/.3d-7pt.c_kerncraft/
 report.xml
 coverage_report/
 

.gitlab-ci.yml

@@ -13,16 +13,17 @@ tests-and-coverage:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
+  before_script:
+    - pip install -e .
   script:
-    - pip install sympy --upgrade
     - env
     - pip list
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
     - mkdir public
-    - py.test -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
-    - python3 -m coverage xml
+    - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
+    - python -m coverage xml
   tags:
     - docker
     - cuda11
@@ -45,8 +46,10 @@ tests-and-coverage-with-longrun:
   when: manual
   allow_failure: true
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
-  script:
+  before_script:
     - pip install sympy --upgrade
+    - pip install -e .
+  script:
     - env
     - pip list
     - export NUM_CORES=$(nproc --all)
@@ -66,9 +69,12 @@ latest-python:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python
+  before_script:
+    - pip install -e .
   script:
     - env
     - pip list
+    - pip install -e .
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
@@ -107,18 +113,16 @@ ubuntu:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu
   before_script:
-    - apt-get -y remove python3-sympy
     - ln -s /usr/include/locale.h /usr/include/xlocale.h
-    - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' pyproject.toml | sed 's/>/=/g'`
-    # - pip3 install `grep -Eo 'sympy[>=]+[0-9\.]+' setup.py | sed 's/>/=/g'`
+    - pip3 install -e .
   script:
     - export NUM_CORES=$(nproc --all)
     - mkdir -p ~/.config/matplotlib
     - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
     - sed -i 's/--doctest-modules //g' pytest.ini
     - env
-    - pip3 list
-    - pytest-3 -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
+    - pip list
+    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml
   tags:
     - docker
     - cuda11
@@ -134,6 +138,8 @@ ubuntu:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   before_script: &multiarch_before_script
+    # - pip3 install -v .
+    - export PYTHONPATH=src
     - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"
     - sed -i '/^fail_under.*/d' pytest.ini
   script:
@@ -143,7 +149,7 @@ ubuntu:
     - sed -i 's/--doctest-modules //g' pytest.ini
     - env
     - pip3 list
-    - pytest-3 -v -n $NUM_CORES --junitxml=report.xml pystencils_tests/test_*vec*.py pystencils_tests/test_random.py pystencils_tests/test_half_precision.py
+    - python3 -m pytest -v -n $NUM_CORES --junitxml=report.xml tests/test_*vec*.py tests/test_random.py tests/test_half_precision.py
   tags:
     - docker
     - AVX
@@ -199,6 +205,8 @@ minimal-conda:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
+  before_script:
+    - pip install -e .
   script:
     - python quicktest.py
   tags:
@@ -212,6 +220,8 @@ minimal-sympy-master:
     variables:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda
+  before_script:
+    - pip install -e .
   script:
     - python -m pip install --upgrade git+https://github.com/sympy/sympy.git
     - python quicktest.py
@@ -275,7 +285,7 @@ flake8-lint:
       - $ENABLE_NIGHTLY_BUILDS
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
   script:
-    - flake8 pystencils
+    - flake8 src/pystencils
   tags:
     - docker
 
@@ -283,8 +293,9 @@ flake8-lint:
 build-documentation:
   stage: test
   image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation
+  before_script:
+    - pip install -e .
   script:
-    - export PYTHONPATH=`pwd`
     - mkdir html_doc
     - sphinx-build -b html doc  html_doc
     - sphinx-build -W -b html doc  html_doc

conftest.py

@@ -8,7 +8,7 @@ import nbformat
 import pytest
 from nbconvert import PythonExporter
 
-from pystencils.boundaries.createindexlistcython import *  # NOQA
+from pystencils.boundaries.createindexlist import *  # NOQA
 # Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
 # at the same time
 from pystencils.cpu import cpujit
@@ -40,48 +40,48 @@ def add_path_to_ignore(path):
 
 
 collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py"),
-                  os.path.join(SCRIPT_FOLDER, "pystencils", "opencl", "opencl.autoinit")]
-add_path_to_ignore('pystencils_tests/benchmark')
+                  os.path.join(SCRIPT_FOLDER, "src", "pystencils", "opencl", "opencl.autoinit")]
+add_path_to_ignore('tests/benchmark')
 add_path_to_ignore('_local_tmp')
 
 
 try:
     import cupy
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_gpu.py")]
-    add_path_to_ignore('pystencils/gpu')
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_gpu.py")]
+    add_path_to_ignore('src/pystencils/gpu')
 
 try:
     import waLBerla
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_aligned_array.py"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_datahandling_parallel.py"),
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_aligned_array.py"),
+                       os.path.join(SCRIPT_FOLDER, "tests/test_datahandling_parallel.py"),
                        os.path.join(SCRIPT_FOLDER, "doc/notebooks/03_tutorial_datahandling.ipynb"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/parallel_datahandling.py"),
-                       os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_small_block_benchmark.ipynb")]
+                       os.path.join(SCRIPT_FOLDER, "src/pystencils/datahandling/parallel_datahandling.py"),
+                       os.path.join(SCRIPT_FOLDER, "tests/test_small_block_benchmark.ipynb")]
 
 try:
     import blitzdb
 except ImportError:
-    add_path_to_ignore('pystencils/runhelper')
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_parameterstudy.py")]
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_json_serializer.py")]
+    add_path_to_ignore('src/pystencils/runhelper')
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_parameterstudy.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "tests/test_json_serializer.py")]
 
 try:
     import islpy
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/integer_set_analysis.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/integer_set_analysis.py")]
 
 try:
     import graphviz
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/backends/dot.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/backends/dot.py")]
     collect_ignore += [os.path.join(SCRIPT_FOLDER, "doc/notebooks/01_tutorial_getting_started.ipynb")]
 
 try:
     import pyevtk
 except ImportError:
-    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/datahandling/vtk.py")]
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "src/pystencils/datahandling/vtk.py")]
 
 collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
 

pyproject.toml

@@ -12,7 +12,7 @@ authors = [
 ]
 license = { file = "COPYING.txt" }
 requires-python = ">=3.10"
-dependencies = ["sympy>=1.6,<=1.11.1", "numpy>=1.8.0", "appdirs", "joblib"]
+dependencies = ["sympy>=1.6,<=1.11.1", "numpy>=1.8.0", "appdirs", "joblib", "pyyaml"]
 classifiers = [
     "Development Status :: 4 - Beta",
     "Framework :: Jupyter",
@@ -62,12 +62,14 @@ tests = [
     'nbformat',
     'nbconvert',
     'ipython',
+    'matplotlib',
+    'py-cpuinfo',
     'randomgen>=1.18',
 ]
 
 [build-system]
 requires = [
-    "setuptools>=69",
+    "setuptools>=61",
     "versioneer>=0.29",
     "tomli; python_version < '3.11'",
     # 'Cython'
@@ -84,7 +86,7 @@ pystencils = [
 ]
 
 [tool.setuptools.packages.find]
-where = ["."]
+where = ["src"]
 include = ["pystencils", "pystencils.*"]
 namespaces = false
 
@@ -94,7 +96,7 @@ namespaces = false
 # resulting files.
 VCS = "git"
 style = "pep440"
-versionfile_source = "pystencils/_version.py"
+versionfile_source = "src/pystencils/_version.py"
 versionfile_build = "pystencils/_version.py"
 tag_prefix = "release/"
 parentdir_prefix = "pystencils-"

pytest.ini

 [pytest]
+testpaths = src tests doc/notebooks
+pythonpath = src
 python_files = test_*.py *_test.py scenario_*.py
 norecursedirs = *.egg-info .git .cache .ipynb_checkpoints htmlcov
 addopts = --doctest-modules --durations=20  --cov-config pytest.ini
@@ -17,20 +19,21 @@ filterwarnings =
 
 [run]
 branch = True
-source = pystencils
-         pystencils_tests
+source = src/pystencils
+         tests
 
 omit = doc/*
-       pystencils_tests/*
+       tests/*
        setup.py
+       quicktest.py
        conftest.py
        versioneer.py
-       pystencils/jupytersetup.py
-       pystencils/cpu/msvc_detection.py
-       pystencils/sympy_gmpy_bug_workaround.py
-       pystencils/cache.py
-       pystencils/pacxx/benchmark.py
-       pystencils/_version.py
+       src/pystencils/jupytersetup.py
+       src/pystencils/cpu/msvc_detection.py
+       src/pystencils/sympy_gmpy_bug_workaround.py
+       src/pystencils/cache.py
+       src/pystencils/pacxx/benchmark.py
+       src/pystencils/_version.py
        venv/
 
 [report]

quicktest.py

@@ -2,7 +2,7 @@
 
 from contextlib import redirect_stdout
 import io
-from pystencils_tests.test_quicktests import (
+from tests.test_quicktests import (
     test_basic_kernel,
     test_basic_blocking_staggered,
     test_basic_vectorization,

setup.py

-import os
+from setuptools import setup, __version__ as setuptools_version
 
-from setuptools import Extension, setup
+if int(setuptools_version.split('.')[0]) < 61:
+    raise Exception(
+        "[ERROR] pystencils requires at least setuptools version 61 to install.\n"
+        "If this error occurs during an installation via pip, it is likely that there is a conflict between "
+        "versions of setuptools installed by pip and the system package manager. "
+        "In this case, it is recommended to install pystencils into a virtual environment instead."
+    )
 
 import versioneer