Added CI and test files

.flake8

+[flake8]
+max-line-length=120
+exclude=pystencils/jupytersetup.py,
+        pystencils/plot2d.py
+        pystencils/session.py
+ignore = W293 W503 W291

.gitignore

+__pycache__
+.ipynb_checkpoints
+.coverage
+*.pyc
+*.vti
+/build
+/dist
+/*.egg-info
+.cache
+_build
+/.idea
+.cache
+_local_tmp
\ No newline at end of file

.gitlab-ci.yml

+stages:
+  - test
+  - deploy
+
+
+# --------------------------  Tests ------------------------------------------------------------------------------------
+
+# Normal test - runs on every commit all but "long run" tests
+tests-and-coverage:
+  stage: test
+  except:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  image: i10git.cs.fau.de:5005/software/pystencils/full
+  script:
+    - export NUM_CORES=$(nproc --all)
+    - mkdir -p ~/.config/matplotlib
+    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
+    - mkdir public
+    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
+  tags:
+    - docker
+    - cuda
+    - AVX
+  artifacts:
+    when: always
+    paths:
+      - coverage_report
+
+# Nightly test  - runs "long run" jobs only
+test-longrun:
+  stage: test
+  only:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  image: i10git.cs.fau.de:5005/software/pystencils/full
+  script:
+    - export NUM_CORES=$(nproc --all)
+    - mkdir -p ~/.config/matplotlib
+    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
+    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=.
+  tags:
+    - docker
+    - cuda
+    - AVX
+  artifacts:
+    paths:
+      - coverage_report
+
+# Minimal tests in windows environment
+minimal-windows:
+  stage: test
+  except:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  tags:
+    - win
+  script:
+    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate
+    - source activate pystencils_dev
+    - env
+    - conda env list
+    - python -c "import numpy"
+    - python setup.py quicktest
+
+minimal-ubuntu:
+  stage: test
+  except:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  image: i10git.cs.fau.de:5005/software/pystencils/minimal_ubuntu
+  script:
+    - python3 setup.py quicktest
+  tags:
+    - docker
+
+minimal-conda:
+  stage: test
+  except:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  image: i10git.cs.fau.de:5005/software/pystencils/minimal_conda
+  script:
+    - python setup.py quicktest
+  tags:
+    - docker
+
+
+# -------------------- Linter & Documentation --------------------------------------------------------------------------
+
+
+flake8-lint:
+  stage: test
+  except:
+    variables:
+      - $ENABLE_NIGHTLY_BUILDS
+  image: i10git.cs.fau.de:5005/software/pystencils/full
+  script:
+    - flake8 pystencils
+  tags:
+    - docker
+    - cuda
+
+
+build-documentation:
+  stage: test
+  image: i10git.cs.fau.de:5005/software/pystencils/full
+  script:
+    - export PYTHONPATH=`pwd`
+    - mkdir html_doc
+    - sphinx-build -W -b html doc  html_doc
+  tags:
+    - docker
+    - cuda
+  artifacts:
+    paths:
+      - html_doc
+
+
+pages:
+  image: i10git.cs.fau.de:5005/software/pystencils/full
+  stage: deploy
+  script:
+    - ls -l
+    - mv coverage_report html_doc
+    - mv html_doc public  # folder has to be named "public" for gitlab to publish it
+  artifacts:
+    paths:
+      - public
+  tags:
+    - docker
+  only:
+    - master@software/pystencils

conftest.py

+import os
+import pytest
+import tempfile
+import runpy
+import sys
+# Trigger config file reading / creation once - to avoid race conditions when multiple instances are creating it
+# at the same time
+from pystencils.cpu import cpujit
+
+# trigger cython imports - there seems to be a problem when multiple processes try to compile the same cython file
+# at the same time
+try:
+    import pyximport
+    pyximport.install(language_level=3)
+except ImportError:
+    pass
+from pystencils.boundaries.createindexlistcython import *  # NOQA
+
+
+SCRIPT_FOLDER = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.abspath('pystencils'))
+
+
+def add_path_to_ignore(path):
+    if not os.path.exists(path):
+        return
+    global collect_ignore
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, path, f) for f in os.listdir(os.path.join(SCRIPT_FOLDER, path))]
+
+
+collect_ignore = [os.path.join(SCRIPT_FOLDER, "doc", "conf.py")]
+add_path_to_ignore('pystencils_tests/benchmark')
+add_path_to_ignore('_local_tmp')
+
+
+try:
+    import pycuda
+except ImportError:
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils/pystencils_tests/test_cudagpu.py")]
+    add_path_to_ignore('pystencils/gpucuda')
+
+try:
+    import llvmlite
+except ImportError:
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, 'pystencils_tests/backends/llvm.py')]
+    add_path_to_ignore('pystencils/llvm')
+
+try:
+    import kerncraft
+except ImportError:
+    collect_ignore += [os.path.join(SCRIPT_FOLDER, "pystencils_tests/test_kerncraft_coupling.py")]
+    add_path_to_ignore('pystencils/kerncraft_coupling')
+
+try:
+    import blitzdb
+except ImportError:
+    add_path_to_ignore('pystencils/runhelper')
+
+
+collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup.py')]
+
+for root, sub_dirs, files in os.walk('.'):
+    for f in files:
+        if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
+            collect_ignore.append(f)
+
+
+import nbformat
+from nbconvert import PythonExporter
+
+
+class IPythonMockup:
+    def run_line_magic(self, *args, **kwargs):
+        pass
+
+    def run_cell_magic(self, *args, **kwargs):
+        pass
+
+    def magic(self, *args, **kwargs):
+        pass
+
+    def __bool__(self):
+        return False
+
+
+class IPyNbTest(pytest.Item):
+    def __init__(self, name, parent, code):
+        super(IPyNbTest, self).__init__(name, parent)
+        self.code = code
+        self.add_marker('notebook')
+
+    def runtest(self):
+        global_dict = {'get_ipython': lambda: IPythonMockup(),
+                       'is_test_run': True}
+
+        # disable matplotlib output
+        exec("import matplotlib.pyplot as p; "
+             "p.switch_backend('Template')", global_dict)
+
+        # in notebooks there is an implicit plt.show() - if this is not called a warning is shown when the next
+        # plot is created. This warning is suppressed here
+        exec("import warnings;"
+             "warnings.filterwarnings('ignore', 'Adding an axes using the same arguments as a previous.*')",
+             global_dict)
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.code.encode())
+            f.flush()
+            runpy.run_path(f.name, init_globals=global_dict, run_name=self.name)
+
+
+class IPyNbFile(pytest.File):
+    def collect(self):
+        exporter = PythonExporter()
+        exporter.exclude_markdown = True
+        exporter.exclude_input_prompt = True
+
+        notebook_contents = self.fspath.open()
+        notebook = nbformat.read(notebook_contents, 4)
+        code, _ = exporter.from_notebook_node(notebook)
+        yield IPyNbTest(self.name, self, code)
+
+    def teardown(self):
+        pass
+
+
+def pytest_collect_file(path, parent):
+    glob_exprs = ["*demo*.ipynb", "*tutorial*.ipynb", "test_*.ipynb"]
+    if any(path.fnmatch(g) for g in glob_exprs):
+        return IPyNbFile(path, parent)

doc/conf.py

 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
+import datetime
+import sphinx_rtd_theme
 import os
 import sys
 
-sys.path.insert(0, os.path.abspath('..'))
-sys.path.insert(0, os.path.abspath('../..'))
-from sphinx_doc_conf import *
+sys.path.insert(0, os.path.abspath('.'))
+from version_from_git import version_number_from_git
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.napoleon',
+    'nbsphinx',
+    'sphinxcontrib.bibtex',
+    'sphinx_autodoc_typehints',
+]
+
+add_module_names = False
+templates_path = ['_templates']
+source_suffix = '.rst'
+master_doc = 'index'
+
+copyright = '{}, Martin Bauer'.format(datetime.datetime.now().year)
+author = 'Martin Bauer'
+version = version_number_from_git()
+release = version_number_from_git()
+language = None
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '**.ipynb_checkpoints']
+default_role = 'any'
+pygments_style = 'sphinx'
+todo_include_todos = False
+
+# Options for HTML output
+
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+html_theme = 'sphinx_rtd_theme'
+htmlhelp_basename = 'pystencilsdoc'
+html_sidebars = {'**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html']}
+
+# NbSphinx configuration
+nbsphinx_execute = 'never'
+nbsphinx_codecell_lexer = 'python3'
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'python': ('https://docs.python.org/3.6', None),
+                       'numpy': ('https://docs.scipy.org/doc/numpy/', None),
+                       'matplotlib': ('https://matplotlib.org/', None),
+                       'sympy': ('https://docs.sympy.org/latest/', None),
+                       }
+
+autodoc_member_order = 'bysource'
 
 project = 'pystencils'
 html_logo = "img/logo.png"

doc/version_from_git.py

+import subprocess
+
+def version_number_from_git(tag_prefix='release/', sha_length=10, version_format="{version}.dev{commits}+{sha}"):
+
+    def get_released_versions():
+        tags = sorted(subprocess.getoutput('git tag').split('\n'))
+        versions = [t[len(tag_prefix):] for t in tags if t.startswith(tag_prefix)]
+        return versions
+
+    def tag_from_version(v):
+        return tag_prefix + v
+
+    def increment_version(v):
+        parsed_version = [int(i) for i in v.split('.')]
+        parsed_version[-1] += 1
+        return '.'.join(str(i) for i in parsed_version)
+
+    latest_release = get_released_versions()[-1]
+    commits_since_tag = subprocess.getoutput('git rev-list {}..HEAD --count'.format(tag_from_version(latest_release)))
+    sha = subprocess.getoutput('git rev-parse HEAD')[:sha_length]
+    is_dirty = len(subprocess.getoutput("git status --untracked-files=no -s")) > 0
+
+    if int(commits_since_tag) == 0:
+        version_string = latest_release
+    else:
+        next_version = increment_version(latest_release)
+        version_string = version_format.format(version=next_version, commits=commits_since_tag, sha=sha)
+
+    if is_dirty:
+        version_string += ".dirty"
+    return version_string

pystencils_tests/test_datahandling.py

 import numpy as np
 import os
 from tempfile import TemporaryDirectory
-
 import pystencils as ps
 from pystencils import create_kernel, create_data_handling
 
@@ -196,11 +195,15 @@ def test_kernel():
     for domain_shape in [(4, 5), (3, 4, 5)]:
         dh = create_data_handling(domain_size=domain_shape, periodicity=True)
         kernel_execution_jacobi(dh, test_gpu=True)
-
-        dh = create_data_handling(domain_size=domain_shape, periodicity=True)
-        kernel_execution_jacobi(dh, test_gpu=False)
         reduction(dh)
 
+        try:
+            import pycuda
+            dh = create_data_handling(domain_size=domain_shape, periodicity=True)
+            kernel_execution_jacobi(dh, test_gpu=False)
+        except ImportError:
+            pass
+
 
 def test_vtk_output():
     for domain_shape in [(4, 5), (3, 4, 5)]:

pystencils_tests/test_field_equality.ipynb

@@ -164,13 +164,13 @@
      "output_type": "stream",
      "text": [
       "Field Accesses:\n",
-      "   - f[0], hash 2177071761647211096, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (fshape_f[0],), (fstride_f[0],), -2638709558778433189, <FieldType.GENERIC: 0>, 'f'), 0)\n",
-      "   - f[0], hash -219035921004479174, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (fshape_f[0],), (fstride_f[0],), 1379426851108887372, <FieldType.GENERIC: 0>, 'f'), 0)\n",
+      "   - f[0], hash -3276894289571194847, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (_size_f_0,), (_stride_f_0,), 3146377891102027609, <FieldType.GENERIC: 0>, 'f', None), 0)\n",
+      "   - f[0], hash -1516451775709390846, offsets (0,), index (), (('f_C', ('commutative', True)), ((0,), (_size_f_0,), (_stride_f_0,), -1421177580377734245, <FieldType.GENERIC: 0>, 'f', None), 0)\n",
       "\n",
       "   -> 0,1  f[0] == f[0]: False\n",
       "Fields\n",
-      "  - f, 139911303819560, shape (fshape_f[0],), strides (fstride_f[0],), double, FieldType.GENERIC, layout (0,)\n",
-      "  - f, 139911303820008, shape (fshape_f[0],), strides (fstride_f[0],), float, FieldType.GENERIC, layout (0,)\n",
+      "  - f, 140548694371968, shape (_size_f_0,), strides (_stride_f_0,), double, FieldType.GENERIC, layout (0,)\n",
+      "  - f, 140548693963104, shape (_size_f_0,), strides (_stride_f_0,), float, FieldType.GENERIC, layout (0,)\n",
       "\n",
       "  - f == f: False, ids equal False, hash equal False\n"
      ]
@@ -179,52 +179,6 @@
    "source": [
     "print_field_accesses_debug(f1.center * f2.center)"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Custom fields"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from lbmpy.sparse.update_rule_sparse import *"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAEcAAAAgBAMAAACyb3jOAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEJl2IquJVETdZu8yu83OyatpAAAACXBIWXMAAA7EAAAOxAGVKw4bAAABN0lEQVQ4EcWQsUoDQRCGJ6A5CcFAsNDqLGJhF3yC4AukEITgE9iIICmCRV4hjZVIAr6AkDoQ8gByTao0aWwPLZR0OjO7MzdLbkErr7j5d75v93YOwD2vvm6V+lpbi57GMNRPRkVjUMQwJf8jNc86UHonAnKny+FtuUTAS5XNIVjpTsZj4KXkHbv3QgBUYpC8MKmOAM7HF2qpRGDn9LONpHb1cKwCBZEC0CDVPCKBBd25MTCqZMGQnFovY/Umzx/z/I0zgm/XBbjmvLtmgC89CUEq0pJzieSA2zuNSQ6wVNlEJA9Y2vuISB6wVMW7pVmymk14WVzcAR67v49QhyDvwMkCaKKnI+wFknMUkPTcikgC9N+kmfxxf4yU8ReoZLNwqSphw2bhVM3YccluiJ70Zyn4dLD7t4sf071mgtNBAg8AAAAASUVORK5CYII=\n",
-      "text/latex": [
-       "$${{f}_{\\mathbf{{l}_{1}^{1}}}^{1}} + {{f}_{\\mathbf{{l}_{1}}}^{1}}$$"
-      ],
-      "text/plain": [
-       "f_000035D373 + f_000035D373"
-      ]
-     },
-     "execution_count": 13,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "list_field = create_symbolic_list('l', 10, 2, np.float64)\n",
-    "normal_field = ps.fields(\"f: [2D]\")\n",
-    "normal_field.field_type = ps.FieldType.CUSTOM\n",
-    "\n",
-    "t1 = normal_field.absolute_access( (list_field[1](0),), (1,))\n",
-    "t2 = normal_field.absolute_access( (list_field[1](1),), (1,))\n",
-    "t1 + t2"
-   ]
   }
  ],
  "metadata": {
@@ -243,7 +197,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.6"
+   "version": "3.6.8"
   }
  },
  "nbformat": 4,

pystencils_tests/test_finite_differences.py

@@ -25,6 +25,7 @@ def test_spatial_2d_unit_sum():
             _, coefficients = stencil_coefficients(discretized)
             assert sum(coefficients) == 0
 
+
 def test_spatial_1d_unit_sum():
     f = ps.fields("f: double[1D]")
     h = sp.symbols("h")

pytest.ini

+[pytest]
+python_files = test_*.py *_test.py scenario_*.py
+norecursedirs = *.egg-info .git .cache .ipynb_checkpoints htmlcov
+addopts = --doctest-modules --durations=20  --cov-config pytest.ini
+
+[run]
+branch = True
+source = pystencils
+         pystencils_tests
+
+omit = doc/*
+       pystencils_tests/*
+       setup.py
+       conftest.py
+       pystencils/jupytersetup.py
+       pystencils/cpu/msvc_detection.py
+       pystencils/sympy_gmpy_bug_workaround.py
+       pystencils/cache.py
+       pystencils/pacxx/benchmark.py
+
+[report]
+exclude_lines =
+       # Have to re-enable the standard pragma
+       pragma: no cover
+
+       def __repr__
+
+       # Don't complain if tests don't hit defensive assertion code:
+       raise AssertionError
+       raise NotImplementedError
+       #raise ValueError
+
+       # Don't complain if non-runnable code isn't run:
+       if 0:
+       if False:
+       if __name__ == .__main__.:
+
+skip_covered = True
+fail_under = 74
+
+[html]
+directory = coverage_report

setup.py

 import os
 import sys
+import io
 from setuptools import setup, find_packages
-sys.path.insert(0, os.path.abspath('..'))
-from custom_pypi_index.pypi_index import get_current_dev_version_from_git
+import distutils
+from contextlib import redirect_stdout
+from importlib import import_module
+sys.path.insert(0, os.path.abspath('doc'))
+from version_from_git import version_number_from_git
+
+
+quick_tests = [
+    'test_datahandling.test_kernel',
+    'test_blocking_staggered.test_blocking_staggered',
+    'test_blocking_staggered.test_blocking_staggered',
+    'test_vectorization.test_vectorization_variable_size',
+]
+
+
+class SimpleTestRunner(distutils.cmd.Command):
+    """A custom command to run selected tests"""
+
+    description = 'run some quick tests'
+    user_options = []
+
+    @staticmethod
+    def _run_tests_in_module(test):
+        """Short test runner function - to work also if py.test is not installed."""
+        test = 'pystencils_tests.' + test
+        mod, function_name = test.rsplit('.', 1)
+        if isinstance(mod, str):
+            mod = import_module(mod)
+
+        func = getattr(mod, function_name)
+        print("   -> %s in %s" % (function_name, mod.__name__))
+        with redirect_stdout(io.StringIO()):
+            func()
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        """Run command."""
+        for test in quick_tests:
+            self._run_tests_in_module(test)
 
 
 setup(name='pystencils',
-      version=get_current_dev_version_from_git(),
+      version=version_number_from_git(),
       description='Python Stencil Compiler based on sympy as numpy',
       author='Martin Bauer',
       license='AGPLv3',
@@ -14,6 +57,7 @@ setup(name='pystencils',
       url='https://i10git.cs.fau.de/software/pystencils/',
       packages=['pystencils'] + ['pystencils.' + s for s in find_packages('pystencils')],
       install_requires=['sympy>=1.1', 'numpy', 'appdirs', 'joblib'],
+      package_data={'pystencils': ['include/*.h']},
       classifiers=[
           'Development Status :: 4 - Beta',
           'Framework :: Jupyter',
@@ -31,6 +75,9 @@ setup(name='pystencils',
           'doc': ['sphinx', 'sphinx_rtd_theme', 'nbsphinx',
                   'sphinxcontrib-bibtex', 'sphinx_autodoc_typehints', 'pandoc'],
       },
-      tests_require=['pytest', 'pytest-cov', 'pytest-xdist', 'flake8'],
+      tests_require=['pytest', 'pytest-cov', 'pytest-xdist', 'flake8', 'nbformat', 'nbconvert', 'ipython'],
       python_requires=">=3.6",
+      cmdclass={
+          'quicktest': SimpleTestRunner
+      }
       )