pytest and coverage improvements

- notebooks are considered in coverage analysis now
   -> custom notebook runner plugin adapted
- added images for documentation

.coveragerc

@@ -3,4 +3,15 @@ branch = True
 source = pystencils
          lbmpy
          pyphasefield
+omit = setup_lbmpy.py
+       setup_lbmpy_walberla.py
+       setup_pyphasefield.py
+       setup_pystencils.py
+       doc/*
+       benchmark/*
+       pyphasefield_tests/*
+       lbmpy_tests/*
+       pystencils_tests/*
+       lbmpy_benchmarks/*
+       conftest.py
 

.gitignore

@@ -6,3 +6,6 @@ __pycache__
 /build
 /dist
 /*.egg-info
+.cache
+_build
+/.idea

.gitlab-ci.yml

@@ -2,12 +2,18 @@
 tests-and-coverage:
   image: i10git.cs.fau.de:5005/software/pystencils
   script:
-    - py.test --doctest-modules -v --durations=10 -n 4 --cov-config .coveragerc --cov=.  -m "not longrun"
+    - export NUM_CORES=$(nproc --all)
+    - mkdir -p ~/.config/matplotlib
+    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc
+    - py.test -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun"
+
   tags:
     - docker
     - cuda
     - AVX
-
+  artifacts:
+    paths:
+      - htmlcov
 
 build-documentation:
   image: i10git.cs.fau.de:5005/software/pystencils

Dockerfile

@@ -27,7 +27,7 @@ ENV LC_ALL en_US.UTF-8
 RUN conda uninstall --yes gmpy2 sympy
 
 RUN conda install --yes -c lssfau \
-            appdirs pytest pytest-cov jupyter matplotlib joblib pillow scipy runipy sphinx sphinx_rtd_theme\
+            appdirs pytest pytest-cov jupyter matplotlib joblib pillow scipy sphinx sphinx_rtd_theme\
             llvmlite walberla gitpython pyqt
 
 RUN pip install pytest-xdist graphviz ipy_table \

conftest.py

@@ -73,19 +73,40 @@ collect_ignore += [os.path.join(SCRIPT_FOLDER, 'setup_pystencils.py'),
                    os.path.join(SCRIPT_FOLDER, 'setup_pyphasefield.py'),
                    os.path.join(SCRIPT_FOLDER, 'setup_lbmpy_walberla.py')]
 
-from runipy.notebook_runner import NotebookRunner
-from nbformat import read as readNotebook
+for root, sub_dirs, files in os.walk('.'):
+    for f in files:
+        if f.endswith(".ipynb") and not any(f.startswith(k) for k in ['demo', 'tutorial', 'test', 'doc']):
+            collect_ignore.append(f)
+
+
+
+import nbformat
+from nbconvert import PythonExporter
+
+
+class IPythonMockup:
+    def run_line_magic(self, *args, **kwargs):
+        pass
+
+    def run_cell_magic(self, *args, **kwargs):
+        pass
+
+    def magic(self, *args, **kwargs):
+        pass
+
+    def __bool__(self):
+        return False
+
 
 class IPyNbTest(pytest.Item):
-    def __init__(self, name, parent, runner):
+    def __init__(self, name, parent, code):
         super(IPyNbTest, self).__init__(name, parent)
-        self.runner = runner
+        self.code = code
 
     def runtest(self):
-        try:
-            self.runner.run_notebook()
-        except NotImplementedError:  # skip exceptions triggered by jupyter widgets
-            pass
+        globalDict = {'get_ipython': lambda: IPythonMockup()}
+        exec("import matplotlib.pyplot as p; p.switch_backend('Template')", globalDict)  # disable matplotlib output
+        exec(self.code, globalDict)
 
     def repr_failure(self, excinfo):
         """ called when self.runtest() raises an exception. """
@@ -94,12 +115,17 @@ class IPyNbTest(pytest.Item):
 
 class IPyNbFile(pytest.File):
     def collect(self):
+        exporter = PythonExporter()
+        exporter.exclude_markdown = True
+        exporter.exclude_input_prompt = True
+
         notebookContents = self.fspath.open()
-        self.runner = NotebookRunner(readNotebook(notebookContents, 3))
-        yield IPyNbTest(self.name, self, self.runner)
+        notebook = nbformat.read(notebookContents, 4)
+        code, _ = exporter.from_notebook_node(notebook)
+        yield IPyNbTest(self.name, self, code)
 
     def teardown(self):
-        self.runner.shutdown_kernel()
+        pass
 
 
 def pytest_collect_file(path, parent):

doc/index.rst

@@ -9,7 +9,7 @@ This repository consists of three modules:
 The pystencils module if the basis for the two advanced modules *lbmpy* (lattice Boltzmann methods)
 and *pyphasefield* (phase-field models with finite differences).
 
-   .. image:: moduleOverview.svg
+   .. image:: img/moduleOverview.svg
 
 
 Table of contents

doc/jupyterhub/GetDemoNotebooksOfNewestVersion.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Get demo notebooks of newest version\n",
+    "\n",
+    "The interactive demo site your currently on, regularly updates the pystencils/lbmpy installation.\n",
+    "These updates might introduce API changes, so the demo and tutorial notebooks in your home directory that have been created there on your first login may be outdated.\n",
+    "\n",
+    "We do not overwrite them automatically, since you might have changed them or created new notebooks in these demo folders. \n",
+    "\n",
+    "Run this notebook to get the newest demos and tutorials. Your old folder is moved - not deleted."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import shutil, glob, os, datetime, socket\n",
+    "\n",
+    "newNotebooks = \"/opt/notebooks\"\n",
+    "home = os.path.expanduser(\"~\")\n",
+    "\n",
+    "if os.path.exists(newNotebooks): # safeguard - when executed in another environment\n",
+    "    backupPath = os.path.join(home, 'backup_' + str(datetime.datetime.now()))\n",
+    "\n",
+    "    os.mkdir(backupPath)\n",
+    "    \n",
+    "    for file in glob.glob(os.path.join(home, '*')):\n",
+    "        shutil.move(file, backupPath)    \n",
+    "    print(\"Files have been moved from your home to \", backupPath)\n",
+    "    \n",
+    "    os.system(\"cp -r %s/* %s\" % (newNotebooks,home))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

doc/modules/getting_started.ipynb

@@ -74,7 +74,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.2"
+   "version": "3.6.4"
   }
  },
  "nbformat": 4,

doc/modules/lbmpy/containerScenario.svg

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="215.91653mm"
+   height="136.50279mm"
+   viewBox="0 0 765.05858 483.67131"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="containerScenario.svg">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.98994949"
+     inkscape:cx="356.63429"
+     inkscape:cy="453.36919"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     fit-margin-top="20"
+     fit-margin-left="20"
+     fit-margin-right="20"
+     fit-margin-bottom="20"
+     inkscape:window-width="2560"
+     inkscape:window-height="1371"
+     inkscape:window-x="4480"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(-5.3692855,-59.353126)">
+    <g
+       id="g4213"
+       transform="translate(0,-7.7074907e-5)"
+       inkscape:export-xdpi="92.410004"
+       inkscape:export-ydpi="92.410004">
+      <rect
+         ry="0"
+         y="130.21935"
+         x="80"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="252.61903"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-3"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="425.23807"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-6"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="597.85712"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-93"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+    </g>
+    <g
+       transform="translate(0.2760347,98.7415)"
+       id="g4213-6"
+       inkscape:export-xdpi="92.410004"
+       inkscape:export-ydpi="92.410004">
+      <rect
+         ry="0"
+         y="130.21935"
+         x="80"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-0"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="252.61903"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-3-6"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="425.23807"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-6-2"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="597.85712"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-93-6"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+    </g>
+    <g
+       transform="translate(-1.7442682,197.48308)"
+       id="g4213-1"
+       inkscape:export-xdpi="92.410004"
+       inkscape:export-ydpi="92.410004">
+      <rect
+         ry="0"
+         y="130.21935"
+         x="80"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-8"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="252.61903"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-3-7"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="425.23807"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-6-9"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="597.85712"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-93-2"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+    </g>
+    <g
+       transform="translate(-3.7645728,296.22465)"
+       id="g4213-0"
+       inkscape:export-xdpi="92.410004"
+       inkscape:export-ydpi="92.410004">
+      <rect
+         ry="0"
+         y="130.21935"
+         x="80"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-2"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="252.61903"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-3-3"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="425.23807"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-6-7"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+      <rect
+         ry="0"
+         y="130.21935"
+         x="597.85712"
+         height="45.714287"
+         width="101.42857"
+         id="rect4136-93-5"
+         style="opacity:1;fill:#000000;fill-opacity:1;stroke:none;stroke-width:3;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1" />
+    </g>
+  </g>
+</svg>