Compare revisions

[flake8]

max-line-length=120

exclude=pystencils/jupyter.py,

        pystencils/plot.py

        pystencils/session.py

ignore = W293 W503 W291 C901

exclude=src/pystencils/jupyter.py,

        src/pystencils/plot.py

        src/pystencils/session.py

ignore = W293 W503 W291 C901 E741

src/pystencils/_version.py export-subst

__pycache__

.ipynb_checkpoints

.coverage

.coverage*

*.pyc

*.vti

/build

/dist

/*.egg-info

*.egg-info

.cache

_build

/html_doc

/.idea

.vscode

.cache

_local_tmp

RELEASE-VERSION

test-report

src/pystencils/boundaries/createindexlistcython.c

src/pystencils/boundaries/createindexlistcython.*.so

tests/tmp

tests/var

tests/kerncraft_inputs/.2d-5pt.c_kerncraft/

tests/kerncraft_inputs/.3d-7pt.c_kerncraft/

report.xml

coverage_report/

# macOS

**/.DS_Store

*.uuid

stages:

  - pretest

  - test

  - nightly

  - docs

  - deploy

# --------------------------  Templates ------------------------------------------------------------------------------------

# Base configuration for jobs meant to run at every commit

.every-commit:

  rules:

    - if: $CI_PIPELINE_SOURCE != "schedule"

# Configuration for jobs meant to run on each commit to pycodegen/pystencils/master

.every-commit-master:

  rules:

    - if: '$CI_PIPELINE_SOURCE != "schedule" && $CI_PROJECT_PATH == "pycodegen/pystencils" && $CI_COMMIT_BRANCH == "master"'

# Base configuration for jobs meant to run at a schedule

.scheduled:

  rules:

    - if: $CI_PIPELINE_SOURCE == "schedule"

# --------------------------  Tests ------------------------------------------------------------------------------------

# Normal test - runs on every commit all but "long run" tests

tests-and-coverage:

  stage: test

  except:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  stage: pretest

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3

  before_script:

    - pip install -e .

  script:

    - env

    - pip list

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - mkdir public

    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. -m "not longrun" --html test-report/index.html

    - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml

    - python -m coverage xml

  tags:

    - docker

    - cuda

    - cuda11

    - AVX

  coverage: /Total coverage:\s\d+.\d+\%/

  artifacts:

    when: always

    paths:

      - coverage_report

      - test-report

    reports:

      coverage_report:

        coverage_format: cobertura

        path: coverage.xml

      junit: report.xml

# Nightly test  - runs "long run" jobs only

test-longrun:

# Normal test with longruns

tests-and-coverage-with-longrun:

  stage: test

  only:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  when: manual

  allow_failure: true

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3

  before_script:

    - pip install sympy --upgrade

    - pip install -e .

  script:

    - env

    - pip list

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - py.test -v -n $NUM_CORES --cov-report html --cov-report term --cov=. --html test-report/index.html

    - mkdir public

    - py.test -v -n $NUM_CORES

  tags:

    - docker

    - cuda11

    - AVX

# pipeline with latest python version

latest-python:

  stage: test

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python

  before_script:

    - pip install -e .

  script:

    - env

    - pip list

    - pip install -e .

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - mkdir public

    - py.test -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml

  tags:

    - docker

    - cuda

    - AVX

  artifacts:

    paths:

      - coverage_report

      - test-report

    when: always

    reports:

      junit: report.xml

# Minimal tests in windows environment

minimal-windows:

#minimal-windows:

#  stage: test

#  tags:

#    - win

#  script:

#    - export NUM_CORES=$(nproc --all)

#    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate

#    - source activate pystencils

#    - pip install joblib

#    - pip list

#    - python -c "import numpy"

#    - py.test -v -m "not (notebook or longrun)"

ubuntu:

  stage: test

  except:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  tags:

    - win

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ubuntu

  before_script:

    - ln -s /usr/include/locale.h /usr/include/xlocale.h

    - pip3 install -e .

  script:

    - source /cygdrive/c/Users/build/Miniconda3/Scripts/activate

    - source activate pystencils_dev

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - sed -i 's/--doctest-modules //g' pytest.ini

    - env

    - conda env list

    - python -c "import numpy"

    - python setup.py quicktest

    - pip list

    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml

  tags:

    - docker

    - cuda11

    - AVX

  artifacts:

    when: always

    reports:

      junit: report.xml

minimal-ubuntu:

.multiarch_template:

  stage: test

  except:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_ubuntu

  extends: .every-commit

  allow_failure: true

  before_script: &multiarch_before_script

    # - pip3 install -v .

    - export PYTHONPATH=src

    - python3 -c "import pystencils as ps; ps.cpu.cpujit.read_config()"

    - sed -i '/^fail_under.*/d' pytest.ini

  script:

    - python3 setup.py quicktest

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - sed -i 's/--doctest-modules //g' pytest.ini

    - env

    - pip3 list

    - python3 -m pytest -v -n $NUM_CORES --reruns 2 --cov-report html --cov-report xml --cov=. --junitxml=report.xml tests/test_*vec*.py tests/test_random.py tests/test_half_precision.py

    - python3 -m coverage xml

  tags:

    - docker

    - multiarch

  artifacts:

    when: always

    paths:

      - coverage_report

    reports:

      coverage_report:

        coverage_format: cobertura

        path: coverage.xml

      junit: report.xml

minimal-conda:

  stage: test

  except:

arm64v8:

  extends: .multiarch_template

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64

  variables:

    QEMU_CPU: "cortex-a76"

  before_script:

    - *multiarch_before_script

ppc64le:

  extends: .multiarch_template

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/ppc64le

  before_script:

    - *multiarch_before_script

arm64v9:

  # SVE support is still unreliable in GCC 13 (incorrect code for fixed-width vectors, internal compiler errors).

  # For half precision Clang is necessary

  extends: .multiarch_template

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/arm64

  variables:

      - $ENABLE_NIGHTLY_BUILDS

    # disable pointer authentication to speed up emulation by 3x

    QEMU_CPU: "max,pauth-impdef=on"

  before_script:

    - *multiarch_before_script

    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json

riscv64:

  # RISC-V vector extension support is incomplete in GCC 13.

  extends: .multiarch_template

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/riscv64

  variables:

    # explicitly set SIMD as detection requires QEMU >= 8.1

    PYSTENCILS_SIMD: "rvv"

    QEMU_CPU: "rv64,v=true,zicboz=true"

  before_script:

    - *multiarch_before_script

    - sed -i 's/march=native/march=rv64imfdvzicboz/g' ~/.config/pystencils/config.json

    - sed -i s/g\+\+/clang++/g ~/.config/pystencils/config.json

minimal-conda:

  stage: pretest

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda

  before_script:

    - pip install -e .

  script:

    - python setup.py quicktest

    - python quicktest.py

  tags:

    - docker

    - cuda

minimal-sympy-master:

  stage: test

  except:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/minimal_conda

  before_script:

    - pip install -e .

  script:

    - python -m pip install --upgrade git+https://github.com/sympy/sympy.git

    - python setup.py quicktest

    - python quicktest.py

  allow_failure: true

  tags:

    - docker

    - cuda

pycodegen-integration:

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  stage: test

  when: manual

  allow_failure: true

  script:

    - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@i10git.cs.fau.de/pycodegen/pycodegen.git

    - cd pycodegen

    - git fetch test

    - git reset --hard $CI_COMMIT_SHA

    - cd ..

    - export PYTHONPATH=`pwd`/pystencils:`pwd`/lbmpy:`pwd`/pygrandchem:`pwd`/pystencils_walberla:`pwd`/lbmpy_walberla

    - pip install -e pystencils/

    - pip install -e lbmpy/

    - cmake --version

    - ./install_walberla.sh

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - cd pystencils

    - py.test -v -n $NUM_CORES .

    - py.test -v -n $NUM_CORES --junitxml=report.xml .

    - cd ../lbmpy

    - py.test -v -n $NUM_CORES .

    - cd ../pygrandchem

    - py.test -v -n $NUM_CORES .

    - py.test -v -n $NUM_CORES --junitxml=report.xml .

    - cd ../walberla/build/

    - make CodegenJacobiCPU CodegenJacobiGPU MicroBenchmarkGpuLbm LbCodeGenerationExample

    - make -j $NUM_CORES CodegenJacobiCPU CodegenJacobiGPU CodegenPoissonCPU CodegenPoissonGPU MicroBenchmarkGpuLbm LbCodeGenerationExample

    - make -j $NUM_CORES multiphaseCPU multiphaseGPU FluctuatingMRT FlowAroundSphereCodeGen FieldLayoutAndVectorizationTest GeneratedOutflowBC

    - cd apps/benchmarks/UniformGridGPU

    - make -j $NUM_CORES

    - cd ../UniformGridCPU

    - make -j $NUM_CORES

  tags:

    - docker

    - cuda

    - cuda11

    - AVX

  artifacts:

    when: always

    reports:

      junit: pycodegen/*/report.xml

# -------------------- Scheduled Tasks --------------------------------------------------------------------------

# Nightly test against the latest (pre-release) version of SymPy published on PyPI

nightly-sympy:

  stage: nightly

  needs: []

  extends: .scheduled

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/latest_python

  before_script:

    - pip install -e .

    - pip install --upgrade --pre sympy

  script:

    - env

    - pip list

    - export NUM_CORES=$(nproc --all)

    - mkdir -p ~/.config/matplotlib

    - echo "backend:template" > ~/.config/matplotlib/matplotlibrc

    - mkdir public

    - pytest -v -n $NUM_CORES -m "not longrun" --junitxml=report.xml

  tags:

    - docker

    - AVX

    - cuda

  artifacts:

    when: always

    reports:

      junit: report.xml

# -------------------- Linter & Documentation --------------------------------------------------------------------------

flake8-lint:

  stage: test

  except:

    variables:

      - $ENABLE_NIGHTLY_BUILDS

  stage: pretest

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  script:

    - flake8 pystencils

    - flake8 src/pystencils

  tags:

    - docker

    - cuda

build-documentation:

  stage: test

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  stage: docs

  extends: .every-commit

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/documentation

  needs: []

  before_script:

    - pip install -e .

  script:

    - export PYTHONPATH=`pwd`

    - mkdir html_doc

    - sphinx-build -b html doc  html_doc

    - sphinx-build -W -b html doc  html_doc

  tags:

    - docker

    - cuda

  artifacts:

    paths:

      - html_doc

pages:

  image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full

  extends: .every-commit-master

  stage: deploy

  needs: ["tests-and-coverage", "build-documentation"]

  script:

    - ls -l

    - mv coverage_report html_doc

      - public

  tags:

    - docker

  only:

    - master@pycodegen/pystencils

line_length=100

balanced_wrapping=True

multi_line_output=4

known_third_party=sympy

-------------

  - Martin Bauer <martin.bauer@fau.de>

  - Markus Holzer <markus.holzer@fau.de>

  - Stephan Seitz <stephan.seitz@fau.de>

  - Michael Kuron <mkuron@icp.uni-stuttgart.de>

  - Jan Hönig <jan.hoenig@fau.de>

  - Nils Kohl <nils.kohl@fau.de>

  - Julian Hammer <julian.hammer@fau.de>

  - Christian Godenschwager <christian.godenschwager@fau.de>

  - Markus Holzer <markus.holzer@fau.de>

  - Michael Kuron <mkuron@icp.uni-stuttgart.de>

  - Nils Kohl <nils.kohl@fau.de>

  - Frederik Hennig <frederik.hennig@fau.de>

  - Dominik Ernst <dominik.ernst@fau.de>

  - João Victor Tozatti Risso <joaovictortr@protonmail.com>

  - Christian Godenschwager <christian.godenschwager@fau.de>

  - Dominik Thoennes <dominik.thoennes@fau.de>

# Change Log

## Unreleased

### Removed

* LLVM backend because it was not used much and not good integrated in pystencils.

* OpenCL backend because it was not used much and not good integrated in pystencils.

# Contributing

Contributions to pystencils are always welcome, and they are greatly appreciated!

A list of open problems can be found [here]( https://i10git.cs.fau.de/pycodegen/pystencils/-/issues).

Of course, it is also always appreciated to bring own ideas and problems to the community!

Please submit all contributions to the official [GitLab repository](https://i10git.cs.fau.de/pycodegen/pystencils) in the form of a Merge Request. Please do not submit git diffs or files containing the changes.

There also exists a GitHub repository, which is only a mirror to the GitLab repository. Contributions to the GitHub repository are not considered.

`pystencils` is an open-source python package under the license of  AGPL3. Thus we consider the act of contributing to the code by submitting a Merge Request as the "Sign off" or agreement to the AGPL3 license.

You can contribute in many different ways:

## Types of Contributions

### Report Bugs

Report bugs at [https://i10git.cs.fau.de/pycodegen/pystencils/-/issues](https://i10git.cs.fau.de/pycodegen/pystencils/-/issues).

For pystencils, it is often necessary to provide the python and [SymPy](https://www.sympy.org/en/index.html) versions used and hardware information like the

processor architecture and the compiler version used to compile the generated kernels.

### Fix Issues

Look through the GitLab issues. Different tags are indicating the status of the issues.

The "bug" tag indicates problems with pystencils, while the "feature" tag shows ideas that should be added in the future.

### Write Documentation

The documentation of pystencils can be found [here](https://pycodegen.pages.i10git.cs.fau.de/pystencils/). Jupyter notebooks are used to provide an

interactive start to pystencils. It is always appreciated if new document notebooks are provided

since this helps others a lot.

## Get Started!

Ready to contribute? Here is how to set up `pystencils` for local development.

1. Fork the `pystencils` repo on GitLab.

2. Clone your fork locally:

```bash

    $ git clone https://i10git.cs.fau.de/your-name/pystencils

```

3. Install your local copy into a virtualenv. It is also recommended to use anaconda or miniconda to manage the python environments.

```bash

    $ mkvirtualenv pystencils

    $ cd pystencils/

    $ pip install -e .

```

4. Create a branch for local development:

```bash

    $ git checkout -b name-of-your-bugfix-or-feature

```

   Now you can make your changes locally.

5. When you're done making changes, check that your changes pass flake8 and the

   tests

```bash

    $ flake8 pystencils

    $ py.test -v -n $NUM_CORES -m "not longrun" .

```

   To get all packages needed for development, a requirements list can be found [here](https://i10git.cs.fau.de/pycodegen/pycodegen/-/blob/master/conda_environment_dev.yml). This includes flake8 and pytest.

6. Commit your changes and push your branch to GitHub::

```bash

    $ git add .

    $ git commit -m "Your detailed description of your changes."

    $ git push origin name-of-your-bugfix-or-feature

```

7. Submit a Merge Request on GitLab.

## Merge Request Guidelines

Before you submit a Merge Request, check that it meets these guidelines:

1. All functionality that is implemented through this Merge Request should be covered by unit tests. These are implemented in `pystencil_tests`

2. If the Merge Request adds functionality, the docs should be updated. Put your new functionality into a function with a docstring.

3. If you have a maintainer status for `pystencils`, you can merge Merge Requests to the master branch. However, every Merge Request needs to be reviewed by another developer. Thus it is not allowed to merge a Merge Request, which is submitted by oneself.

## Tips

To run a subset of tests:

```bash

$ py.test my_test.py

```

 No newline at end of file

include README.md

include COPYING.txt

include RELEASE-VERSION

include AUTHORS.txt

include CONTRIBUTING.md

include CHANGELOG.md

==========

[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/mabau/pystencils/master?filepath=doc%2Fnotebooks)

[![Docs](https://img.shields.io/badge/read-the_docs-brightgreen.svg)](http://pycodegen.pages.walberla.net/pystencils)

[![Docs](https://img.shields.io/badge/read-the_docs-brightgreen.svg)](https://pycodegen.pages.i10git.cs.fau.de/pystencils)

[![pypi-package](https://badge.fury.io/py/pystencils.svg)](https://badge.fury.io/py/pystencils)

[![pipeline status](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/pipeline.svg)](https://i10git.cs.fau.de/pycodegen/pystencils/commits/master)

[![coverage report](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/coverage.svg)](http://pycodegen.pages.walberla.net/pystencils/coverage_report)

[![coverage report](https://i10git.cs.fau.de/pycodegen/pystencils/badges/master/coverage.svg)](http://pycodegen.pages.i10git.cs.fau.de/pystencils/coverage_report)

Run blazingly fast stencil codes on numpy arrays.

*pystencils* uses sympy to define stencil operations, that can be executed on numpy arrays.

Exploiting the stencil structure makes *pystencils* run faster than normal numpy code and even as Cython and numba,

[as demonstrated in this notebook](http://pycodegen.pages.walberla.net/pystencils/notebooks/demo_benchmark.html).

[as demonstrated in this notebook](https://pycodegen.pages.i10git.cs.fau.de/pystencils/notebooks/demo_benchmark.html).

Here is a code snippet that computes the average of neighboring cells:

It comes with automatic finite difference discretization for PDEs:

```python

import pystencils as ps

import sympy as sp

c, v = ps.fields("c, v(2): [2D]")

adv_diff_pde = ps.fd.transient(c) - ps.fd.diffusion(c, sp.symbols("D")) + ps.fd.advection(c, v)

discretize = ps.fd.Discretization2ndOrder(dx=1, dt=0.01)

discretization = discretize(adv_diff_pde)

```

Look at the [documentation](http://pycodegen.pages.walberla.net/pystencils) to learn more.

Installation

------------

Without `[interactive]` you get a minimal version with very little dependencies.

All options:

-  `gpu`: use this if an Nvidia GPU is available and CUDA is installed

-  `pyopencl`: basic OpenCL support (experimental)

- `gpu`: use this if an NVIDIA or AMD GPU is available and CUDA or ROCm is installed

- `alltrafos`: pulls in additional dependencies for loop simplification e.g. libisl

- `bench_db`: functionality to store benchmark result in object databases

- `interactive`: installs dependencies to work in Jupyter including image I/O, plotting etc.

- `autodiff`: enable derivation of adjoint kernels and generation of Torch/Tensorflow operations

- `doc`: packages to build documentation

Options can be combined e.g.

pip install pystencils[interactive, gpu, doc]

```

pystencils is also fully compatible with Windows machines. If working with visual studio and cupy makes sure to run example files first to ensure that cupy can find the compiler's executable.

Documentation

-------------

Read the docs [here](http://pycodegen.pages.walberla.net/pystencils) and

check out the Jupyter notebooks in `doc/notebooks`.

Read the docs [here](https://pycodegen.pages.i10git.cs.fau.de/pystencils) and

check out the Jupyter notebooks in `doc/notebooks`. The **Changelog** of pystencils can be found [here](https://i10git.cs.fau.de/pycodegen/pystencils/-/blob/master/CHANGELOG.md).

Authors

-------

Many thanks go to the [contributors](https://i10git.cs.fau.de/pycodegen/pystencils/-/blob/master/AUTHORS.txt) of pystencils.

### Please cite us