diff --git a/.gitignore b/.gitignore index 992284e7dccd8d0b523d16017acb1672ad3e2fcb..2129c6416052f4c21fea634c1a6f607ae5d67bc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ __pycache__ .ipynb_checkpoints + .coverage* +coverage.xml + *.pyc *.vti /build diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c58a26bdaa7bbe4282a8683aa9578634c383051..c6287f2377973f410e8b9d7b27a8783b62c08e42 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,7 +203,6 @@ minimal-sympy-master: script: - python -m pip install --upgrade git+https://github.com/sympy/sympy.git - python quicktest.py - allow_failure: true tags: - docker - cuda @@ -255,30 +254,25 @@ pycodegen-integration: # -------------------- Code Quality --------------------------------------------------------------------- - -flake8-lint: +.qa-base: stage: "Code Quality" + image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:alpine + needs: [] except: variables: - $ENABLE_NIGHTLY_BUILDS - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full - script: - - flake8 src/pystencils tags: - docker -mypy-typecheck: - stage: "Code Quality" - except: - variables: - - $ENABLE_NIGHTLY_BUILDS - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full - before_script: - - pip install -e .[tests] +lint: + extends: .qa-base script: - - mypy src/pystencils - tags: - - docker + - nox --session lint + +typecheck: + extends: .qa-base + script: + - nox --session typecheck # -------------------- Unit Tests --------------------------------------------------------------------- @@ -286,18 +280,12 @@ mypy-typecheck: tests-and-coverage: stage: "Unit Tests" needs: [] - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3 - before_script: - - pip install -e .[tests] + image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12.6 script: - - env - - pip list - - export NUM_CORES=$(nproc --all) - mkdir -p ~/.config/matplotlib - echo "backend:template" > ~/.config/matplotlib/matplotlibrc - mkdir public - - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml - - python -m coverage xml + - nox --session "testsuite(cupy12)" tags: - docker - cuda11 @@ -318,14 +306,11 @@ tests-and-coverage: build-documentation: - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3 + image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12.6 stage: docs needs: [] - before_script: - - pip install -e .[doc] script: - - cd docs - - make html SPHINXOPTS="-W --keep-going" + - nox --session docs -- --fail-on-warnings tags: - docker - cuda11 @@ -335,7 +320,7 @@ build-documentation: pages: - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full + image: alpine:latest stage: deploy needs: ["tests-and-coverage", "build-documentation"] script: diff --git a/docs/source/contributing/dev-workflow.md b/docs/source/contributing/dev-workflow.md new file mode 100644 index 0000000000000000000000000000000000000000..2aee09ba2e78bd0041ec6d2d2860385514240ecf --- /dev/null +++ b/docs/source/contributing/dev-workflow.md @@ -0,0 +1,160 @@ +# Development Workflow + +This page contains instructions on how to get started with developing pystencils. + +## Prepare the Git Repository + +The pystencils Git repository is hosted at [i10git.cs.fau.de](https://i10git.cs.fau.de), the GitLab instance of the +[Chair for Systems Simulation](https://www.cs10.tf.fau.de/) at [FAU Erlangen-Nürnberg](https://fau.de). +In order to contribute code to pystencils, you will need to acquire an account there; to do so, +please follow the instructions on the GitLab landing page. + +### Create a Fork + +Only the core developers of pystencils have write-access to the primary repository. +To contribute, you will therefore have to create a fork of that repository +by navigating to the [repository page](https://i10git.cs.fau.de/pycodegen/pystencils) +and selecting *Fork* there. +In this fork, you may freely create branches and develop code, which may later be merged to a primary branch +via merge requests. + +### Create a Local Clone + +Once you have a fork of the repository, you can clone it to your local machine using the git command-line. + +:::{note} +To clone via SSH, which is recommended, you will first have to [register an SSH key](https://docs.gitlab.com/ee/user/ssh.html). +::: + +Open up a shell and navigate to a directory you want to work in. +Then, enter + +```bash +git clone git@i10git.cs.fau.de:<your-username>/pystencils.git +``` + +to clone your fork of pystencils. + +:::{note} +To keep up to date with the upstream repository, you can add it as a secondary remote to your clone: +```bash +git remote add upstream git@i10git.cs.fau.de:pycodegen/pystencils.git +``` +You can point your clone's `master` branch at the upstream master like this: +```bash +git pull --set-upstream upstream master +``` +::: + +## Set Up the Python Environment + +To develop pystencils, you will need at least the following software installed on your machine: + +- Python 3.10 or later: Since pystencils minimal supported version is Python 3.10, we recommend that you work with Python 3.10 directly. +- An up-to-date C++ compiler, used by pystencils to JIT-compile generated code +- [Nox](https://nox.thea.codes/en/stable/), which we use for test automation. + Nox will be used extensively in the instructions on testing below. +- Optionally [CUDA](https://developer.nvidia.com/cuda-toolkit), + if you have an Nvidia or AMD GPU and plan to develop on pystencils' GPU capabilities + +Once you have these, set up a [virtual environment](https://docs.python.org/3/library/venv.html) for development. +This ensures that your system's installation of Python is kept clean, and isolates your development environment +from outside influence. +Use the following commands to create a virtual environment at `.venv` and perform an editable install of pystencils into it: + +```bash +python -m venv .venv +source .venv/bin/activate +export PIP_REQUIRE_VIRTUALENV=true +pip install -e .[dev] +``` + +:::{note} +Setting `PIP_REQUIRE_VIRTUALENV` ensures that pip refuses to install packages globally -- +Consider setting this variable globally in your shell's configuration file. +::: + +You are now ready to go! Create a new git branch to work on, open up an IDE, and start coding. +Make sure your IDE recognizes the virtual environment you created, though. + +## Static Code Analysis + +### PEP8 Code Style + +We use [flake8](https://github.com/PyCQA/flake8/tree/main) to check our code for compliance with the +[PEP8](https://peps.python.org/pep-0008/) code style. +You can either run `flake8` directly, or through Nox, to analyze your code with respect to style errors: + +::::{grid} +:::{grid-item} +```bash +nox -s lint +``` +::: +:::{grid-item} +```bash +flake8 src/pystencils +``` +::: +:::: + +### Static Type Checking + +New code added to pystencils is required to carry type annotations, +and its types are checked using [mypy](https://mypy.readthedocs.io/en/stable/index.html#). +To discover type errors, run *mypy* either directly or via Nox: + +::::{grid} +:::{grid-item} +```bash +nox -s typecheck +``` +::: +:::{grid-item} +```bash +mypy src/pystencils +``` +::: +:::: + +:::{note} +Type checking is currently restricted to the `codegen`, `jit`, `backend`, and `types` modules, +since most code in the remaining modules is significantly older and is not comprehensively +type-annotated. As more modules are updated with type annotations, this list will expand in the future. +If you think a new module is ready to be type-checked, add an exception clause for it in the `mypy.ini` file. +::: + +## Running the Test Suite + +Pystencils comes with an extensive and steadily growing suite of unit tests. +To run the testsuite, you may invoke a variant of the Nox `testsuite` session. +There are multiple different versions of the `testsuite` session, depending on whether you are testing with our +without CUDA, or which version of Python you wish to test with. +You can list the available sessions using `nox -l`. +Select one of the `testsuite` variants and run it via `nox -s "testsuite(<variant>)"`, e.g. +``` +nox -s "testsuite(cpu)" +``` +for the CPU-only suite. + +During the testsuite run, coverage information is collected and displayed using [coverage.py](https://coverage.readthedocs.io/en/7.6.10/). +You can display a detailed overview of code coverage by opening the generated `htmlcov/index.html` page. + +## Building the Documentation + +The pystencils documentation pages are written in MyST Markdown and ReStructuredText, +located at the `docs` folder, and built using Sphinx. +To build the documentation pages of pystencils, simply run the `docs` Nox session: +```bash +nox -s docs +``` + +This will emit the generated HTML pages to `docs/build/html`. +The `docs` session permits two parameters to customize its execution: + - `--clean`: Clean the page generator's output before building + - `--fail-on-warnings`: Have the build fail (finish with a nonzero exit code) if Sphinx emits any warnings. + +You must pass any of these to the session command *after a pair of dashes* (`--`); e.g.: +```bash +nox -s docs -- --clean +``` diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md new file mode 100644 index 0000000000000000000000000000000000000000..39e68b06f4304224fc37e60c8d7611f03ce12d17 --- /dev/null +++ b/docs/source/contributing/index.md @@ -0,0 +1,12 @@ +# Contributor Guide + +Welcome to the Contributor's Guide to pystencils! +If you are interested in contributing to the development of pystencils, this is the place to start. + +Pystencils is an open-source package licensed under the [AGPL v3](https://www.gnu.org/licenses/agpl-3.0.en.html). +As such, the act of contributing to pystencils by submitting a merge request is taken as agreement to the terms of the licence. + +:::{toctree} +:maxdepth: 2 +dev-workflow +::: diff --git a/docs/source/index.rst b/docs/source/index.rst index 6aa09bdbdf74e070f03d450d37ea501230d88c02..5ddec09f2bb89619cfbc2b6c05c8dcc2bb3108a4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -95,8 +95,9 @@ Topics .. toctree:: :maxdepth: 1 - :caption: Advanced + :caption: Topics + contributing/index migration backend/index diff --git a/mypy.ini b/mypy.ini index e89adf9f5eaa918753f94ffbf6ba00b1be6e39cd..cc23a503a2da6c9849d3a41e82fe8ceb8de13b43 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,6 +11,12 @@ ignore_errors = False [mypy-pystencils.types.*] ignore_errors = False +[mypy-pystencils.codegen.*] +ignore_errors = False + +[mypy-pystencils.jit.*] +ignore_errors = False + [mypy-setuptools.*] ignore_missing_imports=true @@ -22,3 +28,6 @@ ignore_missing_imports=true [mypy-cupy.*] ignore_missing_imports=true + +[mypy-cpuinfo.*] +ignore_missing_imports=true diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000000000000000000000000000000000..1b882478ac18bfc634e117926c8a0c6b9b94fbc8 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,134 @@ +from __future__ import annotations +from typing import Sequence + +import os +import nox +import subprocess +import re + +nox.options.sessions = ["lint", "typecheck", "testsuite"] + + +def get_cuda_version(session: nox.Session) -> None | tuple[int, ...]: + query_args = ["nvcc", "--version"] + + try: + query_result = subprocess.run(query_args, capture_output=True) + except FileNotFoundError: + return None + + matches = re.findall(r"release \d+\.\d+", str(query_result.stdout)) + if matches: + match = matches[0] + version_string = match.split()[-1] + try: + return tuple(int(v) for v in version_string.split(".")) + except ValueError: + pass + + session.warn("nvcc was found, but I am unable to determine the CUDA version.") + return None + + +def install_cupy( + session: nox.Session, cupy_version: str, skip_if_no_cuda: bool = False +): + if cupy_version is not None: + cuda_version = get_cuda_version(session) + if cuda_version is None or cuda_version[0] not in (11, 12): + if skip_if_no_cuda: + session.skip( + "No compatible installation of CUDA found - Need either CUDA 11 or 12" + ) + else: + session.warn( + "Running without cupy: no compatbile installation of CUDA found. Need either CUDA 11 or 12." + ) + return + + cuda_major = cuda_version[0] + cupy_package = f"cupy-cuda{cuda_major}x=={cupy_version}" + session.install(cupy_package) + + +def check_external_doc_dependencies(session: nox.Session): + dot_args = ["dot", "--version"] + try: + _ = subprocess.run(dot_args, capture_output=True) + except FileNotFoundError: + session.error( + "Unable to build documentation: " + "Command `dot` from the `graphviz` package (https://www.graphviz.org/) is not available" + ) + + +def editable_install(session: nox.Session, opts: Sequence[str] = ()): + if opts: + opts_str = "[" + ",".join(opts) + "]" + else: + opts_str = "" + session.install("-e", f".{opts_str}") + + +@nox.session(python="3.10", tags=["qa", "code-quality"]) +def lint(session: nox.Session): + """Lint code using flake8""" + + session.install("flake8") + session.run("flake8", "src/pystencils") + + +@nox.session(python="3.10", tags=["qa", "code-quality"]) +def typecheck(session: nox.Session): + """Run MyPy for static type checking""" + editable_install(session) + session.install("mypy") + session.run("mypy", "src/pystencils") + + +@nox.parametrize("cupy_version", [None, "12", "13"], ids=["cpu", "cupy12", "cupy13"]) +@nox.session(python="3.10", tags=["test"]) +def testsuite(session: nox.Session, cupy_version: str | None): + if cupy_version is not None: + install_cupy(session, cupy_version, skip_if_no_cuda=True) + + editable_install(session, ["alltrafos", "use_cython", "interactive", "testsuite"]) + + num_cores = os.cpu_count() + + session.run( + "pytest", + "-v", + "-n", + str(num_cores), + "--cov-report=term", + "--cov=.", + "-m", + "not longrun", + "--html", + "test-report/index.html", + "--junitxml=report.xml", + ) + session.run("coverage", "html") + session.run("coverage", "xml") + + +@nox.session(python=["3.10"], tags=["docs"]) +def docs(session: nox.Session): + """Build the documentation pages""" + check_external_doc_dependencies(session) + install_cupy(session, "12.3") + editable_install(session, ["doc"]) + + env = {} + + session_args = session.posargs + if "--fail-on-warnings" in session_args: + env["SPHINXOPTS"] = "-W --keep-going" + + session.chdir("docs") + + if "--clean" in session_args: + session.run("make", "clean", external=True) + + session.run("make", "html", external=True, env=env) diff --git a/pyproject.toml b/pyproject.toml index d9a33c9d7b148541c5c7ce4e21aac7c6565d1600..59e71b8db2d6156c25aebfcdeb88af5652dacc8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,11 @@ interactive = [ use_cython = [ 'Cython' ] +dev = [ + "flake8", + "mypy", + "black", +] doc = [ 'sphinx', 'pydata-sphinx-theme==0.15.4', @@ -52,9 +57,12 @@ doc = [ 'sphinx_autodoc_typehints', 'pandoc', 'sphinx_design', - 'myst-nb' + 'myst-nb', + 'matplotlib', + 'ipywidgets', + 'graphviz', ] -tests = [ +testsuite = [ 'pytest', 'pytest-cov', 'pytest-html', @@ -68,6 +76,7 @@ tests = [ 'matplotlib', 'py-cpuinfo', 'randomgen>=1.18', + 'scipy' ] [build-system] diff --git a/pytest.ini b/pytest.ini index 281eaa21ec0b07aabfbb1a16c3ad9938946e6371..707a43b4548e99e8e6862e1b48a1844e4318b55e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -23,6 +23,7 @@ filterwarnings = ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc':DeprecationWarning ignore:Animation was deleted without rendering anything:UserWarning +# Coverage Configuration [run] branch = True source = src/pystencils @@ -31,6 +32,7 @@ source = src/pystencils omit = doc/* tests/* setup.py + noxfile.py quicktest.py conftest.py versioneer.py diff --git a/src/pystencils/codegen/config.py b/src/pystencils/codegen/config.py index 01161620c4a86355ba69daf613b1d468835e58c1..3a7647907b82a4ee1ddbb72c2c700e42c7547f69 100644 --- a/src/pystencils/codegen/config.py +++ b/src/pystencils/codegen/config.py @@ -448,6 +448,7 @@ class CreateKernelConfig: if cpu_openmp is not None: _deprecated_option("cpu_openmp", "cpu_optim.openmp") + deprecated_omp: OpenMpConfig | bool match cpu_openmp: case True: deprecated_omp = OpenMpConfig() diff --git a/src/pystencils/codegen/driver.py b/src/pystencils/codegen/driver.py index 7bdec96cc0bd32eac08365b24186d319c27fb36a..28b685b55e4a41c0a80f512025ca23ecdf8c92b9 100644 --- a/src/pystencils/codegen/driver.py +++ b/src/pystencils/codegen/driver.py @@ -116,6 +116,7 @@ class DefaultKernelCreationDriver: self._target = self._cfg.get_target() self._platform = self._get_platform() + self._intermediates: CodegenIntermediates | None if retain_intermediates: self._intermediates = CodegenIntermediates() else: