From 1f5a7b44ee6d46a130369cc4c48de77428f661c5 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 14:10:14 +0100
Subject: [PATCH 01/13] introduce Nox and refactor CI
---
.gitignore | 3 ++
.gitlab-ci.yml | 33 +++++++++------------
noxfile.py | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++
pyproject.toml | 7 ++++-
4 files changed, 101 insertions(+), 21 deletions(-)
create mode 100644 noxfile.py
diff --git a/.gitignore b/.gitignore
index 992284e7d..2129c6416 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
__pycache__
.ipynb_checkpoints
+
.coverage*
+coverage.xml
+
*.pyc
*.vti
/build
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6c58a26bd..901eee0ec 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -255,30 +255,25 @@ pycodegen-integration:
# -------------------- Code Quality ---------------------------------------------------------------------
-
-flake8-lint:
+.qa-base:
stage: "Code Quality"
+ image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:alpine
+ needs: []
except:
variables:
- $ENABLE_NIGHTLY_BUILDS
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
- script:
- - flake8 src/pystencils
tags:
- docker
-mypy-typecheck:
- stage: "Code Quality"
- except:
- variables:
- - $ENABLE_NIGHTLY_BUILDS
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
- before_script:
- - pip install -e .[tests]
+lint:
+ extends: .qa-base
script:
- - mypy src/pystencils
- tags:
- - docker
+ - nox --session lint
+
+typecheck:
+ extends: .qa-base
+ script:
+ - nox --session typecheck
# -------------------- Unit Tests ---------------------------------------------------------------------
@@ -286,18 +281,16 @@ mypy-typecheck:
tests-and-coverage:
stage: "Unit Tests"
needs: []
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3
+ image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12
before_script:
- pip install -e .[tests]
script:
- env
- pip list
- - export NUM_CORES=$(nproc --all)
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
- - pytest -v -n $NUM_CORES --cov-report html --cov-report xml --cov-report term --cov=. -m "not longrun" --html test-report/index.html --junitxml=report.xml
- - python -m coverage xml
+ - nox --session "testsuite(cupy12)"
tags:
- docker
- cuda11
diff --git a/noxfile.py b/noxfile.py
new file mode 100644
index 000000000..876d7a914
--- /dev/null
+++ b/noxfile.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+from typing import Sequence
+
+import os
+import nox
+import subprocess
+
+nox.options.sessions = ["lint", "typecheck", "testsuite"]
+
+
+def get_cuda_version() -> None | tuple[int, ...]:
+ smi_args = ["nvidia-smi", "--version"]
+
+ try:
+ result = subprocess.run(smi_args, capture_output=True)
+ except FileNotFoundError:
+ return None
+
+ smi_output = str(result.stdout).splitlines()
+ cuda_version = smi_output[-1].split(":")[1].strip()
+ return tuple(int(v) for v in cuda_version.split("."))
+
+
+def editable_install(session: nox.Session, opts: Sequence[str] = ()):
+ if opts:
+ opts_str = "[" + ",".join(opts) + "]"
+ else:
+ opts_str = ""
+ session.install("-e", f".{opts_str}")
+
+
+@nox.session(python="3.10", tags=["qa", "code-quality"])
+def lint(session: nox.Session):
+ """Lint code using flake8"""
+
+ session.install("flake8")
+ session.run("flake8", "src/pystencils")
+
+
+@nox.session(python="3.10", tags=["qa", "code-quality"])
+def typecheck(session: nox.Session):
+ """Run MyPy for static type checking"""
+ editable_install(session)
+ session.install("mypy")
+ session.run("mypy", "src/pystencils")
+
+
+@nox.parametrize(
+ "cupy_version",
+ [None, "12", "13"],
+ ids=["cpu", "cupy12", "cupy13"]
+)
+@nox.session(python="3.10", tags=["test"])
+def testsuite(session: nox.Session, cupy_version: str | None):
+ if cupy_version is not None:
+ cuda_version = get_cuda_version()
+ if cuda_version is None or cuda_version[0] < 11:
+ session.skip("No compatible installation of CUDA found - Need at least CUDA 11")
+
+ cuda_major = cuda_version[0]
+ cupy_package = f"cupy-cuda{cuda_major}=={cupy_version}"
+ session.install(cupy_package)
+
+ editable_install(session, ["alltrafos", "use_cython", "interactive", "testsuite"])
+
+ num_cores = os.cpu_count()
+
+ session.run(
+ "pytest",
+ "-v",
+ "-n", str(num_cores),
+ "--cov-report=term",
+ "--cov=.",
+ "-m", "not longrun",
+ "--html", "test-report/index.html",
+ "--junitxml=report.xml"
+ )
+ session.run("coverage", "html")
+ session.run("coverage", "xml")
diff --git a/pyproject.toml b/pyproject.toml
index d9a33c9d7..4f4cad8e7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -44,6 +44,11 @@ interactive = [
use_cython = [
'Cython'
]
+dev = [
+ "flake8",
+ "mypy",
+ "black",
+]
doc = [
'sphinx',
'pydata-sphinx-theme==0.15.4',
@@ -54,7 +59,7 @@ doc = [
'sphinx_design',
'myst-nb'
]
-tests = [
+testsuite = [
'pytest',
'pytest-cov',
'pytest-html',
--
GitLab
From c92190d2b7b5ceb79f659ca40eb501ca1a3bdf29 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 14:59:24 +0100
Subject: [PATCH 02/13] Add docs session to nox; adapt CI
---
.gitlab-ci.yml | 9 +++------
noxfile.py | 40 ++++++++++++++++++++++++++++------------
pyproject.toml | 4 +++-
3 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 901eee0ec..0ab7cf354 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -311,14 +311,11 @@ tests-and-coverage:
build-documentation:
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full:cupy12.3
+ image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12
stage: docs
needs: []
- before_script:
- - pip install -e .[doc]
script:
- - cd docs
- - make html SPHINXOPTS="-W --keep-going"
+ - nox --session docs
tags:
- docker
- cuda11
@@ -328,7 +325,7 @@ build-documentation:
pages:
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full
+ image: alpine:latest
stage: deploy
needs: ["tests-and-coverage", "build-documentation"]
script:
diff --git a/noxfile.py b/noxfile.py
index 876d7a914..a4b110191 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -15,7 +15,7 @@ def get_cuda_version() -> None | tuple[int, ...]:
result = subprocess.run(smi_args, capture_output=True)
except FileNotFoundError:
return None
-
+
smi_output = str(result.stdout).splitlines()
cuda_version = smi_output[-1].split(":")[1].strip()
return tuple(int(v) for v in cuda_version.split("."))
@@ -45,18 +45,16 @@ def typecheck(session: nox.Session):
session.run("mypy", "src/pystencils")
-@nox.parametrize(
- "cupy_version",
- [None, "12", "13"],
- ids=["cpu", "cupy12", "cupy13"]
-)
+@nox.parametrize("cupy_version", [None, "12", "13"], ids=["cpu", "cupy12", "cupy13"])
@nox.session(python="3.10", tags=["test"])
def testsuite(session: nox.Session, cupy_version: str | None):
if cupy_version is not None:
cuda_version = get_cuda_version()
if cuda_version is None or cuda_version[0] < 11:
- session.skip("No compatible installation of CUDA found - Need at least CUDA 11")
-
+ session.skip(
+ "No compatible installation of CUDA found - Need at least CUDA 11"
+ )
+
cuda_major = cuda_version[0]
cupy_package = f"cupy-cuda{cuda_major}=={cupy_version}"
session.install(cupy_package)
@@ -68,12 +66,30 @@ def testsuite(session: nox.Session, cupy_version: str | None):
session.run(
"pytest",
"-v",
- "-n", str(num_cores),
+ "-n",
+ str(num_cores),
"--cov-report=term",
"--cov=.",
- "-m", "not longrun",
- "--html", "test-report/index.html",
- "--junitxml=report.xml"
+ "-m",
+ "not longrun",
+ "--html",
+ "test-report/index.html",
+ "--junitxml=report.xml",
)
session.run("coverage", "html")
session.run("coverage", "xml")
+
+
+@nox.session(python=["3.10"], tags=["docs"])
+def docs(session: nox.Session):
+ """Build the documentation pages"""
+ editable_install(session, ["doc"])
+ session.chdir("docs")
+ session.run(
+ "make",
+ "html",
+ external=True,
+ env={
+ "SPHINXOPTS": "-W --keep-going"
+ }
+ )
diff --git a/pyproject.toml b/pyproject.toml
index 4f4cad8e7..c0404a720 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,9 @@ doc = [
'sphinx_autodoc_typehints',
'pandoc',
'sphinx_design',
- 'myst-nb'
+ 'myst-nb',
+ 'matplotlib',
+ 'graphviz',
]
testsuite = [
'pytest',
--
GitLab
From fba41f476ee5913c9c4a1cdafa95355565ac1617 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 15:06:28 +0100
Subject: [PATCH 03/13] fix cuda container name
---
.gitlab-ci.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0ab7cf354..efdfd8b58 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -281,7 +281,7 @@ typecheck:
tests-and-coverage:
stage: "Unit Tests"
needs: []
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12
+ image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12.6
before_script:
- pip install -e .[tests]
script:
@@ -311,7 +311,7 @@ tests-and-coverage:
build-documentation:
- image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12
+ image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12.6
stage: docs
needs: []
script:
--
GitLab
From 3509483f7ff7d0803bc777af07bc1893d01ce718 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 15:41:52 +0100
Subject: [PATCH 04/13] change get_cuda_version to query version from nvcc
---
noxfile.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index a4b110191..db8398b00 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -4,21 +4,26 @@ from typing import Sequence
import os
import nox
import subprocess
+import re
nox.options.sessions = ["lint", "typecheck", "testsuite"]
def get_cuda_version() -> None | tuple[int, ...]:
- smi_args = ["nvidia-smi", "--version"]
-
+ query_args = ["nvcc", "--version"]
+
try:
- result = subprocess.run(smi_args, capture_output=True)
+ query_result = subprocess.run(query_args, capture_output=True)
except FileNotFoundError:
return None
- smi_output = str(result.stdout).splitlines()
- cuda_version = smi_output[-1].split(":")[1].strip()
- return tuple(int(v) for v in cuda_version.split("."))
+ matches = re.findall(r"release \d+\.\d+", str(query_result.stdout))
+ if matches:
+ match = matches[0]
+ version_string = match.split()[-1]
+ return tuple(int(v) for v in version_string.split("."))
+ else:
+ return None
def editable_install(session: nox.Session, opts: Sequence[str] = ()):
--
GitLab
From 9e3e8af0a3dc53e056155c0f01f1d5d260f3622c Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 19:28:37 +0100
Subject: [PATCH 05/13] fix cupy package name
---
noxfile.py | 23 +++++++++--------------
1 file changed, 9 insertions(+), 14 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index db8398b00..aa9ad3043 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -4,26 +4,21 @@ from typing import Sequence
import os
import nox
import subprocess
-import re
nox.options.sessions = ["lint", "typecheck", "testsuite"]
def get_cuda_version() -> None | tuple[int, ...]:
- query_args = ["nvcc", "--version"]
-
+ smi_args = ["nvidia-smi", "--version"]
+
try:
- query_result = subprocess.run(query_args, capture_output=True)
+ result = subprocess.run(smi_args, capture_output=True)
except FileNotFoundError:
return None
- matches = re.findall(r"release \d+\.\d+", str(query_result.stdout))
- if matches:
- match = matches[0]
- version_string = match.split()[-1]
- return tuple(int(v) for v in version_string.split("."))
- else:
- return None
+ smi_output = str(result.stdout).splitlines()
+ cuda_version = smi_output[-1].split(":")[1].strip()
+ return tuple(int(v) for v in cuda_version.split("."))
def editable_install(session: nox.Session, opts: Sequence[str] = ()):
@@ -55,13 +50,13 @@ def typecheck(session: nox.Session):
def testsuite(session: nox.Session, cupy_version: str | None):
if cupy_version is not None:
cuda_version = get_cuda_version()
- if cuda_version is None or cuda_version[0] < 11:
+ if cuda_version is None or cuda_version[0] not in (11, 12):
session.skip(
- "No compatible installation of CUDA found - Need at least CUDA 11"
+ "No compatible installation of CUDA found - Need either CUDA 11 or 12"
)
cuda_major = cuda_version[0]
- cupy_package = f"cupy-cuda{cuda_major}=={cupy_version}"
+ cupy_package = f"cupy-cuda{cuda_major}x=={cupy_version}"
session.install(cupy_package)
editable_install(session, ["alltrafos", "use_cython", "interactive", "testsuite"])
--
GitLab
From 90fd75bbe972478f96cb27287c67dd069d34ac66 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 19:38:22 +0100
Subject: [PATCH 06/13] use nvcc again to get cuda version
---
noxfile.py | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index aa9ad3043..d54815c43 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -4,21 +4,26 @@ from typing import Sequence
import os
import nox
import subprocess
+import re
nox.options.sessions = ["lint", "typecheck", "testsuite"]
def get_cuda_version() -> None | tuple[int, ...]:
- smi_args = ["nvidia-smi", "--version"]
-
+ query_args = ["nvcc", "--version"]
+
try:
- result = subprocess.run(smi_args, capture_output=True)
+ query_result = subprocess.run(query_args, capture_output=True)
except FileNotFoundError:
return None
- smi_output = str(result.stdout).splitlines()
- cuda_version = smi_output[-1].split(":")[1].strip()
- return tuple(int(v) for v in cuda_version.split("."))
+ matches = re.findall(r"release \d+\.\d+", str(query_result.stdout))
+ if matches:
+ match = matches[0]
+ version_string = match.split()[-1]
+ return tuple(int(v) for v in version_string.split("."))
+ else:
+ return None
def editable_install(session: nox.Session, opts: Sequence[str] = ()):
--
GitLab
From 1c0f1c30cafcb19fd70edd56cc2315e7de803c5e Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 20:14:18 +0100
Subject: [PATCH 07/13] clean up testsuite ci task
---
.gitlab-ci.yml | 4 ----
1 file changed, 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index efdfd8b58..02ce29a55 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -282,11 +282,7 @@ tests-and-coverage:
stage: "Unit Tests"
needs: []
image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:ubuntu24.04-cuda12.6
- before_script:
- - pip install -e .[tests]
script:
- - env
- - pip list
- mkdir -p ~/.config/matplotlib
- echo "backend:template" > ~/.config/matplotlib/matplotlibrc
- mkdir public
--
GitLab
From 23272e171a434197507a3dfcab5df10ee7facb12 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Mon, 13 Jan 2025 20:18:04 +0100
Subject: [PATCH 08/13] add missing scipy dependency
---
.gitlab-ci.yml | 1 -
pyproject.toml | 1 +
2 files changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 02ce29a55..4cb4fd0c5 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -203,7 +203,6 @@ minimal-sympy-master:
script:
- python -m pip install --upgrade git+https://github.com/sympy/sympy.git
- python quicktest.py
- allow_failure: true
tags:
- docker
- cuda
diff --git a/pyproject.toml b/pyproject.toml
index c0404a720..c540ca1e9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -75,6 +75,7 @@ testsuite = [
'matplotlib',
'py-cpuinfo',
'randomgen>=1.18',
+ 'scipy'
]
[build-system]
--
GitLab
From 65a865b7209be505301ce1cba23c7bb758590d71 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Tue, 14 Jan 2025 09:50:38 +0100
Subject: [PATCH 09/13] Install cupy in docs session
---
noxfile.py | 55 ++++++++++++++++++++++++++++++++----------------------
pytest.ini | 2 ++
2 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/noxfile.py b/noxfile.py
index d54815c43..121f4f575 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -9,9 +9,9 @@ import re
nox.options.sessions = ["lint", "typecheck", "testsuite"]
-def get_cuda_version() -> None | tuple[int, ...]:
+def get_cuda_version(session: nox.Session) -> None | tuple[int, ...]:
query_args = ["nvcc", "--version"]
-
+
try:
query_result = subprocess.run(query_args, capture_output=True)
except FileNotFoundError:
@@ -21,9 +21,34 @@ def get_cuda_version() -> None | tuple[int, ...]:
if matches:
match = matches[0]
version_string = match.split()[-1]
- return tuple(int(v) for v in version_string.split("."))
- else:
- return None
+ try:
+ return tuple(int(v) for v in version_string.split("."))
+ except ValueError:
+ pass
+
+ session.warn("nvcc was found, but I am unable to determine the CUDA version.")
+ return None
+
+
+def install_cupy(
+ session: nox.Session, cupy_version: str, skip_if_no_cuda: bool = False
+):
+ if cupy_version is not None:
+ cuda_version = get_cuda_version(session)
+ if cuda_version is None or cuda_version[0] not in (11, 12):
+ if skip_if_no_cuda:
+ session.skip(
+ "No compatible installation of CUDA found - Need either CUDA 11 or 12"
+ )
+ else:
+ session.warn(
+ "Running without cupy: no compatbile installation of CUDA found. Need either CUDA 11 or 12."
+ )
+ return
+
+ cuda_major = cuda_version[0]
+ cupy_package = f"cupy-cuda{cuda_major}x=={cupy_version}"
+ session.install(cupy_package)
def editable_install(session: nox.Session, opts: Sequence[str] = ()):
@@ -54,15 +79,7 @@ def typecheck(session: nox.Session):
@nox.session(python="3.10", tags=["test"])
def testsuite(session: nox.Session, cupy_version: str | None):
if cupy_version is not None:
- cuda_version = get_cuda_version()
- if cuda_version is None or cuda_version[0] not in (11, 12):
- session.skip(
- "No compatible installation of CUDA found - Need either CUDA 11 or 12"
- )
-
- cuda_major = cuda_version[0]
- cupy_package = f"cupy-cuda{cuda_major}x=={cupy_version}"
- session.install(cupy_package)
+ install_cupy(session, cupy_version, skip_if_no_cuda=True)
editable_install(session, ["alltrafos", "use_cython", "interactive", "testsuite"])
@@ -88,13 +105,7 @@ def testsuite(session: nox.Session, cupy_version: str | None):
@nox.session(python=["3.10"], tags=["docs"])
def docs(session: nox.Session):
"""Build the documentation pages"""
+ install_cupy(session, "12.3")
editable_install(session, ["doc"])
session.chdir("docs")
- session.run(
- "make",
- "html",
- external=True,
- env={
- "SPHINXOPTS": "-W --keep-going"
- }
- )
+ session.run("make", "html", external=True, env={"SPHINXOPTS": "-W --keep-going"})
diff --git a/pytest.ini b/pytest.ini
index 281eaa21e..707a43b45 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -23,6 +23,7 @@ filterwarnings =
ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc':DeprecationWarning
ignore:Animation was deleted without rendering anything:UserWarning
+# Coverage Configuration
[run]
branch = True
source = src/pystencils
@@ -31,6 +32,7 @@ source = src/pystencils
omit = doc/*
tests/*
setup.py
+ noxfile.py
quicktest.py
conftest.py
versioneer.py
--
GitLab
From fa65b06403889cb9dbdfece32726d6865030170a Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Tue, 14 Jan 2025 12:46:23 +0100
Subject: [PATCH 10/13] check for graphviz installation in noxfile
---
noxfile.py | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/noxfile.py b/noxfile.py
index 121f4f575..6797b0687 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -25,7 +25,7 @@ def get_cuda_version(session: nox.Session) -> None | tuple[int, ...]:
return tuple(int(v) for v in version_string.split("."))
except ValueError:
pass
-
+
session.warn("nvcc was found, but I am unable to determine the CUDA version.")
return None
@@ -51,6 +51,17 @@ def install_cupy(
session.install(cupy_package)
+def check_external_doc_dependencies(session: nox.Session):
+ dot_args = ["dot", "--version"]
+ try:
+ _ = subprocess.run(dot_args)
+ except FileNotFoundError:
+ session.error(
+ "Unable to build documentation: "
+ "command `dot` from the `graphviz` package (https://www.graphviz.org/) could not be found."
+ )
+
+
def editable_install(session: nox.Session, opts: Sequence[str] = ()):
if opts:
opts_str = "[" + ",".join(opts) + "]"
@@ -105,6 +116,7 @@ def testsuite(session: nox.Session, cupy_version: str | None):
@nox.session(python=["3.10"], tags=["docs"])
def docs(session: nox.Session):
"""Build the documentation pages"""
+ check_external_doc_dependencies(session)
install_cupy(session, "12.3")
editable_install(session, ["doc"])
session.chdir("docs")
--
GitLab
From 0a25ca692280ad284014f5550af04d9f0f055982 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Tue, 14 Jan 2025 12:47:16 +0100
Subject: [PATCH 11/13] fix error message
---
noxfile.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/noxfile.py b/noxfile.py
index 6797b0687..18b533554 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -58,7 +58,7 @@ def check_external_doc_dependencies(session: nox.Session):
except FileNotFoundError:
session.error(
"Unable to build documentation: "
- "command `dot` from the `graphviz` package (https://www.graphviz.org/) could not be found."
+ "Command `dot` from the `graphviz` package (https://www.graphviz.org/) is not available"
)
--
GitLab
From 75d0f8ce7cbac89c19b64abf633105b4b01c02aa Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Tue, 14 Jan 2025 14:12:24 +0100
Subject: [PATCH 12/13] add ipywidgets dependency to doc group. Add options for
docs session.
---
.gitlab-ci.yml | 2 +-
noxfile.py | 15 +++++++++++++--
pyproject.toml | 1 +
3 files changed, 15 insertions(+), 3 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 4cb4fd0c5..c6287f237 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -310,7 +310,7 @@ build-documentation:
stage: docs
needs: []
script:
- - nox --session docs
+ - nox --session docs -- --fail-on-warnings
tags:
- docker
- cuda11
diff --git a/noxfile.py b/noxfile.py
index 18b533554..1b882478a 100644
--- a/noxfile.py
+++ b/noxfile.py
@@ -54,7 +54,7 @@ def install_cupy(
def check_external_doc_dependencies(session: nox.Session):
dot_args = ["dot", "--version"]
try:
- _ = subprocess.run(dot_args)
+ _ = subprocess.run(dot_args, capture_output=True)
except FileNotFoundError:
session.error(
"Unable to build documentation: "
@@ -119,5 +119,16 @@ def docs(session: nox.Session):
check_external_doc_dependencies(session)
install_cupy(session, "12.3")
editable_install(session, ["doc"])
+
+ env = {}
+
+ session_args = session.posargs
+ if "--fail-on-warnings" in session_args:
+ env["SPHINXOPTS"] = "-W --keep-going"
+
session.chdir("docs")
- session.run("make", "html", external=True, env={"SPHINXOPTS": "-W --keep-going"})
+
+ if "--clean" in session_args:
+ session.run("make", "clean", external=True)
+
+ session.run("make", "html", external=True, env=env)
diff --git a/pyproject.toml b/pyproject.toml
index c540ca1e9..59e71b8db 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -59,6 +59,7 @@ doc = [
'sphinx_design',
'myst-nb',
'matplotlib',
+ 'ipywidgets',
'graphviz',
]
testsuite = [
--
GitLab
From 1f411899445681f448eb552f7176032c8545f838 Mon Sep 17 00:00:00 2001
From: Frederik Hennig <frederik.hennig@fau.de>
Date: Tue, 14 Jan 2025 15:56:09 +0100
Subject: [PATCH 13/13] Start writing a Contribution Guide. Fix mypy scope and
some small type errors.
---
docs/source/contributing/dev-workflow.md | 160 +++++++++++++++++++++++
docs/source/contributing/index.md | 12 ++
docs/source/index.rst | 3 +-
mypy.ini | 9 ++
src/pystencils/codegen/config.py | 1 +
src/pystencils/codegen/driver.py | 1 +
6 files changed, 185 insertions(+), 1 deletion(-)
create mode 100644 docs/source/contributing/dev-workflow.md
create mode 100644 docs/source/contributing/index.md
diff --git a/docs/source/contributing/dev-workflow.md b/docs/source/contributing/dev-workflow.md
new file mode 100644
index 000000000..2aee09ba2
--- /dev/null
+++ b/docs/source/contributing/dev-workflow.md
@@ -0,0 +1,160 @@
+# Development Workflow
+
+This page contains instructions on how to get started with developing pystencils.
+
+## Prepare the Git Repository
+
+The pystencils Git repository is hosted at [i10git.cs.fau.de](https://i10git.cs.fau.de), the GitLab instance of the
+[Chair for Systems Simulation](https://www.cs10.tf.fau.de/) at [FAU Erlangen-Nürnberg](https://fau.de).
+In order to contribute code to pystencils, you will need to acquire an account there; to do so,
+please follow the instructions on the GitLab landing page.
+
+### Create a Fork
+
+Only the core developers of pystencils have write-access to the primary repository.
+To contribute, you will therefore have to create a fork of that repository
+by navigating to the [repository page](https://i10git.cs.fau.de/pycodegen/pystencils)
+and selecting *Fork* there.
+In this fork, you may freely create branches and develop code, which may later be merged to a primary branch
+via merge requests.
+
+### Create a Local Clone
+
+Once you have a fork of the repository, you can clone it to your local machine using the git command-line.
+
+:::{note}
+To clone via SSH, which is recommended, you will first have to [register an SSH key](https://docs.gitlab.com/ee/user/ssh.html).
+:::
+
+Open up a shell and navigate to a directory you want to work in.
+Then, enter
+
+```bash
+git clone git@i10git.cs.fau.de:<your-username>/pystencils.git
+```
+
+to clone your fork of pystencils.
+
+:::{note}
+To keep up to date with the upstream repository, you can add it as a secondary remote to your clone:
+```bash
+git remote add upstream git@i10git.cs.fau.de:pycodegen/pystencils.git
+```
+You can point your clone's `master` branch at the upstream master like this:
+```bash
+git pull --set-upstream upstream master
+```
+:::
+
+## Set Up the Python Environment
+
+To develop pystencils, you will need at least the following software installed on your machine:
+
+- Python 3.10 or later: Since pystencils minimal supported version is Python 3.10, we recommend that you work with Python 3.10 directly.
+- An up-to-date C++ compiler, used by pystencils to JIT-compile generated code
+- [Nox](https://nox.thea.codes/en/stable/), which we use for test automation.
+ Nox will be used extensively in the instructions on testing below.
+- Optionally [CUDA](https://developer.nvidia.com/cuda-toolkit),
+ if you have an Nvidia or AMD GPU and plan to develop on pystencils' GPU capabilities
+
+Once you have these, set up a [virtual environment](https://docs.python.org/3/library/venv.html) for development.
+This ensures that your system's installation of Python is kept clean, and isolates your development environment
+from outside influence.
+Use the following commands to create a virtual environment at `.venv` and perform an editable install of pystencils into it:
+
+```bash
+python -m venv .venv
+source .venv/bin/activate
+export PIP_REQUIRE_VIRTUALENV=true
+pip install -e .[dev]
+```
+
+:::{note}
+Setting `PIP_REQUIRE_VIRTUALENV` ensures that pip refuses to install packages globally --
+Consider setting this variable globally in your shell's configuration file.
+:::
+
+You are now ready to go! Create a new git branch to work on, open up an IDE, and start coding.
+Make sure your IDE recognizes the virtual environment you created, though.
+
+## Static Code Analysis
+
+### PEP8 Code Style
+
+We use [flake8](https://github.com/PyCQA/flake8/tree/main) to check our code for compliance with the
+[PEP8](https://peps.python.org/pep-0008/) code style.
+You can either run `flake8` directly, or through Nox, to analyze your code with respect to style errors:
+
+::::{grid}
+:::{grid-item}
+```bash
+nox -s lint
+```
+:::
+:::{grid-item}
+```bash
+flake8 src/pystencils
+```
+:::
+::::
+
+### Static Type Checking
+
+New code added to pystencils is required to carry type annotations,
+and its types are checked using [mypy](https://mypy.readthedocs.io/en/stable/index.html#).
+To discover type errors, run *mypy* either directly or via Nox:
+
+::::{grid}
+:::{grid-item}
+```bash
+nox -s typecheck
+```
+:::
+:::{grid-item}
+```bash
+mypy src/pystencils
+```
+:::
+::::
+
+:::{note}
+Type checking is currently restricted to the `codegen`, `jit`, `backend`, and `types` modules,
+since most code in the remaining modules is significantly older and is not comprehensively
+type-annotated. As more modules are updated with type annotations, this list will expand in the future.
+If you think a new module is ready to be type-checked, add an exception clause for it in the `mypy.ini` file.
+:::
+
+## Running the Test Suite
+
+Pystencils comes with an extensive and steadily growing suite of unit tests.
+To run the testsuite, you may invoke a variant of the Nox `testsuite` session.
+There are multiple different versions of the `testsuite` session, depending on whether you are testing with our
+without CUDA, or which version of Python you wish to test with.
+You can list the available sessions using `nox -l`.
+Select one of the `testsuite` variants and run it via `nox -s "testsuite(<variant>)"`, e.g.
+```
+nox -s "testsuite(cpu)"
+```
+for the CPU-only suite.
+
+During the testsuite run, coverage information is collected and displayed using [coverage.py](https://coverage.readthedocs.io/en/7.6.10/).
+You can display a detailed overview of code coverage by opening the generated `htmlcov/index.html` page.
+
+## Building the Documentation
+
+The pystencils documentation pages are written in MyST Markdown and ReStructuredText,
+located at the `docs` folder, and built using Sphinx.
+To build the documentation pages of pystencils, simply run the `docs` Nox session:
+```bash
+nox -s docs
+```
+
+This will emit the generated HTML pages to `docs/build/html`.
+The `docs` session permits two parameters to customize its execution:
+ - `--clean`: Clean the page generator's output before building
+ - `--fail-on-warnings`: Have the build fail (finish with a nonzero exit code) if Sphinx emits any warnings.
+
+You must pass any of these to the session command *after a pair of dashes* (`--`); e.g.:
+```bash
+nox -s docs -- --clean
+```
diff --git a/docs/source/contributing/index.md b/docs/source/contributing/index.md
new file mode 100644
index 000000000..39e68b06f
--- /dev/null
+++ b/docs/source/contributing/index.md
@@ -0,0 +1,12 @@
+# Contributor Guide
+
+Welcome to the Contributor's Guide to pystencils!
+If you are interested in contributing to the development of pystencils, this is the place to start.
+
+Pystencils is an open-source package licensed under the [AGPL v3](https://www.gnu.org/licenses/agpl-3.0.en.html).
+As such, the act of contributing to pystencils by submitting a merge request is taken as agreement to the terms of the licence.
+
+:::{toctree}
+:maxdepth: 2
+dev-workflow
+:::
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 6aa09bdbd..5ddec09f2 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -95,8 +95,9 @@ Topics
.. toctree::
:maxdepth: 1
- :caption: Advanced
+ :caption: Topics
+ contributing/index
migration
backend/index
diff --git a/mypy.ini b/mypy.ini
index e89adf9f5..cc23a503a 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -11,6 +11,12 @@ ignore_errors = False
[mypy-pystencils.types.*]
ignore_errors = False
+[mypy-pystencils.codegen.*]
+ignore_errors = False
+
+[mypy-pystencils.jit.*]
+ignore_errors = False
+
[mypy-setuptools.*]
ignore_missing_imports=true
@@ -22,3 +28,6 @@ ignore_missing_imports=true
[mypy-cupy.*]
ignore_missing_imports=true
+
+[mypy-cpuinfo.*]
+ignore_missing_imports=true
diff --git a/src/pystencils/codegen/config.py b/src/pystencils/codegen/config.py
index 01161620c..3a7647907 100644
--- a/src/pystencils/codegen/config.py
+++ b/src/pystencils/codegen/config.py
@@ -448,6 +448,7 @@ class CreateKernelConfig:
if cpu_openmp is not None:
_deprecated_option("cpu_openmp", "cpu_optim.openmp")
+ deprecated_omp: OpenMpConfig | bool
match cpu_openmp:
case True:
deprecated_omp = OpenMpConfig()
diff --git a/src/pystencils/codegen/driver.py b/src/pystencils/codegen/driver.py
index 7bdec96cc..28b685b55 100644
--- a/src/pystencils/codegen/driver.py
+++ b/src/pystencils/codegen/driver.py
@@ -116,6 +116,7 @@ class DefaultKernelCreationDriver:
self._target = self._cfg.get_target()
self._platform = self._get_platform()
+ self._intermediates: CodegenIntermediates | None
if retain_intermediates:
self._intermediates = CodegenIntermediates()
else:
--
GitLab