diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a89284abf85cf76c9af147b477e958b61e5bc96e..b89453ec319c92bf5a7048f6b787c621771525d0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,44 +4,31 @@ stages: - "Documentation" - deploy +.nox-base: + image: i10git.cs.fau.de:5005/pycodegen/pycodegen/nox:alpine + tags: + - docker + linter: + extends: .nox-base stage: "Code Quality" needs: [] - except: - variables: - - $ENABLE_NIGHTLY_BUILDS - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full script: - - flake8 src/pystencilssfg - tags: - - docker + - nox --session lint typechecker: + extends: .nox-base stage: "Code Quality" needs: [] - except: - variables: - - $ENABLE_NIGHTLY_BUILDS - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full script: - - pip install mypy - - mypy src/pystencilssfg - tags: - - docker + - nox --session typecheck testsuite: + extends: .nox-base stage: "Tests" - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full needs: [] - tags: - - docker - before_script: - - pip install "git+https://i10git.cs.fau.de/pycodegen/pystencils.git@v2.0-dev" - - pip install -e .[tests] script: - - pytest -v --cov=src/pystencilssfg --cov-report=term --cov-config=pyproject.toml - - coverage html - - coverage xml + - nox --session testsuite coverage: '/TOTAL.*\s+(\d+%)$/' artifacts: when: always @@ -54,23 +41,17 @@ testsuite: path: coverage.xml build-documentation: + extends: .nox-base stage: "Documentation" - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full needs: [] - before_script: - - pip install "git+https://i10git.cs.fau.de/pycodegen/pystencils.git@v2.0-dev" - - pip install -e .[docs] script: - - cd docs - - make html - tags: - - docker + - nox --session docs artifacts: paths: - docs/build/html pages: - image: i10git.cs.fau.de:5005/pycodegen/pycodegen/full + image: alpine:latest stage: deploy script: - ls -l diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbcaaf155503325f1282cd74f7ba013ee5693221..e89a6e598cff4c6da65bbdf773237e891536995c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,19 @@ As such, any submission of contributions via merge requests is considered as agr ## Developing `pystencils-sfg` +### Prequesites + +To develop pystencils-sfg, you will need at least these packages: + + - Python 3.10 + - Git + - A C++ compiler supporting at least C++20 (gcc >= 10, or clang >= 10) + - GNU Make + - CMake + - Nox + +Before continuing, make sure that the above packages are installed on your machine. + ### Fork and Clone To work within the `pystencils-sfg` source tree, first create a *fork* of this repository @@ -29,28 +42,29 @@ source .venv/bin/activate pip install -e . ``` +If you have [nox](https://nox.thea.codes/en/stable/) installed, you can also set up your virtual environment +by running `nox --session dev_env`. + ### Code Style and Type Checking To contribute, please adhere to the Python code style set by [PEP 8](https://peps.python.org/pep-0008/). -For consistency, format all your source files using the [black](https://pypi.org/project/black/) formatter. -Use flake8 to check your code style: +For consistency, format all your source files using the [black](https://pypi.org/project/black/) formatter, +and check them regularily using the `flake8` linter through Nox: ```shell -flake8 src/pystencilssfg +nox --session lint ``` Further, `pystencils-sfg` is being fully type-checked using [MyPy](https://www.mypy-lang.org/). All submitted code should contain type annotations ([PEP 484](https://peps.python.org/pep-0484/)) and must be correctly statically typed. -Before each commit, check your types by calling +Regularily check your code for type errors using ```shell -mypy src/pystencilssfg +nox --session typecheck ``` Both `flake8` and `mypy` are also run in the integration pipeline. -You can automate the code quality checks by running them via a git pre-commit hook. -Such a hook can be installed using the [`install_git_hooks.sh`](install_git_hooks.sh) script located at the project root. ### Test Your Code @@ -65,3 +79,11 @@ In [tests/generator_scripts](tests/generator_scripts), a framework is provided t for successful execution, correctness, and compilability of their output. Read the documentation within [test_generator_scripts.py](tests/generator_scripts/test_generator_scripts.py) for more information. + +Run the test suite by calling it through Nox: + +```shell +nox --session testsuite +``` + +This will also collect coverage information and produce a coverage report as a HTML site placed in the `htmlcov` folder. diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000000000000000000000000000000000000..915ab84dcd70663a7c2c7644212e6aded5484d50 --- /dev/null +++ b/noxfile.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +from typing import Sequence +import nox + +nox.options.sessions = ["lint", "typecheck", "testsuite"] + + +def add_pystencils_git(session: nox.Session): + """Clone the pystencils 2.0 development branch and install it in the current session""" + cache_dir = session.cache_dir + + pystencils_dir = cache_dir / "pystencils" + if not pystencils_dir.exists(): + session.run_install( + "git", + "clone", + "--branch", + "v2.0-dev", + "--single-branch", + "https://i10git.cs.fau.de/pycodegen/pystencils.git", + pystencils_dir, + external=True, + ) + session.install("-e", str(pystencils_dir)) + + +def editable_install(session: nox.Session, opts: Sequence[str] = ()): + add_pystencils_git(session) + if opts: + opts_str = "[" + ",".join(opts) + "]" + else: + opts_str = "" + session.install("-e", f".{opts_str}") + + +@nox.session(python="3.10", tags=["qa", "code-quality"]) +def lint(session: nox.Session): + """Lint code using flake8""" + + session.install("flake8") + session.run("flake8", "src/pystencilssfg") + + +@nox.session(python="3.10", tags=["qa", "code-quality"]) +def typecheck(session: nox.Session): + """Run MyPy for static type checking""" + editable_install(session) + session.install("mypy") + session.run("mypy", "src/pystencilssfg") + + +@nox.session(python=["3.10"], tags=["tests"]) +def testsuite(session: nox.Session): + """Run the testsuite and measure coverage.""" + editable_install(session, ["testsuite"]) + session.run( + "pytest", + "-v", + "--cov=src/pystencilssfg", + "--cov-report=term", + "--cov-config=pyproject.toml", + ) + session.run("coverage", "html") + session.run("coverage", "xml") + + +@nox.session(python=["3.10"], tags=["docs"]) +def docs(session: nox.Session): + """Build the documentation pages""" + editable_install(session, ["docs"]) + session.chdir("docs") + session.run("make", "html", external=True) + + +@nox.session() +def dev_env(session: nox.Session): + """Set up the development environment at .venv""" + + session.install("virtualenv") + session.run("virtualenv", ".venv", "--prompt", "pystencils-sfg") + session.run( + ".venv/bin/pip", + "install", + "git+https://i10git.cs.fau.de/pycodegen/pystencils.git@v2.0-dev", + external=True, + ) + session.run(".venv/bin/pip", "install", "-e", ".[dev]", external=True) diff --git a/pyproject.toml b/pyproject.toml index b787da5c7180e32c6751bc8c78c2532f693382e2..da36a11c4a19d510faadc491045485594790c535 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,10 +23,15 @@ requires = [ build-backend = "setuptools.build_meta" [project.optional-dependencies] -tests = [ - "flake8>=6.1.0", - "mypy>=1.7.0", +dev = [ + "flake8", + "mypy", "black", + "clang-format", +] +testsuite = [ + "pytest", + "pytest-cov", "pyyaml", "requests", "fasteners", @@ -60,7 +65,7 @@ omit = [ [tool.coverage.report] exclude_also = [ - "\\.\\.\\.", + "\\.\\.\\.\n", "if TYPE_CHECKING:", "@(abc\\.)?abstractmethod", ] diff --git a/src/pystencilssfg/cmake/modules/PystencilsSfg.cmake b/src/pystencilssfg/cmake/modules/PystencilsSfg.cmake index 27659789625dc1df93bb59bde5af6260bb46e7e1..cd1f1baf2b8da061bd0596b76c28abf42d4fc5eb 100644 --- a/src/pystencilssfg/cmake/modules/PystencilsSfg.cmake +++ b/src/pystencilssfg/cmake/modules/PystencilsSfg.cmake @@ -20,6 +20,9 @@ function(_pssfg_add_gen_source target script) OUTPUT_VARIABLE generatedSources RESULT_VARIABLE _pssfg_result ERROR_VARIABLE _pssfg_stderr) + execute_process(COMMAND ${Python_EXECUTABLE} -c "from pystencils.include import get_pystencils_include_path; print(get_pystencils_include_path(), end='')" + OUTPUT_VARIABLE _Pystencils_INCLUDE_DIR) + if(NOT (${_pssfg_result} EQUAL 0)) message( FATAL_ERROR ${_pssfg_stderr} ) endif() @@ -37,7 +40,7 @@ function(_pssfg_add_gen_source target script) WORKING_DIRECTORY "${generatedSourcesDir}") target_sources(${target} PRIVATE ${generatedSourcesAbsolute}) - target_include_directories(${target} PRIVATE ${PystencilsSfg_GENERATED_SOURCES_DIR}) + target_include_directories(${target} PRIVATE ${PystencilsSfg_GENERATED_SOURCES_DIR} ${_Pystencils_INCLUDE_DIR}) endfunction() diff --git a/src/pystencilssfg/composer/basic_composer.py b/src/pystencilssfg/composer/basic_composer.py index 133f504b140ee36dd7db7c7c01d8bb710fa9c8c5..b96d559a1733ec69b6f40db04f41437cc80e3ad7 100644 --- a/src/pystencilssfg/composer/basic_composer.py +++ b/src/pystencilssfg/composer/basic_composer.py @@ -6,7 +6,7 @@ import sympy as sp from functools import reduce from pystencils import Field -from pystencils.backend import KernelFunction +from pystencils.codegen import Kernel from pystencils.types import create_type, UserTypeSpec from ..context import SfgContext @@ -231,7 +231,7 @@ class SfgBasicComposer(SfgIComposer): return cls def kernel_function( - self, name: str, ast_or_kernel_handle: KernelFunction | SfgKernelHandle + self, name: str, ast_or_kernel_handle: Kernel | SfgKernelHandle ): """Create a function comprising just a single kernel call. @@ -241,7 +241,7 @@ class SfgBasicComposer(SfgIComposer): if self._ctx.get_function(name) is not None: raise ValueError(f"Function {name} already exists.") - if isinstance(ast_or_kernel_handle, KernelFunction): + if isinstance(ast_or_kernel_handle, Kernel): khandle = self._ctx.default_kernel_namespace.add(ast_or_kernel_handle) tree = SfgKernelCallNode(khandle) elif isinstance(ast_or_kernel_handle, SfgKernelHandle): diff --git a/src/pystencilssfg/emission/printers.py b/src/pystencilssfg/emission/printers.py index 3dc4a8f74177c4de16158b1e6f314be25cca9c18..9d7c97e7ce732066c91eda3e3cbf887dcb552f77 100644 --- a/src/pystencilssfg/emission/printers.py +++ b/src/pystencilssfg/emission/printers.py @@ -3,7 +3,7 @@ from __future__ import annotations from textwrap import indent from itertools import chain, repeat, cycle -from pystencils import KernelFunction +from pystencils.codegen import Kernel from pystencils.backend.emission import emit_code from ..context import SfgContext @@ -233,8 +233,8 @@ class SfgImplPrinter(SfgGeneralPrinter): code += f"\n}} // namespace {kns.name}\n" return code - @visit.case(KernelFunction) - def kernel(self, kfunc: KernelFunction) -> str: + @visit.case(Kernel) + def kernel(self, kfunc: Kernel) -> str: return emit_code(kfunc) @visit.case(SfgFunction) diff --git a/src/pystencilssfg/ir/call_tree.py b/src/pystencilssfg/ir/call_tree.py index c6f4951db4397d356c80340ae99f4bf2b8ef1b8e..a5d2c5a35b1795817305515b74797c2bf3f2b91b 100644 --- a/src/pystencilssfg/ir/call_tree.py +++ b/src/pystencilssfg/ir/call_tree.py @@ -226,10 +226,10 @@ class SfgCudaKernelInvocation(SfgCallTreeLeaf): depends: set[SfgVar], ): from pystencils import Target - from pystencils.backend.kernelfunction import GpuKernelFunction + from pystencils.codegen import GpuKernel func = kernel_handle.get_kernel_function() - if not (isinstance(func, GpuKernelFunction) and func.target == Target.CUDA): + if not (isinstance(func, GpuKernel) and func.target == Target.CUDA): raise ValueError( "An `SfgCudaKernelInvocation` node can only call a CUDA kernel." ) diff --git a/src/pystencilssfg/ir/postprocessing.py b/src/pystencilssfg/ir/postprocessing.py index d9d59911464e885e528cba6fa1b0e9ad9f8511b8..aa3cd2732f62f5b9b50131b4e1ae1b48aa23e4ce 100644 --- a/src/pystencilssfg/ir/postprocessing.py +++ b/src/pystencilssfg/ir/postprocessing.py @@ -10,7 +10,7 @@ import sympy as sp from pystencils import Field from pystencils.types import deconstify, PsType -from pystencils.backend.properties import FieldBasePtr, FieldShape, FieldStride +from pystencils.codegen.properties import FieldBasePtr, FieldShape, FieldStride from ..exceptions import SfgException diff --git a/src/pystencilssfg/ir/source_components.py b/src/pystencilssfg/ir/source_components.py index 13c4b5092e2d5926ecdd549eab45737bf05fc625..ea43ac8e06cd7520c75eb266c8ff9008ca7132a0 100644 --- a/src/pystencilssfg/ir/source_components.py +++ b/src/pystencilssfg/ir/source_components.py @@ -7,10 +7,7 @@ from dataclasses import replace from itertools import chain from pystencils import CreateKernelConfig, create_kernel, Field -from pystencils.backend.kernelfunction import ( - KernelFunction, - KernelParameter, -) +from pystencils.codegen import Kernel, Parameter from pystencils.types import PsType, PsCustomType from ..lang import SfgVar, HeaderFile, void @@ -68,7 +65,7 @@ class SfgKernelNamespace: def __init__(self, ctx: SfgContext, name: str): self._ctx = ctx self._name = name - self._kernel_functions: dict[str, KernelFunction] = dict() + self._kernel_functions: dict[str, Kernel] = dict() @property def name(self): @@ -78,7 +75,7 @@ class SfgKernelNamespace: def kernel_functions(self): yield from self._kernel_functions.values() - def get_kernel_function(self, khandle: SfgKernelHandle) -> KernelFunction: + def get_kernel_function(self, khandle: SfgKernelHandle) -> Kernel: if khandle.kernel_namespace is not self: raise ValueError( f"Kernel handle does not belong to this namespace: {khandle}" @@ -86,7 +83,7 @@ class SfgKernelNamespace: return self._kernel_functions[khandle.kernel_name] - def add(self, kernel: KernelFunction, name: str | None = None): + def add(self, kernel: Kernel, name: str | None = None): """Adds an existing pystencils AST to this namespace. If a name is specified, the AST's function name is changed.""" if name is not None: @@ -142,7 +139,7 @@ class SfgKernelHandle: ctx: SfgContext, name: str, namespace: SfgKernelNamespace, - parameters: Sequence[KernelParameter], + parameters: Sequence[Parameter], ): self._ctx = ctx self._name = name @@ -186,11 +183,11 @@ class SfgKernelHandle: def fields(self): return self._fields - def get_kernel_function(self) -> KernelFunction: + def get_kernel_function(self) -> Kernel: return self._namespace.get_kernel_function(self) -SymbolLike_T = TypeVar("SymbolLike_T", bound=KernelParameter) +SymbolLike_T = TypeVar("SymbolLike_T", bound=Parameter) class SfgKernelParamVar(SfgVar): @@ -198,12 +195,12 @@ class SfgKernelParamVar(SfgVar): """Cast pystencils- or SymPy-native symbol-like objects as a `SfgVar`.""" - def __init__(self, param: KernelParameter): + def __init__(self, param: Parameter): self._param = param super().__init__(param.name, param.dtype) @property - def wrapped(self) -> KernelParameter: + def wrapped(self) -> Parameter: return self._param def _args(self):