Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision
  • fhennig/devel
  • master
  • rangersbach/c-interfacing
  • v0.1a1
  • v0.1a2
  • v0.1a3
  • v0.1a4
7 results

Target

Select target project
  • ob28imeq/pystencils-sfg
  • brendan-waters/pystencils-sfg
  • pycodegen/pystencils-sfg
3 results
Select Git revision
  • frontend-cleanup
  • lbwelding-features
  • master
  • refactor-indexing-params
  • unit_tests
  • v0.1a1
  • v0.1a2
  • v0.1a3
  • v0.1a4
9 results
Show changes
Showing
with 866 additions and 347 deletions
from pystencilssfg import SourceFileGenerator
with SourceFileGenerator() as sfg:
sfg.include("<vector>")
sfg.include("<span>")
sfg.include("custom_header.hpp")
from pystencilssfg import SourceFileGenerator
import pystencils as ps
import sympy as sp
with SourceFileGenerator() as sfg:
# Define a copy kernel
src, dst = ps.fields("src, dst: [1D]")
c = sp.Symbol("c")
@ps.kernel
def scale():
dst.center @= c * src.center()
# Add it to the file
scale_kernel = sfg.kernels.create(scale, "scale")
from pystencilssfg import SourceFileGenerator
import pystencils as ps
import sympy as sp
with SourceFileGenerator() as sfg:
# Define a copy kernel
src, dst = ps.fields("src, dst: [1D]")
c = sp.Symbol("c")
@ps.kernel
def scale():
dst.center @= c * src.center()
# Add it to the file
scale_kernel = sfg.kernels.create(scale, "scale")
# start
# ... see above ...
sfg.function("scale_kernel")(
sfg.call(scale_kernel)
)
# end
from pystencilssfg import SourceFileGenerator
import pystencils as ps
import sympy as sp
with SourceFileGenerator() as sfg:
# Define a copy kernel
src, dst = ps.fields("src, dst: double[1D]")
c = sp.Symbol("c")
@ps.kernel
def scale():
dst.center @= c * src.center()
# Add it to the file
scale_kernel = sfg.kernels.create(scale, "scale")
# start
import pystencilssfg.lang.cpp.std as std
sfg.include("<span>")
sfg.function("scale_kernel")(
sfg.map_field(src, std.vector.from_field(src)),
sfg.map_field(dst, std.span.from_field(dst)),
sfg.call(scale_kernel)
)
# end
from pystencilssfg import SourceFileGenerator
from argparse import ArgumentParser
parser = ArgumentParser()
# set up parser ...
with SourceFileGenerator(keep_unknown_argv=True) as sfg:
args = parser.parse_args(sfg.context.argv)
...
from pystencilssfg import SourceFileGenerator, SfgConfig
cfg = SfgConfig()
cfg.output_directory = "gen_src"
cfg.codestyle.indent_width = 4
with SourceFileGenerator(cfg) as sfg:
...
---
file_format: mystnb
kernelspec:
name: python3
---
(composer_guide)=
# How To Use the Composer
```{code-cell} ipython3
:tags: [remove-cell]
import sys
from pathlib import Path
mockup_path = Path("../_util").resolve()
sys.path.append(str(mockup_path))
from sfg_monkeypatch import DocsPatchedGenerator # monkeypatch SFG for docs
from pystencilssfg import SourceFileGenerator
```
The *composer API* is the interface by which C++ code is constructed in pystencils-sfg.
It is exposed through the ubiquitous *composer object* returned by the `SourceFileGenerator`
upon entry into its managed region.
This guide is meant to illustrate the various constructions possible through the composer,
starting from things as simple as `#include` directives and plain code strings,
up to entire classes and their members.
## Basic Functionality
### Prelude Comment
You can equip your generated files with a prelude comment that will be printed at their very top:
```{code-cell} ipython3
import datetime
now = datetime.datetime.now()
with SourceFileGenerator() as sfg:
sfg.prelude(f"This file was generated using pystencils-sfg at {now}.")
```
### `#include` Directives
Use `sfg.include` to add `#include` directives to your generated files.
For a system-header include, delimit the header name with `<>`.
If the directive should be printed not into the header, but the implementation file,
set `private = True`:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.include("my_header.hpp")
sfg.include("<memory>")
sfg.include("detail_header.hpp", private=True)
```
### Plain Code Strings
It is always possible to print out plain code strings verbatim.
Use `sfg.code()` to write code directly to the generated header file.
To emit the code to the implementation file instead, use `sfg.code(..., impl=True)`.
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.code("int THE_ANSWER;")
sfg.code("int THE_ANSWER = 42;", impl=True)
```
## Defining Functions
Free functions can be declared and defined using the `sfg.function` sequencer.
It uses *builder syntax* to declare the various properties of the function in arbitrary
order via a sequence of calls. This sequence must end with a plain pair of parentheses `( ... )`
within which the function body will be defined.
For example, the following will create a function `getValue` with return type `int32` which is marked with the `nodiscard`
attribute:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.function("getValue").returns("int32").attr("nodiscard")(
"return 42;"
)
```
For a list of all available function qualifiers, see the reference of {any}`SfgFunctionSequencer`.
### Populate the Function Body
The function body sequencer takes an arbitrary list of arguments of different types
which are then interpreted as C++ code.
The simplest case are plain strings, which will be printed out verbatim,
in order, each string argument on its own line:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.function("factorial").params(
sfg.var("n", "uint64")
).returns("uint64")(
"if(n == 0) return 1;",
"else return n * factorial(n - 1);"
)
```
However, to make life easier, the composer API offers various APIs to model C++ code programmatically.
:::{note}
Observe that the code generated from the above snippet contains line breaks after the `if()` and `else` keywords
that where not part of the input.
This happens because `pystencils-sfg` passes its generated code through `clang-format` for beautification.
:::
#### Conditionals
To emit an if-else conditional statement, use {any}`sfg.branch <SfgBasicComposer.branch>`.
The syntax of `sfg.branch` mimics the C++ `if () {} else {}` construct by a sequence of
two (or three, with an `else`-branch) pairs of parentheses:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.function("factorial").params(
sfg.var("n", "uint64")
).returns("uint64")(
sfg.branch("n == 0")( # Condition
# then-block
"return 1;"
)(
# else-block
"return n * factorial(n - 1);"
)
)
```
#### Variables and Automatic Collection of Function Parameters
Pystencils-sfg's versatile expression system can keep track of free variables
in a function body, and then automatically exposes these variables as function parameters.
To cast a code string as an expression depending on variables, we need to do two things:
- Create an object for each variable using {any}`sfg.var <SfgBasicComposer.var>`.
This method takes the name and data type of the variable.
- Create the expression through {any}`sfg.expr <SfgBasicComposer.expr>` by interpolating a
Python format string (see {any}`str.format`) with variables or other expressions.
For example, here's the expression in the `else`-block of the `factorial` function modelled this way:
```{code-block} python
n = sfg.var("n", "uint64")
...
sfg.expr("return {0} * factorial({0} - 1);", n)
```
Using this, we can omit the manually specified parameter list for `factorial`:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
n = sfg.var("n", "uint64")
sfg.function("factorial").returns("uint64")(
sfg.branch(sfg.expr("{} == 0", n))( # Condition
# then-block
"return 1;"
)(
# else-block, with interpolated expression
sfg.expr("return {0} * factorial({0} - 1);", n)
)
)
```
#### Manual Parameter Lists
When function parameters are collected from the function body, the composer will always order them
alphabetically. If this is not desired, e.g. if a generated function is expected to have a specific interface
with a fixed parameter order, you will need to specify the parameter list manually using `.params(...)`.
#### Variables of C++ Class Type
`sfg.var` should only be used for the most basic data types: it parses its second argument as a data type using
{any}`create_type <pystencils.types.create_type>`, which is restricted to primitive and fixed-width C types.
For more complex C++ classes, class templates, and their APIs, pystencils-sfg provides its own modelling system,
implemented in `pystencilssfg.lang`.
This system is used, for instance, by `pystencilssfg.lang.cpp.std`, which mirrors (a small part of) the C++ standard library.
:::{seealso}
[](#how_to_cpp_api_modelling)
:::
To create a variable of a class template represented using the `pystencilssfg.lang` modelling system,
first instantiate the class (with any template arguments, as well as optional `const` and `ref` qualifiers)
and then call `var` on it:
```{code-cell} ipython3
from pystencilssfg.lang.cpp import std
data = std.vector("float64", const=True, ref=True).var("data")
str(data), str(data.dtype)
```
#### Initializing Variables
To emit an initializer statement for a variable, use `sfg.init`:
```{code-block} python
from pystencilssfg.lang.cpp import std
result = std.tuple("int32", "int32").var("result")
n, m = sfg.vars("n, m", "int32")
sfg.init(result)(
sfg.expr("{} / {}", n, m),
sfg.expr("{} % {}", n, m)
)
```
This will be recognized by the parameter collector:
variables that are defined using `init` before they are used will be considered *bound*
and will not end up in the function signature.
Also, any variables passed to the braced initializer-expression (by themselves or inside `sfg.expr`)
will be found and tracked by the parameter collector:
```{code-cell} ipython3
from pystencilssfg.lang.cpp import std
with SourceFileGenerator() as sfg:
result = std.tuple("int32", "int32").var("result")
n, m = sfg.vars("n, m", "int32")
sfg.function("div_rem").params(n, m).returns(result.dtype)(
sfg.init(result)(
sfg.expr("{} / {}", n, m),
sfg.expr("{} % {}", n, m)
),
sfg.expr("return {}", result)
)
```
(how_to_namespaces)=
## Namespaces
C++ uses namespaces to structure code and group entities.
By default, pystencils-sfg emits all code into the global namespace.
For instructions on how to change the outermost namespace used by the `SourceFileGenerator`,
see [](#how_to_generator_scripts_config).
Starting from the outermost namespace, nested namespaces can be entered and exited during
code generation.
To enter a new namespace, use `sfg.namespace` in one of two ways:
- Simply calling `sfg.namespace("my_namespace")` and ignoring its return value will cause the
generator script to use the given namespace for the rest of its execution;
- Calling `sfg.namespace("my_namespace")` in a `with` statement will activate the given namespace
only for the duration of the managed block.
To illustrate, the following snippet activates the namespace `mylib::generated` for the entire
length of the generator script, and then enters and exits the nested namespace `mylib::generated::detail`:
```{code-cell} ipython3
with SourceFileGenerator() as sfg:
sfg.namespace("mylib::generated")
sfg.code("/* Generated code in outer namespace */")
with sfg.namespace("detail"):
sfg.code("/* Implementation details in the inner namespace */")
sfg.code("/* More code in the outer namespace */")
```
## Kernels and Parameter Mappings
The original purpose of pystencils-sfg is to simplify the embedding of *pystencils*-generated
numerical kernels into C++ applications.
This section discusses how to register kernels with the source file generator,
how to call them in wrapper code,
and how to automatically map symbolic pystencils fields onto nd-array data structures.
### Registering Kernels
In the generated files, kernels are organized in *kernel namespaces*.
The composer gives us access to the default kernel namespace (`<current_namespace>::kernels`)
via `sfg.kernels`.
To add a kernel,
- either pass its assignments and the pystencils code generator configuration directly to {any}`kernels.create() <KernelsAdder.create>`,
- or create the kernel separately through {any}`pystencils.create_kernel <pystencils.codegen.create_kernel>` and register it using
{any}`kernels.add() <KernelsAdder.add>`.
Both functions return a kernel handle, through which the kernel may later be invoked.
You may create and access custom-named kernel namespaces using {any}`sfg.kernel_namespace() <SfgBasicComposer.kernel_namespace>`.
This gives you a {any}`KernelsAdder` object with the same interface as `sfg.kernels`.
:::{note}
A kernel namespace is not a regular namespace; if you attempt to create both a regular and a kernel namespace with the same name,
the composer will raise an error.
:::
Here's an example with two kernels being registered in different kernel namespace,
once using `add`, and once using `create`.
```{code-cell} ipython3
import pystencils as ps
with SourceFileGenerator() as sfg:
# Create symbolic fields
f, g = ps.fields("f, g: double[2D]")
# Define and create the first kernel
asm1 = ps.Assignment(f(0), g(0))
cfg1 = ps.CreateKernelConfig()
cfg1.cpu.openmp.enable = True
khandle_1 = sfg.kernels.create(asm1, "first_kernel", cfg1)
# Define the second kernel and its codegen configuration
asm2 = ps.Assignment(f(0), 3.0 * g(0))
cfg2 = ps.CreateKernelConfig(target=ps.Target.CUDA)
# Create and register the second kernel at a custom namespace
kernel2 = ps.create_kernel(asm2, cfg2)
khandle_2 = sfg.kernel_namespace("gpu_kernels").add(kernel2, "second_kernel")
```
### Writing Kernel Wrapper Functions
By default, kernel definitions are only visible in the generated implementation file;
kernels are supposed to not be called directly, but through wrapper functions.
This serves to hide their fairly lenghty and complicated low-level function interface.
#### Invoking CPU Kernels
To call a CPU kernel from a function, use `sfg.call` on a kernel handle:
```{code-block} python
sfg.function("kernel_wrapper")(
sfg.call(khandle)
)
```
This will expose all parameters of the kernel into the wrapper function and, in turn,
cause them to be added to its signature.
We don't want to expose this complexity, but instead hide it by using appropriate data structures.
The next section explains how that is achieved in pystencils-sfg.
#### Mapping Fields to Data Structures
Pystencils kernels operate on n-dimensional contiguous or strided arrays,
There exist many classes with diverse APIs modelling such arrays throughout the scientific
computing landscape, including [Kokkos Views][kokkos_view], [C++ std::mdspan][mdspan],
[SYCL buffers][sycl_buffer], and many framework-specific custom-built classes.
Using the protocols behind {any}`sfg.map_field <SfgBasicComposer.map_field>`,
it is possible to automatically emit code
that extracts the indexing information required by a kernel from any of these classes,
as long as a suitable API reflection is available.
:::{seealso}
[](#field_data_structure_reflection) for instructions on how to set up field API
reflection for a custom nd-array data structure.
:::
Pystencils-sfg natively provides field extraction for a number of C++ STL-classes,
such as `std::vector` and `std::span` (for 1D fields) and `std::mdspan`.
Import any of them from `pystencilssfg.lang.cpp.std` and create an instance for a given
field using `.from_field()`.
Then, inside the wrapper function, pass the symbolic field and its associated data structure to
{any}`sfg.map_field <SfgBasicComposer.map_field>`.
before calling the kernel:
```{code-cell} ipython3
import pystencils as ps
from pystencilssfg.lang.cpp import std
with SourceFileGenerator() as sfg:
# Create symbolic fields
f, g = ps.fields("f, g: double[1D]")
# Create data structure reflections
f_vec = std.vector.from_field(f)
g_span = std.span.from_field(g)
# Create the kernel
asm = ps.Assignment(f(0), g(0))
khandle = sfg.kernels.create(asm, "my_kernel")
# Create the wrapper function
sfg.function("call_my_kernel")(
sfg.map_field(f, f_vec),
sfg.map_field(g, g_span),
sfg.call(khandle)
)
```
## GPU Kernels
Pystencils also allows us to generate kernels for the CUDA and HIP GPU programming models.
This section describes how to generate GPU kernels through pystencils-sfg;
how to invoke them with various launch configurations,
and how GPU execution streams are reflected.
### Generate and Invoke CUDA and HIP Kernels
To generate a kernel targetting either of these, set the
{any}`target <pystencils.codegen.config.CreateKernelConfig.target>`
code generator option to either `Target.CUDA` or `Target.HIP`.
After registering a GPU kernel,
its invocation can be rendered using {any}`sfg.gpu_invoke <SfgGpuComposer.gpu_invoke>`.
Here is an example using CUDA:
```{code-cell} ipython3
from pystencilssfg import SfgConfig
sfg_config = SfgConfig()
sfg_config.extensions.impl = "cu"
with SourceFileGenerator(sfg_config) as sfg:
# Configure the code generator to use CUDA
cfg = ps.CreateKernelConfig(target=ps.Target.CUDA)
# Create fields, assemble assignments
f, g = ps.fields("f, g: double[128, 128]")
asm = ps.Assignment(f(0), g(0))
# Register kernel
khandle = sfg.kernels.create(asm, "gpu_kernel", cfg)
# Invoke it
sfg.function("kernel_wrapper")(
sfg.gpu_invoke(khandle)
)
```
In this snippet, we used the [generator configuration](#how_to_generator_scripts_config)
to change the suffix of the generated implementation file to `.cu`.
When investigating the generated `.cu` file, you can see that the GPU launch configuration parameters
*grid size* and *block size* are being computed automatically from the array sizes.
This behavior can be changed by modifying options in the {any}`gpu <pystencils.codegen.config.GpuOptions>`
category of the `CreateKernelConfig`.
### Adapting the Launch Configuration
GPU kernel invocations usually require the user to provide a launch grid, defined
by the GPU thread block size and the number of blocks on the grid.
In the simplest case (seen above), pystencils-sfg will emit code that automatically
computes these parameters from the size of the arrays passed to the kernel,
using a default block size defined by pystencils.
The code generator also permits customization of the launch configuration.
You may provide a custom block size to override the default, in which case the
grid size will still be computed by dividing the array sizes by your block size.
Otherwise, you can also fully take over control of both block and grid size.
For both cases, instructions are given in the following.
#### User-Defined Block Size for Auto-Computed Grid Size
To merely modify the block size argument while still automatically inferring the grid size,
pass a variable or expression of type `dim3` to the `block_size` parameter of `gpu_invoke`.
Pystencils-sfg exposes two versions of `dim3`, which differ primarily in their associated
runtime headers:
- {any}`pystencilssfg.lang.gpu.cuda.dim3 <CudaAPI.dim3>` for CUDA, and
- {any}`pystencilssfg.lang.gpu.hip.dim3 <HipAPI.dim3>` for HIP.
The following snippet selects the correct `dim3` type according to the kernel target;
it then creates a variable of that type and turns that into an argument to the kernel invocation:
```{code-cell} ipython3
:tags: [remove-cell]
target = ps.Target.HIP
cfg = ps.CreateKernelConfig(target=target)
f, g = ps.fields("f, g: double[128, 128]")
asm = ps.Assignment(f(0), g(0))
```
```{code-cell} ipython3
from pystencilssfg.lang.gpu import hip
with SourceFileGenerator(sfg_config) as sfg:
# ... define kernel ...
khandle = sfg.kernels.create(asm, "gpu_kernel", cfg)
# Select dim3 reflection
match target:
case ps.Target.CUDA:
from pystencilssfg.lang.gpu import cuda as gpu_api
case ps.Target.HIP:
from pystencilssfg.lang.gpu import hip as gpu_api
# Create dim3 variable and pass it to kernel invocation
block_size = gpu_api.dim3(const=True).var("block_size")
sfg.function("kernel_wrapper")(
sfg.gpu_invoke(khandle, block_size=block_size)
)
```
#### Manual Launch Configurations
To take full control of the launch configuration, we must disable its automatic inferrence
by setting the {any}`gpu.manual_launch_grid <pystencils.codegen.config.GpuOptions.manual_launch_grid>`
code generator option to `True`.
Then, we must pass `dim3` arguments for both `block_size` and `grid_size` to the kernel invocation:
```{code-cell} ipython3
from pystencilssfg.lang.gpu import hip
with SourceFileGenerator(sfg_config) as sfg:
# ... define kernel ...
# Configure for manual launch config
cfg = ps.CreateKernelConfig(target=ps.Target.CUDA)
cfg.gpu.manual_launch_grid = True
# Register kernel
khandle = sfg.kernels.create(asm, "gpu_kernel", cfg)
# Create dim3 variables
from pystencilssfg.lang.gpu import cuda
block_size = cuda.dim3(const=True).var("block_size")
grid_size = cuda.dim3(const=True).var("grid_size")
sfg.function("kernel_wrapper")(
sfg.gpu_invoke(khandle, block_size=block_size, grid_size=grid_size)
)
```
### Using Streams
CUDA and HIP kernels can be enqueued into streams for concurrent execution.
This is mirrored in pystencils-sfg;
all overloads of `gpu_invoke` take an optional `stream` argument.
The `stream_t` data types of both CUDA and HIP are made available
through the respective API reflections:
- {any}`lang.gpu.cuda.stream_t <CudaAPI.stream_t>` reflects `cudaStream_t`, and
- {any}`lang.gpu.hip.stream_t <HipAPI.stream_t>` reflects `hipStream_t`.
Here is an example that creates a variable of the HIP stream type
and passes it to `gpu_invoke`:
```{code-cell} ipython3
:tags: [remove-cell]
cfg = ps.CreateKernelConfig(target=ps.Target.HIP)
f, g = ps.fields("f, g: double[128, 128]")
asm = ps.Assignment(f(0), g(0))
```
```{code-cell} ipython3
from pystencilssfg.lang.gpu import hip
with SourceFileGenerator(sfg_config) as sfg:
# ... define kernel ...
khandle = sfg.kernels.create(asm, "gpu_kernel", cfg)
stream = hip.stream_t(const=True).var("stream")
sfg.function("kernel_wrapper")(
sfg.gpu_invoke(khandle, stream=stream)
)
```
:::{admonition} To Do
- Defining classes, their fields constructors, and methods
:::
[kokkos_view]: https://kokkos.org/kokkos-core-wiki/ProgrammingGuide/View.html
[mdspan]: https://en.cppreference.com/w/cpp/container/mdspan
[sycl_buffer]: https://registry.khronos.org/SYCL/specs/sycl-2020/html/sycl-2020.html#subsec:buffers
(guide_project_integration)=
# Project and Build System Integration
(config_module)=
## Project-Wide Settings using Configuration Modules
When embedding *pystencils-sfg* into a C++ project or build system,
you might want to set a project-wide base configuration for all generator scripts.
In addition, it might be necessary to pass various details about the project
and build setup to the generator scripts.
Both can be achieved by the use of a *configuration module*.
A configuration module is a Python file that defines up to two functions:
- `def configure_sfg(cfg: SfgConfig)` is called to set up the project-wide base configuration.
It takes an {any}`SfgConfig` object which it may modify to establish the project-wide option set.
- `def project_info() -> Any` is called by *pystencils-sfg* to retrieve an object that encapsulates
any custom project-specific information.
This information is passed on to the generator scripts through
the {any}`sfg.context.project_info <SfgContext.project_info>` attribute.
An example configuration module might look like this:
```Python
from pystencilssfg import SfgConfig
def configure_sfg(cfg: SfgConfig):
cfg.extensions.header = "h++"
cfg.extensions.impl = "c++"
cfg.clang_format.code_style = "llvm"
...
def project_info():
return {
"project_name": "my-project",
"float_precision": "float32",
"use_cuda": False,
...
}
```
Here, `project_info` returns a dictionary, but this is just for illustration;
the function may return any type of arbitrarily complex objects.
For improved API safety, {any}`dataclasses` might be a good tool for setting up
project info objects.
When invoking a generator script, the path to the current configuration module must be passed to it
using the `--sfg-config-module` command-line parameter.
This can be automated by an adequately set up build system, such as GNU Make or CMake.
If you are using pystencils-sfg with CMake through the provided CMake module,
[see below](#cmake_set_config_module) on how to specify a configuration module for your project.
(cmake_integration)=
## CMake Integration
*pystencils-sfg* is shipped with a CMake module for on-the-fly code generation during the CMake build process.
### Add the module
To include the module in your CMake source tree, you must first add the pystencils-sfg *Find-module*
to your CMake module path.
To create the Find-module, navigate to the directory it should be placed in and run the following command:
```shell
sfg-cli cmake make-find-module
```
This will create the `FindPystencilsSfg.cmake` file.
Make sure that its containing directory is added to the CMake module path.
To load pystencils-sfg into CMake, we first need to set the Python interpreter
of the environment SFG is installed in.
There are several ways of doing this:
#### Set Python via a Find-Module Hint
Set the `PystencilsSfg_PYTHON_PATH` hint variable inside your `CMakeLists.txt` to point at the
Python executable which should be used to invoke pystencils-sfg, e.g.:
```CMake
set(PystencilsSfg_PYTHON_PATH ${CMAKE_SOURCE_DIR}/.venv/bin/python)
```
This is the recommended way, especially when other parts of your project also use Python.
#### Set Python via a Cache Variable
On the command line or in a [CMake configure preset](https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html),
set the `PystencilsSfg_PYTHON_INTERPRETER` cache variable to point at the Python executable to be used to invoke pystencils-sfg;
e.g.:
```bash
cmake -S . -B build -DPystencilsSfg_PYTHON_INTERPRETER=`pwd`/.venv/bin/python
```
If both the cache variable and the `PystencilsSfg_PYTHON_PATH` hint are set, the cache variable takes precedence,
so you can use the cache variable to override the hint.
#### Automatically Find a Python Installation
If none of the above is provided, pystencils-sfg will invoke [FindPython](https://cmake.org/cmake/help/latest/module/FindPython.html)
to determine the Python interpreter it should use.
You can affect this process through any of the hints listed in the `FindPython` documentation.
#### Find pystencils-sfg
Finally, call `find_package( PystencilsSfg )` from your `CMakeLists.txt` to load the SFG module.
If SFG as a dependency is not optional, add the `REQUIRED` flag such that the call will fail if
the package cannot be found.
(cmake_add_generator_scripts)=
### Adding Generator Scripts
The primary interaction point in CMake is the function `pystencilssfg_generate_target_sources`,
with the following signature:
```CMake
pystencilssfg_generate_target_sources( <target>
SCRIPTS script1.py [script2.py ...]
[SCRIPT_ARGS arg1 [arg2 ...]]
[DEPENDS dependency1.py [dependency2.py...]]
[FILE_EXTENSIONS <header-extension> <impl-extension>]
[CONFIG_MODULE <path-to-config-module.py>]
[OUTPUT_DIRECTORY <output-directory>]
[HEADER_ONLY]
)
```
It registers the generator scripts `script1.py [script2.py ...]` to be executed at compile time using `add_custom_command`
and adds their output files to the specified `<target>`.
Any changes in the generator scripts, or any listed dependency, will trigger regeneration.
The function takes the following options:
- `SCRIPTS`: A list of generator scripts
- `SCRIPT_ARGS`: A list of custom command line arguments passed to the generator scripts; see [](#custom_cli_args)
- `DEPENDS`: A list of dependencies for the generator scripts
- `FILE_EXTENSION`: The desired extensions for the generated files
- `CONFIG_MODULE`: Set the configuration module for all scripts registered with this call.
If set, this overrides the value of `PystencilsSfg_CONFIG_MODULE`
in the current scope (see [](#cmake_set_config_module))
- `OUTPUT_DIRECTORY`: Custom output directory for generated files. If `OUTPUT_DIRECTORY` is a relative path,
it will be interpreted relative to the current build directory.
- `HEADER_ONLY`: If this option is set, instruct the generator scripts to only generate header files
(see {any}`SfgConfig.header_only`).
If `OUTPUT_DIRECTORY` is *not* specified, any C++ header files generated by the above call
can be included in any files belonging to `target` via:
```C++
#include "gen/<file1.hpp>"
#include "gen/<file2.hpp>"
/* ... */
```
:::{attention}
If you change the code generator output directory using the `OUTPUT_DIRECTORY` argument,
you are yourself responsible for placing that directory--or any of its parents--on the
include path of your target.
:::
(cmake_set_config_module)=
### Set a Configuration Module
There are two ways of specifying a [configuration module](#config_module) for generator scripts
registered with CMake:
- To set a configuration module for scripts registered with a single call to `pystencilssfg_generate_target_sources`,
use the `CONFIG_MODULE` function parameter (see [](#cmake_add_generator_scripts)).
- To set a config module for all generator scripts within the current CMake directory and its subdirectories,
set the scoped variable `PystencilsSfg_CONFIG_MODULE` to point at the respective Python file, e.g.
`set( PystencilsSfg_CONFIG_MODULE ProjectConfig.py )`.
You might want to populate your configuration module with information about the current
build setup and environment.
For this purpose, take a look at the
[configure_file](https://cmake.org/cmake/help/latest/command/configure_file.html) CMake function.
(guide:tips_n_tricks)=
# Tips and Tricks
## Make CLion treat generated files as project sources ## Make CLion treat generated files as project sources
......
## Namespaces
Conceptually, there exist two different kinds of namespaces: *kernel namespaces* for the generated kernels,
and a single *code namespace* for all the generated code.
Both get mapped to standard C++ namespaces, in the end, but they fulfill different purposes in the code generator.
*Kernel namespaces* are used for grouping generated kernels together, e.g. to avoid name collisions.
If, for example, a code generation script combines kernels and functions produced by different components, each
component may create its own kernel namespace to isolate its kernels.
The *code namespace*, in contrast, envelops all the generated code. Its fully qualified name is built from two parts:
- The *outer namespace* is defined in the [generator configuration][pystencilssfg.SfgConfiguration], typically by
the global project configuration;
- The *inner namespace* is defined by the code generation script, e.g. via [`SfgComposer.namespace`][pystencilssfg.SfgComposer.namespace].
These namespaces will finally occur in the generated implementation file as:
```C++
namespace outer_namespace::inner_namespace {
namespace kernels {
/* kernel definitions */
} // namespace kernels
/* function definitions */
} // namespace outer_namespace::inner_namespace
```
## Command Line Interface
*pystencils-sfg* exposes not one, but two command line interfaces:
The *global CLI* offers a few tools meant to be used by build systems,
while the *generator script* command line interface is meant for a build system to communicate
with the code generator during on-the-fly generation.
### Global CLI
The global CLI may be accessed either through the `sfg-cli` shell command, or using `python -m pystencilssfg`.
### Generator Script CLI
The [SourceFileGenerator][pystencilssfg.SourceFileGenerator] evaluates a generator script's command line arguments,
which can be supplied by the user, but more frequently by the build system.
## CMake Integration
*pystencils-sfg* is shipped with a CMake module for on-the-fly code generation during the CMake build process.
### Add the module
To include the module in your CMake source tree, a separate find module is provided.
You can use the global CLI to obtain the find module; simply run
```shell
sfg-cli cmake make-find-module
```
to create the file `FindPystencilsSfg.cmake` in the current directory.
Add it to the CMake module path, and load the *pystencils-sfg* module via *find_package*:
```CMake
find_package( PystencilsSfg )
```
Make sure to set the `Python_ROOT_DIR` cache variable to point to the correct Python interpreter
(i.e. the virtual environment you have installed *pystencils-sfg* into).
### Add generator scripts
The primary interaction point in CMake is the function `pystencilssfg_generate_target_sources`,
with the following signature:
```CMake
pystencilssfg_generate_target_sources( <target>
SCRIPTS script1.py [script2.py ...]
[DEPENDS dependency1.py [dependency2.py...]]
[FILE_EXTENSIONS <header-extension> <impl-extension>]
[HEADER_ONLY])
```
It registers the generator scripts `script1.py [script2.py ...]` to be executed at compile time using `add_custom_command`
and adds their output files to the specified `<target>`.
Any changes in the generator scripts, or any listed dependency, will trigger regeneration.
The function takes the following options:
- `SCRIPTS`: A list of generator scripts
- `DEPENDS`: A list of dependencies for the generator scripts
- `FILE_EXTENSION`: The desired extensions for the generated files
- `HEADER_ONLY`: Toggles header-only code generation
### Include generated files
The `pystencils-sfg` CMake module creates a subfolder `sfg_sources/gen` at the root of the build tree
and writes all generated source files into it. The directory `sfg_sources` is added to the project's include
path, such that generated header files for a target `<target>` may be included via:
```C++
#include "gen/<target>/kernels.h"
```
### Project Configuration
The *pystencils-sfg* CMake module reads the scoped variable `PystencilsSfg_CONFIGURATOR_SCRIPT` to find
the *configuration module* that should be passed to the generator scripts.
Generator scripts are the primary way *pystencils-sfg* is meant to be used.
A generator script is a single Python script, say `kernels.py`, which contains *pystencils-sfg*
code at the top level such that, when executed, it emits source code to a pair of files `kernels.h`
and `kernels.cpp`. This guide describes how to write such a generator script, its structure, and how
it can be used to generate code.
This page gives a general overview over the code generation process, but introduces only the
convenient high-level interface provided by the [SourceFileGenerator][pystencilssfg.SourceFileGenerator]
and [SfgComposer][pystencilssfg.SfgComposer] classes.
For a more in-depth look into building source files, and about using *pystencils-sfg* outside
of a generator script, please take a look at the [In-Depth Guide](building.md).
## Anatomy
The code generation process in a generator script is controlled by the
[SourceFileGenerator][pystencilssfg.SourceFileGenerator] context manager.
It configures the code generator by combining configuration options from the
environment (e.g. a CMake build system) with options specified in the script,
and infers the names of the output files from the script's name.
It then prepares and returns a code generation [context][pystencilssfg.SfgContext].
This context may then be passed to a [composer][pystencilssfg.SfgComposer],
which provides a convenient interface for constructing the source files.
To start, place the following code in a Python script, e.g. `kernels.py`:
```Python
from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer
sfg_config = SfgConfiguration()
with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx)
```
The source file is constructed within the context manager's managed region.
During execution of the script, when the region ends, a header/source file pair
`kernels.h` and `kernels.cpp` will be written to the file system next to your script.
Execute the script as-is and inspect the generated files, which will of course
still be empty.
A few notes on configuration:
- The [SourceFileGenerator][pystencilssfg.SourceFileGenerator] parses the script's command line arguments
for configuration options (refer to [CLI and Build System Integration](cli_and_build_system.md)).
If you intend to use command-line parameters in your
generation script, use [`sfg.context.argv`][pystencilssfg.SfgContext.argv] instead of `sys.argv`.
There, all arguments meant for the code generator are already removed.
- The code generator's configuration is consolidated from a global project configuration which may
be provided by the build system; a number of command line arguments; and the
[SfgConfiguration][pystencilssfg.SfgConfiguration] provided in the script.
The project configuration may safely be overridden by the latter two; however, conflicts
between command-line arguments and the configuration defined in the script will cause
an exception to be thrown.
## Using the Composer
The object `sfg` constructed in above snippet is an instance of [SfgComposer][pystencilssfg.SfgComposer].
The composer is the central part of the user front-end of *pystencils-sfg*.
It provides an interface for constructing source files that attempts to closely mimic
C++ syntactic structures within Python.
Here is an overview of its various functions:
### Includes and Definitions
With [`SfgComposer.include`][pystencilssfg.SfgComposer.include], the code generator can be instructed
to include header files.
```Python
with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx)
# ...
sfg.include("<vector>")
sfg.incldue("custom_header.h")
```
### Adding Kernels
`pystencils`-generated kernels are managed in
[kernel namespaces][pystencilssfg.source_components.SfgKernelNamespace].
The default kernel namespace is called `kernels` and is available via
[`SfgComposer.kernels`][pystencilssfg.SfgComposer.kernels].
Adding an existing `pystencils` AST, or creating one from a list of assignments, is possible
through [`add`][pystencilssfg.source_components.SfgKernelNamespace.add]
and [`create`][pystencilssfg.source_components.SfgKernelNamespace.create].
The latter is a wrapper around
[`pystencils.create_kernel`](
https://pycodegen.pages.i10git.cs.fau.de/pystencils/sphinx/kernel_compile_and_call.html#pystencils.create_kernel
).
Both functions return a [kernel handle][pystencilssfg.source_components.SfgKernelHandle]
through which the kernel can be accessed, e.g. for calling it in a function.
If required, use [`SfgComposer.kernel_namespace`][pystencilssfg.SfgComposer.kernel_namespace]
to access other kernel namespaces than the default one.
```Python
with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx)
# ...
ast = ps.create_kernel(assignments, config)
khandle = sfg.kernels.add(ast, "kernel_a")
# is equivalent to
khandle = sfg.kernels.create(assignments, "kernel_a", config)
# You may use a different namespace
nspace = sfg.kernel_namespace("group_of_kernels")
nspace.create(assignments, "kernel_a", config)
```
### Building Functions
[Functions][pystencilssfg.source_components.SfgFunction] form the link between your `pystencils` kernels
and your C++ framework. A function in *pystencils-sfg* translates to a simple C++ function, and should
fulfill just the following tasks:
- Extract kernel parameters (pointers, sizes, strides, numerical coefficients)
from C++ objects (like fields, vectors, other data containers)
- Call one or more kernels in sequence or in conditional branches
It is the philosophy of this project that anything more complicated than this should happen in handwritten
code; these generated functions are merely meant to close the remaining gap.
The composer provides an interface for constructing functions that tries to mimic the look of the generated C++
code.
Use [`SfgComposer.function`][pystencilssfg.SfgComposer.function] to create a function,
and [`SfgComposer.call`][pystencilssfg.SfgComposer.call] to call a kernel by its handle:
```Python
with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx)
# ...
sfg.function("MyFunction")(
sfg.call(khandle)
)
```
Note the special syntax: To mimic the look of a C++ function, the composer uses a sequence of two calls
to construct the function.
The function body may further be populated with the following things:
#### Parameter Mappings
Extract kernel parameters from C++ objects:
- [`map_param`][pystencilssfg.SfgComposer.map_param]: Add a single line of code to define one parameter
depending on one other.
- [`map_field`][pystencilssfg.SfgComposer.map_field] maps a pystencils
[`Field`](https://pycodegen.pages.i10git.cs.fau.de/pystencils/sphinx/field.html)
to a field data structure providing the necessary pointers, sizes and stride information.
The field data structure must be provided as an instance of a subclass of
[`SrcField`][pystencilssfg.source_concepts.SrcField].
Currently, *pystencils-sfg* provides mappings to
[`std::vector`](https://en.cppreference.com/w/cpp/container/vector)
(via [`std_vector_ref`][pystencilssfg.source_concepts.cpp.std_vector_ref])
and
[`std::mdspan`](https://en.cppreference.com/w/cpp/container/mdspan)
(via [`mdspan_ref`][pystencilssfg.source_concepts.cpp.mdspan_ref])
from the C++ standard library.
- [`map_vector`][pystencilssfg.SfgComposer.map_vector] maps a sequence of scalar numerical values
(given as `pystencils.TypedSymbol`s) to a vector data type. Currently, only `std::vector` is provided.
#### Conditional Branches
A conditonal branch may be added with [`SfgComposer.branch`][pystencilssfg.SfgComposer.branch]
using a special syntax:
```Python
with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx)
# ...
sfg.function("myFunction")(
# ...
sfg.branch("condition")(
# then-body
)(
# else-body (may be omitted)
)
)
```
\ No newline at end of file
# User Guides
These pages provide an overview of how to use the pystencils Source File Generator.
A basic understanding of [pystencils](https://pycodegen.pages.i10git.cs.fau.de/pystencils/index.html)
is required.
## Guides
- [Writing Generator Scripts](generator_scripts.md) explains about the primary interface of *pystencils-sfg*:
Generator scripts, which are Python scripts that, when executed, emit *pystencils*-generated code to a header/source
file pair with the same name as the script.
- [In-Depth: Building Source Files](building.md)
- [CLI and Build System Integration](cli_and_build_system.md)
- [Tips And Tricks](tips_n_tricks.md): A collection of various tricks that might come in handy when working with *pystencils-sfg*.
\ No newline at end of file
#!/bin/bash #!/bin/bash
echo "[Pre-Commit] Checking code style" echo "[Pre-Commit] Checking code style"
pdm run flake8 src/pystencilssfg flake8 src/pystencilssfg
status=$? status=$?
if [ ${status} != 0 ]; then if [ ${status} != 0 ]; then
...@@ -11,7 +11,7 @@ else ...@@ -11,7 +11,7 @@ else
fi fi
echo "[Pre-Commit] Checking types" echo "[Pre-Commit] Checking types"
pdm run mypy src/pystencilssfg mypy src/pystencilssfg
status=$? status=$?
if [ ${status} != 0 ]; then if [ ${status} != 0 ]; then
exit 1 exit 1
......
...@@ -11,7 +11,7 @@ execute_process( COMMAND sfg-cli cmake make-find-module ...@@ -11,7 +11,7 @@ execute_process( COMMAND sfg-cli cmake make-find-module
find_package( PystencilsSfg REQUIRED ) find_package( PystencilsSfg REQUIRED )
set( PystencilsSfg_CONFIGURATOR_SCRIPT codegen_config.py ) set( PystencilsSfg_CONFIG_MODULE codegen_config.py )
add_library( genlib ) add_library( genlib )
pystencilssfg_generate_target_sources( genlib SCRIPTS kernels.py FILE_EXTENSIONS .h .cpp ) pystencilssfg_generate_target_sources( genlib SCRIPTS kernels.py FILE_EXTENSIONS .h .cpp )
......
from sys import stderr from pystencilssfg import SfgConfig
from pystencilssfg import SfgConfiguration
def sfg_config():
print("sfg_config() called!", file=stderr)
project_info = { def configure(cfg: SfgConfig):
'B': 'A' cfg.extensions.header = "h++"
} cfg.extensions.impl = "c++"
return SfgConfiguration(
header_extension='hpp',
impl_extension='cpp',
outer_namespace='cmake_demo',
project_info=project_info
)
...@@ -5,7 +5,7 @@ import sympy as sp ...@@ -5,7 +5,7 @@ import sympy as sp
from pystencils import fields, kernel from pystencils import fields, kernel
from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer
from pystencilssfg.source_concepts.cpp import mdspan_ref from pystencilssfg.lang.cpp import mdspan_ref
sfg_config = SfgConfiguration( sfg_config = SfgConfiguration(
outer_namespace="make_demo" outer_namespace="make_demo"
...@@ -22,7 +22,7 @@ Author: Frederik Hennig <frederik.hennig@fau.de>""") ...@@ -22,7 +22,7 @@ Author: Frederik Hennig <frederik.hennig@fau.de>""")
sfg.namespace("jacobi") sfg.namespace("jacobi")
u_src, u_dst, f = fields("u_src, u_dst, f(1) : double[2D]", layout="fzyx") u_src, u_dst, f = fields("u_src, u_dst, f : double[2D]", layout="fzyx")
h = sp.Symbol("h") h = sp.Symbol("h")
@kernel @kernel
......
...@@ -9,13 +9,11 @@ with SourceFileGenerator() as ctx: ...@@ -9,13 +9,11 @@ with SourceFileGenerator() as ctx:
lb_config = LBMConfig(streaming_pattern='esotwist') lb_config = LBMConfig(streaming_pattern='esotwist')
lb_ast_even = create_lb_ast(lbm_config=lb_config, timestep=Timestep.EVEN) lb_ast_even = create_lb_ast(lbm_config=lb_config, timestep=Timestep.EVEN)
lb_ast_even.function_name = "streamCollide_even"
lb_ast_odd = create_lb_ast(lbm_config=lb_config, timestep=Timestep.ODD) lb_ast_odd = create_lb_ast(lbm_config=lb_config, timestep=Timestep.ODD)
lb_ast_odd.function_name = "streamCollide_odd"
kernel_even = sfg.kernels.add(lb_ast_even) kernel_even = sfg.kernels.add(lb_ast_even, "lb_even")
kernel_odd = sfg.kernels.add(lb_ast_odd) kernel_odd = sfg.kernels.add(lb_ast_odd, "lb_odd")
sfg.function("myFunction")( sfg.function("myFunction")(
sfg.branch("(timestep & 1) ^ 1")( sfg.branch("(timestep & 1) ^ 1")(
......
...@@ -2,7 +2,6 @@ ...@@ -2,7 +2,6 @@
from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer
from pystencilssfg.configuration import SfgCodeStyle from pystencilssfg.configuration import SfgCodeStyle
from pystencilssfg.composer import SfgClassComposer from pystencilssfg.composer import SfgClassComposer
from pystencilssfg.source_concepts import SrcObject
from pystencils import fields, kernel from pystencils import fields, kernel
...@@ -19,7 +18,6 @@ f, g = fields("f, g(1): double[2D]") ...@@ -19,7 +18,6 @@ f, g = fields("f, g(1): double[2D]")
with SourceFileGenerator(sfg_config) as ctx: with SourceFileGenerator(sfg_config) as ctx:
sfg = SfgComposer(ctx) sfg = SfgComposer(ctx)
c = SfgClassComposer(ctx)
@kernel @kernel
def assignments(): def assignments():
...@@ -27,29 +25,29 @@ with SourceFileGenerator(sfg_config) as ctx: ...@@ -27,29 +25,29 @@ with SourceFileGenerator(sfg_config) as ctx:
khandle = sfg.kernels.create(assignments) khandle = sfg.kernels.create(assignments)
c.struct("DataStruct")( sfg.struct("DataStruct")(
SrcObject("coord", "uint32_t"), sfg.var("coord", "uint32_t"),
SrcObject("value", "float") sfg.var("value", "float")
), ),
c.klass("MyClass", bases=("MyBaseClass",))( sfg.klass("MyClass", bases=("MyBaseClass",))(
# class body sequencer # class body sequencer
c.constructor(SrcObject("a", "int")) sfg.constructor(sfg.var("a", "int"))
.init("a_(a)") .init("a_(a)")
.body( .body(
'cout << "Hi!" << endl;' 'cout << "Hi!" << endl;'
), ),
c.private( sfg.private(
c.var("a_", "int"), sfg.var("a_", "int"),
c.method("getX", returns="int")( sfg.method("getX", returns="int")(
"return 2.0;" "return 2.0;"
) )
), ),
c.public( sfg.public(
"using xtype = uint8_t;" "using xtype = uint8_t;"
) )
) )
# type: ignore # type: ignore
from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer from pystencilssfg import SourceFileGenerator, SfgConfiguration, SfgComposer
from pystencilssfg.configuration import SfgCodeStyle from pystencilssfg.configuration import SfgCodeStyle
from pystencilssfg.types import SrcType from pystencils.types import PsCustomType
from pystencilssfg.source_concepts import SrcObject from pystencilssfg.ir.source_components import SfgClass, SfgMemberVariable, SfgConstructor, SfgMethod
from pystencilssfg.source_components import SfgClass, SfgMemberVariable, SfgConstructor, SfgMethod, SfgVisibility
from pystencils import fields, kernel from pystencils import fields, kernel
...@@ -38,7 +37,7 @@ with SourceFileGenerator(sfg_config) as ctx: ...@@ -38,7 +37,7 @@ with SourceFileGenerator(sfg_config) as ctx:
sfg.seq( sfg.seq(
"return -1.0;" "return -1.0;"
), ),
return_type=SrcType("double"), return_type="double",
inline=True, inline=True,
const=True const=True
)) ))
...@@ -48,20 +47,20 @@ with SourceFileGenerator(sfg_config) as ctx: ...@@ -48,20 +47,20 @@ with SourceFileGenerator(sfg_config) as ctx:
sfg.seq( sfg.seq(
"return 2.0f;" "return 2.0f;"
), ),
return_type=SrcType("float"), return_type="float",
inline=False, inline=False,
const=True const=True
)) ))
cls.default.append_member( cls.default.append_member(
SfgMemberVariable( SfgMemberVariable(
"stuff", "std::vector< int >" "stuff", PsCustomType("std::vector< int > &")
) )
) )
cls.default.append_member( cls.default.append_member(
SfgConstructor( SfgConstructor(
[SrcObject("stuff", "std::vector< int > &")], [sfg.var("stuff", PsCustomType("std::vector< int > &"))],
["stuff_(stuff)"] ["stuff_(stuff)"]
) )
) )
......