tests/generator_scripts/source/StlContainers1D.harness.cpp

+#include "StlContainers1D.hpp"
+
+#include <iostream>
+#include <span>
+#include <vector>
+#include <random>
+#include <cmath>
+#include <memory>
+
+#ifdef NDEBUG
+#undef NDEBUG
+#include <cassert>
+#define NDEBUG
+#else
+#include <cassert>
+#endif
+
+namespace StlContainers1D
+{
+    constexpr size_t N{974};
+    constexpr double one_third { 1.0 / 3.0 };
+
+    void test_vector_kernel()
+    {
+        std::random_device rd;
+        std::mt19937 gen{ rd() };
+        std::uniform_real_distribution<double> distrib{-1.0, 1.0};
+
+        std::vector<double> src;
+        std::vector<double> dst;
+
+        src.resize(N);
+        dst.resize(N);
+
+        for (size_t i = 0; i < N; ++i)
+        {
+            src[i] = distrib(gen);
+            dst[i] = 0.0;
+        }
+
+        gen::averageVector(dst, src);
+
+        for (size_t i = 1; i < N - 1; ++i)
+        {
+            const double desired = one_third * ( src[i - 1] + src[i] + src[i + 1] );
+            assert( std::abs(desired - dst[i]) < 1e-12 );
+        }
+    }
+
+    void test_span_kernel()
+    {
+        std::random_device rd;
+        std::mt19937 gen{ rd() };
+        std::uniform_real_distribution<double> distrib{-1.0, 1.0};
+
+        auto src_data = std::make_unique< double[] >(N);
+        auto dst_data = std::make_unique< double[] >(N);
+
+        std::span< double > src{ src_data.get(), N };
+        std::span< double > dst{ dst_data.get(), N };
+
+        for (size_t i = 0; i < N; ++i)
+        {
+            src[i] = distrib(gen);
+            dst[i] = 0.0;
+        }
+
+        gen::averageSpan(dst, src);
+
+        for (size_t i = 1; i < N - 1; ++i)
+        {
+            const double desired = one_third * ( src[i - 1] + src[i] + src[i + 1] );
+            assert( std::abs(desired - dst[i]) < 1e-12 );
+        }
+    }
+
+}
+
+
+int main(void)
+{
+    StlContainers1D::test_vector_kernel();
+    StlContainers1D::test_span_kernel();
+    return 0;
+}

tests/generator_scripts/source/StlContainers1D.py

+import pystencils as ps
+import sympy as sp
+
+from pystencilssfg import SourceFileGenerator
+from pystencilssfg.lang.cpp import std
+
+
+with SourceFileGenerator() as sfg:
+    sfg.namespace("StlContainers1D::gen")
+
+    src, dst = ps.fields("src, dst: double[1D]")
+
+    asms = [ps.Assignment(dst[0], sp.Rational(1, 3) * (src[-1] + src[0] + src[1]))]
+
+    kernel = sfg.kernels.create(asms, "average")
+
+    sfg.function("averageVector")(
+        sfg.map_field(src, std.vector.from_field(src)),
+        sfg.map_field(dst, std.vector.from_field(dst)),
+        sfg.call(kernel),
+    )
+
+    sfg.function("averageSpan")(
+        sfg.map_field(src, std.span.from_field(src)),
+        sfg.map_field(dst, std.span.from_field(dst)),
+        sfg.call(kernel),
+    )

tests/generator_scripts/source/SyclBuffers.harness.cpp

+#include <sycl/sycl.hpp>
+#include <iostream>
+
+#include "SyclBuffers.hpp"
+
+int main(void) {
+    sycl::queue queue;
+    {
+        sycl::range< 2 > domainSize { 64, 64 };
+        const double h { 1.0 / 63.0 };
+        sycl::buffer< double, 2 > u { domainSize };
+        sycl::buffer< double, 2 > uTmp { domainSize };
+        sycl::buffer< double, 2 > f { domainSize };
+
+        queue.submit([&](sycl::handler & cgh){
+            sycl::accessor uAcc { u, cgh };
+            sycl::accessor uTmpAcc { uTmp, cgh };
+            sycl::accessor fAcc { f, cgh };
+
+            gen::jacobiUpdate(fAcc, h, cgh, domainSize, uTmpAcc, uAcc);
+        });
+    }
+}

tests/generator_scripts/source/SyclBuffers.py

+import pystencils as ps
+import sympy as sp
+from pystencilssfg import SourceFileGenerator
+import pystencilssfg.extensions.sycl as sycl
+
+
+with SourceFileGenerator() as sfg:
+    sfg = sycl.SyclComposer(sfg)
+    sfg.namespace("gen")
+
+    u_src, u_dst, f = ps.fields("u_src, u_dst, f : double[2D]", layout="fzyx")
+    h = sp.Symbol("h")
+
+    jacobi_update = [
+        ps.Assignment(
+            u_dst.center(),
+            (h**2 * f[0, 0] + u_src[1, 0] + u_src[-1, 0] + u_src[0, 1] + u_src[0, -1])
+            / 4,
+        )
+    ]
+
+    kernel_config = ps.CreateKernelConfig(target=ps.Target.SYCL)
+    jacobi_kernel = sfg.kernels.create(jacobi_update, config=kernel_config)
+
+    cgh = sfg.sycl_handler("handler")
+    rang = sfg.sycl_range(2, "range")
+    mappings = [
+        sfg.map_field(u_src, sycl.accessor.from_field(u_src)),
+        sfg.map_field(u_dst, sycl.accessor.from_field(u_dst)),
+        sfg.map_field(f, sycl.accessor.from_field(f)),
+    ]
+
+    sfg.function("jacobiUpdate")(
+        cgh.parallel_for(rang)(*mappings, sfg.call(jacobi_kernel)),
+    )

tests/generator_scripts/source/SyclKernels.py

+import sympy as sp
+import pystencils as ps
+
+from pystencilssfg import SourceFileGenerator, SfgConfig
+from pystencilssfg.extensions.sycl import SyclComposer
+
+cfg = SfgConfig()
+cfg.header_only = True
+
+with SourceFileGenerator(cfg) as sfg:
+    sfg = SyclComposer(sfg)
+
+    u_src, u_dst, f = ps.fields("u_src, u_dst, f: double[10, 10]", layout="fzyx")
+    h = sp.Rational(1, 4)
+
+    @ps.kernel
+    def poisson_jacobi():
+        u_dst[0,0] @= (h**2 * f[0, 0] + u_src[1, 0] + u_src[-1, 0] + u_src[0, 1] + u_src[0, -1]) / 4
+
+    gen_config = ps.CreateKernelConfig(
+        target=ps.Target.SYCL
+    )
+
+    poisson_kernel = sfg.kernels.create(poisson_jacobi, config=gen_config)
+
+    cgh = sfg.sycl_handler("cgh")
+    rang = sfg.sycl_range(2, "range")
+
+    sfg.function("invoke_parallel_for")(
+        cgh.parallel_for(rang)(
+            sfg.call(poisson_kernel)
+        )
+    )

tests/generator_scripts/source/TestConfigModule.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator() as sfg:
+    ctx = sfg.context
+
+    assert ctx.outer_namespace == "myproject"
+    assert ctx.codestyle.indent_width == 3
+
+    assert not ctx.argv
+    assert isinstance(ctx.project_info, dict)
+    assert ctx.project_info == {
+        "use_openmp": True,
+        "use_cuda": True,
+        "float_format": "float32",
+    }

tests/generator_scripts/source/TestExtraCommandLineArgs.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator(keep_unknown_argv=True) as sfg:
+    ctx = sfg.context
+
+    assert ctx.argv == ["--precision", "float32", "test1", "test2"]

tests/generator_scripts/source/TestIllegalArgs.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator() as sfg:
+    ...

tests/generator_scripts/source/TestIncludeSorting.py

+from pystencilssfg import SourceFileGenerator, SfgConfig
+from pystencilssfg.lang import HeaderFile
+
+
+def sortkey(h: HeaderFile):
+    try:
+        return [
+            "memory",
+            "vector",
+            "array"
+        ].index(h.filepath)
+    except ValueError:
+        return 100
+
+
+cfg = SfgConfig()
+cfg.codestyle.includes_sorting_key = sortkey
+
+
+with SourceFileGenerator(cfg) as sfg:
+    sfg.include("<array>")
+    sfg.include("<memory>")
+    sfg.include("<vector>")

tests/generator_scripts/source/VectorExtraction.harness.cpp

+#include "VectorExtraction.hpp"
+#include <experimental/mdspan>
+#include <memory>
+#include <vector>
+
+#undef NDEBUG
+#include <cassert>
+
+namespace stdex = std::experimental;
+
+using extents_t = stdex::extents<std::int64_t, std::dynamic_extent, std::dynamic_extent, 3>;
+using vector_field_t = stdex::mdspan<double, extents_t, stdex::layout_right>;
+constexpr size_t N{41};
+
+int main(void)
+{
+    auto u_data = std::make_unique<double[]>(N * N * 3);
+    vector_field_t u_field{u_data.get(), extents_t{N, N}};
+    std::vector<double> v{3.1, 3.2, 3.4};
+
+    gen::invoke(u_field, v);
+
+    for (size_t j = 0; j < N; ++j)
+        for (size_t i = 0; i < N; ++i)
+        {
+            assert(u_field(j, i, 0) == v[0]);
+            assert(u_field(j, i, 1) == v[1]);
+            assert(u_field(j, i, 2) == v[2]);
+        }
+}
\ No newline at end of file

tests/generator_scripts/source/VectorExtraction.py

+from pystencilssfg import SourceFileGenerator
+from pystencilssfg.lang.cpp import std
+import pystencils as ps
+import sympy as sp
+
+std.mdspan.configure(namespace="std::experimental", header="<experimental/mdspan>")
+
+with SourceFileGenerator() as sfg:
+    sfg.namespace("gen")
+
+    u_field = ps.fields("u(3): double[2D]", layout="c")
+    u = sp.symbols("u_:3")
+
+    asms = [ps.Assignment(u_field(i), u[i]) for i in range(3)]
+    ker = sfg.kernels.create(asms)
+
+    sfg.function("invoke")(
+        sfg.map_field(u_field, std.mdspan.from_field(u_field, layout_policy="layout_right")),
+        sfg.map_vector(u, std.vector("double", const=True, ref=True).var("vel")),
+        sfg.call(ker)
+    )

tests/generator_scripts/source/config/TestConfigModule_cfg.py

+from pystencilssfg import SfgConfig
+
+
+def configure_sfg(cfg: SfgConfig):
+    cfg.outer_namespace = "myproject"
+    cfg.codestyle.indent_width = 3
+
+
+def project_info():
+    return {
+        "use_openmp": True,
+        "use_cuda": True,
+        "float_format": "float32",
+    }

tests/generator_scripts/test_generator_scripts.py

+"""Test suite for generator scripts.
+
+For more information, refer to the `README.md` file in the same directory.
+"""
+
+import pytest
+
+import pathlib
+import yaml
+import re
+import shutil
+import warnings
+import subprocess
+
+from pystencils.include import get_pystencils_include_path
+
+THIS_DIR = pathlib.Path(__file__).parent
+
+DEPS_DIR = THIS_DIR / "deps"
+MDSPAN_QUAL_PATH = "mdspan-mdspan-0.6.0/include/"
+
+PYSTENCILS_RT_INCLUDE_PATH = get_pystencils_include_path()
+
+TEST_INDEX = THIS_DIR / "index.yaml"
+SOURCE_DIR = THIS_DIR / "source"
+EXPECTED_DIR = THIS_DIR / "expected"
+CXX_INCLUDE_FLAGS = [
+    "-I",
+    f"{DEPS_DIR}/{MDSPAN_QUAL_PATH}",
+    "-I",
+    PYSTENCILS_RT_INCLUDE_PATH,
+]
+
+
+def prepare_deps():
+    mdspan_archive_url = (
+        "https://github.com/kokkos/mdspan/archive/refs/tags/mdspan-0.6.0.zip"
+    )
+    mdspan_path = DEPS_DIR / MDSPAN_QUAL_PATH
+
+    import fasteners
+
+    with fasteners.InterProcessLock(THIS_DIR / ".get-deps.lock"):
+        if not mdspan_path.exists():
+            DEPS_DIR.mkdir(parents=True, exist_ok=True)
+            print("Downloading mdspan reference implementation...")
+
+            import requests
+            import tempfile
+            from zipfile import ZipFile
+
+            server_resp = requests.get(mdspan_archive_url)
+            with tempfile.TemporaryFile() as tmpfile:
+                tmpfile.write(server_resp.content)
+
+                with ZipFile(tmpfile) as archive:
+                    for name in archive.namelist():
+                        if name.startswith(MDSPAN_QUAL_PATH):
+                            archive.extract(name, DEPS_DIR)
+
+
+class GenScriptTest:
+    @classmethod
+    def make(cls, name, test_description: dict):
+        return pytest.param(cls(name, test_description), id=f"{name}.py")
+
+    def __init__(self, name: str, test_description: dict):
+        self._name = name
+        script_path = SOURCE_DIR / f"{name}.py"
+
+        if not script_path.exists():
+            raise ValueError(f"Script {script_path.name} does not exist.")
+
+        self._script_path = script_path
+
+        self._output_dir: pathlib.Path
+        self._script_args = []
+
+        expected_extensions = ["cpp", "hpp"]
+
+        sfg_args: dict = test_description.get("sfg-args", dict())
+
+        if (header_only := sfg_args.get("header-only", None)) is not None:
+            if header_only:
+                expected_extensions = ["hpp"]
+                self._script_args += ["--sfg-header-only"]
+            else:
+                self._script_args += ["--no--sfg-header-only"]
+
+        if (file_exts := sfg_args.get("file-extensions", None)) is not None:
+            expected_extensions = file_exts
+            self._script_args += ["--sfg-file-extensions", ",".join(file_exts)]
+
+        if (config_module := sfg_args.get("config-module", None)) is not None:
+            config_module = SOURCE_DIR / config_module
+            self._script_args += ["--sfg-config-module", str(config_module)]
+
+        self._script_args += test_description.get("extra-args", [])
+
+        self._expected_extensions = test_description.get(
+            "expected-output", expected_extensions
+        )
+
+        self._expect_failure = test_description.get("expect-failure", False)
+
+        self._expected_files: set[str] = set()
+        self._files_to_compile: list[str] = []
+
+        for ext in self._expected_extensions:
+            fname = f"{self._name}.{ext}"
+            self._expected_files.add(fname)
+            if ext in ("cpp", "cxx", "c++", "cu", "hip"):
+                self._files_to_compile.append(fname)
+
+        compile_descr: dict = test_description.get("compile", dict())
+        cxx_compiler: str = compile_descr.get("cxx", "g++")
+
+        skip_if_no_compiler: bool = compile_descr.get("skip-if-not-found", False)
+        cxx_options: list[str] = compile_descr.get(
+            "cxx-flags", ["--std=c++20", "-Wall", "-Werror"]
+        )
+        link_options: list[str] = compile_descr.get("link-flags", [])
+
+        self._compile_cmd: list[str] | None
+        self._link_cmd: list[str] | None
+
+        if shutil.which(cxx_compiler) is None:
+            if skip_if_no_compiler:
+                warnings.warn(
+                    f"[Test/{self._name}] Requested compiler {cxx_compiler} is not available."
+                )
+                self._compile_cmd = self._link_cmd = None
+            else:
+                pytest.fail(f"Requested compiler {cxx_compiler} is not available.")
+        else:
+            self._compile_cmd = (
+                [cxx_compiler] + cxx_options + CXX_INCLUDE_FLAGS + ["-c"]
+            )
+            self._link_cmd = [cxx_compiler] + link_options
+
+        self._expect_code: dict = test_description.get("expect-code", dict())
+
+        harness_file = SOURCE_DIR / f"{self._name}.harness.cpp"
+        if harness_file.exists():
+            self._harness = harness_file
+        else:
+            self._harness = None
+
+    def run(self, output_dir: pathlib.Path):
+        self._output_dir = output_dir
+
+        self.run_script()
+
+        if self._expect_failure:
+            return
+
+        self.check_output_files()
+        self.compile_files()
+        self.run_harness()
+
+    def run_script(self):
+        args = [
+            "python",
+            str(self._script_path),
+            "--sfg-output-dir",
+            str(self._output_dir),
+        ] + list(self._script_args)
+        result = subprocess.run(args)
+
+        if self._expect_failure:
+            if result.returncode == 0:
+                pytest.fail(
+                    f"Generator script {self._script_path.name} was expected to fail, but didn't."
+                )
+        elif result.returncode != 0:
+            pytest.fail(f"Generator script {self._script_path.name} failed.")
+
+    def check_output_files(self):
+        output_files = set(p.name for p in self._output_dir.iterdir())
+        assert output_files == self._expected_files
+
+        for fp in self._output_dir.iterdir():
+            self.check_file(fp)
+
+    def check_file(self, actual_file: pathlib.Path):
+        with actual_file.open("r") as f:
+            actual_code = f.read()
+
+        extension = actual_file.name.split(".")[1]
+        if (expectations := self._expect_code.get(extension, None)) is not None:
+            for expectation in expectations:
+                if isinstance(expectation, str):
+                    assert (
+                        expectation in actual_code
+                    ), f"Did not find expected code string in contents of {actual_file.name}:\n{expectation}"
+                elif isinstance(expectation, dict):
+                    if (regex := expectation.get("regex", None)) is not None:
+                        if expectation.get("strip-whitespace", False):
+                            regex = "".join(regex.split())
+                        matcher = re.compile(regex)
+                        count = expectation.get("count", 1)
+                        findings = matcher.findall(actual_code)
+                        assert len(findings) == count, (
+                            f"Regex {regex} matched incorrect number of times in generated code in {actual_file.name}:"
+                            f"\nExpected {count}, got {len(findings)}"
+                        )
+
+    def compile_files(self):
+        if self._compile_cmd is None:
+            return
+
+        #   Check if output compiles
+        for file in self._files_to_compile:
+            compile_args = self._compile_cmd + [file]
+            compile_result = subprocess.run(compile_args, cwd=str(self._output_dir))
+
+            if compile_result.returncode != 0:
+                cmd_str = " ".join(compile_args)
+                pytest.fail(
+                    "Compilation of generated files failed: \n"
+                    f"    Command: {cmd_str}"
+                )
+
+        if self._harness is not None:
+            compile_args = self._compile_cmd + [
+                "-I",
+                str(self._output_dir),
+                str(self._harness),
+            ]
+            compile_result = subprocess.run(compile_args, cwd=str(self._output_dir))
+
+            if compile_result.returncode != 0:
+                pytest.fail(f"Compilation of test harness for {self._name} failed.")
+
+    def run_harness(self):
+        if self._compile_cmd is None:
+            return
+
+        #   Run after `compile`; i.e. for all compilable generated files, objects are already present
+        if self._harness is not None:
+            objects = self._output_dir.glob("*.o")
+            linker_args = self._link_cmd + [str(obj) for obj in objects]
+            linker_result = subprocess.run(linker_args, cwd=str(self._output_dir))
+
+            if linker_result.returncode != 0:
+                pytest.fail(f"Linking to test harness for {self._name} failed.")
+
+            exe_args = "./a.out"
+            exe_result = subprocess.run(exe_args, cwd=str(self._output_dir))
+            if exe_result.returncode != 0:
+                pytest.fail(f"Execution of test harness for {self._name} failed.")
+
+
+def discover() -> list[GenScriptTest]:
+    with TEST_INDEX.open() as indexfile:
+        index = yaml.safe_load(indexfile.read())
+
+    tests = []
+    for name, descr in index.items():
+        if descr is None:
+            descr = dict()
+        tests.append(GenScriptTest.make(name, descr))
+    return tests
+
+
+DISCOVERED_TESTS = discover()
+
+
+@pytest.mark.parametrize("test_descriptor", DISCOVERED_TESTS)
+def test_generator_script(test_descriptor: GenScriptTest, tmp_path):
+    prepare_deps()
+    test_descriptor.run(tmp_path)

tests/integration/cmake_project/.gitignore

+FindPystencilsSfg.cmake
\ No newline at end of file

tests/integration/cmake_project/CMakeLists.txt

+cmake_minimum_required( VERSION 3.22 )
+project( sfg_cmake_project_test )
+
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR})
+
+find_package( PystencilsSfg REQUIRED )
+
+if(NOT ${PystencilsSfg_FOUND})
+    message( FATAL_ERROR "PystencilsSfg_FOUND was not set even though find_package returned successfully. This is an error." )
+endif()
+
+set( UseGlobalCfgModule OFF CACHE BOOL "Specify config module globally" )
+set( UseLocalCfgModule OFF CACHE BOOL "Specify config module locally" )
+
+if( $CACHE{UseGlobalCfgModule} )
+    set( PystencilsSfg_CONFIG_MODULE ${CMAKE_CURRENT_SOURCE_DIR}/gen_config.py )
+endif()
+
+add_executable( TestApp TestApp.cpp )
+
+if( $CACHE{UseLocalCfgModule} )
+    pystencilssfg_generate_target_sources(
+        TestApp
+        SCRIPTS GenTest.py
+        CONFIG_MODULE ${CMAKE_CURRENT_SOURCE_DIR}/gen_config.py
+    )
+else()
+    pystencilssfg_generate_target_sources(
+        TestApp
+        SCRIPTS GenTest.py
+    )
+endif()
+
+pystencilssfg_generate_target_sources(
+    TestApp
+    SCRIPTS CliTest.py
+    SCRIPT_ARGS apples bananas unicorns
+    HEADER_ONLY
+)
+
+pystencilssfg_generate_target_sources(
+    TestApp
+    SCRIPTS CustomDirTest.py
+    OUTPUT_DIRECTORY my-output
+    HEADER_ONLY
+)

tests/integration/cmake_project/CliTest.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator(keep_unknown_argv=True) as sfg:
+    sfg.include("<string>")
+    for i, arg in enumerate(sfg.context.argv):
+        sfg.code(f"constexpr std::string arg{i} = \"{arg}\";")

tests/integration/cmake_project/CustomDirTest.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator() as sfg:
+    sfg.code("#define NOTHING")

tests/integration/cmake_project/GenTest.py

+from pystencilssfg import SourceFileGenerator
+
+with SourceFileGenerator() as sfg:
+    sfg.namespace("gen")
+    retval = 42 if sfg.context.project_info is None else sfg.context.project_info
+
+    sfg.function("getValue", return_type="int")(
+        f"return {retval};"
+    )

tests/integration/cmake_project/TestApp.cpp

+#include "gen/GenTest.hpp"
+
+int main(void) {
+    return int( gen::getValue() );
+}

tests/integration/cmake_project/gen_config.py

+from pystencilssfg import SfgConfig
+
+
+def configure_sfg(cfg: SfgConfig):
+    cfg.extensions.impl = "c++"
+
+
+def project_info():
+    return 31