diff --git a/dashboards/dashboard_walberla.py b/dashboards/dashboard_walberla.py
index 2b58fbff33f56d505e65b8a616d07c549f2d2664..e1257443d521069ebd494da6b58bd864706b28d5 100644
--- a/dashboards/dashboard_walberla.py
+++ b/dashboards/dashboard_walberla.py
@@ -1,49 +1,51 @@
-from dashboards.dashboard_base import (DashboardOptions,
- build_dashboard,
- get_commit_annotation,
- pack_in_row,
- Repeat,
- get_grid_pos)
-
-
-from dashboards.panels import PanelInfos, get_time_series_panel
-from dashboards.variables import get_dashboard_variable, Filter
-from dashboards.overrides import get_color_regex_override
-
-from dashboards.influx_queries import join_variable_and
-from dashboards.legends import Units
-
from cbutil.ncu_keys import (
- memory_write_data_key,
- memory_read_data_key,
+ dp_key,
+ memory_bandwidth_key,
memory_data_key,
- memory_write_bandwidth_key,
memory_read_bandwidth_key,
- memory_bandwidth_key,
- runtime_key,
+ memory_read_data_key,
+ memory_write_bandwidth_key,
+ memory_write_data_key,
operational_intensity_key,
p_max_key,
- dp_key,
+ runtime_key,
+)
+from dashboards.dashboard_base import (
+ DashboardOptions,
+ Repeat,
+ build_dashboard,
+ get_commit_annotation,
+ get_grid_pos,
+ pack_in_row,
+)
+from dashboards.influx_queries import join_variable_and
+from dashboards.legends import Units
+from dashboards.overrides import get_color_regex_override
+from dashboards.panels import PanelInfos, get_time_series_panel
+from dashboards.variables import (
+ Filter,
+ get_dashboard_variable,
+ get_time_dependend_dashboard_variable,
)
def dashboard_uniformgridcpu():
- data_source = 'InfluxDB-1'
- arch = 'CPU'
- measurment_name = f'UniformGrid{arch}'
+ data_source = "InfluxDB-1"
+ arch = "CPU"
+ measurment_name = f"UniformGrid{arch}"
row_repeat = "host"
options = DashboardOptions(
- title=f'Uniform Grid {arch}',
+ title=f"Uniform Grid {arch}",
description=f"Benchmark dashboard for the Uniform Grid {arch} Benchmark from walberla",
- tags=[arch, 'benchmark', 'walberla', 'Uniform Grid'],
+ tags=[arch, "benchmark", "walberla", "Uniform Grid"],
timezone="browser",
)
filters = [
Filter(row_repeat),
- Filter("project_id", multi=True, default_value='walberla/walberla'),
- Filter("branch", multi=True, default_value='master'),
+ Filter("project_id", multi=True, default_value="walberla/walberla"),
+ Filter("branch", multi=True, default_value="master"),
Filter("collisionSetup"),
Filter("mpi_num_processes"),
Filter("streamingPattern"),
@@ -60,27 +62,48 @@ def dashboard_uniformgridcpu():
Filter("periodic_2"),
]
- fields = [PanelInfos("Runtime (RDTSC) [s] STAT Max", Units.seconds),
- PanelInfos("mlupsPerProcess", Units.number),
- PanelInfos("DP [MFLOP/s] STAT Sum", Units.mflop_sec),
- PanelInfos("Operational intensity STAT Sum", Units.flop_per_byte),
- PanelInfos('AVX DP [MFLOP/s] STAT Sum" / "DP [MFLOP/s] STAT Sum', Units.percent),
- PanelInfos('Memory bandwidth [MBytes/s] STAT Sum', Units.mbytes_per_second),
- PanelInfos('Memory write bandwidth [MBytes/s] STAT Sum', Units.mbytes_per_second),
- PanelInfos('Memory read bandwidth [MBytes/s] STAT Sum', Units.mbytes_per_second),
- PanelInfos('Memory data volume [GBytes] STAT Sum', Units.gigabyte),
- PanelInfos('Memory read data volume [GBytes] STAT Sum', Units.gigabyte),
- PanelInfos('Memory write data volume [GBytes] STAT Sum', Units.gigabyte),
- PanelInfos('Energy [J] STAT Sum', Units.joule),
- PanelInfos('Power [W] STAT Sum', Units.watt),
- PanelInfos('Clock [MHz] STAT Avg', Units.megahertz), ]
-
- filter_vars = [get_dashboard_variable(filter, measurment_name, data_source) for filter in filters]
+ fields = [
+ PanelInfos("Runtime (RDTSC) [s] STAT Max", Units.seconds),
+ PanelInfos("mlupsPerProcess", Units.number),
+ PanelInfos("DP [MFLOP/s] STAT Sum", Units.mflop_sec),
+ PanelInfos("Operational intensity STAT Sum", Units.flop_per_byte),
+ PanelInfos(
+ 'AVX DP [MFLOP/s] STAT Sum" / "DP [MFLOP/s] STAT Sum', Units.percent
+ ),
+ PanelInfos(
+ "Memory bandwidth [MBytes/s] STAT Sum", Units.mbytes_per_second),
+ PanelInfos(
+ "Memory write bandwidth [MBytes/s] STAT Sum", Units.mbytes_per_second
+ ),
+ PanelInfos(
+ "Memory read bandwidth [MBytes/s] STAT Sum", Units.mbytes_per_second
+ ),
+ PanelInfos("Memory data volume [GBytes] STAT Sum", Units.gigabyte),
+ PanelInfos(
+ "Memory read data volume [GBytes] STAT Sum", Units.gigabyte),
+ PanelInfos(
+ "Memory write data volume [GBytes] STAT Sum", Units.gigabyte),
+ PanelInfos("Energy [J] STAT Sum", Units.joule),
+ PanelInfos("Power [W] STAT Sum", Units.watt),
+ PanelInfos("Clock [MHz] STAT Avg", Units.megahertz),
+ ]
+
+ filter_vars = [
+ get_dashboard_variable(filter, measurment_name, data_source)
+ for filter in filters[1:]
+ ]
+ filter_vars = [
+ get_time_dependend_dashboard_variable(
+ filters[0], measurment_name, data_source, inner_field_key="mlupsPerProcess"
+ ),
+ *filter_vars,
+ ]
row_repeat_var = [fv for fv in filter_vars if fv.name == row_repeat][0]
where = join_variable_and([f.name for f in filters])
- annotations = get_commit_annotation(data_source, "red", "commits", measurment_name)
+ annotations = get_commit_annotation(
+ data_source, "red", "commits", measurment_name)
panels = [
get_time_series_panel(
@@ -90,36 +113,36 @@ def dashboard_uniformgridcpu():
where=where,
group_by=[f.name for f in filters],
)
- for field in fields]
+ for field in fields
+ ]
row = pack_in_row(
title=f"{row_repeat}: ${row_repeat_var.name}",
panels=[*panels],
- repeat=Repeat('v', row_repeat_var.name),
+ repeat=Repeat("v", row_repeat_var.name),
+ )
+ return build_dashboard(
+ options, rows=[row], templating=[*filter_vars], annotations=annotations
)
- return build_dashboard(options,
- rows=[row],
- templating=[*filter_vars],
- annotations=annotations)
def dashboard_uniformgridgpu():
- data_source = 'InfluxDB-1'
- arch = 'GPU'
- measurment_name = f'UniformGrid{arch}'
+ data_source = "InfluxDB-1"
+ arch = "GPU"
+ measurment_name = f"UniformGrid{arch}"
row_repeat = "host"
options = DashboardOptions(
- title=f'Uniform Grid {arch}',
+ title=f"Uniform Grid {arch}",
description=f"Benchmark dashboard for the Uniform Grid {arch} Benchmark from walberla",
- tags=[arch, 'benchmark', 'walberla', 'Uniform Grid'],
+ tags=[arch, "benchmark", "walberla", "Uniform Grid"],
timezone="browser",
)
filters = [
- Filter("project_id", multi=True, default_value='walberla/walberla'),
- Filter("branch", multi=True, default_value='master'),
- Filter("host", multi=False, default_value='medusa'),
+ Filter("host", multi=False, default_value="medusa"),
+ Filter("project_id", multi=True, default_value="walberla/walberla"),
+ Filter("branch", multi=True, default_value="master"),
Filter("GPU"),
Filter("collisionSetup"),
Filter("mpi_num_processes"),
@@ -140,14 +163,20 @@ def dashboard_uniformgridgpu():
Filter("periodic_2"),
]
- fields = [PanelInfos("mlupsPerProcess", Units.number), ]
+ fields = [
+ PanelInfos("mlupsPerProcess", Units.number),
+ ]
- filter_vars = [get_dashboard_variable(filter, measurment_name, data_source) for filter in filters]
+ filter_vars = [
+ get_dashboard_variable(filter, measurment_name, data_source)
+ for filter in filters
+ ]
row_repeat_var = [fv for fv in filter_vars if fv.name == row_repeat][0]
where = join_variable_and([f.name for f in filters])
- annotations = get_commit_annotation(data_source, "red", "commits", measurment_name)
+ annotations = get_commit_annotation(
+ data_source, "red", "commits", measurment_name)
panels = [
get_time_series_panel(
@@ -157,36 +186,36 @@ def dashboard_uniformgridgpu():
where=where,
group_by=[f.name for f in filters],
)
- for field in fields]
+ for field in fields
+ ]
row = pack_in_row(
title=f"{row_repeat}: ${row_repeat_var.name}",
panels=[*panels],
- repeat=Repeat('v', row_repeat_var.name),
+ repeat=Repeat("v", row_repeat_var.name),
+ )
+ return build_dashboard(
+ options, rows=[row], templating=[*filter_vars], annotations=annotations
)
- return build_dashboard(options,
- rows=[row],
- templating=[*filter_vars],
- annotations=annotations)
def dashboard_uniformgridgpu_profile():
- data_source = 'InfluxDB-1'
- arch = 'GPU'
- measurment_name = f'UniformGrid{arch}_profile'
+ data_source = "InfluxDB-1"
+ arch = "GPU"
+ measurment_name = f"UniformGrid{arch}_profile"
row_repeat = "host"
options = DashboardOptions(
- title=f'Uniform Grid {arch} GPU Profiling',
+ title=f"Uniform Grid {arch} GPU Profiling",
description=f"Benchmark dashboard for the Uniform Grid {arch} Benchmark from walberla",
- tags=[arch, 'benchmark', 'walberla', 'Uniform Grid', 'profiling'],
+ tags=[arch, "benchmark", "walberla", "Uniform Grid", "profiling"],
timezone="browser",
)
filters = [
- Filter("project_id", multi=True, default_value='walberla/walberla'),
- Filter("branch", multi=True, default_value='master'),
- Filter("host", multi=False, default_value='medusa'),
+ Filter("host", multi=False, default_value="medusa"),
+ Filter("project_id", multi=True, default_value="walberla/walberla"),
+ Filter("branch", multi=True, default_value="master"),
Filter("GPU"),
Filter("collisionSetup"),
Filter("mpi_num_processes"),
@@ -207,25 +236,30 @@ def dashboard_uniformgridgpu_profile():
Filter("periodic_2"),
]
- fields = [PanelInfos(runtime_key, Units.seconds),
- PanelInfos(dp_key, Units.mflop_sec),
- PanelInfos(p_max_key, Units.mflop_sec),
- PanelInfos(f'{dp_key}"/"{p_max_key}', Units.percent),
- PanelInfos(operational_intensity_key, Units.flop_per_byte),
- PanelInfos(memory_bandwidth_key, Units.mbytes_per_second),
- PanelInfos(memory_write_bandwidth_key, Units.mbytes_per_second),
- PanelInfos(memory_read_bandwidth_key, Units.mbytes_per_second),
- PanelInfos(memory_data_key, Units.gigabyte),
- PanelInfos(memory_write_data_key, Units.gigabyte),
- PanelInfos(memory_read_data_key, Units.gigabyte),
- ]
-
- filter_vars = [get_dashboard_variable(filter, measurment_name, data_source) for filter in filters]
+ fields = [
+ PanelInfos(runtime_key, Units.seconds),
+ PanelInfos(dp_key, Units.mflop_sec),
+ PanelInfos(p_max_key, Units.mflop_sec),
+ PanelInfos(f'{dp_key}"/"{p_max_key}', Units.percent),
+ PanelInfos(operational_intensity_key, Units.flop_per_byte),
+ PanelInfos(memory_bandwidth_key, Units.mbytes_per_second),
+ PanelInfos(memory_write_bandwidth_key, Units.mbytes_per_second),
+ PanelInfos(memory_read_bandwidth_key, Units.mbytes_per_second),
+ PanelInfos(memory_data_key, Units.gigabyte),
+ PanelInfos(memory_write_data_key, Units.gigabyte),
+ PanelInfos(memory_read_data_key, Units.gigabyte),
+ ]
+
+ filter_vars = [
+ get_dashboard_variable(filter, measurment_name, data_source)
+ for filter in filters
+ ]
row_repeat_var = [fv for fv in filter_vars if fv.name == row_repeat][0]
where = join_variable_and([f.name for f in filters])
- annotations = get_commit_annotation(data_source, "red", "commits", measurment_name)
+ annotations = get_commit_annotation(
+ data_source, "red", "commits", measurment_name)
panels = [
get_time_series_panel(
@@ -235,44 +269,50 @@ def dashboard_uniformgridgpu_profile():
where=where,
group_by=[f.name for f in filters],
)
- for field in fields]
+ for field in fields
+ ]
row = pack_in_row(
title=f"{row_repeat}: ${row_repeat_var.name}",
panels=[*panels],
- repeat=Repeat('v', row_repeat_var.name),
+ repeat=Repeat("v", row_repeat_var.name),
+ )
+ return build_dashboard(
+ options, rows=[row], templating=[*filter_vars], annotations=annotations
)
- return build_dashboard(options,
- rows=[row],
- templating=[*filter_vars],
- annotations=annotations)
def dashboard_fslbmgravitywave():
- data_source = 'InfluxDB-1'
- arch = 'CPU'
- measurment_name = 'FSLBM_gravitywave'
+ data_source = "InfluxDB-1"
+ arch = "CPU"
+ measurment_name = "FSLBM_gravitywave"
options = DashboardOptions(
- title='Free Surface LBM - Gravity Wave',
+ title="Free Surface LBM - Gravity Wave",
description="Benchmark dashboard for the FSBLM Gravitywave Benchmark from walberla",
- tags=[arch, 'benchmark', 'walberla', 'FSLBM'],
+ tags=[arch, "benchmark", "walberla", "FSLBM"],
timezone="browser",
)
filters = [
Filter("host"),
- Filter("project_id", multi=True, default_value='walberla/walberla'),
- Filter("branch", multi=True, default_value='master'),
+ Filter("project_id", multi=True, default_value="walberla/walberla"),
+ Filter("branch", multi=True, default_value="master"),
Filter("numMPIProcesses"),
]
- fields = [PanelInfos("simulationTime", Units.seconds), ]
+ fields = [
+ PanelInfos("simulationTime", Units.seconds),
+ ]
- filter_vars = [get_dashboard_variable(filter, measurment_name, data_source) for filter in filters]
+ filter_vars = [
+ get_dashboard_variable(filter, measurment_name, data_source)
+ for filter in filters
+ ]
where = join_variable_and([f.name for f in filters])
- annotations = get_commit_annotation(data_source, "red", "commits", measurment_name)
+ annotations = get_commit_annotation(
+ data_source, "red", "commits", measurment_name)
groups = ["Communication", "Sweep", "MPI Barrier"]
colors = ["red", "green", "blue"]
overrides = [
@@ -287,9 +327,10 @@ def dashboard_fslbmgravitywave():
measurment_name,
where=where,
group_by=[f.name for f in filters],
- gridPos=get_grid_pos(12, 24, 0, idx*(13))
+ gridPos=get_grid_pos(12, 24, 0, idx * (13)),
)
- for idx, field in enumerate(fields)]
+ for idx, field in enumerate(fields)
+ ]
panels.append(
get_time_series_panel(
PanelInfos("/.*_percentage/", Units.percent),
@@ -297,7 +338,7 @@ def dashboard_fslbmgravitywave():
measurment_name,
where=where,
group_by=[f.name for f in filters],
- gridPos=get_grid_pos(12, 12, 0, len(fields)*(13)),
+ gridPos=get_grid_pos(12, 12, 0, len(fields) * (13)),
drawStyle="bars",
stacking={"mode": "normal"},
fillOpacity=70,
@@ -305,7 +346,8 @@ def dashboard_fslbmgravitywave():
tooltipMode="single",
overrides=overrides,
colorMode="fixed",
- ))
+ )
+ )
panels.append(
get_time_series_panel(
@@ -314,13 +356,14 @@ def dashboard_fslbmgravitywave():
measurment_name,
where=where,
group_by=[f.name for f in filters],
- gridPos=get_grid_pos(12, 12, 12, len(fields)*(13)),
+ gridPos=get_grid_pos(12, 12, 12, len(fields) * (13)),
drawStyle="bars",
stacking={"mode": "normal"},
fillOpacity=70,
gradientMode="Opacity",
tooltipMode="single",
- ))
+ )
+ )
panels.append(
get_time_series_panel(
@@ -329,13 +372,14 @@ def dashboard_fslbmgravitywave():
measurment_name,
where=where,
group_by=[f.name for f in filters],
- gridPos=get_grid_pos(12, 12, 0, (len(fields)+1)*(13)),
+ gridPos=get_grid_pos(12, 12, 0, (len(fields) + 1) * (13)),
drawStyle="bars",
stacking={"mode": "normal"},
fillOpacity=70,
gradientMode="Opacity",
tooltipMode="single",
- ))
+ )
+ )
panels.append(
get_time_series_panel(
@@ -344,15 +388,15 @@ def dashboard_fslbmgravitywave():
measurment_name,
where=where,
group_by=[f.name for f in filters],
- gridPos=get_grid_pos(12, 12, 12, (len(fields)+1)*(13)),
+ gridPos=get_grid_pos(12, 12, 12, (len(fields) + 1) * (13)),
drawStyle="bars",
stacking={"mode": "normal"},
fillOpacity=70,
gradientMode="Opacity",
tooltipMode="single",
- ))
+ )
+ )
- return build_dashboard(options,
- panels=panels,
- templating=filter_vars,
- annotations=annotations)
+ return build_dashboard(
+ options, panels=panels, templating=filter_vars, annotations=annotations
+ )
diff --git a/dashboards/influx_queries.py b/dashboards/influx_queries.py
index ac09048759f0f3174670ed217ee250eb8c40a478..d2113cfa50d8c7a1d4d9df3e3362c15ccbb4a8ae 100644
--- a/dashboards/influx_queries.py
+++ b/dashboards/influx_queries.py
@@ -14,25 +14,25 @@ class Query:
def _get_select(self):
if self.select_string:
- return f'SELECT \"{self.select_}\" '
+ return f'SELECT "{self.select_}" '
else:
- return f'SELECT {self.select_} '
+ return f"SELECT {self.select_} "
def _get_from(self):
if self.from_string:
- return f'FROM \"{self.from_}\" '
+ return f'FROM "{self.from_}" '
else:
- return f'FROM {self.from_} '
+ return f"FROM {self.from_} "
def _get_where(self):
- return f'WHERE ({self.where_}) AND $timeFilter '
+ return f"WHERE ({self.where_}) AND $timeFilter "
def _get_group_by(self):
if self.group_by:
- group_by = ', '.join(f'"{tag}"' for tag in self.group_by)
- return f'GROUP BY {group_by}'
+ group_by = ", ".join(f'"{tag}"' for tag in self.group_by)
+ return f"GROUP BY {group_by}"
else:
- return ''
+ return ""
def __str__(self):
ret = self._get_select()
@@ -49,7 +49,15 @@ def show_tag_values(table: str, key_name: str) -> str:
if table != "":
from_part = f'FROM "{table}" '
- return f"{base} {from_part}WITH key = \"{key_name}\""
+ return f'{base} {from_part}WITH key = "{key_name}"'
+
+
+def get_tag_values(table: str, key_name: str, *, inner_field_key="") -> str:
+ inner_select = "*"
+ if inner_field_key != "":
+ inner_select = f'"{inner_field_key}", "{key_name}"'
+ inner_query = f'SELECT {inner_select} FROM "{table}" WHERE $timeFilter'
+ return f'SELECT distinct("{key_name}") FROM ({inner_query}) GROUP BY {key_name}'
def get_variable_condition(variable_name: str, *, tag_key: str = None) -> str:
@@ -75,12 +83,11 @@ def join_conditions(conditions: List[str], operators: Union[List[str], str]):
ret = conditions[0]
for op, cond in zip(ops, conditions[1:]):
- ret += f' {op} {cond}'
+ ret += f" {op} {cond}"
return ret
def join_variable_and(variable_names: List[str]) -> str:
return join_conditions(
- [get_variable_condition(name) for name in variable_names],
- "AND"
+ [get_variable_condition(name) for name in variable_names], "AND"
)
diff --git a/dashboards/variables.py b/dashboards/variables.py
index 22a6d44a5368339179aaa566e76ef89383a7fddb..63dc836494edb2daf61dc0fdc8b82c4a45de7b9f 100644
--- a/dashboards/variables.py
+++ b/dashboards/variables.py
@@ -1,13 +1,15 @@
from grafanalib.core import Template
from collections import namedtuple
-from dashboards.influx_queries import show_tag_values
+from dashboards.influx_queries import show_tag_values, get_tag_values
-Filter = namedtuple("Filter", ("name", "multi", "default_value"), defaults=('', True, ''))
+Filter = namedtuple(
+ "Filter", ("name", "multi", "default_value"), defaults=("", True, "")
+)
-def get_measurement_filter(name, data_source,
- *, filter_pattern: str = None,
- **kwargs) -> Template:
+def get_measurement_filter(
+ name, data_source, *, filter_pattern: str = None, **kwargs
+) -> Template:
query = "SHOW MEASUREMENTS"
if filter_pattern:
query += f" WITH MEASUREMENT =~ /{filter_pattern}/"
@@ -16,21 +18,49 @@ def get_measurement_filter(name, data_source,
def get_dashboard_variable(filter: Filter, measurment_name: str, data_source: str):
query = show_tag_values(measurment_name, filter.name)
- kwargs = {'includeAll': filter.multi, 'multi': filter.multi, }
+ kwargs = {
+ "includeAll": filter.multi,
+ "multi": filter.multi,
+ }
if filter.default_value:
- kwargs.update({'default': filter.default_value})
+ kwargs.update({"default": filter.default_value})
return get_dashboard_variable_query(filter.name, query, data_source, **kwargs)
-def get_dashboard_variable_query(name: str, template_query: str,
- dataSource: str, *,
- includeAll=True,
- multi=True,
- **kwargs) -> Template:
- return Template(name,
- template_query,
- dataSource=dataSource,
- type="query",
- includeAll=includeAll,
- multi=multi,
- **kwargs)
+def get_time_dependend_dashboard_variable(
+ filter: Filter,
+ measurment_name: str,
+ data_source: str,
+ *,
+ inner_field_key: str = "",
+):
+ query = get_tag_values(
+ measurment_name, filter.name, inner_field_key=inner_field_key
+ )
+ kwargs = {
+ "includeAll": filter.multi,
+ "multi": filter.multi,
+ }
+ if filter.default_value:
+ kwargs.update({"default": filter.default_value})
+ return get_dashboard_variable_query(filter.name, query, data_source, **kwargs)
+
+
+def get_dashboard_variable_query(
+ name: str,
+ template_query: str,
+ dataSource: str,
+ *,
+ includeAll=True,
+ multi=True,
+ **kwargs,
+) -> Template:
+ return Template(
+ name,
+ template_query,
+ dataSource=dataSource,
+ type="query",
+ includeAll=includeAll,
+ multi=multi,
+ **kwargs,
+ )