diff --git a/dashboards/dashboard_base.py b/dashboards/dashboard_base.py
index d6dfa481871e7d258219f46ef1ddfe0461cb6cae..ccbc81c197a6a187a144a2b5f15ea9549c215fe7 100644
--- a/dashboards/dashboard_base.py
+++ b/dashboards/dashboard_base.py
@@ -14,10 +14,11 @@ from dashboards.annotations import Annotation
from dashboards.influx_queries import Query
-def get_influx_target(target_query: str, result_format: str = 'time_series', **kwargs) -> InfluxDBTarget:
+def get_influx_target(target_query: str, result_format: str = 'time_series', alias: str = "", **kwargs) -> InfluxDBTarget:
return InfluxDBTarget(
query=target_query,
format=result_format,
+ alias=alias,
**{k: v for k, v in kwargs.items() if v is not None},
)
diff --git a/dashboards/dashboard_walberla.py b/dashboards/dashboard_walberla.py
index 501f453887a6a3ba685ebdbce3c5e021847ba558..78410e304b84aa44fb2ee2f64c43e13f168ba795 100644
--- a/dashboards/dashboard_walberla.py
+++ b/dashboards/dashboard_walberla.py
@@ -18,7 +18,7 @@ from dashboards.dashboard_base import (
get_grid_pos,
pack_in_row,
)
-from dashboards.influx_queries import join_variable_and, get_variable_condition_with_tag
+from dashboards.influx_queries import join_variable_and, get_variable_condition_with_tag, get_variable_tag, join_conditions, get_variable_condition, get_variable_condition_without_regex
from dashboards.legends import Units
from dashboards.overrides import get_color_regex_override
from dashboards.panels import PanelInfos, get_time_series_panel, get_table_panel, get_bar_chart_panel
@@ -28,6 +28,11 @@ from dashboards.variables import (
get_time_dependend_dashboard_variable,
get_field_keys_dashboard_variable,
)
+from dashboards.influx_queries import Query
+
+
+def is_regex(name):
+ return name[0] == "/" and name[-1] == "/"
def dashboard_uniformgridcpu():
@@ -307,10 +312,57 @@ def dashboard_uniformgridcpu_relativeperformance():
Filter("BandwidthBenchmark", multi=True, default_value="bw_stream"),
]
+ fieldConfig = {
+ "defaults": {
+ "custom": {
+ "lineWidth": 1,
+ "fillOpacity": 80,
+ "gradientMode": "none",
+ "axisPlacement": "auto",
+ "axisLabel": "",
+ "axisColorMode": "text",
+ "axisBorderShow": False,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "axisCenteredZero": False,
+ "hideFrom": {
+ "tooltip": False,
+ "viz": False,
+ "legend": False
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "color": {
+ "mode": "palette-classic"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": None
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "fieldMinMax": True,
+ "max": 1,
+ "unit": "percentunit"
+ },
+ "overrides": []
+ }
+
fields = [
PanelInfos("Relative Performance", Units.number, absthreshold=80),
PanelInfos("Bandwidth", Units.number, absthreshold=80),
- PanelInfos("Relative Performance", Units.mbytes_per_second, absthreshold=80),
+ PanelInfos("Relative Performance", Units.percent, absthreshold=80),
]
filter_vars = [
@@ -326,57 +378,252 @@ def dashboard_uniformgridcpu_relativeperformance():
row_repeat_var = [fv for fv in filter_vars if fv.name == row_repeat][0]
- where = [
- join_variable_and([f.name for f in filters]),
- get_variable_condition_with_tag(filters[0].name),
- join_variable_and([f.name for f in filters])
- ]
+ where0A2 = join_conditions(
+ [
+ get_variable_condition_with_tag(filters[i].name) for i in range(2)
+ ] + [
+ get_variable_condition_without_regex(filters[i].name) for i in [2]
+ ] + [
+ get_variable_condition(filters[3].name)
+ ] + [
+ get_variable_condition_without_regex(filters[i].name) for i in [4]
+ ], "AND"
+ )
+ where0B2 = get_variable_condition_with_tag(filters[0].name)
+ where1A = get_variable_condition_with_tag(filters[0].name)
+ where2A2 = join_conditions(
+ [
+ get_variable_condition_with_tag(filters[i].name) for i in range(2)
+ ] + [
+ get_variable_condition_without_regex(filters[i].name) for i in [2]
+ ] + [
+ get_variable_condition(filters[3].name)
+ ] + [
+ get_variable_condition_without_regex(filters[i].name) for i in [4]
+ ], "AND"
+ )
+ where2B2 = get_variable_condition_with_tag(filters[0].name)
+
+ group_by0A2 = ["host", "project_id", "branch", "collisionSetup", "mpi_num_processes", "streamingPattern", "timeStepStrategy", "stencil", "blocks_0", "blocks_1", "blocks_2", "cellsPerBlock_0", "cellsPerBlock_1", "cellsPerBlock_2", "periodic_0", "periodic_1", "periodic_2"]
+ group_by0B2 = [get_variable_tag(filters[0].name)]
+ group_by1A = [filters[0].name]
+ group_by2A2 = ["host", "project_id", "branch", "collisionSetup", "mpi_num_processes", "streamingPattern", "timeStepStrategy", "stencil", "blocks_0", "blocks_1", "blocks_2", "cellsPerBlock_0", "cellsPerBlock_1", "cellsPerBlock_2", "periodic_0", "periodic_1", "periodic_2"]
+ group_by2B2 = [get_variable_tag(filters[0].name)]
+
annotations = get_commit_annotation(
data_source, "red", "commits", measurment_name2)
- selected_columns = [
- ", ".join([
- "host",
- "collisionSetup",
- "MLUPSperProcess",
- "mpi_num_processes"]),
- ", ".join([
+ selected_columns0A1 = ", ".join([
+ '"host"',
+ '"collisionSetup"',
+ '"MLUPSperProcess"',
+ '"mpi_num_processes"'])
+ selected_columns0A2 = ", ".join([
+ '(last("mlupsPerProcess")) as "MLUPSperProcess"',
+ '"mpi_num_processes"'])
+ selected_columns0B1 = ", ".join([
+ '"host"',
+ '"P_max"'])
+ selected_columns0B2 = ", ".join([
+ 'last($BandwidthBenchmark)/(27*2*8) as "P_max"'])
+ selected_columns1A = ", ".join([
'last($BandwidthBenchmark) as "BW[MByte/s]"',
- 'last($BandwidthBenchmark) / (27 * 2 * 8) as "P_max[MLUPS]"']),
- ", ".join([
- "host",
- "collisionSetup",
- "MLUPSperProcess",
- "mpi_num_processes"]),
- ]
+ 'last($BandwidthBenchmark)/(27*2*8) as "P_max[MLUPS]"'])
+ selected_columns2A1 = ", ".join([
+ '"host"',
+ '"collisionSetup"',
+ '"MLUPSperProcess"',
+ '"mpi_num_processes"'])
+ selected_columns2A2 = ", ".join([
+ '(last("mlupsPerProcess")) as "MLUPSperProcess"',
+ '"mpi_num_processes"'])
+ selected_columns2B1 = ", ".join([
+ '"host"',
+ '"P_max"'])
+ selected_columns2B2 = ", ".join([
+ 'last($BandwidthBenchmark)/(27*2*8) as "P_max"'])
+
+
+ query0A2 = Query(select_=selected_columns0A2,
+ from_=measurment_name,
+ where_=where0A2,
+ group_by=group_by0A2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query0A1 = Query(selected_columns0A1,
+ from_=query0A2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query0B2 = Query(select_=selected_columns0B2,
+ from_=measurment_name2,
+ where_=where0B2,
+ group_by=group_by0B2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query0B1 = Query(select_=selected_columns0B1,
+ from_=query0B2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+
+ query1A = Query(select_=selected_columns1A,
+ from_=measurment_name2,
+ where_=where1A,
+ group_by=group_by1A,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+
+ query2A2 = Query(select_=selected_columns2A2,
+ from_=measurment_name,
+ where_=where2A2,
+ group_by=group_by2A2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query2A1 = Query(selected_columns2A1,
+ from_=query2A2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query2B2 = Query(select_=selected_columns2B2,
+ from_=measurment_name2,
+ where_=where2B2,
+ group_by=group_by2B2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+ query2B1 = Query(select_=selected_columns2B1,
+ from_=query2B2,
+ from_string=not is_regex(measurment_name),
+ select_string=False)
+
+ alias0A = "$tag_collisionSetup"
+ alias2A = "$tag_collisionSetup"
+
+
+ transformations0 = [
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "host",
+ "mode": "outerTabular"
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "alias": "P",
+ "binary": {
+ "left": "MLUPSperProcess",
+ "operator": "*",
+ "right": "mpi_num_processes"
+ },
+ "mode": "binary",
+ "reduce": {
+ "reducer": "sum"
+ },
+ "replaceFields": False
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "binary": {
+ "left": "P",
+ "operator": "/",
+ "right": "P_max"
+ },
+ "mode": "binary",
+ "reduce": {
+ "reducer": "sum"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "Time 1",
+ "collisionSetup",
+ "P",
+ "P / P_max",
+ "MLUPSperProcess"
+ ]
+ }
+ }
+ }
+ ]
+
+ transformations2 = [
+ {
+ "id": "joinByField",
+ "options": {
+ "byField": "host",
+ "mode": "outerTabular"
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "alias": "P",
+ "binary": {
+ "left": "MLUPSperProcess",
+ "operator": "*",
+ "right": "mpi_num_processes"
+ },
+ "mode": "binary",
+ "reduce": {
+ "reducer": "sum"
+ },
+ "replaceFields": False
+ }
+ },
+ {
+ "id": "calculateField",
+ "options": {
+ "binary": {
+ "left": "P",
+ "operator": "/",
+ "right": "P_max"
+ },
+ "mode": "binary",
+ "reduce": {
+ "reducer": "sum"
+ }
+ }
+ },
+ {
+ "id": "filterFieldsByName",
+ "options": {
+ "include": {
+ "names": [
+ "collisionSetup",
+ "P / P_max"
+ ]
+ }
+ }
+ }
+ ]
panels = [
# Panel 0
get_table_panel(
fields[0],
- selected_columns[0],
data_source,
- measurment_name,
- where=where[0],
- group_by=[f.name for f in filters],
+ [query0A1, query0B1],
+ alias_list = [alias0A, ""],
+ transformations=transformations0,
),
# Panel 1
get_table_panel(
fields[1],
- selected_columns[1],
data_source,
- measurment_name2,
- where=where[1],
- group_by=[filters[0].name],
+ [query1A],
),
# Panel 2
get_bar_chart_panel(
fields[2],
- selected_columns[2],
data_source,
- measurment_name,
- where=where[2],
- group_by=[f.name for f in filters],
+ [query2A1, query2B1],
+ alias_list = [alias2A, ""],
+ transformations=transformations2,
),
]
diff --git a/dashboards/influx_queries.py b/dashboards/influx_queries.py
index 7bd23de2fb11aad7acc1460de54204a98b777326..cc7045ae35b925094a85c1875fcfafa4aa6aeb9d 100644
--- a/dashboards/influx_queries.py
+++ b/dashboards/influx_queries.py
@@ -6,8 +6,8 @@ from typing import List, Union
@dataclass
class Query:
select_: str
- from_: str
- where_: str
+ from_: Union[str, 'Query']
+ where_: str = None
group_by: List[str] = field(default_factory=list)
select_string: bool = True
from_string: bool = True
@@ -19,17 +19,22 @@ class Query:
return f"SELECT {self.select_} "
def _get_from(self):
- if self.from_string:
+ if isinstance(self.from_, Query):
+ return f'FROM ({self.from_}) '
+ elif self.from_string:
return f'FROM "{self.from_}" '
else:
return f"FROM {self.from_} "
def _get_where(self):
- return f"WHERE ({self.where_}) "
+ if self.where_:
+ return f"WHERE ({self.where_}) "
+ else:
+ return ""
def _get_group_by(self):
if self.group_by:
- group_by = ", ".join(f'"{tag}"' for tag in self.group_by)
+ group_by = ", ".join(tag if tag.endswith("::tag") else f'"{tag}"' for tag in self.group_by)
return f"GROUP BY {group_by}"
else:
return ""
@@ -80,6 +85,7 @@ def get_variable_condition(variable_name: str, *, tag_key: str = None) -> str:
raise ValueError("Empty variable name")
return f'"{clean_lhs}" =~ /^${{{clean_rhs}:regex}}$/'
+
def get_variable_condition_with_tag(variable_name: str, *, tag_key: str = None) -> str:
clean_rhs = variable_name.strip()
if tag_key:
@@ -91,6 +97,28 @@ def get_variable_condition_with_tag(variable_name: str, *, tag_key: str = None)
return f'"{clean_lhs}"::tag =~ /^${clean_rhs}$/'
+def get_variable_condition_without_regex(variable_name: str, *, tag_key: str = None) -> str:
+ clean_rhs = variable_name.strip()
+ if tag_key:
+ clean_lhs = tag_key.strip()
+ else:
+ clean_lhs = clean_rhs
+ if not clean_rhs:
+ raise ValueError("Empty variable name")
+ return f'"{clean_lhs}" =~ /^${clean_rhs}$/'
+
+
+def get_variable_tag(variable_name: str, *, tag_key: str = None) -> str:
+ clean_rhs = variable_name.strip()
+ if tag_key:
+ clean_lhs = tag_key.strip()
+ else:
+ clean_lhs = clean_rhs
+ if not clean_rhs:
+ raise ValueError("Empty variable name")
+ return f'"{clean_lhs}"::tag'
+
+
def join_conditions(conditions: List[str], operators: Union[List[str], str], include_time_filter: bool = True):
ops = operators
if isinstance(operators, str):
diff --git a/dashboards/panels.py b/dashboards/panels.py
index 1b153e073f39c882294950ecee62b8936c79670a..bfe23ccb2b86cc82c89ffbd1ce874793fb0dfe2c 100644
--- a/dashboards/panels.py
+++ b/dashboards/panels.py
@@ -96,20 +96,19 @@ def get_stat_panel(title: str,
def get_table_panel(panel_infos: PanelInfos,
- selected_columns: str,
data_source: str,
- measurment_name: str,
+ query_list: list[Query],
*,
- where=None,
- group_by=None,
+ result_format_list: list[str] = None,
+ alias_list: list[str] = None,
+ transformations=[],
overrides=None,
**kwargs):
- query = Query(select_=selected_columns,
- from_=measurment_name,
- where_=where,
- group_by=group_by,
- from_string=not is_regex(measurment_name),
- select_string=False)
+ if not alias_list:
+ alias_list = [''] * len(query_list)
+ if not result_format_list:
+ result_format_list = ['table'] * len(query_list)
+ targets = [get_influx_target(str(query), result_format=result_format, alias=alias) for query, result_format, alias in zip(query_list, result_format_list, alias_list)]
new_kwargs = {**kwargs}
if panel_infos.absthreshold is not None:
new_kwargs.update({'thresholdType': 'absolute',
@@ -117,12 +116,12 @@ def get_table_panel(panel_infos: PanelInfos,
Threshold('red', index=1, value=float(panel_infos.absthreshold), op='lt'), ],
}
)
- result_format = ['table']
return Table(
title=panel_infos.name,
dataSource=data_source,
- targets=[get_influx_target(str(query), result_format[0])],
+ targets=targets,
+ transformations=transformations,
unit=panel_infos.unit,
overrides=overrides,
**new_kwargs,
@@ -131,20 +130,19 @@ def get_table_panel(panel_infos: PanelInfos,
def get_bar_chart_panel(panel_infos: PanelInfos,
- selected_columns: str,
data_source: str,
- measurment_name: str,
+ query_list: list[Query],
*,
- where=None,
- group_by=None,
+ result_format_list: list[str] = None,
+ alias_list: list[str] = None,
+ transformations=[],
overrides=None,
**kwargs):
- query = Query(select_=selected_columns,
- from_=measurment_name,
- where_=where,
- group_by=group_by,
- from_string=not is_regex(measurment_name),
- select_string=False)
+ if not alias_list:
+ alias_list = [''] * len(query_list)
+ if not result_format_list:
+ result_format_list = ['table'] * len(query_list)
+ targets = [get_influx_target(str(query), result_format=result_format, alias=alias) for query, result_format, alias in zip(query_list, result_format_list, alias_list)]
new_kwargs = {**kwargs}
if panel_infos.absthreshold is not None:
new_kwargs.update({'thresholdType': 'absolute',
@@ -152,12 +150,14 @@ def get_bar_chart_panel(panel_infos: PanelInfos,
Threshold('red', index=1, value=float(panel_infos.absthreshold), op='lt'), ],
}
)
- result_format = ['table']
+
return BarChart(
title=panel_infos.name,
dataSource=data_source,
- targets=[get_influx_target(str(query), result_format[0])],
+ targets=targets,
+ transformations=transformations,
+ xTickLabelRotation=-45,
**new_kwargs,
)