Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
from pathlib import Path
import json
import numpy as np
import pandas as pd
import plotly.express as px
from typing import Union, Literal
image_height = 500
def load_data(bench_suite):
for bench_file in Path(f"benchmarks/results").glob(f"*{bench_suite}*.json"):
(_, _, jvm, run) = bench_file.stem.split("-")
for data in json.loads(bench_file.read_bytes()):
benchmark = data["benchmark"].replace(
f"benchmarks.{bench_suite}.", "")
if "params" in data:
benchmark += "(" + ",".join(data["params"].values()) + ")"
for fork, times in enumerate(data["primaryMetric"]["rawData"]):
for time in times:
yield (jvm, benchmark, int(run), int(fork), time)
Mode = Union[Literal["bar"], Literal["strip"], Literal["box"]]
Orientation = Union[Literal["h"], Literal["v"]]
def make_bars_graph(bench_suite, mode: Mode, orientation: Orientation):
df = pd.DataFrame(
load_data(bench_suite),
columns=("jvm", "benchmark", "run", "fork", "time")
)
if mode == "bar":
df = df.drop(columns=["run", "fork"])
df = df.groupby(["jvm", "benchmark"])
df = df.agg([("median", np.median), ("min", np.min), ("max", np.max)])
df = df.reset_index()
df.columns = [' '.join(col).strip() for col in df.columns.values]
df["time error"] = df["time median"] - df["time min"]
df["time error minus"] = df["time max"] - df["time median"]
df = df.sort_values(by=["benchmark"], ascending=False)
args = {
"x" if orientation == "h" else "y": "time median" if mode == "bar" else "time",
"y" if orientation == "h" else "x": "benchmark",
"color": "jvm",
"orientation": orientation,
"color_discrete_sequence": px.colors.qualitative.Pastel1,
"template": "plotly_white"
}
if mode == "bar":
args |= {
"barmode": "group",
"error_x" if orientation == "h" else "error_y": "time error",
"error_x_minus" if orientation == "h" else "error_y_minus": "time error minus"
}
elif mode == "box":
args |= {
"points": "all"
}
fig = getattr(px, mode)(df, **args)
layout_args = {
"title": bench_suite,
"xaxis_title" if orientation == "h" else "yaxis_title": "Throughput [ops/s] (higher is better)",
"yaxis_title" if orientation == "h" else "xaxis_title": "Benchmark"
}
fig.update_layout(**layout_args)
fig.write_html(
f"benchmarks/graphs/{bench_suite}-{mode}-{orientation}.html",
include_plotlyjs='cdn'
)
fig.write_image(
f"benchmarks/graphs/{bench_suite}-{mode}-{orientation}.svg",
height=image_height,
width=image_height * 2
)
make_bars_graph("SumBenchmark", "bar", "h")