import json
import sys
from pathlib import Path
from pprint import pprint
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from matplotlib import cm, dates
from matplotlib.lines import Line2D
indir = Path(sys.argv[1])
outdir = Path(sys.argv[2])
json_paths = list(Path(indir).rglob("*.json"))
print(f"Found {len(json_paths)} JSON files")
def get_benchmarks(paths):
benchmarks = []
num_benchmarks = 0
for path in paths:
with open(path) as file:
jsn = json.load(file)
system = jsn["machine_info"]["system"]
python = jsn["machine_info"]["python_version"]
if len(python.split(".")) == 3:
python = python.rpartition(".")[0]
tstamp = jsn["datetime"]
bmarks = jsn["benchmarks"]
for benchmark in bmarks:
num_benchmarks += 1
fullname = benchmark["fullname"]
included = ["min", "mean"]
for stat, value in benchmark["stats"].items():
if stat not in included:
continue
benchmarks.append(
{
"system": system,
"python": python,
"time": tstamp,
"case": fullname,
"stat": stat,
"value": value,
}
)
print("Found", num_benchmarks, "benchmarks")
return benchmarks
# create data frame and save to CSV
benchmarks_df = pd.DataFrame(get_benchmarks(json_paths))
benchmarks_df["time"] = pd.to_datetime(benchmarks_df["time"])
benchmarks_df.to_csv(str(outdir / "benchmarks.csv"), index=False)
def matplotlib_plot(stats):
nstats = len(stats)
fig, axs = plt.subplots(nstats, 1, sharex=True)
# color-code according to python version
pythons = np.unique(benchmarks_df["python"])
colors = dict(zip(pythons, cm.jet(np.linspace(0, 1, len(pythons)))))
# markers according to system
systems = np.unique(benchmarks_df["system"])
markers = dict(zip(systems, ["x", "o", "s"])) # osx, linux, windows
benchmarks_df["marker"] = benchmarks_df["system"].apply(lambda x: markers[x])
for i, (stat_name, stat_group) in enumerate(stats):
stat_df = pd.DataFrame(stat_group)
ax = axs[i] if nstats > 1 else axs
ax.set_title(stat_name)
ax.tick_params(axis="x", rotation=45)
ax.xaxis.set_major_locator(dates.DayLocator(interval=1))
ax.xaxis.set_major_formatter(dates.DateFormatter("\n%m-%d-%Y"))
for si, system in enumerate(systems):
ssub = stat_df[stat_df["system"] == system]
marker = markers[system]
for pi, python in enumerate(pythons):
psub = ssub[ssub["python"] == python]
color = colors[python]
ax.scatter(psub["time"], psub["value"], color=color, marker=marker)
ax.plot(psub["time"], psub["value"], linestyle="dotted", color=color)
# configure legend
patches = []
for system in systems:
for python in pythons:
patches.append(
Line2D(
[0],
[0],
color=colors[python],
marker=markers[system],
label=f"{system} Python{python}",
)
)
leg = plt.legend(
handles=patches,
loc="upper left",
ncol=3,
bbox_to_anchor=(0, 0),
framealpha=0.5,
bbox_transform=ax.transAxes,
)
for lh in leg.legendHandles:
lh.set_alpha(0.5)
fig.suptitle(case_name)
plt.ylabel("ms")
fig.tight_layout()
fig.set_size_inches(8, 8)
return fig
def seaborn_plot(stats):
nstats = len(stats)
fig, axs = plt.subplots(nstats, 1, sharex=True)
for i, (stat_name, stat_group) in enumerate(stats):
stat_df = pd.DataFrame(stat_group)
ax = axs[i] if nstats > 1 else axs
ax.tick_params(axis="x", rotation=45)
sp = sns.scatterplot(
x="time",
y="value",
,
hue="python",
data=stat_df,
ax=ax,
palette="YlOrBr",
)
sp.set(xlabel=None)
ax.set_title(stat_name)
ax.get_legend().remove()
ax.set_ylabel("ms")
fig.suptitle(case_name)
fig.tight_layout()
plt.subplots_adjust(left=0.3)
plt.legend(loc="lower left", framealpha=0.3, bbox_to_anchor=(-0.45, -0.6))
return fig
# create and save plots
cases = benchmarks_df.groupby("case")
for case_name, case in cases:
stats = pd.DataFrame(case).groupby("stat")
case_name = str(case_name).replace("/", "_").replace(":", "_")
fig = seaborn_plot(stats)
plt.savefig(str(outdir / f"{case_name}.png"))