RustPython/scripts/pyperformance/pr_diff.py at main · Stars1233/RustPython · GitHub

Repository navigation

History

266 lines (225 loc) · 10.5 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

#!/usr/bin/env python3

"""Render a base/head/CPython pyperformance comparison as a PR comment.

`compare.py` puts one catalog next to one other catalog, which is the right

shape for "how far is RustPython from CPython". A pull request needs a third

column -- the base commit -- and it needs all three measured *in the same CI

job*, back to back on one runner: a ratio between two numbers that were

measured on different machines can move by more than the change under review

did, so a cross-run comparison cannot tell a real regression from a runner

that happened to be slower that day.

Reads `<results-dir>/<label>/catalog.json` for the three labels and writes a

Markdown comment body (and optionally the same data as JSON).

Usage:

python3 scripts/pyperformance/pr_diff.py \\

--cpython cpython --base base --head head \\

--results-dir "$RUNNER_TEMP/pyperf-results" \\

--md diff.md --json diff.json

"""

from __future__ import annotations

import argparse

import json

import statistics

from pathlib import Path

from compare import load_catalog, parse_mean_seconds

MARKER = ""

# Below this, a head/base difference is indistinguishable from run-to-run

# noise on a shared CI runner, so it is rendered as "~" rather than a number

# that invites over-reading.

NOISE = 0.03

def fmt_ratio(value: float | None) -> str:

return "-" if value is None else f"{value:.2f}x"

def fmt_change(base: float | None, head: float | None) -> str:

"""Head vs. base as a percentage, blanked out inside the noise floor."""

if not base or not head:

return "-"

change = head / base - 1.0

if abs(change) < NOISE:

return "~"

return f"{change * 100:+.1f}%"

def parse_args() -> argparse.Namespace:

parser = argparse.ArgumentParser(

description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter

)

parser.add_argument("--cpython", default="cpython", help="CPython catalog label")

parser.add_argument("--base", default="base", help="base-commit catalog label")

parser.add_argument("--head", default="head", help="head-commit catalog label")

parser.add_argument(

"--results-dir",

type=Path,

required=True,

help="parent directory containing <label>/catalog.json",

)

parser.add_argument("--md", type=Path, required=True, help="Markdown output file")

parser.add_argument("--json", type=Path, help="optional JSON output file")

parser.add_argument(

"--cpython-version",

default="CPython",

help="version string to label the CPython column with",

)

return parser.parse_args()

def load_base_catalog(results_dir: Path, label: str) -> tuple[dict, bool]:

"""Load the base-commit catalog, if any.

The base commit is the column that makes this a review aid rather than a

status report, but it is also the one that can go missing: it may fail to

build, or predate something the benchmarks now need. The caller falls

back to head against CPython alone when nothing here produced a usable

time, rather than failing the whole comparison.

"""

base = {}

if (results_dir / label / "catalog.json").exists():

base = load_catalog(results_dir, label)

have_base = any(parse_mean_seconds(r.get("mean")) for r in base.values())

return base, have_base

def build_rows(cpython: dict, base: dict, head: dict) -> list[dict]:

rows = []

for name in sorted(set(cpython) | set(base) | set(head)):

c = parse_mean_seconds((cpython.get(name) or {}).get("mean"))

b = parse_mean_seconds((base.get(name) or {}).get("mean"))

h = parse_mean_seconds((head.get(name) or {}).get("mean"))

rows.append(

{

"benchmark": name,

"cpython_mean": (cpython.get(name) or {}).get("mean") or "",

"base_mean": (base.get(name) or {}).get("mean") or "",

"head_mean": (head.get(name) or {}).get("mean") or "",

"base_status": (base.get(name) or {}).get("status", "missing"),

"head_status": (head.get(name) or {}).get("status", "missing"),

"base_vs_cpython": (b / c) if (b and c) else None,

"head_vs_cpython": (h / c) if (h and c) else None,

"head_vs_base": (h / b) if (h and b) else None,

}

)

return rows

def render_intro(have_base: bool, cpython_version: str) -> list[str]:

if have_base:

return [

"All three interpreters were measured back to back in this one job, "

"so they share the exact same CPU and kernel. Ratios are "

f"`time / {cpython_version} time` -- lower is better. A "

f"head/base difference smaller than {NOISE:.0%} is shown as `~`; CI "

"runners are not quiet enough to read more into it than that."

]

return [

"No usable result for the base commit -- it did not build, or none "

"of its benchmarks produced a time -- so this is head against "

f"{cpython_version} alone, both measured back to back in this "

"one job. Ratios are `time / {} time`; lower is better.".format(cpython_version)

]

def render_summary_panel(

rows: list[dict], have_base: bool, cpython_version: str

) -> list[str]:

"""The "| | base | head |" mini-table: pass counts, the median slowdown

vs. CPython, and the head/base geometric mean.

Each row is independently gated on having the data it needs: the two

medians are restricted to benchmarks with a valid ratio on *both* sides,

so a benchmark that regressed to failure on only one side can't drop out

of only that side's population and skew the comparison; "Benchmarks

passed" and the geometric mean don't depend on CPython at all, so they

must not disappear just because no benchmark had a usable CPython

comparison.

"""

if not have_base:

return []

common_vs_cpython = [

r for r in rows if r["base_vs_cpython"] and r["head_vs_cpython"]

]

base_ratios = [r["base_vs_cpython"] for r in common_vs_cpython]

head_ratios = [r["head_vs_cpython"] for r in common_vs_cpython]

base_passed = sum(1 for r in rows if r["base_status"] == "ok")

head_passed = sum(1 for r in rows if r["head_status"] == "ok")

both = [r for r in rows if r["head_vs_base"]]

out = ["| | base | head |", "| --- | ---: | ---: |"]

if base_ratios and head_ratios:

out.append(

f"| Median slowdown vs. {cpython_version} "

f"({len(common_vs_cpython)} common) "

f"| {statistics.median(base_ratios):.2f}x "

f"| {statistics.median(head_ratios):.2f}x |"

)

out.append(f"| Benchmarks passed | {base_passed} | {head_passed} |")

if both:

geo = statistics.geometric_mean([r["head_vs_base"] for r in both])

out.append(f"| Geometric mean, head/base ({len(both)} common) | | {geo:.3f} |")

out.append("")

return out

def render_table(rows: list[dict], have_base: bool, cpython_version: str) -> list[str]:

if have_base:

out = [

f"| Benchmark | {cpython_version} | base | head "

f"| base/{cpython_version} | head/{cpython_version} "

"| head vs. base |",

"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",

]

else:

out = [

f"| Benchmark | {cpython_version} | head | head/{cpython_version} |",

"| --- | ---: | ---: | ---: |",

]

for r in sorted(rows, key=lambda r: r["head_vs_base"] or r["head_vs_cpython"] or 0):

# A row with no mean on either side but a fail/timeout status is still

# worth showing -- e.g. a C-extension gap that fails identically on

# base and head. Only a benchmark never attempted on either commit

# (no entry in that catalog at all) has nothing to report.

if r["base_status"] == "missing" and r["head_status"] == "missing":

continue

head_cell = r["head_mean"] or "({})".format(r["head_status"])

if not have_base:

out.append(

f"| {r['benchmark']} | {r['cpython_mean'] or '-'} | {head_cell} "

f"| {fmt_ratio(r['head_vs_cpython'])} |"

)

continue

base_cell = r["base_mean"] or "({})".format(r["base_status"])

out.append(

f"| {r['benchmark']} | {r['cpython_mean'] or '-'} "

f"| {base_cell} | {head_cell} "

f"| {fmt_ratio(r['base_vs_cpython'])} | {fmt_ratio(r['head_vs_cpython'])} "

f"| {fmt_change(1.0, r['head_vs_base'])} |"

)

return out

def find_transitions(rows: list[dict]) -> tuple[list[str], list[str]]:

"""Benchmarks whose pass/fail status flipped between base and head.

A transition claim needs an actual *fail/timeout* status on the other

side, not just a missing entry: a benchmark never attempted there (e.g.

a partial --benchmarks run) isn't a regression or a fix, just no data

point. This intentionally doesn't gate on `have_base` (whether *any*

base benchmark produced a timing) -- a benchmark can still validly flip

from failing to passing even when every other base benchmark failed too.

"""

only_head = [

r["benchmark"]

for r in rows

if r["head_status"] == "ok" and r["base_status"] in ("fail", "timeout")

]

only_base = [

r["benchmark"]

for r in rows

if r["base_status"] == "ok" and r["head_status"] in ("fail", "timeout")

]

return only_head, only_base

def main() -> None:

args = parse_args()

cpython = load_catalog(args.results_dir, args.cpython)

head = load_catalog(args.results_dir, args.head)

base, have_base = load_base_catalog(args.results_dir, args.base)

rows = build_rows(cpython, base, head)

title = (

"base vs. head vs. CPython (same runner)"

if have_base

else "head vs. CPython (same runner)"

)

out = [MARKER, f"### pyperformance: {title}", ""]

out += render_intro(have_base, args.cpython_version)

out.append("")

out += render_summary_panel(rows, have_base, args.cpython_version)

out += render_table(rows, have_base, args.cpython_version)

only_head, only_base = find_transitions(rows)

if only_head:

out += ["", f"Newly passing on head: {', '.join(only_head)}."]

if only_base:

out += ["", f"No longer passing on head: {', '.join(only_base)}."]

args.md.write_text("\n".join(out) + "\n")

if args.json:

args.json.write_text(json.dumps(rows, indent=2) + "\n")

if __name__ == "__main__":

main()

RustPython/scripts/pyperformance/pr_diff.py at main · Stars1233/RustPython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

pr_diff.py

Latest commit

History

Breadcrumbs

pr_diff.py

File metadata and controls