RustPython/scripts/pyperformance/run_all.py at main · RustPython/RustPython · GitHub

Repository navigation

History

executable file

549 lines (490 loc) · 20 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

505

506

507

508

509

510

511

512

513

514

515

516

517

518

519

520

521

522

523

524

525

526

527

528

529

530

531

532

533

534

535

536

537

538

539

540

541

542

543

544

545

546

547

548

549

#!/usr/bin/env python3

"""Run the upstream pyperformance benchmark suite (https://github.com/python/pyperformance)

against a Python executable -- RustPython, a real CPython, or both -- and

catalog which benchmarks pass, fail, or time out under each.

Background

----------

pyperformance cannot be installed as-is on RustPython: its runtime dependency

`pyperf` hard-depends on `psutil`, a C-extension package. RustPython has no

CPython C-API / extension-module loading support (no `_imp.create_dynamic` /

`_imp.exec_dynamic`, no compiler config vars such as LDCXXSHARED in

`_sysconfigdata`), so building or loading any C extension fails.

Workaround: `pyperf` itself already disables psutil usage on interpreters

that report `Py_GIL_DISABLED=1` (see `pyperf._utils.USE_PSUTIL`), which is

exactly what RustPython reports (it has no GIL). So a functionality-free,

pure-Python "psutil" stub (see stub_psutil/) is enough to satisfy pip's

dependency resolution -- pyperf never actually calls into it at runtime on

RustPython. This unblocks any *pure-Python* pyperformance benchmark; any

benchmark whose own workload (not just pyperf) requires a real C extension

(e.g. lxml, numpy, greenlet) will still fail, and that failure is a genuine

finding: it marks a real C-extension gap in RustPython, not a tooling

artifact of this script. A real CPython target doesn't need this workaround

at all -- pass --no-psutil-stub for it, so it installs and uses the genuine

psutil.

This script:

1. Ensures a *host* CPython venv with pyperformance installed (pyperformance

itself only runs under a real CPython; the --python target is just the

interpreter it benchmarks, which may be that same CPython or RustPython).

2. Builds the pure-Python psutil stub wheel once (skipped with --no-psutil-stub).

3. Runs every benchmark pyperformance knows about, one at a time, against

the given --python target, with a timeout per benchmark.

4. Writes a JSON + Markdown catalog of the results under results/<label>/.

Usage

-----

python3 scripts/pyperformance/run_all.py \\

--python target/release/rustpython --label rustpython

python3 scripts/pyperformance/run_all.py \\

--python "$(command -v python3.13)" --label cpython3.13 --no-psutil-stub

Then compare two labels' catalogs with compare.py (see its docstring).

Re-running resumes from results/<label>/catalog.json by default (skips

benchmarks already recorded); pass --force to redo everything.

"""

from __future__ import annotations

import argparse

import hashlib

import json

import os

import re

import shutil

import signal

import subprocess

from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent

REPO_ROOT = SCRIPT_DIR.parent.parent

STUB_PSUTIL_DIR = SCRIPT_DIR / "stub_psutil"

DEFAULT_CACHE_DIR = REPO_ROOT / "target" / "pyperformance"

DEFAULT_OUT_DIR = SCRIPT_DIR / "results"

DEFAULT_TIMEOUT = 180

def log(msg: str) -> None:

print(f"[pyperformance-runner] {msg}", flush=True)

def executable_fingerprint(path: Path) -> str:

"""Hash of the target executable's contents -- and, when RUSTPYTHONPATH

points it at a stdlib copy, that stdlib's contents too, since that's the

other half of what pyperf actually runs. Used to invalidate a resumed

catalog when a local rebuild (or a different target under the same

--label) replaces either one from under the earlier results.

"""

digest = hashlib.sha256()

digest.update(path.read_bytes())

rustpythonpath = os.environ.get("RUSTPYTHONPATH")

if rustpythonpath:

for file in sorted(Path(rustpythonpath).rglob("*")):

if file.is_file():

digest.update(str(file.relative_to(rustpythonpath)).encode())

digest.update(file.read_bytes())

return digest.hexdigest()

def find_host_python() -> str:

for candidate in (

"python3.13",

"python3.12",

"python3.11",

"python3.10",

"python3",

path = shutil.which(candidate)

if path:

return path

raise SystemExit("No usable host CPython found on PATH (need python3.10+)")

def ensure_host_venv(cache_dir: Path) -> Path:

venv_dir = cache_dir / "host-venv"

pip_marker = venv_dir / "pyvenv.cfg"

pyperf_installed = False

if pip_marker.exists():

pip_bin = venv_dir / "bin" / "pip"

result = subprocess.run(

[str(pip_bin), "show", "pyperformance"],

capture_output=True,

text=True,

)

pyperf_installed = result.returncode == 0

if not pyperf_installed:

log(

f"setting up host venv at {venv_dir} (this runs pyperformance's CLI; "

"RustPython is only the --python target it benchmarks)"

)

shutil.rmtree(venv_dir, ignore_errors=True)

venv_dir.parent.mkdir(parents=True, exist_ok=True)

host_python = find_host_python()

subprocess.run([host_python, "-m", "venv", str(venv_dir)], check=True)

pip_bin = venv_dir / "bin" / "pip"

subprocess.run([str(pip_bin), "install", "-q", "-U", "pip"], check=True)

subprocess.run([str(pip_bin), "install", "-q", "pyperformance"], check=True)

return venv_dir

def ensure_stub_psutil_wheel(host_venv: Path, cache_dir: Path) -> Path:

stub_pkgs_dir = cache_dir / "stub-pkgs"

existing = list(stub_pkgs_dir.glob("psutil-*.whl"))

if existing:

return stub_pkgs_dir

log(

"building pure-Python psutil stub wheel (see scripts/pyperformance/stub_psutil/)"

)

stub_pkgs_dir.mkdir(parents=True, exist_ok=True)

pip_bin = host_venv / "bin" / "pip"

subprocess.run(

[

str(pip_bin),

"wheel",

str(STUB_PSUTIL_DIR),

"-w",

str(stub_pkgs_dir),

"--no-deps",

"-q",

check=True,

)

return stub_pkgs_dir

def list_benchmarks(host_venv: Path) -> list[str]:

pyperformance_bin = host_venv / "bin" / "pyperformance"

result = subprocess.run(

[str(pyperformance_bin), "list"],

capture_output=True,

text=True,

check=True,

)

names = []

for line in result.stdout.splitlines():

line = line.strip()

if line.startswith("- "):

names.append(line[2:].strip())

return names

MEAN_RE = re.compile(r"Mean \+- std dev:\s*(.+)")

FAILED_BUILD_RE = re.compile(r"Failed to build (\S+)")

FAILED_WHEEL_RE = re.compile(r"Failed building wheel for (\S+)")

LDCXXSHARED_RE = re.compile(

r"Unexpected None in config vars: \[('LDCXXSHARED'[^\]]*)\]"

)

# The root-cause exception raised by the *benchmark script itself* (running under

# RustPython), as opposed to pyperformance's own wrapper exceptions

# (e.g. "RuntimeError: Benchmark died", "RuntimeError: ... failed with exit code ...")

# which are always the LAST exception in the combined output, not the first.

PYPERFORMANCE_WRAPPER_ERRORS = {

"Benchmark died",

}

EXCEPTION_LINE_RE = re.compile(

r"^(?:\w+\.)*(\w*(?:Error|Exception)): (.+)$", re.MULTILINE

)

def classify_failure(combined_output: str) -> str:

if LDCXXSHARED_RE.search(combined_output):

pkg_match = FAILED_WHEEL_RE.search(combined_output) or FAILED_BUILD_RE.search(

combined_output

)

pkg = pkg_match.group(1) if pkg_match else "unknown package"

return f"C-extension build blocked (no compiler config / no CPython C-API) building {pkg}"

pkg_match = FAILED_WHEEL_RE.search(combined_output) or FAILED_BUILD_RE.search(

combined_output

)

if pkg_match:

return f"failed to build/install {pkg_match.group(1)}"

# Prefer the first real exception in the log: it's raised by the benchmark

# script running under RustPython, before pyperformance's own wrapper

# exceptions (always last) obscure it with a generic "Benchmark died".

for exc_type, exc_msg in EXCEPTION_LINE_RE.findall(combined_output):

if exc_msg.strip() in PYPERFORMANCE_WRAPPER_ERRORS:

continue

if "failed with exit code" in exc_msg:

continue

return f"{exc_type}: {exc_msg.strip()}"[:300]

m = re.search(r"^(ERROR: .*)$", combined_output, re.MULTILINE)

if m:

return m.group(1)[:200]

tail = "\n".join(combined_output.strip().splitlines()[-5:])

return tail[:400] or "unknown failure"

def run_one_benchmark(

pyperformance_bin: Path,

target_python: Path,

bench: str,

work_dir: Path,

out_dir: Path,

stub_pkgs_dir: Path | None,

timeout: int,

extra_args: list[str],

) -> dict:

result_json = out_dir / "raw" / f"{bench}.json"

result_json.parent.mkdir(parents=True, exist_ok=True)

if result_json.exists():

result_json.unlink()

env = os.environ.copy()

cmd = [

str(pyperformance_bin),

"run",

"--python",

str(target_python),

"-b",

bench,

"-o",

str(result_json),

*extra_args,

]

inherited = []

if stub_pkgs_dir is not None:

# Work around the target interpreter's lack of a C-extension psutil (see

# module docstring). Real CPython doesn't need this -- pass

# stub_pkgs_dir=None for it so it installs and uses the real psutil.

env["PIP_FIND_LINKS"] = str(stub_pkgs_dir)

inherited.append("PIP_FIND_LINKS")

if "RUSTPYTHONPATH" in env:

# A RustPython binary copied away from its checkout (as CI does when it

# keeps one build of each commit around) can only find the stdlib

# through this variable, and pyperf re-executes the target interpreter

# in a venv of its own, so it has to survive that hop too.

inherited.append("RUSTPYTHONPATH")

if inherited:

# One comma-separated flag, not one flag per variable: pyperf's

# `--inherit-environ` is a plain (non-appending) option, so repeating it

# keeps only the last name.

cmd += ["--inherit-environ", ",".join(inherited)]

log(f"running {bench} ...")

# A new session makes this process the leader of its own process group, so

# a timeout can kill the whole group -- pyperf re-execs the target

# interpreter as a child, and `subprocess.run`'s own timeout handling only

# ever reaches the direct child, leaving that descendant free to keep

# burning CPU into the next benchmark's measurement.

proc = subprocess.Popen(

cmd,

cwd=work_dir,

env=env,

stdout=subprocess.PIPE,

stderr=subprocess.PIPE,

text=True,

start_new_session=True,

)

try:

stdout, stderr = proc.communicate(timeout=timeout)

except subprocess.TimeoutExpired:

os.killpg(proc.pid, signal.SIGKILL)

proc.communicate() # reap the process, discard its output

return {

"benchmark": bench,

"status": "timeout",

"detail": f"exceeded {timeout}s timeout",

"mean": None,

}

combined = (stdout or "") + "\n" + (stderr or "")

if proc.returncode == 0 and result_json.exists():

mean_match = MEAN_RE.search(combined)

return {

"benchmark": bench,

"status": "ok",

"detail": None,

"mean": mean_match.group(1).strip() if mean_match else None,

}

return {

"benchmark": bench,

"status": "fail",

"detail": classify_failure(combined),

"mean": None,

}

def write_catalog(results: list[dict], out_dir: Path, label: str) -> None:

out_dir.mkdir(parents=True, exist_ok=True)

catalog_json = out_dir / "catalog.json"

# Write-then-rename rather than truncate-in-place: this is called after

# every single benchmark, so an interruption mid-write must not leave

# catalog.json half-written -- that would both lose every earlier result

# in the file and make the next run's json.loads() fail outright.

tmp_path = out_dir / "catalog.json.tmp"

tmp_path.write_text(json.dumps(results, indent=2, sort_keys=False) + "\n")

tmp_path.replace(catalog_json)

ok = [r for r in results if r["status"] == "ok"]

fail = [r for r in results if r["status"] == "fail"]

timeout = [r for r in results if r["status"] == "timeout"]

lines = []

lines.append(f"# pyperformance on {label} -- benchmark catalog")

lines.append("")

lines.append(

f"Generated by `scripts/pyperformance/run_all.py --label {label}`. "

f"`{label}` was benchmarked as the `--python` target of upstream "

"`pyperformance`. See the script docstring for the psutil-stub workaround "

"used for interpreters without CPython C-API / C-extension support."

)

lines.append("")

lines.append(

f"**{len(ok)} passed, {len(fail)} failed, {len(timeout)} timed out** "

f"out of {len(results)} benchmarks."

)

lines.append("")

lines.append(f"| Benchmark | Status | Mean ({label}) | Notes |")

lines.append("|---|---|---|---|")

for r in sorted(results, key=lambda r: (r["status"] != "ok", r["benchmark"])):

status_label = {"ok": "✅ pass", "fail": "❌ fail", "timeout": "⏱ timeout"}[

r["status"]

]

mean = r["mean"] or ""

detail = (r["detail"] or "").replace("|", "\\|").replace("\n", " ")

lines.append(f"| {r['benchmark']} | {status_label} | {mean} | {detail} |")

lines.append("")

catalog_md = out_dir / "CATALOG.md"

catalog_md.write_text("\n".join(lines) + "\n")

log(f"wrote {catalog_json} and {catalog_md}")

def parse_args() -> argparse.Namespace:

parser = argparse.ArgumentParser(

description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter

)

parser.add_argument(

"--python",

"--rustpython",

dest="python",

type=Path,

default=REPO_ROOT / "target" / "release" / "rustpython",

help="path to the Python (or RustPython) executable to benchmark "

"(default: target/release/rustpython)",

)

parser.add_argument(

"--label",

type=str,

default=None,

help="name for this target, used for the results subdirectory and report "

"headings (default: the executable's basename, e.g. 'rustpython' or "

"'python3.13')",

)

parser.add_argument(

"--no-psutil-stub",

action="store_true",

help="don't inject the pure-Python psutil stub -- use this for a real "

"CPython target, which can install and use the genuine psutil",

)

parser.add_argument(

"--timeout",

type=int,

default=DEFAULT_TIMEOUT,

help="per-benchmark timeout in seconds (default: %(default)s)",

)

parser.add_argument(

"--out",

type=Path,

default=DEFAULT_OUT_DIR,

help="parent directory to write <label>/catalog.json and <label>/CATALOG.md into",

)

parser.add_argument(

"--cache-dir",

type=Path,

default=DEFAULT_CACHE_DIR,

help="directory for the host venv and stub wheel cache (default: target/pyperformance)",

)

parser.add_argument(

"--benchmarks",

type=str,

default=None,

help="comma-separated list of benchmarks to run (default: everything pyperformance knows)",

)

parser.add_argument(

"--fast",

action="store_true",

default=True,

help="pass --fast to pyperformance run (default: on, for a quicker catalog run)",

)

parser.add_argument(

"--rigorous",

action="store_true",

help="pass --rigorous to pyperformance run instead of --fast",

)

parser.add_argument(

"--force",

action="store_true",

help="re-run benchmarks even if already present in an existing catalog.json",

)

return parser.parse_args()

def resolve_target_python(python_arg: Path) -> Path:

target_python = shutil.which(str(python_arg)) or str(python_arg)

target_python = Path(target_python)

if not target_python.exists():

raise SystemExit(

f"Python executable not found: {python_arg} "

"(for RustPython, build it first, e.g. `cargo build --release --features ssl`)"

)

return target_python.resolve()

def load_resumable_catalog(

out_dir: Path, target_python: Path, label: str

) -> dict[str, dict]:

"""Load catalog.json to resume from, keyed by benchmark name -- unless

the target executable (or the stdlib RUSTPYTHONPATH points it at) has

changed since that catalog was written, in which case it is invalidated

(and every benchmark re-run) instead of trusted.

"""

out_dir.mkdir(parents=True, exist_ok=True)

fingerprint_path = out_dir / ".executable-fingerprint"

catalog_json = out_dir / "catalog.json"

current_fingerprint = executable_fingerprint(target_python)

stale = (

fingerprint_path.exists()

and fingerprint_path.read_text().strip() != current_fingerprint

)

all_results: dict[str, dict] = {}

if stale:

log(

f"target executable for label {label!r} changed since the last run; "

"ignoring the existing catalog and re-running all benchmarks"

)

# Persist the invalidated (empty) catalog before recording the new

# fingerprint below -- otherwise a crash between the two writes would

# leave a fingerprint that matches the *new* executable pointing at a

# catalog.json still full of results measured against the old one.

write_catalog([], out_dir, label)

elif catalog_json.exists():

for r in json.loads(catalog_json.read_text()):

all_results[r["benchmark"]] = r

fingerprint_path.write_text(current_fingerprint + "\n")

return all_results

def main() -> None:

args = parse_args()

target_python = resolve_target_python(args.python)

label = args.label or target_python.name

# Every benchmark subprocess runs with cwd=work_dir (below); a relative

# --out, --cache-dir, or RUSTPYTHONPATH would then resolve against that

# directory instead of the one this script was invoked from.

args.out = args.out.resolve()

args.cache_dir = args.cache_dir.resolve()

if os.environ.get("RUSTPYTHONPATH"):

os.environ["RUSTPYTHONPATH"] = str(Path(os.environ["RUSTPYTHONPATH"]).resolve())

out_dir = args.out / label

extra_args = ["--rigorous"] if args.rigorous else ["--fast"]

host_venv = ensure_host_venv(args.cache_dir)

stub_pkgs_dir = (

None

if args.no_psutil_stub

else ensure_stub_psutil_wheel(host_venv, args.cache_dir)

)

pyperformance_bin = host_venv / "bin" / "pyperformance"

benchmarks = (

[b.strip() for b in args.benchmarks.split(",") if b.strip()]

if args.benchmarks

else list_benchmarks(host_venv)

)

log(f"{len(benchmarks)} benchmarks to run against {label} ({target_python})")

work_dir = args.cache_dir / "work" / label

work_dir.mkdir(parents=True, exist_ok=True)

# Starts as *every* benchmark previously recorded (not just this

# invocation's --benchmarks subset), so a partial/targeted re-run never

# drops earlier results from catalog.json/CATALOG.md -- it only updates

# the entries it actually re-ran.

all_results = load_resumable_catalog(out_dir, target_python, label)

for bench in benchmarks:

if bench in all_results and not args.force:

log(f"skipping {bench} (already in catalog; use --force to redo)")

continue

r = run_one_benchmark(

pyperformance_bin,

target_python,

bench,

work_dir,

out_dir,

stub_pkgs_dir,

args.timeout,

extra_args,

)

log(

f" -> {bench}: {r['status']}"

+ (f" ({r['detail']})" if r["detail"] else "")

)

all_results[bench] = r

write_catalog(

list(all_results.values()), out_dir, label

) # incremental, so a crash keeps progress

write_catalog(list(all_results.values()), out_dir, label)

ran = [all_results[b] for b in benchmarks]

ok = sum(1 for r in ran if r["status"] == "ok")

log(

f"done: {ok}/{len(ran)} requested benchmarks passed "

f"({len(all_results)} total in catalog for {label})"

)

if __name__ == "__main__":

main()

RustPython/scripts/pyperformance/run_all.py at main · RustPython/RustPython · GitHub

Repository navigation

Expand file tree

Breadcrumbs

run_all.py

Latest commit

History

Breadcrumbs

run_all.py

File metadata and controls