on:
pull_request:
types: [labeled, opened, synchronize, reopened]
workflow_dispatch:
name: pyperformance
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: true
env:
CARGO_INCREMENTAL: 0
CARGO_TERM_COLOR: always
CI: true
# Pure-Python benchmarks only: anything whose workload needs a real C
# extension cannot install under RustPython at all, and the async/SSL ones
# are far too slow here to be worth a runner's time. Keep this list short --
# every entry is paid for three times (CPython, base, head) in one job.
BENCHMARKS: >-
chaos,comprehensions,coroutines,deepcopy,deltablue,fannkuch,float,
generators,go,hexiom,json_dumps,json_loads,logging,mdp,nbody,nqueens,
pickle_pure_python,raytrace,regex_compile,regex_v8,richards,scimark,
spectral_norm,telco,typing_runtime_protocols,unpack_sequence,
unpickle_pure_python,xml_etree
# A benchmark that needs longer than this on the base commit is one this
# comparison cannot afford to wait for; it shows up as a failure in the
# table, which is itself the interesting signal.
BENCH_TIMEOUT: 300
jobs:
compare:
# `skip:ci` mirrors the other workflows; `run:pyperformance` opts a PR in,
# because the job costs well over an hour and most PRs do not need it.
if: >-
github.event_name == 'workflow_dispatch' ||
(!contains(github.event.pull_request.labels.*.name, 'skip:ci') &&
contains(github.event.pull_request.labels.*.name, 'run:pyperformance'))
name: base vs. head vs. CPython
runs-on: ubuntu-latest
timeout-minutes: 300
permissions:
contents: read
# Honoured for a pull request opened from this repo, which is then
# commented on directly by the last step; a fork's run is handed
# read-only whatever is asked for here and goes the artifact route.
pull-requests: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 0
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
# Both the third interpreter under test and the host pyperformance
# itself runs on (it only runs under a real CPython; RustPython is just
# the `--python` target it drives).
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
id: cpython
with:
python-version: "3.14"
- name: Restore cache
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
restore-keys: pyperformance-
continue-on-error: true
# The base commit may predate these scripts, and `git checkout `
# below would then delete them; run them from a copy outside the tree.
- name: Copy the benchmark scripts out of the checkout
run: cp -r scripts/pyperformance "${RUNNER_TEMP}/pyperformance-scripts"
# Each build is copied aside with the stdlib of its own commit, so the
# second does not overwrite the first and a PR that touches `Lib/` is
# still compared against the base commit's `Lib/`. A binary moved away
# from its checkout finds the stdlib only through RUSTPYTHONPATH, which
# `run_all.py` forwards through pyperf into the venv it builds.
- name: Build head
run: |
cargo build --release --locked
mkdir -p "${RUNNER_TEMP}/head"
cp target/release/rustpython "${RUNNER_TEMP}/head/rustpython"
cp -r Lib "${RUNNER_TEMP}/head/Lib"
- name: Build base
continue-on-error: true # a base that cannot build leaves head vs. CPython
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
run: |
git checkout --quiet "${BASE_SHA}" --
cargo build --release --locked
mkdir -p "${RUNNER_TEMP}/base"
cp target/release/rustpython "${RUNNER_TEMP}/base/rustpython"
cp -r Lib "${RUNNER_TEMP}/base/Lib"
- name: Restore head checkout
env:
HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: git checkout --quiet "${HEAD_SHA}" --
- name: Save cache
if: always()
uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
target/
key: pyperformance-${{ hashFiles('**/Cargo.lock') }}
continue-on-error: true
# The three runs are deliberately adjacent, with no build between them,
# so nothing but the interpreter differs across the comparison.
- name: Measure CPython
env:
CPYTHON: ${{ steps.cpython.outputs.python-path }}
run: |
python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
--python "${CPYTHON}" \
--label cpython --no-psutil-stub \
--benchmarks "${BENCHMARKS//[$'\n' ]/}" \
--timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
- name: Measure base
continue-on-error: true # as above: the table falls back to head alone
env:
RUSTPYTHONPATH: ${{ runner.temp }}/base/Lib
run: |
python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
--python "${RUNNER_TEMP}/base/rustpython" \
--label base \
--benchmarks "${BENCHMARKS//[$'\n' ]/}" \
--timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
- name: Measure head
env:
RUSTPYTHONPATH: ${{ runner.temp }}/head/Lib
run: |
python3 "${RUNNER_TEMP}/pyperformance-scripts/run_all.py" \
--python "${RUNNER_TEMP}/head/rustpython" \
--label head \
--benchmarks "${BENCHMARKS//[$'\n' ]/}" \
--timeout "${BENCH_TIMEOUT}" --out "${RUNNER_TEMP}/pyperf-results"
- name: Render the comparison
env:
CPYTHON_VERSION: ${{ steps.cpython.outputs.python-version }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
python3 "${RUNNER_TEMP}/pyperformance-scripts/pr_diff.py" \
--results-dir "${RUNNER_TEMP}/pyperf-results" \
--cpython-version "CPython ${CPYTHON_VERSION}" \
--md diff.md --json diff.json
echo "${PR_NUMBER}" > pr_number.txt
# A `pull_request` run triggered from a fork carries a read-only token no
# matter what this workflow grants, so the comment is posted from
# `pyperformance-comment.yaml`, which `workflow_run` runs in the base
# repo's context. This artifact is the hand-off.
- name: Upload the comparison
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: pyperformance-diff
path: |
diff.md
diff.json
pr_number.txt
retention-days: 14
if-no-files-found: ignore
# A pull request opened from this repo can be commented on from here,
# which puts the table on the PR in the same run instead of waiting for
# the companion workflow. Forks fall through to that workflow instead.
- name: Post or update the PR comment
if: >-
github.event_name == 'pull_request' &&
github.event.pull_request.head.repo.full_name == github.repository
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const fs = require('fs');
const body = fs.readFileSync('diff.md', 'utf8');
const marker = '';
const comments = await github.paginate(github.rest.issues.listComments, {
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
});
const existing = comments.find((c) => c.body && c.body.includes(marker));
if (existing) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existing.id,
body,
});
} else {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.payload.pull_request.number,
body,
});
}