FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

benchmark: add --analyze mode to compare.js · nodejs/node@74234ee · GitHub

/ node Public

Commit 74234ee

Browse files
jasnell
authored and
aduh95
committed
benchmark: add --analyze mode to compare.js
Add an --analyze flag that performs statistical analysis directly after benchmarks complete, eliminating the need for R and compare.R. When --analyze is specified, compare.js collects the rate data during the run and prints a statistical summary table instead of CSV output. The table matches the format of compare.R: improvement percentage, significance stars (* p<0.05, ** p<0.01, *** p<0.001), and confidence intervals at three risk levels. Also adds a --max-regression N option that causes the compare.js to exit with 1 (error) when the `--new` is N% slower. Useful for CI use to detect regressions. Uses the histogram API's welchTest() and cohensD() methods introduced in the previous commit. Benchmark rates are scaled to integers for HdrHistogram recording; the --scale option (default 1000) controls the multiplier for precision. Usage: node benchmark/compare.js --old ./node-old --new ./node-new \ --analyze url Signed-off-by: James M Snell <jasnell@gmail.com> Assisted-by: Opencode/Opus PR-URL: #65416 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
1 parent 9e8e908 commit 74234ee

3 files changed

Lines changed: 288 additions & 26 deletions

File tree

‎benchmark/_benchmark_progress.js‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,10 @@ function getTime(diff) {
2525
// A run is an item in the job queue: { binary, filename, iter }
2626
// A config is an item in the subqueue: { binary, filename, iter, configs }
2727
class BenchmarkProgress {
28-
constructor(queue, benchmarks) {
28+
constructor(queue, benchmarks, options = {}) {
2929
this.queue = queue; // Scheduled runs.
3030
this.benchmarks = benchmarks; // Filenames of scheduled benchmarks.
31+
this.analyze = !!options.analyze; // stdout is not piped, but unused.
3132
this.completedRuns = 0; // Number of completed runs.
3233
this.scheduledRuns = queue.length; // Number of scheduled runs.
3334
// Time when starting to run benchmarks.
@@ -107,7 +108,10 @@ class BenchmarkProgress {
107108
}
108109

109110
updateProgress() {
110-
if (!process.stderr.isTTY || process.stdout.isTTY) {
111+
// Progress renders on stderr when stdout is piped (not a TTY).
112+
// In --analyze mode, stdout is the terminal but is unused during
113+
// the run, so treat it the same as piped.
114+
if (!process.stderr.isTTY || (process.stdout.isTTY && !this.analyze)) {
111115
return;
112116
}
113117
readline.clearLine(process.stderr);

‎benchmark/compare.js‎

Lines changed: 230 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
1313
Run each benchmark in the <category> directory many times using two different
1414
node versions. More than one <category> directory can be specified.
1515
The output is formatted as csv, which can be processed using for
16-
example 'compare.R'.
16+
example 'compare.R'. Use --analyze to perform statistical analysis
17+
directly without R.
1718
1819
--new ./new-node-binary new node binary (required)
1920
--old ./old-node-binary old node binary (required)
@@ -24,20 +25,33 @@ const cli = new CLI(`usage: ./node compare.js [options] [--] <category> ...
2425
repeated)
2526
--set variable=value set benchmark variable (can be repeated)
2627
--no-progress don't show benchmark progress indicator
28+
--analyze perform statistical analysis after benchmarks
29+
complete (Welch's t-test, effect size) instead
30+
of printing csv output
31+
--scale 1000 rate-to-integer multiplier for histogram
32+
precision when using --analyze (default: 1000)
33+
--max-regression N exit with code 1 if any statistically
34+
significant regression exceeds N% (implies
35+
--analyze)
2736
2837
Examples:
2938
--set CPUSET=0 Runs benchmarks on CPU core 0.
3039
--set CPUSET=0-2 Specifies that benchmarks should run on CPU cores 0 to 2.
3140
3241
Note: The CPUSET format should match the specifications of the 'taskset' command
33-
`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress'] });
42+
`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress', 'analyze'] });
3443

3544
if (!cli.optional.new || !cli.optional.old) {
3645
cli.abort(cli.usage);
3746
}
3847

3948
const binaries = ['old', 'new'];
4049
const runs = cli.optional.runs ? parseInt(cli.optional.runs, 10) : 30;
50+
const maxRegression = cli.optional['max-regression'] ?
51+
parseFloat(cli.optional['max-regression']) :
52+
0;
53+
const analyze = !!cli.optional.analyze || maxRegression > 0;
54+
const scale = cli.optional.scale ? parseInt(cli.optional.scale, 10) : 1000;
4155
const benchmarks = cli.benchmarks();
4256

4357
if (benchmarks.length === 0) {
@@ -46,6 +60,9 @@ if (benchmarks.length === 0) {
4660
return;
4761
}
4862

63+
// When --analyze is set, collect results for statistical analysis.
64+
const results = analyze ? new Map() : null;
65+
4966
// Create queue from the benchmarks list such both node versions are tested
5067
// `runs` amount of times each.
5168
// Note: BenchmarkProgress relies on this order to estimate
@@ -61,15 +78,17 @@ for (const filename of benchmarks) {
6178
}
6279
// queue.length = binary.length * runs * benchmarks.length
6380

64-
// Print csv header
65-
console.log('"binary","filename","configuration","rate","time"');
81+
// Print csv header (unless analyzing inline).
82+
if (!analyze) {
83+
console.log('"binary","filename","configuration","rate","time"');
84+
}
6685

6786
const kStartOfQueue = 0;
6887

6988
const showProgress = !cli.optional['no-progress'];
7089
let progress;
7190
if (showProgress) {
72-
progress = new BenchmarkProgress(queue, benchmarks);
91+
progress = new BenchmarkProgress(queue, benchmarks, { analyze });
7392
progress.startQueue(kStartOfQueue);
7493
}
7594

@@ -99,11 +118,20 @@ if (showProgress) {
99118
conf += ` ${key}=${inspect(data.conf[key])}`;
100119
}
101120
conf = conf.slice(1);
102-
// Escape quotes (") for correct csv formatting
103-
conf = conf.replace(/"/g, '""');
104121

105-
console.log(`"${job.binary}","${job.filename}","${conf}",` +
106-
`${data.rate},${data.time}`);
122+
if (analyze) {
123+
// Collect results for post-run analysis.
124+
const name = `${job.filename} ${conf}`;
125+
if (!results.has(name)) {
126+
results.set(name, { old: [], new: [] });
127+
}
128+
results.get(name)[job.binary].push(data.rate);
129+
} else {
130+
// Escape quotes (") for correct csv formatting
131+
conf = conf.replace(/"/g, '""');
132+
console.log(`"${job.binary}","${job.filename}","${conf}",` +
133+
`${data.rate},${data.time}`);
134+
}
107135
if (showProgress) {
108136
// One item in the subqueue has been completed.
109137
progress.completeConfig(data);
@@ -125,6 +153,199 @@ if (showProgress) {
125153
// If there are more benchmarks execute the next
126154
if (i + 1 < queue.length) {
127155
recursive(i + 1);
156+
} else if (analyze) {
157+
printAnalysis(results, scale, maxRegression);
128158
}
129159
});
130160
})(kStartOfQueue);
161+
162+
function printAnalysis(results, scale, maxRegression) {
163+
const { createHistogram } = require('node:perf_hooks');
164+
165+
// Build per-benchmark histograms and run statistical tests.
166+
const rows = [];
167+
let maxNameLen = 0;
168+
169+
let skipped = 0;
170+
171+
for (const [name, { old: oldRates, new: newRates }] of results) {
172+
if (oldRates.length < 2 || newRates.length < 2) {
173+
skipped++;
174+
continue;
175+
}
176+
177+
const hOld = createHistogram({ figures: 3 });
178+
const hNew = createHistogram({ figures: 3 });
179+
180+
for (const r of oldRates) hOld.record(Math.max(1, Math.round(r * scale)));
181+
for (const r of newRates) hNew.record(Math.max(1, Math.round(r * scale)));
182+
183+
const oldMean = oldRates.reduce((a, b) => a + b, 0) / oldRates.length;
184+
const newMean = newRates.reduce((a, b) => a + b, 0) / newRates.length;
185+
const improvement = ((newMean - oldMean) / oldMean) * 100;
186+
187+
// Query the three confidence levels. The p-value and t-statistic
188+
// are the same regardless of the confidence level, so we extract
189+
// them from the first result.
190+
const w95 = hOld.welchTest(hNew, { confidence: 0.95 });
191+
const w99 = hOld.welchTest(hNew, { confidence: 0.99 });
192+
const w999 = hOld.welchTest(hNew, { confidence: 0.999 });
193+
194+
// Significance stars matching compare.R convention.
195+
let stars = '';
196+
if (w95.pValue < 0.001) stars = '***';
197+
else if (w95.pValue < 0.01) stars = ' **';
198+
else if (w95.pValue < 0.05) stars = ' *';
199+
200+
// Confidence intervals expressed as percentage of the old mean.
201+
const ciPct = (w) => {
202+
const half =
203+
(w.confidenceInterval.upper - w.confidenceInterval.lower) / 2;
204+
return (half / (oldMean * scale)) * 100;
205+
};
206+
207+
rows.push({
208+
name,
209+
stars,
210+
improvement,
211+
ci95: ciPct(w95),
212+
ci99: ciPct(w99),
213+
ci999: ciPct(w999),
214+
pValue: w95.pValue,
215+
});
216+
217+
if (name.length > maxNameLen) maxNameLen = name.length;
218+
}
219+
220+
// Print header.
221+
const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));
222+
const rpad = (s, n) => ' '.repeat(Math.max(0, n - s.length)) + s;
223+
224+
console.log(`${pad('', maxNameLen)} confidence` +
225+
` improvement accuracy (*) (**) (***)`);
226+
227+
for (const row of rows) {
228+
const imp = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)} %`;
229+
console.log(
230+
`${pad(row.name, maxNameLen)} ${pad(row.stars, 10)}` +
231+
` ${rpad(imp, 11)}` +
232+
` ±${row.ci95.toFixed(2)}%` +
233+
` ±${row.ci99.toFixed(2)}%` +
234+
` ±${row.ci999.toFixed(2)}%`,
235+
);
236+
}
237+
238+
if (skipped > 0) {
239+
console.log('');
240+
console.log(
241+
`Note: ${skipped} configuration${skipped === 1 ? ' was' : 's were'}` +
242+
` skipped because Welch's t-test requires at least 2 samples per` +
243+
` binary. Use --runs 2 or higher.`,
244+
);
245+
}
246+
247+
// --- Bar chart visualization ---
248+
printChart(rows, maxNameLen);
249+
250+
console.log('');
251+
console.log(
252+
`Rates were scaled by ${scale}x into HdrHistogram (3 significant figures).\n` +
253+
`Use --scale to adjust precision if needed.\n`,
254+
);
255+
console.log(
256+
`Be aware that when doing many comparisons the risk of a false-positive\n` +
257+
`result increases. In this case, there are ${rows.length} comparisons, ` +
258+
`you can thus\nexpect the following amount of false-positive results:\n` +
259+
` ${(rows.length * 0.05).toFixed(2)} false positives, when considering ` +
260+
`a 5% risk acceptance (*, **, ***),\n` +
261+
` ${(rows.length * 0.01).toFixed(2)} false positives, when considering ` +
262+
`a 1% risk acceptance (**, ***),\n` +
263+
` ${(rows.length * 0.001).toFixed(2)} false positives, when considering ` +
264+
`a 0.1% risk acceptance (***)`,
265+
);
266+
267+
// Gate: exit with error if any significant regression exceeds the limit.
268+
if (maxRegression > 0) {
269+
const failures = rows.filter(
270+
(r) => r.stars.trim() !== '' && r.improvement < -maxRegression,
271+
);
272+
if (failures.length > 0) {
273+
console.log('');
274+
console.log(
275+
`FAIL: ${failures.length} benchmark${failures.length === 1 ? '' : 's'}` +
276+
` showed a statistically significant regression exceeding` +
277+
` ${maxRegression}%:`,
278+
);
279+
for (const f of failures) {
280+
console.log(` ${f.name} ${f.improvement.toFixed(2)}%`);
281+
}
282+
process.exitCode = 1;
283+
}
284+
}
285+
}
286+
287+
function printChart(rows, maxNameLen) {
288+
if (rows.length === 0) return;
289+
290+
// Determine the chart scale from the data. The bar region covers
291+
// the range [-maxAbs, +maxAbs] so the zero line sits in the center.
292+
const barWidth = 40;
293+
const halfWidth = barWidth / 2;
294+
let maxAbs = 0;
295+
for (const row of rows) {
296+
const extent = Math.abs(row.improvement) + row.ci95;
297+
if (extent > maxAbs) maxAbs = extent;
298+
}
299+
if (maxAbs === 0) maxAbs = 1;
300+
301+
const pad = (s, n) => s + ' '.repeat(Math.max(0, n - s.length));
302+
303+
// Scale axis labels.
304+
const axisLeft = `-${maxAbs.toFixed(1)}%`;
305+
const axisRight = `+${maxAbs.toFixed(1)}%`;
306+
const axisCenter = '0%';
307+
308+
// Print axis header.
309+
const labelPad = maxNameLen + 5;
310+
const leftLabel = ' '.repeat(labelPad) +
311+
axisLeft +
312+
' '.repeat(Math.max(0, halfWidth - axisLeft.length - Math.floor(axisCenter.length / 2))) +
313+
axisCenter +
314+
' '.repeat(Math.max(0, halfWidth - Math.ceil(axisCenter.length / 2) - axisRight.length)) +
315+
axisRight;
316+
console.log('');
317+
console.log(leftLabel);
318+
319+
for (const row of rows) {
320+
const imp = row.improvement;
321+
const ci = row.ci95;
322+
323+
// Position of the improvement value in the bar region [0, barWidth].
324+
const center = halfWidth;
325+
const impPos = center + (imp / maxAbs) * halfWidth;
326+
327+
// CI extent in bar positions.
328+
const ciLeft = center + ((imp - ci) / maxAbs) * halfWidth;
329+
const ciRight = center + ((imp + ci) / maxAbs) * halfWidth;
330+
331+
// Build the bar character by character.
332+
const chars = [];
333+
for (let x = 0; x < barWidth; x++) {
334+
const pos = x + 0.5; // Center of this character cell.
335+
if (x === Math.floor(center)) {
336+
chars.push('|');
337+
} else if ((imp >= 0 && pos > center && pos <= impPos) ||
338+
(imp < 0 && pos < center && pos >= impPos)) {
339+
chars.push(row.stars ? '\u2588' : '\u2593'); // solid or dark shade
340+
} else if (pos >= ciLeft && pos <= ciRight) {
341+
chars.push('\u2591'); // Light shade for CI region
342+
} else {
343+
chars.push(' ');
344+
}
345+
}
346+
347+
const label = `${row.improvement >= 0 ? '+' : ''}${row.improvement.toFixed(2)}%`;
348+
const sig = row.stars.trim();
349+
console.log(`${pad(row.name, maxNameLen)} ${chars.join('')} ${label} ${sig}`);
350+
}
351+
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL