FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

benchmark: improve `--filter` pattern matching · nodejs/node@ac5de4f · GitHub

/ node Public

Commit ac5de4f

Browse files
mmarchini
authored and
targos
committed
benchmark: improve --filter pattern matching
 * Let users provide more than one pattern by repeating the flag * Add new flag --exclude to exclude patterns * Add tests for --filter * Document --filter This commit also fixes a bug where things like `compare.js --new --old binary --new binary` was acceptable (now the script will exit and print the usage message). PR-URL: #29987 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Denys Otrishko <shishugi@gmail.com> Reviewed-By: Rich Trott <rtrott@gmail.com>
1 parent 50f6541 commit ac5de4f

5 files changed

Lines changed: 152 additions & 7 deletions

File tree

‎benchmark/_cli.js‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ function CLI(usage, settings) {
3434
if (arg === '--') {
3535
// Only items can follow --
3636
mode = 'item';
37-
} else if (['both', 'option'].includes(mode) && arg[0] === '-') {
37+
} else if ('both' === mode && arg[0] === '-') {
3838
// Optional arguments declaration
3939

4040
if (arg[1] === '-') {
@@ -82,17 +82,36 @@ CLI.prototype.abort = function(msg) {
8282

8383
CLI.prototype.benchmarks = function() {
8484
const paths = [];
85-
const filter = this.optional.filter || false;
8685

8786
for (const category of this.items) {
8887
if (benchmarks[category] === undefined)
8988
continue;
9089
for (const scripts of benchmarks[category]) {
91-
if (filter && scripts.lastIndexOf(filter) === -1) continue;
90+
if (this.shouldSkip(scripts)) continue;
9291

9392
paths.push(path.join(category, scripts));
9493
}
9594
}
9695

9796
return paths;
9897
};
98+
99+
CLI.prototype.shouldSkip = function(scripts) {
100+
const filters = this.optional.filter || [];
101+
const excludes = this.optional.exclude || [];
102+
let skip = filters.length > 0;
103+
104+
for (const filter of filters) {
105+
if (scripts.lastIndexOf(filter) !== -1) {
106+
skip = false;
107+
}
108+
}
109+
110+
for (const exclude of excludes) {
111+
if (scripts.lastIndexOf(exclude) !== -1) {
112+
skip = true;
113+
}
114+
}
115+
116+
return skip;
117+
};

‎benchmark/compare.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,13 @@ const cli = CLI(`usage: ./node compare.js [options] [--] <category> ...
1818
--new ./new-node-binary new node binary (required)
1919
--old ./old-node-binary old node binary (required)
2020
--runs 30 number of samples
21-
--filter pattern string to filter benchmark scripts
21+
--filter pattern includes only benchmark scripts matching
22+
<pattern> (can be repeated)
23+
--exclude pattern excludes scripts matching <pattern> (can be
24+
repeated)
2225
--set variable=value set benchmark variable (can be repeated)
2326
--no-progress don't show benchmark progress indicator
24-
`, { arrayArgs: ['set'], boolArgs: ['no-progress'] });
27+
`, { arrayArgs: ['set', 'filter', 'exclude'], boolArgs: ['no-progress'] });
2528

2629
if (!cli.optional.new || !cli.optional.old) {
2730
cli.abort(cli.usage);

‎benchmark/run.js‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,13 @@ const cli = CLI(`usage: ./node run.js [options] [--] <category> ...
88
Run each benchmark in the <category> directory a single time, more than one
99
<category> directory can be specified.
1010
11-
--filter pattern string to filter benchmark scripts
11+
--filter pattern includes only benchmark scripts matching <pattern>
12+
(can be repeated)
13+
--exclude pattern excludes scripts matching <pattern> (can be
14+
repeated)
1215
--set variable=value set benchmark variable (can be repeated)
1316
--format [simple|csv] optional value that specifies the output format
14-
`, { arrayArgs: ['set'] });
17+
`, { arrayArgs: ['set', 'filter', 'exclude'] });
1518
const benchmarks = cli.benchmarks();
1619

1720
if (benchmarks.length === 0) {

‎benchmark/writing-and-running-benchmarks.md‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
* [Running benchmarks](#running-benchmarks)
99
* [Running individual benchmarks](#running-individual-benchmarks)
1010
* [Running all benchmarks](#running-all-benchmarks)
11+
* [Filtering benchmarks](#filtering-benchmarks)
1112
* [Comparing Node.js versions](#comparing-nodejs-versions)
1213
* [Comparing parameters](#comparing-parameters)
1314
* [Running Benchmarks on the CI](#running-benchmarks-on-the-ci)
@@ -149,6 +150,87 @@ It is possible to execute more groups by adding extra process arguments.
149150
$ node benchmark/run.js assert async_hooks
150151
```
151152

153+
#### Filtering benchmarks
154+
155+
`benchmark/run.js` and `benchmark/compare.js` have `--filter pattern` and
156+
`--exclude pattern` options, which can be used to run a subset of benchmarks or
157+
to exclude specific benchmarks from the execution, respectively.
158+
159+
```console
160+
$ node benchmark/run.js --filter "deepequal-b" assert
161+
162+
assert/deepequal-buffer.js
163+
assert/deepequal-buffer.js method="deepEqual" strict=0 len=100 n=20000: 773,200.4995493788
164+
assert/deepequal-buffer.js method="notDeepEqual" strict=0 len=100 n=20000: 964,411.712953848
165+
166+
$ node benchmark/run.js --exclude "deepequal-b" assert
167+
168+
assert/deepequal-map.js
169+
assert/deepequal-map.js method="deepEqual_primitiveOnly" strict=0 len=500 n=500: 20,445.06368453332
170+
assert/deepequal-map.js method="deepEqual_objectOnly" strict=0 len=500 n=500: 1,393.3481642240833
171+
...
172+
173+
assert/deepequal-object.js
174+
assert/deepequal-object.js method="deepEqual" strict=0 size=100 n=5000: 1,053.1950937538475
175+
assert/deepequal-object.js method="notDeepEqual" strict=0 size=100 n=5000: 9,734.193251965213
176+
...
177+
```
178+
179+
`--filter` and `--exclude` can be repeated to provide multiple patterns.
180+
181+
```console
182+
$ node benchmark/run.js --filter "deepequal-b" --filter "deepequal-m" assert
183+
184+
assert/deepequal-buffer.js
185+
assert/deepequal-buffer.js method="deepEqual" strict=0 len=100 n=20000: 773,200.4995493788
186+
assert/deepequal-buffer.js method="notDeepEqual" strict=0 len=100 n=20000: 964,411.712953848
187+
188+
assert/deepequal-map.js
189+
assert/deepequal-map.js method="deepEqual_primitiveOnly" strict=0 len=500 n=500: 20,445.06368453332
190+
assert/deepequal-map.js method="deepEqual_objectOnly" strict=0 len=500 n=500: 1,393.3481642240833
191+
192+
$ node benchmark/run.js --exclude "deepequal-b" --exclude "deepequal-m" assert
193+
194+
assert/deepequal-object.js
195+
assert/deepequal-object.js method="deepEqual" strict=0 size=100 n=5000: 1,053.1950937538475
196+
assert/deepequal-object.js method="notDeepEqual" strict=0 size=100 n=5000: 9,734.193251965213
197+
...
198+
199+
assert/deepequal-prims-and-objs-big-array-set.js
200+
assert/deepequal-prims-and-objs-big-array-set.js method="deepEqual_Array" strict=0 len=20000 n=25 primitive="string": 865.2977195251661
201+
assert/deepequal-prims-and-objs-big-array-set.js method="notDeepEqual_Array" strict=0 len=20000 n=25 primitive="string": 827.8297281403861
202+
assert/deepequal-prims-and-objs-big-array-set.js method="deepEqual_Set" strict=0 len=20000 n=25 primitive="string": 28,826.618268696366
203+
...
204+
```
205+
206+
If `--filter` and `--exclude` are used together, `--filter` is applied first,
207+
and `--exclude` is applied on the result of `--filter`:
208+
209+
```console
210+
$ node benchmark/run.js --filter "bench-" process
211+
212+
process/bench-env.js
213+
process/bench-env.js operation="get" n=1000000: 2,356,946.0770617095
214+
process/bench-env.js operation="set" n=1000000: 1,295,176.3266261867
215+
process/bench-env.js operation="enumerate" n=1000000: 24,592.32231990992
216+
process/bench-env.js operation="query" n=1000000: 3,625,787.2150573144
217+
process/bench-env.js operation="delete" n=1000000: 1,521,131.5742806569
218+
219+
process/bench-hrtime.js
220+
process/bench-hrtime.js type="raw" n=1000000: 13,178,002.113936031
221+
process/bench-hrtime.js type="diff" n=1000000: 11,585,435.712423025
222+
process/bench-hrtime.js type="bigint" n=1000000: 13,342,884.703919787
223+
224+
$ node benchmark/run.js --filter "bench-" --exclude "hrtime" process
225+
226+
process/bench-env.js
227+
process/bench-env.js operation="get" n=1000000: 2,356,946.0770617095
228+
process/bench-env.js operation="set" n=1000000: 1,295,176.3266261867
229+
process/bench-env.js operation="enumerate" n=1000000: 24,592.32231990992
230+
process/bench-env.js operation="query" n=1000000: 3,625,787.2150573144
231+
process/bench-env.js operation="delete" n=1000000: 1,521,131.5742806569
232+
```
233+
152234
### Comparing Node.js versions
153235

154236
To compare the effect of a new Node.js version use the `compare.js` tool. This

‎test/parallel/test-benchmark-cli.js‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
'use strict';
2+
3+
require('../common');
4+
5+
// This tests the CLI parser for our benchmark suite.
6+
7+
const assert = require('assert');
8+
9+
const CLI = require('../../benchmark/_cli.js');
10+
11+
const originalArgv = process.argv;
12+
13+
function testFilterPattern(filters, excludes, filename, expectedResult) {
14+
process.argv = process.argv.concat(...filters.map((p) => ['--filter', p]));
15+
process.argv = process.argv.concat(...excludes.map((p) => ['--exclude', p]));
16+
process.argv = process.argv.concat(['bench']);
17+
18+
const cli = new CLI('', { 'arrayArgs': ['filter', 'exclude'] });
19+
assert.deepStrictEqual(cli.shouldSkip(filename), expectedResult);
20+
21+
process.argv = originalArgv;
22+
}
23+
24+
25+
testFilterPattern([], [], 'foo', false);
26+
27+
testFilterPattern(['foo'], [], 'foo', false);
28+
testFilterPattern(['foo'], [], 'bar', true);
29+
testFilterPattern(['foo', 'bar'], [], 'foo', false);
30+
testFilterPattern(['foo', 'bar'], [], 'bar', false);
31+
32+
testFilterPattern([], ['foo'], 'foo', true);
33+
testFilterPattern([], ['foo'], 'bar', false);
34+
testFilterPattern([], ['foo', 'bar'], 'foo', true);
35+
testFilterPattern([], ['foo', 'bar'], 'bar', true);
36+
37+
testFilterPattern(['foo'], ['bar'], 'foo', false);
38+
testFilterPattern(['foo'], ['bar'], 'foo-bar', true);

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL