FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test_runner: add 'test:summary' event · nodejs/node@b62f2f8 · GitHub

/ node Public

Commit b62f2f8

Browse files
cjihrig
authored and
targos
committed
test_runner: add 'test:summary' event
This commit adds a new 'test:summary' event to the test runner's reporting interface. This new event serves two purposes: - In the future, the test runner internals will no longer need to change the process exit code. This may be important to run() users. Unfortunately, this is a breaking change, so it needs to be changed in a major version. - The reporting interface now has a single event that can identify passing or failing test runs. Refs: #53867 Refs: #54812 PR-URL: #54851 Reviewed-By: Chemi Atlow <chemi@atlow.co.il> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent 9ae80e1 commit b62f2f8

7 files changed

Lines changed: 62 additions & 12 deletions

File tree

‎doc/api/test.md‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3034,6 +3034,31 @@ This event is only emitted if `--test` flag is passed.
30343034
This event is not guaranteed to be emitted in the same order as the tests are
30353035
defined.
30363036

3037+
### Event: `'test:summary'`
3038+
3039+
* `data` {Object}
3040+
* `counts` {Object} An object containing the counts of various test results.
3041+
* `cancelled` {number} The total number of cancelled tests.
3042+
* `failed` {number} The total number of failed tests.
3043+
* `passed` {number} The total number of passed tests.
3044+
* `skipped` {number} The total number of skipped tests.
3045+
* `suites` {number} The total number of suites run.
3046+
* `tests` {number} The total number of tests run, excluding suites.
3047+
* `todo` {number} The total number of TODO tests.
3048+
* `topLevel` {number} The total number of top level tests and suites.
3049+
* `duration_ms` {number} The duration of the test run in milliseconds.
3050+
* `file` {string|undefined} The path of the test file that generated the
3051+
summary. If the summary corresponds to multiple files, this value is
3052+
`undefined`.
3053+
* `success` {boolean} Indicates whether or not the test run is considered
3054+
successful or not. If any error condition occurs, such as a failing test or
3055+
unmet coverage threshold, this value will be set to `false`.
3056+
3057+
Emitted when a test run completes. This event contains metrics pertaining to
3058+
the completed test run, and is useful for determining if a test run passed or
3059+
failed. If process-level test isolation is used, a `'test:summary'` event is
3060+
generated for each test file in addition to a final cumulative summary.
3061+
30373062
### Event: `'test:watch:drained'`
30383063

30393064
Emitted when no more tests are queued for execution in watch mode.

‎lib/internal/main/test_runner.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ if (isUsingInspector() && options.isolation === 'process') {
3131
options.globPatterns = ArrayPrototypeSlice(process.argv, 1);
3232

3333
debug('test runner configuration:', options);
34-
run(options).on('test:fail', (data) => {
35-
if (data.todo === undefined || data.todo === false) {
34+
run(options).on('test:summary', (data) => {
35+
if (!data.success) {
3636
process.exitCode = kGenericUserError;
3737
}
3838
});

‎lib/internal/test_runner/harness.js‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ function createTestTree(rootTestOptions, globalOptions) {
5252
resetCounters() {
5353
harness.counters = {
5454
__proto__: null,
55-
all: 0,
55+
tests: 0,
5656
failed: 0,
5757
passed: 0,
5858
cancelled: 0,
@@ -62,6 +62,7 @@ function createTestTree(rootTestOptions, globalOptions) {
6262
suites: 0,
6363
};
6464
},
65+
success: true,
6566
counters: null,
6667
shouldColorizeTestFiles: shouldColorizeTestFiles(globalOptions.destinations),
6768
teardown: null,
@@ -130,6 +131,7 @@ function createProcessEventHandler(eventName, rootTest) {
130131
}
131132

132133
rootTest.diagnostic(msg);
134+
rootTest.harness.success = false;
133135
process.exitCode = kGenericUserError;
134136
return;
135137
}
@@ -152,6 +154,7 @@ function configureCoverage(rootTest, globalOptions) {
152154
const msg = `Warning: Code coverage could not be enabled. ${err}`;
153155

154156
rootTest.diagnostic(msg);
157+
rootTest.harness.success = false;
155158
process.exitCode = kGenericUserError;
156159
}
157160
}
@@ -167,13 +170,15 @@ function collectCoverage(rootTest, coverage) {
167170
summary = coverage.summary();
168171
} catch (err) {
169172
rootTest.diagnostic(`Warning: Could not report code coverage. ${err}`);
173+
rootTest.harness.success = false;
170174
process.exitCode = kGenericUserError;
171175
}
172176

173177
try {
174178
coverage.cleanup();
175179
} catch (err) {
176180
rootTest.diagnostic(`Warning: Could not clean up code coverage. ${err}`);
181+
rootTest.harness.success = false;
177182
process.exitCode = kGenericUserError;
178183
}
179184

@@ -248,14 +253,16 @@ function lazyBootstrapRoot() {
248253
if (!globalRoot) {
249254
// This is where the test runner is bootstrapped when node:test is used
250255
// without the --test flag or the run() API.
256+
const entryFile = process.argv?.[1];
251257
const rootTestOptions = {
252258
__proto__: null,
253-
entryFile: process.argv?.[1],
259+
entryFile,
260+
loc: entryFile ? [1, 1, entryFile] : undefined,
254261
};
255262
const globalOptions = parseCommandLine();
256263
createTestTree(rootTestOptions, globalOptions);
257-
globalRoot.reporter.on('test:fail', (data) => {
258-
if (data.todo === undefined || data.todo === false) {
264+
globalRoot.reporter.on('test:summary', (data) => {
265+
if (!data.success) {
259266
process.exitCode = kGenericUserError;
260267
}
261268
});

‎lib/internal/test_runner/test.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,14 +1043,15 @@ class Test extends AsyncResource {
10431043
reporter.diagnostic(nesting, loc, diagnostics[i]);
10441044
}
10451045

1046-
reporter.diagnostic(nesting, loc, `tests ${harness.counters.all}`);
1046+
const duration = this.duration();
1047+
reporter.diagnostic(nesting, loc, `tests ${harness.counters.tests}`);
10471048
reporter.diagnostic(nesting, loc, `suites ${harness.counters.suites}`);
10481049
reporter.diagnostic(nesting, loc, `pass ${harness.counters.passed}`);
10491050
reporter.diagnostic(nesting, loc, `fail ${harness.counters.failed}`);
10501051
reporter.diagnostic(nesting, loc, `cancelled ${harness.counters.cancelled}`);
10511052
reporter.diagnostic(nesting, loc, `skipped ${harness.counters.skipped}`);
10521053
reporter.diagnostic(nesting, loc, `todo ${harness.counters.todo}`);
1053-
reporter.diagnostic(nesting, loc, `duration_ms ${this.duration()}`);
1054+
reporter.diagnostic(nesting, loc, `duration_ms ${duration}`);
10541055

10551056
if (coverage) {
10561057
const coverages = [
@@ -1067,6 +1068,7 @@ class Test extends AsyncResource {
10671068
for (let i = 0; i < coverages.length; i++) {
10681069
const { threshold, actual, name } = coverages[i];
10691070
if (actual < threshold) {
1071+
harness.success = false;
10701072
process.exitCode = kGenericUserError;
10711073
reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
10721074
}
@@ -1075,6 +1077,10 @@ class Test extends AsyncResource {
10751077
reporter.coverage(nesting, loc, coverage);
10761078
}
10771079

1080+
reporter.summary(
1081+
nesting, loc?.file, harness.success, harness.counters, duration,
1082+
);
1083+
10781084
if (harness.watching) {
10791085
this.reported = false;
10801086
harness.resetCounters();

‎lib/internal/test_runner/tests_stream.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,16 @@ class TestsStream extends Readable {
132132
});
133133
}
134134

135+
summary(nesting, file, success, counts, duration_ms) {
136+
this[kEmitMessage]('test:summary', {
137+
__proto__: null,
138+
success,
139+
counts,
140+
duration_ms,
141+
file,
142+
});
143+
}
144+
135145
end() {
136146
this.#tryPush(null);
137147
}

‎lib/internal/test_runner/utils.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,12 +357,14 @@ function countCompletedTest(test, harness = test.root.harness) {
357357
harness.counters.todo++;
358358
} else if (test.cancelled) {
359359
harness.counters.cancelled++;
360+
harness.success = false;
360361
} else if (!test.passed) {
361362
harness.counters.failed++;
363+
harness.success = false;
362364
} else {
363365
harness.counters.passed++;
364366
}
365-
harness.counters.all++;
367+
harness.counters.tests++;
366368
}
367369

368370

‎test/parallel/test-runner-reporters.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ describe('node:test reporters', { concurrency: true }, () => {
113113
testFile]);
114114
assert.strictEqual(child.stderr.toString(), '');
115115
const stdout = child.stdout.toString();
116-
assert.match(stdout, /{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/);
116+
assert.match(stdout, /{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:summary":2,"test:diagnostic":\d+}$/);
117117
assert.strictEqual(stdout.slice(0, filename.length + 2), `${filename} {`);
118118
});
119119
});
@@ -125,7 +125,7 @@ describe('node:test reporters', { concurrency: true }, () => {
125125
assert.strictEqual(child.stderr.toString(), '');
126126
assert.match(
127127
child.stdout.toString(),
128-
/^package: reporter-cjs{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
128+
/^package: reporter-cjs{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:summary":2,"test:diagnostic":\d+}$/,
129129
);
130130
});
131131

@@ -136,7 +136,7 @@ describe('node:test reporters', { concurrency: true }, () => {
136136
assert.strictEqual(child.stderr.toString(), '');
137137
assert.match(
138138
child.stdout.toString(),
139-
/^package: reporter-esm{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:diagnostic":\d+}$/,
139+
/^package: reporter-esm{"test:enqueue":5,"test:dequeue":5,"test:complete":5,"test:start":4,"test:pass":2,"test:fail":2,"test:plan":2,"test:summary":2,"test:diagnostic":\d+}$/,
140140
);
141141
});
142142

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL