FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test_runner: add support for coverage via run() · nodejs/node@d7c708a · GitHub

/ node Public

Commit d7c708a

Browse files
atlowChemi
authored and
targos
committed
test_runner: add support for coverage via run()
PR-URL: #53937 Fixes: #53867 Refs: #53924 Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
1 parent 4244f1a commit d7c708a

3 files changed

Lines changed: 247 additions & 1 deletion

File tree

‎doc/api/test.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,6 +1248,9 @@ added:
12481248
- v18.9.0
12491249
- v16.19.0
12501250
changes:
1251+
- version: REPLACEME
1252+
pr-url: https://github.com/nodejs/node/pull/53937
1253+
description: Added coverage options.
12511254
- version: v22.8.0
12521255
pr-url: https://github.com/nodejs/node/pull/53927
12531256
description: Added the `isolation` option.
@@ -1319,6 +1322,29 @@ changes:
13191322
that specifies the index of the shard to run. This option is _required_.
13201323
* `total` {number} is a positive integer that specifies the total number
13211324
of shards to split the test files to. This option is _required_.
1325+
* `coverage` {boolean} enable [code coverage][] collection.
1326+
**Default:** `false`.
1327+
* `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
1328+
using a glob pattern, which can match both absolute and relative file paths.
1329+
This property is only applicable when `coverage` was set to `true`.
1330+
If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
1331+
files must meet **both** criteria to be included in the coverage report.
1332+
**Default:** `undefined`.
1333+
* `coverageIncludeGlobs` {string|Array} Includes specific files in code coverage
1334+
using a glob pattern, which can match both absolute and relative file paths.
1335+
This property is only applicable when `coverage` was set to `true`.
1336+
If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
1337+
files must meet **both** criteria to be included in the coverage report.
1338+
**Default:** `undefined`.
1339+
* `lineCoverage` {number} Require a minimum percent of covered lines. If code
1340+
coverage does not reach the threshold specified, the process will exit with code `1`.
1341+
**Default:** `0`.
1342+
* `branchCoverage` {number} Require a minimum percent of covered branches. If code
1343+
coverage does not reach the threshold specified, the process will exit with code `1`.
1344+
**Default:** `0`.
1345+
* `functionCoverage` {number} Require a minimum percent of covered functions. If code
1346+
coverage does not reach the threshold specified, the process will exit with code `1`.
1347+
**Default:** `0`.
13221348
* Returns: {TestsStream}
13231349

13241350
**Note:** `shard` is used to horizontally parallelize test running across
@@ -3527,6 +3553,7 @@ Can be used to abort test subtasks when the test has been aborted.
35273553
[`run()`]: #runoptions
35283554
[`suite()`]: #suitename-options-fn
35293555
[`test()`]: #testname-options-fn
3556+
[code coverage]: #collecting-code-coverage
35303557
[describe options]: #describename-options-fn
35313558
[it options]: #testname-options-fn
35323559
[stream.compose]: stream.md#streamcomposestreams

‎lib/internal/test_runner/runner.js‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ const {
5555
validateObject,
5656
validateOneOf,
5757
validateInteger,
58+
validateStringArray,
5859
} = require('internal/validators');
5960
const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector');
6061
const { isRegExp } = require('internal/util/types');
@@ -524,7 +525,13 @@ function watchFiles(testFiles, opts) {
524525
function run(options = kEmptyObject) {
525526
validateObject(options, 'options');
526527

527-
let { testNamePatterns, testSkipPatterns, shard } = options;
528+
let {
529+
testNamePatterns,
530+
testSkipPatterns,
531+
shard,
532+
coverageExcludeGlobs,
533+
coverageIncludeGlobs,
534+
} = options;
528535
const {
529536
concurrency,
530537
timeout,
@@ -537,6 +544,10 @@ function run(options = kEmptyObject) {
537544
setup,
538545
only,
539546
globPatterns,
547+
coverage = false,
548+
lineCoverage = 0,
549+
branchCoverage = 0,
550+
functionCoverage = 0,
540551
} = options;
541552

542553
if (files != null) {
@@ -615,6 +626,22 @@ function run(options = kEmptyObject) {
615626
});
616627
}
617628
validateOneOf(isolation, 'options.isolation', ['process', 'none']);
629+
validateBoolean(coverage, 'options.coverage');
630+
if (coverageExcludeGlobs != null) {
631+
if (!ArrayIsArray(coverageExcludeGlobs)) {
632+
coverageExcludeGlobs = [coverageExcludeGlobs];
633+
}
634+
validateStringArray(coverageExcludeGlobs, 'options.coverageExcludeGlobs');
635+
}
636+
if (coverageIncludeGlobs != null) {
637+
if (!ArrayIsArray(coverageIncludeGlobs)) {
638+
coverageIncludeGlobs = [coverageIncludeGlobs];
639+
}
640+
validateStringArray(coverageIncludeGlobs, 'options.coverageIncludeGlobs');
641+
}
642+
validateInteger(lineCoverage, 'options.lineCoverage', 0, 100);
643+
validateInteger(branchCoverage, 'options.branchCoverage', 0, 100);
644+
validateInteger(functionCoverage, 'options.functionCoverage', 0, 100);
618645

619646
const rootTestOptions = { __proto__: null, concurrency, timeout, signal };
620647
const globalOptions = {
@@ -623,6 +650,12 @@ function run(options = kEmptyObject) {
623650
// behavior has relied on it, so removing it must be done in a semver major.
624651
...parseCommandLine(),
625652
setup, // This line can be removed when parseCommandLine() is removed here.
653+
coverage,
654+
coverageExcludeGlobs,
655+
coverageIncludeGlobs,
656+
lineCoverage: lineCoverage,
657+
branchCoverage: branchCoverage,
658+
functionCoverage: functionCoverage,
626659
};
627660
const root = createTestTree(rootTestOptions, globalOptions);
628661

‎test/parallel/test-runner-run-coverage.mjs‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
import * as common from '../common/index.mjs';
2+
import * as fixtures from '../common/fixtures.mjs';
3+
import { describe, it, run } from 'node:test';
4+
import assert from 'node:assert';
5+
import { sep } from 'node:path';
6+
7+
const files = [fixtures.path('test-runner', 'coverage.js')];
8+
const abortedSignal = AbortSignal.abort();
9+
10+
describe('require(\'node:test\').run coverage settings', { concurrency: true }, async () => {
11+
await describe('validation', async () => {
12+
await it('should only allow boolean in options.coverage', async () => {
13+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, '', '1', Promise.resolve(true), []]
14+
.forEach((coverage) => assert.throws(() => run({ coverage }), {
15+
code: 'ERR_INVALID_ARG_TYPE'
16+
}));
17+
});
18+
19+
await it('should only allow string|string[] in options.coverageExcludeGlobs', async () => {
20+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
21+
.forEach((coverageExcludeGlobs) => {
22+
assert.throws(() => run({ coverage: true, coverageExcludeGlobs }), {
23+
code: 'ERR_INVALID_ARG_TYPE'
24+
});
25+
assert.throws(() => run({ coverage: true, coverageExcludeGlobs: [coverageExcludeGlobs] }), {
26+
code: 'ERR_INVALID_ARG_TYPE'
27+
});
28+
});
29+
run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: [''] });
30+
run({ files: [], signal: abortedSignal, coverage: true, coverageExcludeGlobs: '' });
31+
});
32+
33+
await it('should only allow string|string[] in options.coverageIncludeGlobs', async () => {
34+
[Symbol(), {}, () => {}, 0, 1, 0n, 1n, Promise.resolve([]), true, false]
35+
.forEach((coverageIncludeGlobs) => {
36+
assert.throws(() => run({ coverage: true, coverageIncludeGlobs }), {
37+
code: 'ERR_INVALID_ARG_TYPE'
38+
});
39+
assert.throws(() => run({ coverage: true, coverageIncludeGlobs: [coverageIncludeGlobs] }), {
40+
code: 'ERR_INVALID_ARG_TYPE'
41+
});
42+
});
43+
44+
run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: [''] });
45+
run({ files: [], signal: abortedSignal, coverage: true, coverageIncludeGlobs: '' });
46+
});
47+
48+
await it('should only allow an int within range in options.lineCoverage', async () => {
49+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
50+
.forEach((lineCoverage) => {
51+
assert.throws(() => run({ coverage: true, lineCoverage }), {
52+
code: 'ERR_INVALID_ARG_TYPE'
53+
});
54+
assert.throws(() => run({ coverage: true, lineCoverage: [lineCoverage] }), {
55+
code: 'ERR_INVALID_ARG_TYPE'
56+
});
57+
});
58+
assert.throws(() => run({ coverage: true, lineCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
59+
assert.throws(() => run({ coverage: true, lineCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
60+
61+
run({ files: [], signal: abortedSignal, coverage: true, lineCoverage: 0 });
62+
});
63+
64+
await it('should only allow an int within range in options.branchCoverage', async () => {
65+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
66+
.forEach((branchCoverage) => {
67+
assert.throws(() => run({ coverage: true, branchCoverage }), {
68+
code: 'ERR_INVALID_ARG_TYPE'
69+
});
70+
assert.throws(() => run({ coverage: true, branchCoverage: [branchCoverage] }), {
71+
code: 'ERR_INVALID_ARG_TYPE'
72+
});
73+
});
74+
75+
assert.throws(() => run({ coverage: true, branchCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
76+
assert.throws(() => run({ coverage: true, branchCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
77+
78+
run({ files: [], signal: abortedSignal, coverage: true, branchCoverage: 0 });
79+
});
80+
81+
await it('should only allow an int within range in options.functionCoverage', async () => {
82+
[Symbol(), {}, () => {}, [], 0n, 1n, Promise.resolve([]), true, false]
83+
.forEach((functionCoverage) => {
84+
assert.throws(() => run({ coverage: true, functionCoverage }), {
85+
code: 'ERR_INVALID_ARG_TYPE'
86+
});
87+
assert.throws(() => run({ coverage: true, functionCoverage: [functionCoverage] }), {
88+
code: 'ERR_INVALID_ARG_TYPE'
89+
});
90+
});
91+
92+
assert.throws(() => run({ coverage: true, functionCoverage: -1 }), { code: 'ERR_OUT_OF_RANGE' });
93+
assert.throws(() => run({ coverage: true, functionCoverage: 101 }), { code: 'ERR_OUT_OF_RANGE' });
94+
95+
run({ files: [], signal: abortedSignal, coverage: true, functionCoverage: 0 });
96+
});
97+
});
98+
99+
const options = { concurrency: false, skip: !process.features.inspector ? 'inspector disabled' : false };
100+
await describe('run with coverage', options, async () => {
101+
await it('should run with coverage', async () => {
102+
const stream = run({ files, coverage: true });
103+
stream.on('test:fail', common.mustNotCall());
104+
stream.on('test:pass', common.mustCall());
105+
stream.on('test:coverage', common.mustCall());
106+
// eslint-disable-next-line no-unused-vars
107+
for await (const _ of stream);
108+
});
109+
110+
await it('should run with coverage and exclude by glob', async () => {
111+
const stream = run({ files, coverage: true, coverageExcludeGlobs: ['test/*/test-runner/invalid-tap.js'] });
112+
stream.on('test:fail', common.mustNotCall());
113+
stream.on('test:pass', common.mustCall(1));
114+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
115+
const filesPaths = files.map(({ path }) => path);
116+
assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}invalid-tap.js`)), false);
117+
}));
118+
// eslint-disable-next-line no-unused-vars
119+
for await (const _ of stream);
120+
});
121+
122+
await it('should run with coverage and include by glob', async () => {
123+
const stream = run({
124+
files,
125+
coverage: true,
126+
coverageIncludeGlobs: ['test/fixtures/test-runner/coverage.js', 'test/*/v8-coverage/throw.js'],
127+
});
128+
stream.on('test:fail', common.mustNotCall());
129+
stream.on('test:pass', common.mustCall(1));
130+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
131+
const filesPaths = files.map(({ path }) => path);
132+
assert.strictEqual(filesPaths.some((path) => path.includes(`v8-coverage${sep}throw.js`)), true);
133+
}));
134+
// eslint-disable-next-line no-unused-vars
135+
for await (const _ of stream);
136+
});
137+
138+
await it('should run while including and excluding globs', async () => {
139+
const stream = run({
140+
files: [...files, fixtures.path('test-runner/invalid-tap.js')],
141+
coverage: true,
142+
coverageIncludeGlobs: ['test/fixtures/test-runner/*.js'],
143+
coverageExcludeGlobs: ['test/fixtures/test-runner/*-tap.js']
144+
});
145+
stream.on('test:fail', common.mustNotCall());
146+
stream.on('test:pass', common.mustCall(2));
147+
stream.on('test:coverage', common.mustCall(({ summary: { files } }) => {
148+
const filesPaths = files.map(({ path }) => path);
149+
assert.strictEqual(filesPaths.every((path) => !path.includes(`test-runner${sep}invalid-tap.js`)), true);
150+
assert.strictEqual(filesPaths.some((path) => path.includes(`test-runner${sep}coverage.js`)), true);
151+
}));
152+
// eslint-disable-next-line no-unused-vars
153+
for await (const _ of stream);
154+
});
155+
156+
await it('should run with coverage and fail when below line threshold', async () => {
157+
const thresholdErrors = [];
158+
const originalExitCode = process.exitCode;
159+
assert.notStrictEqual(originalExitCode, 1);
160+
const stream = run({ files, coverage: true, lineCoverage: 99, branchCoverage: 99, functionCoverage: 99 });
161+
stream.on('test:fail', common.mustNotCall());
162+
stream.on('test:pass', common.mustCall(1));
163+
stream.on('test:diagnostic', ({ message }) => {
164+
const match = message.match(/Error: \d{2}\.\d{2}% (line|branch|function) coverage does not meet threshold of 99%/);
165+
if (match) {
166+
thresholdErrors.push(match[1]);
167+
}
168+
});
169+
// eslint-disable-next-line no-unused-vars
170+
for await (const _ of stream);
171+
assert.deepStrictEqual(thresholdErrors.sort(), ['branch', 'function', 'line']);
172+
assert.strictEqual(process.exitCode, 1);
173+
process.exitCode = originalExitCode;
174+
});
175+
});
176+
});
177+
178+
179+
// exitHandler doesn't run until after the tests / after hooks finish.
180+
process.on('exit', () => {
181+
assert.strictEqual(process.listeners('uncaughtException').length, 0);
182+
assert.strictEqual(process.listeners('unhandledRejection').length, 0);
183+
assert.strictEqual(process.listeners('beforeExit').length, 0);
184+
assert.strictEqual(process.listeners('SIGINT').length, 0);
185+
assert.strictEqual(process.listeners('SIGTERM').length, 0);
186+
});

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL