FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test_runner: add support for coverage thresholds · nodejs/node@d4310fe · GitHub

/ node Public

Commit d4310fe

Browse files
authored and
RafaelGSS
committed
test_runner: add support for coverage thresholds
Co-Authored-By: Marco Ippolito <marcoippolito54@gmail.com> PR-URL: #54429 Reviewed-By: Matteo Collina <matteo.collina@gmail.com> Reviewed-By: Paolo Insogna <paolo@cowtech.it> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Colin Ihrig <cjihrig@gmail.com> Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
1 parent b5a23c9 commit d4310fe

7 files changed

Lines changed: 207 additions & 0 deletions

File tree

‎doc/api/cli.md‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2218,6 +2218,17 @@ concurrently. If `--experimental-test-isolation` is set to `'none'`, this flag
22182218
is ignored and concurrency is one. Otherwise, concurrency defaults to
22192219
`os.availableParallelism() - 1`.
22202220

2221+
### `--test-coverage-branches=threshold`
2222+
2223+
<!-- YAML
2224+
added: REPLACEME
2225+
-->
2226+
2227+
> Stability: 1 - Experimental
2228+
2229+
Require a minimum percent of covered branches. If code coverage does not reach
2230+
the threshold specified, the process will exit with code `1`.
2231+
22212232
### `--test-coverage-exclude`
22222233

22232234
<!-- YAML
@@ -2235,6 +2246,17 @@ This option may be specified multiple times to exclude multiple glob patterns.
22352246
If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
22362247
files must meet **both** criteria to be included in the coverage report.
22372248

2249+
### `--test-coverage-functions=threshold`
2250+
2251+
<!-- YAML
2252+
added: REPLACEME
2253+
-->
2254+
2255+
> Stability: 1 - Experimental
2256+
2257+
Require a minimum percent of covered functions. If code coverage does not reach
2258+
the threshold specified, the process will exit with code `1`.
2259+
22382260
### `--test-coverage-include`
22392261

22402262
<!-- YAML
@@ -2252,6 +2274,17 @@ This option may be specified multiple times to include multiple glob patterns.
22522274
If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
22532275
files must meet **both** criteria to be included in the coverage report.
22542276

2277+
### `--test-coverage-lines=threshold`
2278+
2279+
<!-- YAML
2280+
added: REPLACEME
2281+
-->
2282+
2283+
> Stability: 1 - Experimental
2284+
2285+
Require a minimum percent of covered lines. If code coverage does not reach
2286+
the threshold specified, the process will exit with code `1`.
2287+
22552288
### `--test-force-exit`
22562289

22572290
<!-- YAML
@@ -3047,8 +3080,11 @@ one is included in the list below.
30473080
* `--secure-heap-min`
30483081
* `--secure-heap`
30493082
* `--snapshot-blob`
3083+
* `--test-coverage-branches`
30503084
* `--test-coverage-exclude`
3085+
* `--test-coverage-functions`
30513086
* `--test-coverage-include`
3087+
* `--test-coverage-lines`
30523088
* `--test-name-pattern`
30533089
* `--test-only`
30543090
* `--test-reporter-destination`

‎doc/node.1‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,12 +450,21 @@ Starts the Node.js command line test runner.
450450
The maximum number of test files that the test runner CLI will execute
451451
concurrently.
452452
.
453+
.It Fl -test-coverage-branches Ns = Ns Ar threshold
454+
Require a minimum threshold for branch coverage (0 - 100).
455+
.
453456
.It Fl -test-coverage-exclude
454457
A glob pattern that excludes matching files from the coverage report
455458
.
459+
.It Fl -test-coverage-functions Ns = Ns Ar threshold
460+
Require a minimum threshold for function coverage (0 - 100).
461+
.
456462
.It Fl -test-coverage-include
457463
A glob pattern that only includes matching files in the coverage report
458464
.
465+
.It Fl -test-coverage-lines Ns = Ns Ar threshold
466+
Require a minimum threshold for line coverage (0 - 100).
467+
.
459468
.It Fl -test-force-exit
460469
Configures the test runner to exit the process once all known tests have
461470
finished executing even if the event loop would otherwise remain active.

‎lib/internal/test_runner/test.js‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const {
1111
FunctionPrototype,
1212
MathMax,
1313
Number,
14+
NumberPrototypeToFixed,
1415
ObjectDefineProperty,
1516
ObjectSeal,
1617
PromisePrototypeThen,
@@ -28,6 +29,7 @@ const {
2829
SymbolDispose,
2930
} = primordials;
3031
const { getCallerLocation } = internalBinding('util');
32+
const { exitCodes: { kGenericUserError } } = internalBinding('errors');
3133
const { addAbortListener } = require('internal/events/abort_listener');
3234
const { queueMicrotask } = require('internal/process/task_queues');
3335
const { AsyncResource } = require('async_hooks');
@@ -1009,6 +1011,25 @@ class Test extends AsyncResource {
10091011

10101012
if (coverage) {
10111013
reporter.coverage(nesting, loc, coverage);
1014+
1015+
const coverages = [
1016+
{ __proto__: null, actual: coverage.totals.coveredLinePercent,
1017+
threshold: this.config.lineCoverage, name: 'line' },
1018+
1019+
{ __proto__: null, actual: coverage.totals.coveredBranchPercent,
1020+
threshold: this.config.branchCoverage, name: 'branch' },
1021+
1022+
{ __proto__: null, actual: coverage.totals.coveredFunctionPercent,
1023+
threshold: this.config.functionCoverage, name: 'function' },
1024+
];
1025+
1026+
for (let i = 0; i < coverages.length; i++) {
1027+
const { threshold, actual, name } = coverages[i];
1028+
if (actual < threshold) {
1029+
process.exitCode = kGenericUserError;
1030+
reporter.diagnostic(nesting, loc, `Error: ${NumberPrototypeToFixed(actual, 2)}% ${name} coverage does not meet threshold of ${threshold}%.`);
1031+
}
1032+
}
10121033
}
10131034

10141035
if (harness.watching) {

‎lib/internal/test_runner/utils.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ const {
3939
kIsNodeError,
4040
} = require('internal/errors');
4141
const { compose } = require('stream');
42+
const { validateInteger } = require('internal/validators');
4243

4344
const coverageColors = {
4445
__proto__: null,
@@ -194,6 +195,9 @@ function parseCommandLine() {
194195
let concurrency;
195196
let coverageExcludeGlobs;
196197
let coverageIncludeGlobs;
198+
let lineCoverage;
199+
let branchCoverage;
200+
let functionCoverage;
197201
let destinations;
198202
let isolation;
199203
let only = getOptionValue('--test-only');
@@ -278,6 +282,14 @@ function parseCommandLine() {
278282
if (coverage) {
279283
coverageExcludeGlobs = getOptionValue('--test-coverage-exclude');
280284
coverageIncludeGlobs = getOptionValue('--test-coverage-include');
285+
286+
branchCoverage = getOptionValue('--test-coverage-branches');
287+
lineCoverage = getOptionValue('--test-coverage-lines');
288+
functionCoverage = getOptionValue('--test-coverage-functions');
289+
290+
validateInteger(branchCoverage, '--test-coverage-branches', 0, 100);
291+
validateInteger(lineCoverage, '--test-coverage-lines', 0, 100);
292+
validateInteger(functionCoverage, '--test-coverage-functions', 0, 100);
281293
}
282294

283295
const setup = reporterScope.bind(async (rootReporter) => {
@@ -299,6 +311,9 @@ function parseCommandLine() {
299311
destinations,
300312
forceExit,
301313
isolation,
314+
branchCoverage,
315+
functionCoverage,
316+
lineCoverage,
302317
only,
303318
reporters,
304319
setup,

‎src/node_options.cc‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -670,6 +670,19 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
670670
AddOption("--experimental-test-coverage",
671671
"enable code coverage in the test runner",
672672
&EnvironmentOptions::test_runner_coverage);
673+
AddOption("--test-coverage-branches",
674+
"the branch coverage minimum threshold",
675+
&EnvironmentOptions::test_coverage_branches,
676+
kAllowedInEnvvar);
677+
AddOption("--test-coverage-functions",
678+
"the function coverage minimum threshold",
679+
&EnvironmentOptions::test_coverage_functions,
680+
kAllowedInEnvvar);
681+
AddOption("--test-coverage-lines",
682+
"the line coverage minimum threshold",
683+
&EnvironmentOptions::test_coverage_lines,
684+
kAllowedInEnvvar);
685+
673686
AddOption("--experimental-test-isolation",
674687
"configures the type of test isolation used in the test runner",
675688
&EnvironmentOptions::test_isolation);

‎src/node_options.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,9 @@ class EnvironmentOptions : public Options {
184184
uint64_t test_runner_timeout = 0;
185185
bool test_runner_coverage = false;
186186
bool test_runner_force_exit = false;
187+
uint64_t test_coverage_branches = 0;
188+
uint64_t test_coverage_functions = 0;
189+
uint64_t test_coverage_lines = 0;
187190
bool test_runner_module_mocks = false;
188191
bool test_runner_snapshots = false;
189192
bool test_runner_update_snapshots = false;

‎test/parallel/test-runner-coverage-thresholds.js‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
'use strict';
2+
const common = require('../common');
3+
const assert = require('node:assert');
4+
const { spawnSync } = require('node:child_process');
5+
const { readdirSync } = require('node:fs');
6+
const { test } = require('node:test');
7+
const fixtures = require('../common/fixtures');
8+
const tmpdir = require('../common/tmpdir');
9+
10+
common.skipIfInspectorDisabled();
11+
tmpdir.refresh();
12+
13+
function findCoverageFileForPid(pid) {
14+
const pattern = `^coverage\\-${pid}\\-(\\d{13})\\-(\\d+)\\.json$`;
15+
const regex = new RegExp(pattern);
16+
17+
return readdirSync(tmpdir.path).find((file) => {
18+
return regex.test(file);
19+
});
20+
}
21+
22+
function getTapCoverageFixtureReport() {
23+
/* eslint-disable @stylistic/js/max-len */
24+
const report = [
25+
'# start of coverage report',
26+
'# -------------------------------------------------------------------------------------------------------------------',
27+
'# file | line % | branch % | funcs % | uncovered lines',
28+
'# -------------------------------------------------------------------------------------------------------------------',
29+
'# test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12-13 16-22 27 39 43-44 61-62 66-67 71-72',
30+
'# test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ',
31+
'# test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5-6',
32+
'# -------------------------------------------------------------------------------------------------------------------',
33+
'# all files | 78.35 | 43.75 | 60.00 |',
34+
'# -------------------------------------------------------------------------------------------------------------------',
35+
'# end of coverage report',
36+
].join('\n');
37+
/* eslint-enable @stylistic/js/max-len */
38+
39+
if (common.isWindows) {
40+
return report.replaceAll('/', '\\');
41+
}
42+
43+
return report;
44+
}
45+
46+
const fixture = fixtures.path('test-runner', 'coverage.js');
47+
const neededArguments = [
48+
'--experimental-test-coverage',
49+
'--test-reporter', 'tap',
50+
];
51+
52+
const coverages = [
53+
{ flag: '--test-coverage-lines', name: 'line', actual: 78.35 },
54+
{ flag: '--test-coverage-functions', name: 'function', actual: 60.00 },
55+
{ flag: '--test-coverage-branches', name: 'branch', actual: 43.75 },
56+
];
57+
58+
for (const coverage of coverages) {
59+
test(`test passing ${coverage.flag}`, async (t) => {
60+
const result = spawnSync(process.execPath, [
61+
...neededArguments,
62+
`${coverage.flag}=25`,
63+
fixture,
64+
]);
65+
66+
const stdout = result.stdout.toString();
67+
assert(stdout.includes(getTapCoverageFixtureReport()));
68+
assert.doesNotMatch(stdout, RegExp(`Error: [\\d\\.]+% ${coverage.name} coverage`));
69+
assert.strictEqual(result.status, 0);
70+
assert(!findCoverageFileForPid(result.pid));
71+
});
72+
73+
test(`test failing ${coverage.flag}`, async (t) => {
74+
const result = spawnSync(process.execPath, [
75+
...neededArguments,
76+
`${coverage.flag}=99`,
77+
fixture,
78+
]);
79+
80+
const stdout = result.stdout.toString();
81+
assert(stdout.includes(getTapCoverageFixtureReport()));
82+
assert.match(stdout, RegExp(`Error: ${coverage.actual.toFixed(2)}% ${coverage.name} coverage does not meet threshold of 99%`));
83+
assert.strictEqual(result.status, 1);
84+
assert(!findCoverageFileForPid(result.pid));
85+
});
86+
87+
test(`test out-of-range ${coverage.flag} (too high)`, async (t) => {
88+
const result = spawnSync(process.execPath, [
89+
...neededArguments,
90+
`${coverage.flag}=101`,
91+
fixture,
92+
]);
93+
94+
assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
95+
assert.strictEqual(result.status, 1);
96+
assert(!findCoverageFileForPid(result.pid));
97+
});
98+
99+
test(`test out-of-range ${coverage.flag} (too low)`, async (t) => {
100+
const result = spawnSync(process.execPath, [
101+
...neededArguments,
102+
`${coverage.flag}=-1`,
103+
fixture,
104+
]);
105+
106+
assert.match(result.stderr.toString(), RegExp(`The value of "${coverage.flag}`));
107+
assert.strictEqual(result.status, 1);
108+
assert(!findCoverageFileForPid(result.pid));
109+
});
110+
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL