| FazBrowse GitHub Viewer | Trending | | Home |
| Tools: [Download Repo ZIP] [Original HTTPS Page] |
1 parent 2b7a7a5 commit e076f78
13 files changed
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -2601,6 +2601,20 @@ changes: | |||
| 2601 | 2601 | The destination for the corresponding test reporter. See the documentation on | |
| 2602 | 2602 | [test reporters][] for more details. | |
| 2603 | 2603 | ||
| 2604 | + ### `--test-rerun-failures` | ||
| 2605 | + | ||
| 2606 | + <!-- YAML | ||
| 2607 | + added: | ||
| 2608 | + - REPLACEME | ||
| 2609 | + --> | ||
| 2610 | + | ||
| 2611 | + A path to a file allowing the test runner to persist the state of the test | ||
| 2612 | + suite between runs. The test runner will use this file to determine which tests | ||
| 2613 | + have already succeeded or failed, allowing for re-running of failed tests | ||
| 2614 | + without having to re-run the entire test suite. The test runner will create this | ||
| 2615 | + file if it does not exist. | ||
| 2616 | + See the documentation on [test reruns][] for more details. | ||
| 2617 | + | ||
| 2604 | 2618 | ### `--test-shard` | |
| 2605 | 2619 | ||
| 2606 | 2620 | <!-- YAML | |
@@ -3461,6 +3475,7 @@ one is included in the list below. | |||
| 3461 | 3475 | * `--test-only` | |
| 3462 | 3476 | * `--test-reporter-destination` | |
| 3463 | 3477 | * `--test-reporter` | |
| 3478 | + * `--test-rerun-failures` | ||
| 3464 | 3479 | * `--test-shard` | |
| 3465 | 3480 | * `--test-skip-pattern` | |
| 3466 | 3481 | * `--throw-deprecation` | |
@@ -4035,6 +4050,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12 | |||
| 4035 | 4050 | [snapshot testing]: test.md#snapshot-testing | |
| 4036 | 4051 | [syntax detection]: packages.md#syntax-detection | |
| 4037 | 4052 | [test reporters]: test.md#test-reporters | |
| 4053 | + [test reruns]: test.md#rerunning-failed-tests | ||
| 4038 | 4054 | [test runner execution model]: test.md#test-runner-execution-model | |
| 4039 | 4055 | [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones | |
| 4040 | 4056 | [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014 | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -153,6 +153,46 @@ test('skip() method with message', (t) => { | |||
| 153 | 153 | }); | |
| 154 | 154 | ``` | |
| 155 | 155 | ||
| 156 | + ## Rerunning failed tests | ||
| 157 | + | ||
| 158 | + The test runner supports persisting the state of the run to a file, allowing | ||
| 159 | + the test runner to rerun failed tests without having to re-run the entire test suite. | ||
| 160 | + Use the [`--test-rerun-failures`][] command-line option to specify a file path where the | ||
| 161 | + state of the run is stored. if the state file does not exist, the test runner will | ||
| 162 | + create it. | ||
| 163 | + the state file is a JSON file that contains an array of run attempts. | ||
| 164 | + Each run attempt is an object mapping successful tests to the attempt they have passed in. | ||
| 165 | + The key identifying a test in this map is the test file path, with the line and column where the test is defined. | ||
| 166 | + in a case where a test defined in a specific location is run multiple times, | ||
| 167 | + for example within a function or a loop, | ||
| 168 | + a counter will be appended to the key, to disambiguate the test runs. | ||
| 169 | + note changing the order of test execution or the location of a test can lead the test runner | ||
| 170 | + to consider tests as passed on a previous attempt, | ||
| 171 | + meaning `--test-rerun-failures` should be used when tests run in a deterministic order. | ||
| 172 | + | ||
| 173 | + example of a state file: | ||
| 174 | + | ||
| 175 | + ```json | ||
| 176 | + [ | ||
| 177 | + { | ||
| 178 | + "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" }, | ||
| 179 | + }, | ||
| 180 | + { | ||
| 181 | + "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" }, | ||
| 182 | + "test.js:20:5": { "passed_on_attempt": 1, "name": "test 2" } | ||
| 183 | + } | ||
| 184 | + ] | ||
| 185 | + ``` | ||
| 186 | + | ||
| 187 | + in this example, there are two run attempts, with two tests defined in `test.js`, | ||
| 188 | + the first test succeeded on the first attempt, and the second test succeeded on the second attempt. | ||
| 189 | + | ||
| 190 | + When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed. | ||
| 191 | + | ||
| 192 | + ```bash | ||
| 193 | + node --test-rerun-failures /path/to/state/file | ||
| 194 | + ``` | ||
| 195 | + | ||
| 156 | 196 | ## TODO tests | |
| 157 | 197 | ||
| 158 | 198 | Individual tests can be marked as flaky or incomplete by passing the `todo` | |
@@ -1342,6 +1382,9 @@ added: | |||
| 1342 | 1382 | - v18.9.0 | |
| 1343 | 1383 | - v16.19.0 | |
| 1344 | 1384 | changes: | |
| 1385 | + - version: REPLACEME | ||
| 1386 | + pr-url: https://github.com/nodejs/node/pull/59443 | ||
| 1387 | + description: Added a rerunFailuresFilePath option. | ||
| 1345 | 1388 | - version: v23.0.0 | |
| 1346 | 1389 | pr-url: https://github.com/nodejs/node/pull/54705 | |
| 1347 | 1390 | description: Added the `cwd` option. | |
@@ -1432,6 +1475,10 @@ changes: | |||
| 1432 | 1475 | that specifies the index of the shard to run. This option is _required_. | |
| 1433 | 1476 | * `total` {number} is a positive integer that specifies the total number | |
| 1434 | 1477 | of shards to split the test files to. This option is _required_. | |
| 1478 | + * `rerunFailuresFilePath` {string} A file path where the test runner will | ||
| 1479 | + store the state of the tests to allow rerunning only the failed tests on a next run. | ||
| 1480 | + see \[Rerunning failed tests]\[] for more information. | ||
| 1481 | + **Default:** `undefined`. | ||
| 1435 | 1482 | * `coverage` {boolean} enable [code coverage][] collection. | |
| 1436 | 1483 | **Default:** `false`. | |
| 1437 | 1484 | * `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage | |
@@ -3219,6 +3266,8 @@ Emitted when a test is enqueued for execution. | |||
| 3219 | 3266 | * `cause` {Error} The actual error thrown by the test. | |
| 3220 | 3267 | * `type` {string|undefined} The type of the test, used to denote whether | |
| 3221 | 3268 | this is a suite. | |
| 3269 | + * `attempt` {number|undefined} The attempt number of the test run, | ||
| 3270 | + present only when using the [`--test-rerun-failures`][] flag. | ||
| 3222 | 3271 | * `file` {string|undefined} The path of the test file, | |
| 3223 | 3272 | `undefined` if test was run through the REPL. | |
| 3224 | 3273 | * `line` {number|undefined} The line number where the test is defined, or | |
@@ -3243,6 +3292,10 @@ The corresponding execution ordered event is `'test:complete'`. | |||
| 3243 | 3292 | * `duration_ms` {number} The duration of the test in milliseconds. | |
| 3244 | 3293 | * `type` {string|undefined} The type of the test, used to denote whether | |
| 3245 | 3294 | this is a suite. | |
| 3295 | + * `attempt` {number|undefined} The attempt number of the test run, | ||
| 3296 | + present only when using the [`--test-rerun-failures`][] flag. | ||
| 3297 | + * `passed_on_attempt` {number|undefined} The attempt number the test passed on, | ||
| 3298 | + present only when using the [`--test-rerun-failures`][] flag. | ||
| 3246 | 3299 | * `file` {string|undefined} The path of the test file, | |
| 3247 | 3300 | `undefined` if test was run through the REPL. | |
| 3248 | 3301 | * `line` {number|undefined} The line number where the test is defined, or | |
@@ -3946,6 +3999,7 @@ Can be used to abort test subtasks when the test has been aborted. | |||
| 3946 | 3999 | [`--test-only`]: cli.md#--test-only | |
| 3947 | 4000 | [`--test-reporter-destination`]: cli.md#--test-reporter-destination | |
| 3948 | 4001 | [`--test-reporter`]: cli.md#--test-reporter | |
| 4002 | + [`--test-rerun-failures`]: cli.md#--test-rerun-failures | ||
| 3949 | 4003 | [`--test-skip-pattern`]: cli.md#--test-skip-pattern | |
| 3950 | 4004 | [`--test-update-snapshots`]: cli.md#--test-update-snapshots | |
| 3951 | 4005 | [`--test`]: cli.md#--test | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -443,6 +443,9 @@ | |||
| 443 | 443 | } | |
| 444 | 444 | ] | |
| 445 | 445 | }, | |
| 446 | + "test-rerun-failures": { | ||
| 447 | + "type": "string" | ||
| 448 | + }, | ||
| 446 | 449 | "test-shard": { | |
| 447 | 450 | "type": "string" | |
| 448 | 451 | }, | |
@@ -695,6 +698,9 @@ | |||
| 695 | 698 | } | |
| 696 | 699 | ] | |
| 697 | 700 | }, | |
| 701 | + "test-rerun-failures": { | ||
| 702 | + "type": "string" | ||
| 703 | + }, | ||
| 698 | 704 | "test-shard": { | |
| 699 | 705 | "type": "string" | |
| 700 | 706 | }, | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -490,6 +490,10 @@ A test reporter to use when running tests. | |||
| 490 | 490 | .It Fl -test-reporter-destination | |
| 491 | 491 | The destination for the corresponding test reporter. | |
| 492 | 492 | . | |
| 493 | + .It Fl -test-rerun-failures | ||
| 494 | + Configures the tests runner to persist the state of tests to allow | ||
| 495 | + rerunning only failed tests. | ||
| 496 | + . | ||
| 493 | 497 | .It Fl -test-only | |
| 494 | 498 | Configures the test runner to only execute top level tests that have the `only` | |
| 495 | 499 | option set. | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -26,7 +26,10 @@ const { | |||
| 26 | 26 | reporterScope, | |
| 27 | 27 | shouldColorizeTestFiles, | |
| 28 | 28 | setupGlobalSetupTeardownFunctions, | |
| 29 | + parsePreviousRuns, | ||
| 29 | 30 | } = require('internal/test_runner/utils'); | |
| 31 | + const { PassThrough, compose } = require('stream'); | ||
| 32 | + const { reportReruns } = require('internal/test_runner/reporter/rerun'); | ||
| 30 | 33 | const { queueMicrotask } = require('internal/process/task_queues'); | |
| 31 | 34 | const { TIMEOUT_MAX } = require('internal/timers'); | |
| 32 | 35 | const { clearInterval, setInterval } = require('timers'); | |
@@ -69,6 +72,7 @@ function createTestTree(rootTestOptions, globalOptions) { | |||
| 69 | 72 | shouldColorizeTestFiles: shouldColorizeTestFiles(globalOptions.destinations), | |
| 70 | 73 | teardown: null, | |
| 71 | 74 | snapshotManager: null, | |
| 75 | + previousRuns: null, | ||
| 72 | 76 | isFilteringByName, | |
| 73 | 77 | isFilteringByOnly, | |
| 74 | 78 | async runBootstrap() { | |
@@ -203,6 +207,25 @@ function collectCoverage(rootTest, coverage) { | |||
| 203 | 207 | return summary; | |
| 204 | 208 | } | |
| 205 | 209 | ||
| 210 | + function setupFailureStateFile(rootTest, globalOptions) { | ||
| 211 | + if (!globalOptions.rerunFailuresFilePath) { | ||
| 212 | + return; | ||
| 213 | + } | ||
| 214 | + rootTest.harness.previousRuns = parsePreviousRuns(globalOptions.rerunFailuresFilePath); | ||
| 215 | + if (rootTest.harness.previousRuns === null) { | ||
| 216 | + rootTest.diagnostic(`Warning: The rerun failures file at ` + | ||
| 217 | + `${globalOptions.rerunFailuresFilePath} is not a valid rerun file. ` + | ||
| 218 | + 'The test runner will not be able to rerun failed tests.'); | ||
| 219 | + rootTest.harness.success = false; | ||
| 220 | + process.exitCode = kGenericUserError; | ||
| 221 | + return; | ||
| 222 | + } | ||
| 223 | + if (!process.env.NODE_TEST_CONTEXT) { | ||
| 224 | + const reporter = reportReruns(rootTest.harness.previousRuns, globalOptions); | ||
| 225 | + compose(rootTest.reporter, reporter).pipe(new PassThrough()); | ||
| 226 | + } | ||
| 227 | + } | ||
| 228 | + | ||
| 206 | 229 | function setupProcessState(root, globalOptions) { | |
| 207 | 230 | const hook = createHook({ | |
| 208 | 231 | __proto__: null, | |
@@ -230,6 +253,9 @@ function setupProcessState(root, globalOptions) { | |||
| 230 | 253 | const rejectionHandler = | |
| 231 | 254 | createProcessEventHandler('unhandledRejection', root); | |
| 232 | 255 | const coverage = configureCoverage(root, globalOptions); | |
| 256 | + | ||
| 257 | + setupFailureStateFile(root, globalOptions); | ||
| 258 | + | ||
| 233 | 259 | const exitHandler = async (kill) => { | |
| 234 | 260 | if (root.subtests.length === 0 && (root.hooks.before.length > 0 || root.hooks.after.length > 0)) { | |
| 235 | 261 | // Run global before/after hooks in case there are no tests | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -0,0 +1,40 @@ | |||
| 1 | + 'use strict'; | ||
| 2 | + | ||
| 3 | + const { | ||
| 4 | + ArrayPrototypePush, | ||
| 5 | + JSONStringify, | ||
| 6 | + } = primordials; | ||
| 7 | + const { relative } = require('path'); | ||
| 8 | + const { writeFileSync } = require('fs'); | ||
| 9 | + | ||
| 10 | + function reportReruns(previousRuns, globalOptions) { | ||
| 11 | + return async function reporter(source) { | ||
| 12 | + const obj = { __proto__: null }; | ||
| 13 | + const disambiguator = { __proto__: null }; | ||
| 14 | + | ||
| 15 | + for await (const { type, data } of source) { | ||
| 16 | + if (type === 'test:pass') { | ||
| 17 | + let identifier = `${relative(globalOptions.cwd, data.file)}:${data.line}:${data.column}`; | ||
| 18 | + if (disambiguator[identifier] !== undefined) { | ||
| 19 | + identifier += `:(${disambiguator[identifier]})`; | ||
| 20 | + disambiguator[identifier] += 1; | ||
| 21 | + } else { | ||
| 22 | + disambiguator[identifier] = 1; | ||
| 23 | + } | ||
| 24 | + obj[identifier] = { | ||
| 25 | + __proto__: null, | ||
| 26 | + name: data.name, | ||
| 27 | + passed_on_attempt: data.details.passed_on_attempt ?? data.details.attempt, | ||
| 28 | + }; | ||
| 29 | + } | ||
| 30 | + } | ||
| 31 | + | ||
| 32 | + ArrayPrototypePush(previousRuns, obj); | ||
| 33 | + writeFileSync(globalOptions.rerunFailuresFilePath, JSONStringify(previousRuns, null, 2), 'utf8'); | ||
| 34 | + }; | ||
| 35 | + }; | ||
| 36 | + | ||
| 37 | + module.exports = { | ||
| 38 | + __proto__: null, | ||
| 39 | + reportReruns, | ||
| 40 | + }; | ||
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit, | |||
| 148 | 148 | only, | |
| 149 | 149 | argv: suppliedArgs, | |
| 150 | 150 | execArgv, | |
| 151 | + rerunFailuresFilePath, | ||
| 151 | 152 | root: { timeout }, | |
| 152 | 153 | cwd }) { | |
| 153 | 154 | const processNodeOptions = getOptionsAsFlagsFromBinding(); | |
@@ -170,6 +171,9 @@ function getRunArgs(path, { forceExit, | |||
| 170 | 171 | if (timeout != null) { | |
| 171 | 172 | ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`); | |
| 172 | 173 | } | |
| 174 | + if (rerunFailuresFilePath) { | ||
| 175 | + ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`); | ||
| 176 | + } | ||
| 173 | 177 | ||
| 174 | 178 | ArrayPrototypePushApply(runArgs, execArgv); | |
| 175 | 179 | ||
@@ -588,6 +592,7 @@ function run(options = kEmptyObject) { | |||
| 588 | 592 | execArgv = [], | |
| 589 | 593 | argv = [], | |
| 590 | 594 | cwd = process.cwd(), | |
| 595 | + rerunFailuresFilePath, | ||
| 591 | 596 | } = options; | |
| 592 | 597 | ||
| 593 | 598 | if (files != null) { | |
@@ -620,6 +625,10 @@ function run(options = kEmptyObject) { | |||
| 620 | 625 | ); | |
| 621 | 626 | } | |
| 622 | 627 | ||
| 628 | + if (rerunFailuresFilePath) { | ||
| 629 | + validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath'); | ||
| 630 | + } | ||
| 631 | + | ||
| 623 | 632 | if (shard != null) { | |
| 624 | 633 | validateObject(shard, 'options.shard'); | |
| 625 | 634 | // Avoid re-evaluating the shard object in case it's a getter | |
@@ -702,6 +711,7 @@ function run(options = kEmptyObject) { | |||
| 702 | 711 | coverage, | |
| 703 | 712 | coverageExcludeGlobs, | |
| 704 | 713 | coverageIncludeGlobs, | |
| 714 | + rerunFailuresFilePath, | ||
| 705 | 715 | lineCoverage: lineCoverage, | |
| 706 | 716 | branchCoverage: branchCoverage, | |
| 707 | 717 | functionCoverage: functionCoverage, | |
@@ -735,6 +745,7 @@ function run(options = kEmptyObject) { | |||
| 735 | 745 | isolation, | |
| 736 | 746 | argv, | |
| 737 | 747 | execArgv, | |
| 748 | + rerunFailuresFilePath, | ||
| 738 | 749 | }; | |
| 739 | 750 | ||
| 740 | 751 | if (isolation === 'process') { | |
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
@@ -71,6 +71,7 @@ const { | |||
| 71 | 71 | } = require('timers'); | |
| 72 | 72 | const { TIMEOUT_MAX } = require('internal/timers'); | |
| 73 | 73 | const { fileURLToPath } = require('internal/url'); | |
| 74 | + const { relative } = require('path'); | ||
| 74 | 75 | const { availableParallelism } = require('os'); | |
| 75 | 76 | const { innerOk } = require('internal/assert/utils'); | |
| 76 | 77 | const { bigint: hrtime } = process.hrtime; | |
@@ -290,6 +291,10 @@ class TestContext { | |||
| 290 | 291 | return this.#test.passed; | |
| 291 | 292 | } | |
| 292 | 293 | ||
| 294 | + get attempt() { | ||
| 295 | + return this.#test.attempt ?? 0; | ||
| 296 | + } | ||
| 297 | + | ||
| 293 | 298 | diagnostic(message) { | |
| 294 | 299 | this.#test.diagnostic(message); | |
| 295 | 300 | } | |
@@ -537,6 +542,7 @@ class Test extends AsyncResource { | |||
| 537 | 542 | this.childNumber = 0; | |
| 538 | 543 | this.timeout = kDefaultTimeout; | |
| 539 | 544 | this.entryFile = entryFile; | |
| 545 | + this.testDisambiguator = new SafeMap(); | ||
| 540 | 546 | } else { | |
| 541 | 547 | const nesting = parent.parent === null ? parent.nesting : | |
| 542 | 548 | parent.nesting + 1; | |
@@ -646,6 +652,8 @@ class Test extends AsyncResource { | |||
| 646 | 652 | this.endTime = null; | |
| 647 | 653 | this.passed = false; | |
| 648 | 654 | this.error = null; | |
| 655 | + this.attempt = undefined; | ||
| 656 | + this.passedAttempt = undefined; | ||
| 649 | 657 | this.message = typeof skip === 'string' ? skip : | |
| 650 | 658 | typeof todo === 'string' ? todo : null; | |
| 651 | 659 | this.activeSubtests = 0; | |
@@ -690,6 +698,23 @@ class Test extends AsyncResource { | |||
| 690 | 698 | this.loc.file = fileURLToPath(this.loc.file); | |
| 691 | 699 | } | |
| 692 | 700 | } | |
| 701 | + | ||
| 702 | + if (this.loc != null && this.root.harness.previousRuns != null) { | ||
| 703 | + let testIdentifier = `${relative(this.config.cwd, this.loc.file)}:${this.loc.line}:${this.loc.column}`; | ||
| 704 | + const disambiguator = this.root.testDisambiguator.get(testIdentifier); | ||
| 705 | + if (disambiguator !== undefined) { | ||
| 706 | + testIdentifier += `:(${disambiguator})`; | ||
| 707 | + this.root.testDisambiguator.set(testIdentifier, disambiguator + 1); | ||
| 708 | + } else { | ||
| 709 | + this.root.testDisambiguator.set(testIdentifier, 1); | ||
| 710 | + } | ||
| 711 | + this.attempt = this.root.harness.previousRuns.length; | ||
| 712 | + const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]?.passed_on_attempt; | ||
| 713 | + if (previousAttempt != null) { | ||
| 714 | + this.passedAttempt = previousAttempt; | ||
| 715 | + this.fn = noop; | ||
| 716 | + } | ||
| 717 | + } | ||
| 693 | 718 | } | |
| 694 | 719 | ||
| 695 | 720 | applyFilters() { | |
@@ -1329,6 +1354,12 @@ class Test extends AsyncResource { | |||
| 1329 | 1354 | if (!this.passed) { | |
| 1330 | 1355 | details.error = this.error; | |
| 1331 | 1356 | } | |
| 1357 | + if (this.attempt !== undefined) { | ||
| 1358 | + details.attempt = this.attempt; | ||
| 1359 | + } | ||
| 1360 | + if (this.passedAttempt !== undefined) { | ||
| 1361 | + details.passed_on_attempt = this.passedAttempt; | ||
| 1362 | + } | ||
| 1332 | 1363 | return { __proto__: null, details, directive }; | |
| 1333 | 1364 | } | |
| 1334 | 1365 | ||
| Back | FazBrowse Home | New Git URL |
0 commit comments