FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test: update WPT runner · nodejs/node@14c7f4a · GitHub

/ node Public

Commit 14c7f4a

Browse files
panva
authored and
juanarbol
committed
test: update WPT runner
PR-URL: #43455 Reviewed-By: Rich Trott <rtrott@gmail.com> Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com> Backport-PR-URL: #44872
1 parent 4368acd commit 14c7f4a

17 files changed

Lines changed: 3622 additions & 132 deletions

‎test/common/wpt.js‎

Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ const fixtures = require('../common/fixtures');
55
const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
8+
const events = require('events');
89
const { inspect } = require('util');
910
const { Worker } = require('worker_threads');
1011

@@ -152,21 +153,30 @@ class WPTTestSpec {
152153
this.filename = filename;
153154

154155
this.requires = new Set();
155-
this.failReasons = [];
156+
this.failedTests = [];
157+
this.flakyTests = [];
156158
this.skipReasons = [];
157159
for (const item of rules) {
158160
if (item.requires.length) {
159161
for (const req of item.requires) {
160162
this.requires.add(req);
161163
}
162164
}
163-
if (item.fail) {
164-
this.failReasons.push(item.fail);
165+
if (Array.isArray(item.fail?.expected)) {
166+
this.failedTests.push(...item.fail.expected);
167+
}
168+
if (Array.isArray(item.fail?.flaky)) {
169+
this.failedTests.push(...item.fail.flaky);
170+
this.flakyTests.push(...item.fail.flaky);
165171
}
166172
if (item.skip) {
167173
this.skipReasons.push(item.skip);
168174
}
169175
}
176+
177+
this.failedTests = [...new Set(this.failedTests)];
178+
this.flakyTests = [...new Set(this.flakyTests)];
179+
this.skipReasons = [...new Set(this.skipReasons)];
170180
}
171181

172182
getRelativePath() {
@@ -368,7 +378,7 @@ class WPTRunner {
368378

369379
// TODO(joyeecheung): work with the upstream to port more tests in .html
370380
// to .js.
371-
runJsTests() {
381+
async runJsTests() {
372382
let queue = [];
373383

374384
// If the tests are run as `node test/wpt/test-something.js subset.any.js`,
@@ -459,6 +469,8 @@ class WPTRunner {
459469
);
460470
this.inProgress.delete(testFileName);
461471
});
472+
473+
await events.once(worker, 'exit').catch(() => {});
462474
}
463475

464476
process.on('exit', () => {
@@ -469,34 +481,72 @@ class WPTRunner {
469481
}
470482
}
471483
inspect.defaultOptions.depth = Infinity;
472-
console.log(this.results);
484+
// Sorts the rules to have consistent output
485+
console.log(JSON.stringify(Object.keys(this.results).sort().reduce(
486+
(obj, key) => {
487+
obj[key] = this.results[key];
488+
return obj;
489+
},
490+
{}
491+
), null, 2));
473492

474493
const failures = [];
475494
let expectedFailures = 0;
476495
let skipped = 0;
477-
for (const key of Object.keys(this.results)) {
478-
const item = this.results[key];
479-
if (item.fail && item.fail.unexpected) {
496+
for (const [key, item] of Object.entries(this.results)) {
497+
if (item.fail?.unexpected) {
480498
failures.push(key);
481499
}
482-
if (item.fail && item.fail.expected) {
500+
if (item.fail?.expected) {
483501
expectedFailures++;
484502
}
485503
if (item.skip) {
486504
skipped++;
487505
}
488506
}
507+
508+
const unexpectedPasses = [];
509+
for (const [key, specMap] of this.specMap) {
510+
// File has no expected failures
511+
if (!specMap.failedTests.length) {
512+
continue;
513+
}
514+
515+
// File was (maybe even conditionally) skipped
516+
if (this.results[key]?.skip) {
517+
continue;
518+
}
519+
520+
// Full check: every expected to fail test is present
521+
if (specMap.failedTests.some((expectedToFail) => {
522+
if (specMap.flakyTests.includes(expectedToFail)) {
523+
return false;
524+
}
525+
return this.results[key]?.fail?.expected?.includes(expectedToFail) !== true;
526+
})) {
527+
unexpectedPasses.push(key);
528+
continue;
529+
}
530+
}
531+
489532
const ran = total - skipped;
490533
const passed = ran - expectedFailures - failures.length;
491534
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
492535
`${passed} passed, ${expectedFailures} expected failures,`,
493-
`${failures.length} unexpected failures`);
536+
`${failures.length} unexpected failures,`,
537+
`${unexpectedPasses.length} unexpected passes`);
494538
if (failures.length > 0) {
495539
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
496540
throw new Error(
497541
`Found ${failures.length} unexpected failures. ` +
498542
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
499543
}
544+
if (unexpectedPasses.length > 0) {
545+
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
546+
throw new Error(
547+
`Found ${unexpectedPasses.length} unexpected passes. ` +
548+
`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
549+
}
500550
});
501551
}
502552

@@ -577,8 +627,9 @@ class WPTRunner {
577627
if (!result[item.status][key]) {
578628
result[item.status][key] = [];
579629
}
580-
if (result[item.status][key].indexOf(item.reason) === -1) {
581-
result[item.status][key].push(item.reason);
630+
const hasName = result[item.status][key].includes(item.name);
631+
if (!hasName) {
632+
result[item.status][key].push(item.name);
582633
}
583634
}
584635
}
@@ -589,10 +640,10 @@ class WPTRunner {
589640

590641
fail(filename, test, status) {
591642
const spec = this.specMap.get(filename);
592-
const expected = !!(spec.failReasons.length);
643+
const expected = spec.failedTests.includes(test.name);
593644
if (expected) {
594645
console.log(`[EXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
595-
console.log(spec.failReasons.join('; '));
646+
console.log(test.message || status);
596647
} else {
597648
console.log(`[UNEXPECTED_FAILURE][${status.toUpperCase()}] ${test.name}`);
598649
}
@@ -604,6 +655,7 @@ class WPTRunner {
604655
` ${require.main.filename} ${filename}`;
605656
console.log(`Command: ${command}\n`);
606657
this.addTestResult(filename, {
658+
name: test.name,
607659
expected,
608660
status: kFail,
609661
reason: test.message || status

‎test/wpt/README.md‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,11 @@ add this to `test/wpt/status/url.json`:
9191

9292
```json
9393
"url-searchparams.any.js": {
94-
"fail": "explain why the test fails, ideally with links"
94+
"fail": {
95+
"expected": [
96+
"test name in the WPT test case, e.g. second argument passed to test()"
97+
]
98+
}
9599
}
96100
```
97101

@@ -155,8 +159,17 @@ expected failures.
155159
// Optional: the test will be skipped with the reason printed
156160
"skip": "explain why we cannot run a test that's supposed to pass",
157161
158-
// Optional: the test will be skipped with the reason printed
159-
"fail": "explain why we the test is expected to fail"
162+
// Optional: failing tests
163+
"fail": {
164+
"note": "You may leave an optional arbitrary note e.g. with TODOs",
165+
"expected": [
166+
"test name in the WPT test case, e.g. second argument passed to test()",
167+
"another test name"
168+
],
169+
"flaky": [
170+
"flaky test name"
171+
]
172+
}
160173
}
161174
}
162175
```

‎test/wpt/status/FileAPI/blob.json‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,47 @@
11
{
2-
"Blob-constructor.any.js": {
3-
"skip": "Depends on File API"
4-
},
52
"Blob-constructor-dom.window.js": {
63
"skip": "Depends on DOM API"
74
},
8-
"Blob-slice.any.js": {
9-
"skip": "Depends on File API"
5+
"Blob-constructor.any.js": {
6+
"fail": {
7+
"note": "Depends on File API",
8+
"expected": [
9+
"A plain object with @@iterator should be treated as a sequence for the blobParts argument.",
10+
"A plain object with @@iterator and a length property should be treated as a sequence for the blobParts argument.",
11+
"A String object should be treated as a sequence for the blobParts argument.",
12+
"A Uint8Array object should be treated as a sequence for the blobParts argument.",
13+
"Getters and value conversions should happen in order until an exception is thrown.",
14+
"Changes to the blobParts array should be reflected in the returned Blob (pop).",
15+
"Changes to the blobParts array should be reflected in the returned Blob (unshift).",
16+
"ToString should be called on elements of the blobParts array.",
17+
"ArrayBuffer elements of the blobParts array should be supported.",
18+
"Passing typed arrays as elements of the blobParts array should work.",
19+
"Passing a Float64Array as element of the blobParts array should work.",
20+
"Array with two blobs",
21+
"Array with two buffers",
22+
"Array with two bufferviews",
23+
"Array with mixed types",
24+
"options properties should be accessed in lexicographic order.",
25+
"Arguments should be evaluated from left to right.",
26+
"Passing null (index 0) for options should use the defaults.",
27+
"Passing null (index 0) for options should use the defaults (with newlines).",
28+
"Passing undefined (index 1) for options should use the defaults.",
29+
"Passing undefined (index 1) for options should use the defaults (with newlines).",
30+
"Passing object \"[object Object]\" (index 2) for options should use the defaults.",
31+
"Passing object \"[object Object]\" (index 2) for options should use the defaults (with newlines).",
32+
"Passing object \"[object Object]\" (index 3) for options should use the defaults.",
33+
"Passing object \"[object Object]\" (index 3) for options should use the defaults (with newlines).",
34+
"Passing object \"/regex/\" (index 4) for options should use the defaults.",
35+
"Passing object \"/regex/\" (index 4) for options should use the defaults (with newlines).",
36+
"Passing function \"function() {}\" (index 5) for options should use the defaults.",
37+
"Passing function \"function() {}\" (index 5) for options should use the defaults (with newlines)."
38+
]
39+
}
1040
},
1141
"Blob-in-worker.worker.js": {
1242
"skip": "Depends on Web Workers API"
43+
},
44+
"Blob-slice.any.js": {
45+
"skip": "Depends on File API"
1346
}
1447
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL