FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test: add WPTRunner support for variants and generating WPT reports · nodejs/node@5f422c4 · GitHub

/ node Public

Commit 5f422c4

Browse files
panva
authored and
juanarbol
committed
test: add WPTRunner support for variants and generating WPT reports
PR-URL: #46498 Reviewed-By: Yagiz Nizipli <yagiz@nizipli.com> Reviewed-By: Richard Lau <rlau@redhat.com> Backport-PR-URL: #46767
1 parent 492fc95 commit 5f422c4

2 files changed

Lines changed: 209 additions & 59 deletions

File tree

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,6 +595,12 @@ test-message: test-build
595595
test-wpt: all
596596
$(PYTHON) tools/test.py $(PARALLEL_ARGS) wpt
597597

598+
.PHONY: test-wpt-report
599+
test-wpt-report:
600+
$(RM) -r out/wpt
601+
mkdir -p out/wpt
602+
WPT_REPORT=1 $(PYTHON) tools/test.py --shell $(NODE) $(PARALLEL_ARGS) wpt
603+
598604
.PHONY: test-simple
599605
test-simple: | cctest # Depends on 'all'.
600606
$(PYTHON) tools/test.py $(PARALLEL_ARGS) parallel sequential

‎test/common/wpt.js‎

Lines changed: 203 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,119 @@ const fs = require('fs');
66
const fsPromises = fs.promises;
77
const path = require('path');
88
const events = require('events');
9+
const os = require('os');
910
const { inspect } = require('util');
1011
const { Worker } = require('worker_threads');
1112

13+
function getBrowserProperties() {
14+
const { node: version } = process.versions; // e.g. 18.13.0, 20.0.0-nightly202302078e6e215481
15+
const release = /^\d+\.\d+\.\d+$/.test(version);
16+
const browser = {
17+
browser_channel: release ? 'stable' : 'experimental',
18+
browser_version: version,
19+
};
20+
21+
return browser;
22+
}
23+
24+
/**
25+
* Return one of three expected values
26+
* https://github.com/web-platform-tests/wpt/blob/1c6ff12/tools/wptrunner/wptrunner/tests/test_update.py#L953-L958
27+
*/
28+
function getOs() {
29+
switch (os.type()) {
30+
case 'Linux':
31+
return 'linux';
32+
case 'Darwin':
33+
return 'mac';
34+
case 'Windows_NT':
35+
return 'win';
36+
default:
37+
throw new Error('Unsupported os.type()');
38+
}
39+
}
40+
41+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3705
42+
function sanitizeUnpairedSurrogates(str) {
43+
return str.replace(
44+
/([\ud800-\udbff]+)(?![\udc00-\udfff])|(^|[^\ud800-\udbff])([\udc00-\udfff]+)/g,
45+
function(_, low, prefix, high) {
46+
let output = prefix || ''; // Prefix may be undefined
47+
const string = low || high; // Only one of these alternates can match
48+
for (let i = 0; i < string.length; i++) {
49+
output += codeUnitStr(string[i]);
50+
}
51+
return output;
52+
});
53+
}
54+
55+
function codeUnitStr(char) {
56+
return 'U+' + char.charCodeAt(0).toString(16);
57+
}
58+
59+
class WPTReport {
60+
constructor() {
61+
this.results = [];
62+
this.time_start = Date.now();
63+
}
64+
65+
addResult(name, status) {
66+
const result = {
67+
test: name,
68+
status,
69+
subtests: [],
70+
addSubtest(name, status, message) {
71+
const subtest = {
72+
status,
73+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
74+
name: sanitizeUnpairedSurrogates(name),
75+
};
76+
if (message) {
77+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
78+
subtest.message = sanitizeUnpairedSurrogates(message);
79+
}
80+
this.subtests.push(subtest);
81+
return subtest;
82+
},
83+
};
84+
this.results.push(result);
85+
return result;
86+
}
87+
88+
write() {
89+
this.time_end = Date.now();
90+
this.results = this.results.filter((result) => {
91+
return result.status === 'SKIP' || result.subtests.length !== 0;
92+
}).map((result) => {
93+
const url = new URL(result.test, 'http://wpt');
94+
url.pathname = url.pathname.replace(/\.js$/, '.html');
95+
result.test = url.href.slice(url.origin.length);
96+
return result;
97+
});
98+
99+
if (fs.existsSync('out/wpt/wptreport.json')) {
100+
const prev = JSON.parse(fs.readFileSync('out/wpt/wptreport.json'));
101+
this.results = [...prev.results, ...this.results];
102+
this.time_start = prev.time_start;
103+
this.time_end = Math.max(this.time_end, prev.time_end);
104+
this.run_info = prev.run_info;
105+
} else {
106+
/**
107+
* Return required and some optional properties
108+
* https://github.com/web-platform-tests/wpt.fyi/blob/60da175/api/README.md?plain=1#L331-L335
109+
*/
110+
this.run_info = {
111+
product: 'node.js',
112+
...getBrowserProperties(),
113+
revision: process.env.WPT_REVISION || 'unknown',
114+
os: getOs(),
115+
};
116+
}
117+
118+
fs.writeFileSync('out/wpt/wptreport.json', JSON.stringify(this));
119+
}
120+
}
121+
12122
// https://github.com/web-platform-tests/wpt/blob/HEAD/resources/testharness.js
13123
// TODO: get rid of this half-baked harness in favor of the one
14124
// pulled from WPT
@@ -313,6 +423,10 @@ class WPTRunner {
313423
this.unexpectedFailures = [];
314424

315425
this.scriptsModifier = null;
426+
427+
if (process.env.WPT_REPORT != null) {
428+
this.report = new WPTReport();
429+
}
316430
}
317431

318432
/**
@@ -339,18 +453,27 @@ class WPTRunner {
339453
this.scriptsModifier = modifier;
340454
}
341455

342-
get fullInitScript() {
343-
if (this.initScript === null && this.dummyGlobalThisScript === null) {
456+
fullInitScript(hasSubsetScript, locationSearchString) {
457+
let { initScript } = this;
458+
if (hasSubsetScript || locationSearchString) {
459+
initScript = `${initScript}\n\n//===\nglobalThis.location ||= {};`;
460+
}
461+
462+
if (locationSearchString) {
463+
initScript = `${initScript}\n\n//===\nglobalThis.location.search = "${locationSearchString}";`;
464+
}
465+
466+
if (initScript === null && this.dummyGlobalThisScript === null) {
344467
return null;
345468
}
346469

347-
if (this.initScript === null) {
470+
if (initScript === null) {
348471
return this.dummyGlobalThisScript;
349472
} else if (this.dummyGlobalThisScript === null) {
350-
return this.initScript;
473+
return initScript;
351474
}
352475

353-
return `${this.dummyGlobalThisScript}\n\n//===\n${this.initScript}`;
476+
return `${this.dummyGlobalThisScript}\n\n//===\n${initScript}`;
354477
}
355478

356479
/**
@@ -398,15 +521,20 @@ class WPTRunner {
398521
for (const spec of queue) {
399522
const testFileName = spec.filename;
400523
const content = spec.getContent();
401-
const meta = spec.title = this.getMeta(content);
524+
const meta = spec.meta = this.getMeta(content);
402525

403526
const absolutePath = spec.getAbsolutePath();
404527
const relativePath = spec.getRelativePath();
405528
const harnessPath = fixtures.path('wpt', 'resources', 'testharness.js');
406529
const scriptsToRun = [];
530+
let hasSubsetScript = false;
531+
407532
// Scripts specified with the `// META: script=` header
408533
if (meta.script) {
409534
for (const script of meta.script) {
535+
if (script === '/common/subset-tests.js' || script === '/common/subset-tests-by-key.js') {
536+
hasSubsetScript = true;
537+
}
410538
const obj = {
411539
filename: this.resource.toRealFilePath(relativePath, script),
412540
code: this.resource.read(relativePath, script, false),
@@ -423,54 +551,65 @@ class WPTRunner {
423551
this.scriptsModifier?.(obj);
424552
scriptsToRun.push(obj);
425553

426-
const workerPath = path.join(__dirname, 'wpt/worker.js');
427-
const worker = new Worker(workerPath, {
428-
execArgv: this.flags,
429-
workerData: {
430-
testRelativePath: relativePath,
431-
wptRunner: __filename,
432-
wptPath: this.path,
433-
initScript: this.fullInitScript,
434-
harness: {
435-
code: fs.readFileSync(harnessPath, 'utf8'),
436-
filename: harnessPath,
554+
/**
555+
* Example test with no META variant
556+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/sign_verify/hmac.https.any.js#L1-L4
557+
*
558+
* Example test with multiple META variants
559+
* https://github.com/nodejs/node/blob/03854f6/test/fixtures/wpt/WebCryptoAPI/generateKey/successes_RSASSA-PKCS1-v1_5.https.any.js#L1-L9
560+
*/
561+
for (const variant of meta.variant || ['']) {
562+
const workerPath = path.join(__dirname, 'wpt/worker.js');
563+
const worker = new Worker(workerPath, {
564+
execArgv: this.flags,
565+
workerData: {
566+
testRelativePath: relativePath,
567+
wptRunner: __filename,
568+
wptPath: this.path,
569+
initScript: this.fullInitScript(hasSubsetScript, variant),
570+
harness: {
571+
code: fs.readFileSync(harnessPath, 'utf8'),
572+
filename: harnessPath,
573+
},
574+
scriptsToRun,
437575
},
438-
scriptsToRun,
439-
},
440-
});
441-
this.workers.set(testFileName, worker);
442-
443-
worker.on('message', (message) => {
444-
switch (message.type) {
445-
case 'result':
446-
return this.resultCallback(testFileName, message.result);
447-
case 'completion':
448-
return this.completionCallback(testFileName, message.status);
449-
default:
450-
throw new Error(`Unexpected message from worker: ${message.type}`);
451-
}
452-
});
576+
});
577+
this.workers.set(testFileName, worker);
578+
579+
let reportResult;
580+
worker.on('message', (message) => {
581+
switch (message.type) {
582+
case 'result':
583+
reportResult ||= this.report?.addResult(`/${relativePath}${variant}`, 'OK');
584+
return this.resultCallback(testFileName, message.result, reportResult);
585+
case 'completion':
586+
return this.completionCallback(testFileName, message.status);
587+
default:
588+
throw new Error(`Unexpected message from worker: ${message.type}`);
589+
}
590+
});
453591

454-
worker.on('error', (err) => {
455-
if (!this.inProgress.has(testFileName)) {
456-
// The test is already finished. Ignore errors that occur after it.
457-
// This can happen normally, for example in timers tests.
458-
return;
459-
}
460-
this.fail(
461-
testFileName,
462-
{
463-
status: NODE_UNCAUGHT,
464-
name: 'evaluation in WPTRunner.runJsTests()',
465-
message: err.message,
466-
stack: inspect(err),
467-
},
468-
kUncaught,
469-
);
470-
this.inProgress.delete(testFileName);
471-
});
592+
worker.on('error', (err) => {
593+
if (!this.inProgress.has(testFileName)) {
594+
// The test is already finished. Ignore errors that occur after it.
595+
// This can happen normally, for example in timers tests.
596+
return;
597+
}
598+
this.fail(
599+
testFileName,
600+
{
601+
status: NODE_UNCAUGHT,
602+
name: 'evaluation in WPTRunner.runJsTests()',
603+
message: err.message,
604+
stack: inspect(err),
605+
},
606+
kUncaught,
607+
);
608+
this.inProgress.delete(testFileName);
609+
});
472610

473-
await events.once(worker, 'exit').catch(() => {});
611+
await events.once(worker, 'exit').catch(() => {});
612+
}
474613
}
475614

476615
process.on('exit', () => {
@@ -530,6 +669,8 @@ class WPTRunner {
530669
}
531670
}
532671

672+
this.report?.write();
673+
533674
const ran = queue.length;
534675
const total = ran + skipped;
535676
const passed = ran - expectedFailures - failures.length;
@@ -554,8 +695,7 @@ class WPTRunner {
554695

555696
getTestTitle(filename) {
556697
const spec = this.specMap.get(filename);
557-
const title = spec.meta && spec.meta.title;
558-
return title ? `${filename} : ${title}` : filename;
698+
return spec.meta?.title || filename;
559699
}
560700

561701
// Map WPT test status to strings
@@ -581,14 +721,14 @@ class WPTRunner {
581721
* @param {string} filename
582722
* @param {Test} test The Test object returned by WPT harness
583723
*/
584-
resultCallback(filename, test) {
724+
resultCallback(filename, test, reportResult) {
585725
const status = this.getTestStatus(test.status);
586726
const title = this.getTestTitle(filename);
587727
console.log(`---- ${title} ----`);
588728
if (status !== kPass) {
589-
this.fail(filename, test, status);
729+
this.fail(filename, test, status, reportResult);
590730
} else {
591-
this.succeed(filename, test, status);
731+
this.succeed(filename, test, status, reportResult);
592732
}
593733
}
594734

@@ -636,11 +776,12 @@ class WPTRunner {
636776
}
637777
}
638778

639-
succeed(filename, test, status) {
779+
succeed(filename, test, status, reportResult) {
640780
console.log(`[${status.toUpperCase()}] ${test.name}`);
781+
reportResult?.addSubtest(test.name, 'PASS');
641782
}
642783

643-
fail(filename, test, status) {
784+
fail(filename, test, status, reportResult) {
644785
const spec = this.specMap.get(filename);
645786
const expected = spec.failedTests.includes(test.name);
646787
if (expected) {
@@ -656,6 +797,9 @@ class WPTRunner {
656797
const command = `${process.execPath} ${process.execArgv}` +
657798
` ${require.main.filename} ${filename}`;
658799
console.log(`Command: ${command}\n`);
800+
801+
reportResult?.addSubtest(test.name, 'FAIL', test.message);
802+
659803
this.addTestResult(filename, {
660804
name: test.name,
661805
expected,
@@ -685,7 +829,7 @@ class WPTRunner {
685829
const parts = match.match(/\/\/ META: ([^=]+?)=(.+)/);
686830
const key = parts[1];
687831
const value = parts[2];
688-
if (key === 'script') {
832+
if (key === 'script' || key === 'variant') {
689833
if (result[key]) {
690834
result[key].push(value);
691835
} else {

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL