FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test: report error wpt test results · nodejs/node@7e3eb02 · GitHub

/ node Public

Commit 7e3eb02

Browse files
legendecas
authored and
targos
committed
test: report error wpt test results
When a wpt test file is exited for uncaught error, its result should be recorded in the `wptreport.json` and uploaded to wpt.fyi. For instance, `html/webappapis/timers/evil-spec-example.any.js` is exited for uncaught error in Node.js but it shows as "MISSING" at https://wpt.fyi/results/html/webappapis/timers?label=master&label=experimental&product=chrome&product=node.js&aligned. PR-URL: #50429 Reviewed-By: Filip Skokan <panva.ip@gmail.com>
1 parent 93e3cc3 commit 7e3eb02

1 file changed

Lines changed: 73 additions & 34 deletions

File tree

‎test/common/wpt.js‎

Lines changed: 73 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -58,46 +58,66 @@ function codeUnitStr(char) {
5858
return 'U+' + char.charCodeAt(0).toString(16);
5959
}
6060

61+
class ReportResult {
62+
constructor(name) {
63+
this.test = name;
64+
this.status = 'OK';
65+
this.subtests = [];
66+
}
67+
68+
addSubtest(name, status, message) {
69+
const subtest = {
70+
status,
71+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
72+
name: sanitizeUnpairedSurrogates(name),
73+
};
74+
if (message) {
75+
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
76+
subtest.message = sanitizeUnpairedSurrogates(message);
77+
}
78+
this.subtests.push(subtest);
79+
return subtest;
80+
}
81+
82+
finish(status) {
83+
this.status = status ?? 'OK';
84+
}
85+
}
86+
87+
// Generates a report that can be uploaded to wpt.fyi.
88+
// Checkout https://github.com/web-platform-tests/wpt.fyi/tree/main/api#results-creation
89+
// for more details.
6190
class WPTReport {
6291
constructor(path) {
6392
this.filename = `report-${path.replaceAll('/', '-')}.json`;
64-
this.results = [];
93+
/** @type {Map<string, ReportResult>} */
94+
this.results = new Map();
6595
this.time_start = Date.now();
6696
}
6797

68-
addResult(name, status) {
69-
const result = {
70-
test: name,
71-
status,
72-
subtests: [],
73-
addSubtest(name, status, message) {
74-
const subtest = {
75-
status,
76-
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L3722
77-
name: sanitizeUnpairedSurrogates(name),
78-
};
79-
if (message) {
80-
// https://github.com/web-platform-tests/wpt/blob/b24eedd/resources/testharness.js#L4506
81-
subtest.message = sanitizeUnpairedSurrogates(message);
82-
}
83-
this.subtests.push(subtest);
84-
return subtest;
85-
},
86-
};
87-
this.results.push(result);
98+
/**
99+
* Get or create a ReportResult for a test spec.
100+
* @param {WPTTestSpec} spec
101+
*/
102+
getResult(spec) {
103+
const name = `/${spec.getRelativePath()}${spec.variant}`;
104+
if (this.results.has(name)) {
105+
return this.results.get(name);
106+
}
107+
const result = new ReportResult(name);
108+
this.results.set(name, result);
88109
return result;
89110
}
90111

91112
write() {
92113
this.time_end = Date.now();
93-
this.results = this.results.filter((result) => {
94-
return result.status === 'SKIP' || result.subtests.length !== 0;
95-
}).map((result) => {
96-
const url = new URL(result.test, 'http://wpt');
97-
url.pathname = url.pathname.replace(/\.js$/, '.html');
98-
result.test = url.href.slice(url.origin.length);
99-
return result;
100-
});
114+
const results = Array.from(this.results.values())
115+
.map((result) => {
116+
const url = new URL(result.test, 'http://wpt');
117+
url.pathname = url.pathname.replace(/\.js$/, '.html');
118+
result.test = url.href.slice(url.origin.length);
119+
return result;
120+
});
101121

102122
/**
103123
* Return required and some optional properties
@@ -110,7 +130,12 @@ class WPTReport {
110130
os: getOs(),
111131
};
112132

113-
fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify(this));
133+
fs.writeFileSync(`out/wpt/${this.filename}`, JSON.stringify({
134+
time_start: this.time_start,
135+
time_end: this.time_end,
136+
run_info: this.run_info,
137+
results: results,
138+
}));
114139
}
115140
}
116141

@@ -642,14 +667,13 @@ class WPTRunner {
642667
this.inProgress.add(spec);
643668
this.workers.set(spec, worker);
644669

645-
let reportResult;
670+
const reportResult = this.report?.getResult(spec);
646671
worker.on('message', (message) => {
647672
switch (message.type) {
648673
case 'result':
649-
reportResult ||= this.report?.addResult(`/${relativePath}${spec.variant}`, 'OK');
650674
return this.resultCallback(spec, message.result, reportResult);
651675
case 'completion':
652-
return this.completionCallback(spec, message.status);
676+
return this.completionCallback(spec, message.status, reportResult);
653677
default:
654678
throw new Error(`Unexpected message from worker: ${message.type}`);
655679
}
@@ -661,6 +685,8 @@ class WPTRunner {
661685
// This can happen normally, for example in timers tests.
662686
return;
663687
}
688+
// Generate a subtest failure for visibility.
689+
// No need to record this synthetic failure with wpt.fyi.
664690
this.fail(
665691
spec,
666692
{
@@ -671,6 +697,8 @@ class WPTRunner {
671697
},
672698
kUncaught,
673699
);
700+
// Mark the whole test as failed in wpt.fyi report.
701+
reportResult?.finish('ERROR');
674702
this.inProgress.delete(spec);
675703
});
676704

@@ -680,7 +708,11 @@ class WPTRunner {
680708

681709
process.on('exit', () => {
682710
for (const spec of this.inProgress) {
711+
// No need to record this synthetic failure with wpt.fyi.
683712
this.fail(spec, { name: 'Incomplete' }, kIncomplete);
713+
// Mark the whole test as failed in wpt.fyi report.
714+
const reportResult = this.report?.getResult(spec);
715+
reportResult?.finish('ERROR');
684716
}
685717
inspect.defaultOptions.depth = Infinity;
686718
// Sorts the rules to have consistent output
@@ -780,6 +812,7 @@ class WPTRunner {
780812
* in one test file).
781813
* @param {WPTTestSpec} spec
782814
* @param {Test} test The Test object returned by WPT harness
815+
* @param {ReportResult} reportResult The report result object
783816
*/
784817
resultCallback(spec, test, reportResult) {
785818
const status = this.getTestStatus(test.status);
@@ -794,13 +827,19 @@ class WPTRunner {
794827
* Report the status of each WPT test (one per file)
795828
* @param {WPTTestSpec} spec
796829
* @param {object} harnessStatus - The status object returned by WPT harness.
830+
* @param {ReportResult} reportResult The report result object
797831
*/
798-
completionCallback(spec, harnessStatus) {
832+
completionCallback(spec, harnessStatus, reportResult) {
799833
const status = this.getTestStatus(harnessStatus.status);
800834

801835
// Treat it like a test case failure
802836
if (status === kTimeout) {
837+
// No need to record this synthetic failure with wpt.fyi.
803838
this.fail(spec, { name: 'WPT testharness timeout' }, kTimeout);
839+
// Mark the whole test as TIMEOUT in wpt.fyi report.
840+
reportResult?.finish('TIMEOUT');
841+
} else {
842+
reportResult?.finish();
804843
}
805844
this.inProgress.delete(spec);
806845
// Always force termination of the worker. Some tests allocate resources

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL