FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test: improve WPT runner summary · nodejs/node@9e21711 · GitHub

/ node Public

Commit 9e21711

Browse files
panva
authored and
aduh95
committed
test: improve WPT runner summary
Signed-off-by: Filip Skokan <panva.ip@gmail.com> PR-URL: #62636 Reviewed-By: Mattias Buelens <mattias@buelens.com> Reviewed-By: Jason Zhang <xzha4350@gmail.com> Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
1 parent ef384fe commit 9e21711

1 file changed

Lines changed: 39 additions & 20 deletions

File tree

‎test/common/wpt.js‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -500,14 +500,16 @@ class StatusLoader {
500500

501501
load() {
502502
const dir = path.join(__dirname, '..', 'wpt');
503-
let statusFile = path.join(dir, 'status', `${this.path}.json`);
504503
let result;
505504

506-
if (fs.existsSync(statusFile)) {
507-
result = JSON.parse(fs.readFileSync(statusFile, 'utf8'));
508-
} else {
509-
statusFile = path.join(dir, 'status', `${this.path}.cjs`);
510-
result = require(statusFile);
505+
try {
506+
this.statusFile = `${this.path}.json`;
507+
const jsonFile = path.join(dir, 'status', this.statusFile);
508+
result = JSON.parse(fs.readFileSync(jsonFile, 'utf8'));
509+
} catch (err) {
510+
if (err?.code !== 'ENOENT') throw err;
511+
this.statusFile = `${this.path}.cjs`;
512+
result = require(path.join(dir, 'status', this.statusFile));
511513
}
512514

513515
this.rules.addRules(result);
@@ -573,13 +575,16 @@ class WPTRunner {
573575

574576
this.status = new StatusLoader(path);
575577
this.status.load();
578+
this.statusFile = this.status.statusFile;
576579
this.specs = new Set(this.status.specs);
577580

578581
this.results = {};
579582
this.inProgress = new Set();
580583
this.workers = new Map();
581584
this.unexpectedFailures = [];
582585

586+
this.subtestCounts = { passed: 0, failed: 0, expectedFailures: 0, skipped: 0, unexpectedPasses: 0 };
587+
583588
if (process.env.WPT_REPORT != null) {
584589
this.report = new WPTReport(path);
585590
}
@@ -775,7 +780,6 @@ class WPTRunner {
775780
const failures = [];
776781
let expectedFailures = 0;
777782
let skipped = 0;
778-
let skippedTests = 0;
779783
for (const [key, item] of Object.entries(this.results)) {
780784
if (item.fail?.unexpected) {
781785
failures.push(key);
@@ -786,9 +790,6 @@ class WPTRunner {
786790
if (item.skip) {
787791
skipped++;
788792
}
789-
if (item.skipTests) {
790-
skippedTests += item.skipTests.length;
791-
}
792793
}
793794

794795
const unexpectedPasses = [];
@@ -823,23 +824,28 @@ class WPTRunner {
823824
// so that results survive if the process is killed.
824825
this.report?.write();
825826

827+
const p = (n, word, suffix = 's') => `${n} ${word}${n === 1 ? '' : suffix}`;
826828
const ran = queue.length;
827829
const total = ran + skipped;
828830
const passed = ran - expectedFailures - failures.length;
831+
const { subtestCounts } = this;
829832
console.log('');
830-
console.log(`Ran ${ran}/${total} tests, ${skipped} skipped,`,
831-
`${passed} passed, ${expectedFailures} expected failures,`,
832-
`${failures.length} unexpected failures,`,
833-
`${unexpectedPasses.length} unexpected passes` +
834-
(skippedTests ? `, ${skippedTests} subtests skipped` : ''));
833+
console.log(`Files: ${ran}/${total} ran, ${passed} passed,`,
834+
`${skipped} skipped, ${p(expectedFailures, 'expected failure')},`,
835+
`${p(failures.length, 'unexpected failure')},`,
836+
`${p(unexpectedPasses.length, 'unexpected pass', 'es')}`);
837+
console.log(`Subtests: ${subtestCounts.passed} passed,`,
838+
`${subtestCounts.skipped} skipped, ${p(subtestCounts.expectedFailures, 'expected failure')},`,
839+
`${p(subtestCounts.failed, 'unexpected failure')},`,
840+
`${p(subtestCounts.unexpectedPasses, 'unexpected pass', 'es')}`);
835841
if (failures.length > 0) {
836-
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
842+
const file = path.join('test', 'wpt', 'status', this.statusFile);
837843
throw new Error(
838844
`Found ${failures.length} unexpected failures. ` +
839845
`Consider updating ${file} for these files:\n${failures.join('\n')}`);
840846
}
841847
if (unexpectedPasses.length > 0) {
842-
const file = path.join('test', 'wpt', 'status', `${this.path}.json`);
848+
const file = path.join('test', 'wpt', 'status', this.statusFile);
843849
throw new Error(
844850
`Found ${unexpectedPasses.length} unexpected passes. ` +
845851
`Consider updating ${file} for these files:\n${unexpectedPasses.join('\n')}`);
@@ -875,7 +881,7 @@ class WPTRunner {
875881
if (status !== kPass) {
876882
this.fail(spec, test, status, reportResult);
877883
} else {
878-
this.succeed(test, status, reportResult);
884+
this.succeed(spec, test, status, reportResult);
879885
}
880886
}
881887

@@ -943,14 +949,22 @@ class WPTRunner {
943949
}
944950
}
945951

946-
succeed(test, status, reportResult) {
947-
console.log(`[${status.toUpperCase()}] ${test.name}`);
952+
succeed(spec, test, status, reportResult) {
953+
const unexpectedPass = spec.failedTests.includes(test.name);
954+
if (unexpectedPass) {
955+
console.log(`[UNEXPECTED_PASS][${status.toUpperCase()}] ${test.name}`);
956+
this.subtestCounts.unexpectedPasses++;
957+
} else {
958+
console.log(`[${status.toUpperCase()}] ${test.name}`);
959+
this.subtestCounts.passed++;
960+
}
948961
reportResult?.addSubtest(test.name, 'PASS');
949962
}
950963

951964
skipTest(spec, test, reportResult) {
952965
console.log(`[SKIP] ${test.name}`);
953966
reportResult?.addSubtest(test.name, 'NOTRUN');
967+
this.subtestCounts.skipped++;
954968
this.addTestResult(spec, {
955969
name: test.name,
956970
status: kSkip,
@@ -973,6 +987,11 @@ class WPTRunner {
973987
console.log(`Command: ${command}\n`);
974988

975989
reportResult?.addSubtest(test.name, 'FAIL', test.message);
990+
if (expected) {
991+
this.subtestCounts.expectedFailures++;
992+
} else {
993+
this.subtestCounts.failed++;
994+
}
976995

977996
this.addTestResult(spec, {
978997
name: test.name,

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL