Unify logic for running parser tests from external suites (#10444)
This commit is contained in:
Nicolò Ribaudo
193
scripts/tests/test262/index.js
Normal file
193
scripts/tests/test262/index.js
Normal file
@@ -0,0 +1,193 @@
|
||||
const path = require("path");
|
||||
const TestStream = require("test262-stream");
|
||||
const TestRunner = require("../utils/parser-test-runner");
|
||||
|
||||
const ignoredFeatures = [
|
||||
"Array.prototype.flat",
|
||||
"Array.prototype.flatMap",
|
||||
"Array.prototype.values",
|
||||
"ArrayBuffer",
|
||||
"async-functions",
|
||||
"async-iteration",
|
||||
"arrow-function",
|
||||
"Atomics",
|
||||
"caller",
|
||||
"class",
|
||||
"computed-property-names",
|
||||
"const",
|
||||
"cross-realm",
|
||||
"DataView",
|
||||
"DataView.prototype.getFloat32",
|
||||
"DataView.prototype.getFloat64",
|
||||
"DataView.prototype.getInt8",
|
||||
"DataView.prototype.getInt16",
|
||||
"DataView.prototype.getInt32",
|
||||
"DataView.prototype.getUint16",
|
||||
"DataView.prototype.getUint32",
|
||||
"DataView.prototype.setUint8",
|
||||
"default-parameters",
|
||||
"destructuring-assignment",
|
||||
"destructuring-binding",
|
||||
"FinalizationGroup",
|
||||
"Float32Array",
|
||||
"Float64Array",
|
||||
"for-of",
|
||||
"generators",
|
||||
"globalThis",
|
||||
"hashbang",
|
||||
"host-gc-required",
|
||||
"Int8Array",
|
||||
"Int32Array",
|
||||
"Intl.DateTimeFormat-datetimestyle",
|
||||
"Intl.DateTimeFormat-dayPeriod",
|
||||
"Intl.DateTimeFormat-fractionalSecondDigits",
|
||||
"Intl.DateTimeFormat-formatRange",
|
||||
"Intl.ListFormat",
|
||||
"Intl.Locale",
|
||||
"Intl.NumberFormat-unified",
|
||||
"Intl.RelativeTimeFormat",
|
||||
"Intl.Segmenter",
|
||||
"IsHTMLDDA",
|
||||
"json-superset",
|
||||
"let",
|
||||
"Map",
|
||||
"new.target",
|
||||
"Object.fromEntries",
|
||||
"Object.is",
|
||||
"object-rest",
|
||||
"object-spread",
|
||||
"optional-catch-binding",
|
||||
"Promise.allSettled",
|
||||
"Promise.prototype.finally",
|
||||
"Proxy",
|
||||
"proxy-missing-checks",
|
||||
"Reflect",
|
||||
"Reflect.construct",
|
||||
"Reflect.set",
|
||||
"Reflect.setPrototypeOf",
|
||||
"regexp-dotall",
|
||||
"regexp-lookbehind",
|
||||
"regexp-named-groups",
|
||||
"regexp-unicode-property-escapes",
|
||||
"rest-parameters",
|
||||
"SharedArrayBuffer",
|
||||
"Set",
|
||||
"String.fromCodePoint",
|
||||
"String.prototype.endsWith",
|
||||
"String.prototype.includes",
|
||||
"String.prototype.matchAll",
|
||||
"String.prototype.trimEnd",
|
||||
"String.prototype.trimStart",
|
||||
"string-trimming",
|
||||
"super",
|
||||
"Symbol",
|
||||
"Symbol.asyncIterator",
|
||||
"Symbol.hasInstance",
|
||||
"Symbol.isConcatSpreadable",
|
||||
"Symbol.iterator",
|
||||
"Symbol.match",
|
||||
"Symbol.matchAll",
|
||||
"Symbol.prototype.description",
|
||||
"Symbol.replace",
|
||||
"Symbol.search",
|
||||
"Symbol.split",
|
||||
"Symbol.species",
|
||||
"Symbol.toPrimitive",
|
||||
"Symbol.toStringTag",
|
||||
"Symbol.unscopables",
|
||||
"tail-call-optimization",
|
||||
"template",
|
||||
"TypedArray",
|
||||
"u180e",
|
||||
"Uint8Array",
|
||||
"Uint8ClampedArray",
|
||||
"Uint16Array",
|
||||
"WeakMap",
|
||||
"WeakSet",
|
||||
"WeakRef",
|
||||
"well-formed-json-stringify",
|
||||
];
|
||||
|
||||
const ignoredTests = ["built-ins/RegExp/", "language/literals/regexp/"];
|
||||
|
||||
const featuresToPlugins = {
|
||||
BigInt: "bigInt",
|
||||
"class-fields-private": "classPrivateProperties",
|
||||
"class-fields-public": "classProperties",
|
||||
"class-methods-private": "classPrivateMethods",
|
||||
"class-static-fields-public": "classProperties",
|
||||
"class-static-fields-private": "classPrivateProperties",
|
||||
"class-static-methods-private": "classPrivateMethods",
|
||||
"dynamic-import": "dynamicImport",
|
||||
"export-star-as-namespace-from-module": "exportNamespaceFrom",
|
||||
"import.meta": "importMeta",
|
||||
"numeric-separator-literal": "numericSeparator",
|
||||
"optional-chaining": "optionalChaining",
|
||||
"top-level-await": "topLevelAwait",
|
||||
};
|
||||
|
||||
const unmappedFeatures = new Set();
|
||||
|
||||
function* getPlugins(features) {
|
||||
if (!features) return;
|
||||
|
||||
for (const f of features) {
|
||||
if (featuresToPlugins[f]) {
|
||||
yield featuresToPlugins[f];
|
||||
} else if (!ignoredFeatures.includes(f)) {
|
||||
unmappedFeatures.add(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const runner = new TestRunner({
|
||||
testDir: path.join(__dirname, "../../../build/test262"),
|
||||
whitelist: path.join(__dirname, "whitelist.txt"),
|
||||
logInterval: 500,
|
||||
shouldUpdate: process.argv.includes("--update-whitelist"),
|
||||
|
||||
async *getTests() {
|
||||
const stream = new TestStream(this.testDir, {
|
||||
omitRuntime: true,
|
||||
});
|
||||
|
||||
for await (const test of stream) {
|
||||
// strip test/
|
||||
const fileName = test.file.substr(5);
|
||||
|
||||
if (ignoredTests.some(start => fileName.startsWith(start))) continue;
|
||||
|
||||
yield {
|
||||
contents: test.contents,
|
||||
fileName,
|
||||
id: `${fileName}(${test.scenario})`,
|
||||
sourceType: test.attrs.flags.module ? "module" : "script",
|
||||
plugins: Array.from(getPlugins(test.attrs.features)),
|
||||
expectedError:
|
||||
!!test.attrs.negative &&
|
||||
(test.attrs.negative.phase === "parse" ||
|
||||
test.attrs.negative.phase === "early"),
|
||||
};
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
runner
|
||||
.run()
|
||||
.then(() => {
|
||||
if (unmappedFeatures.size) {
|
||||
console.log("");
|
||||
console.log(
|
||||
"The following Features are not currently mapped or ignored:"
|
||||
);
|
||||
console.log(
|
||||
Array.from(unmappedFeatures)
|
||||
.join("\n")
|
||||
.replace(/^/gm, " ")
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,126 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
const path = require("path");
|
||||
const chalk = require("chalk");
|
||||
const utils = require("./run_babel_parser_test262_utils");
|
||||
|
||||
const testDir = path.join(__dirname, "../../../build/test262");
|
||||
const whitelistFile = path.join(__dirname, "test262_whitelist.txt");
|
||||
const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1;
|
||||
|
||||
Promise.all([utils.getTests(testDir), utils.getWhitelist(whitelistFile)])
|
||||
.then(function([tests, whitelist]) {
|
||||
const total = tests.length;
|
||||
const reportInc = Math.floor(total / 20);
|
||||
|
||||
console.log(`Now running ${total} tests...`);
|
||||
|
||||
const results = tests.map(function(test, idx) {
|
||||
if (idx % reportInc === 0) {
|
||||
console.log(`> ${Math.round((100 * idx) / total)}% complete`);
|
||||
}
|
||||
|
||||
return utils.runTest(test);
|
||||
});
|
||||
|
||||
return utils.interpret(results, whitelist);
|
||||
})
|
||||
.then(function(summary) {
|
||||
const goodnews = [
|
||||
summary.allowed.success.length + " valid programs parsed without error",
|
||||
summary.allowed.failure.length +
|
||||
" invalid programs produced a parsing error",
|
||||
summary.allowed.falsePositive.length +
|
||||
" invalid programs did not produce a parsing error" +
|
||||
" (and allowed by the whitelist file)",
|
||||
summary.allowed.falseNegative.length +
|
||||
" valid programs produced a parsing error" +
|
||||
" (and allowed by the whitelist file)",
|
||||
];
|
||||
const badnews = [];
|
||||
const badnewsDetails = [];
|
||||
|
||||
void [
|
||||
{
|
||||
tests: summary.disallowed.success,
|
||||
label:
|
||||
"valid programs parsed without error" +
|
||||
" (in violation of the whitelist file)",
|
||||
},
|
||||
{
|
||||
tests: summary.disallowed.failure,
|
||||
label:
|
||||
"invalid programs produced a parsing error" +
|
||||
" (in violation of the whitelist file)",
|
||||
},
|
||||
{
|
||||
tests: summary.disallowed.falsePositive,
|
||||
label:
|
||||
"invalid programs did not produce a parsing error" +
|
||||
" (without a corresponding entry in the whitelist file)",
|
||||
},
|
||||
{
|
||||
tests: summary.disallowed.falseNegative,
|
||||
label:
|
||||
"valid programs produced a parsing error" +
|
||||
" (without a corresponding entry in the whitelist file)",
|
||||
},
|
||||
{
|
||||
tests: summary.unrecognized,
|
||||
label: "non-existent programs specified in the whitelist file",
|
||||
},
|
||||
].forEach(function({ tests, label }) {
|
||||
if (!tests.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const desc = tests.length + " " + label;
|
||||
|
||||
badnews.push(desc);
|
||||
badnewsDetails.push(desc + ":");
|
||||
badnewsDetails.push(
|
||||
...tests.map(function(test) {
|
||||
return test.id || test;
|
||||
})
|
||||
);
|
||||
});
|
||||
|
||||
console.log("Testing complete.");
|
||||
console.log("Summary:");
|
||||
console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ ")));
|
||||
|
||||
if (!summary.passed) {
|
||||
console.log("");
|
||||
console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ ")));
|
||||
console.log("");
|
||||
console.log("Details:");
|
||||
console.log(badnewsDetails.join("\n").replace(/^/gm, " "));
|
||||
}
|
||||
|
||||
if (shouldUpdate) {
|
||||
return utils.updateWhitelist(whitelistFile, summary).then(function() {
|
||||
console.log("");
|
||||
console.log("Whitelist file updated.");
|
||||
});
|
||||
} else {
|
||||
process.exitCode = summary.passed ? 0 : 1;
|
||||
}
|
||||
|
||||
const unmappedFeatures = utils.getUnmappedFeatures();
|
||||
|
||||
if (unmappedFeatures.size) {
|
||||
console.log("");
|
||||
console.log(
|
||||
"The following Features are not currently mapped or ignored:"
|
||||
);
|
||||
console.log(
|
||||
Array.from(unmappedFeatures)
|
||||
.join("\n")
|
||||
.replace(/^/gm, " ")
|
||||
);
|
||||
}
|
||||
})
|
||||
.catch(function(err) {
|
||||
console.error(err);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,297 +0,0 @@
|
||||
"use strict";
|
||||
|
||||
const fs = require("graceful-fs");
|
||||
const promisify = require("util").promisify;
|
||||
const TestStream = require("test262-stream");
|
||||
const pfs = {
|
||||
readFile: promisify(fs.readFile),
|
||||
writeFile: promisify(fs.writeFile),
|
||||
readdir: promisify(fs.readdir),
|
||||
stat: promisify(fs.stat),
|
||||
};
|
||||
|
||||
const parse = require("../../../packages/babel-parser").parse;
|
||||
|
||||
const ignoredFeatures = [
|
||||
"Array.prototype.flat",
|
||||
"Array.prototype.flatMap",
|
||||
"Array.prototype.values",
|
||||
"ArrayBuffer",
|
||||
"async-functions",
|
||||
"async-iteration",
|
||||
"arrow-function",
|
||||
"Atomics",
|
||||
"caller",
|
||||
"class",
|
||||
"computed-property-names",
|
||||
"const",
|
||||
"cross-realm",
|
||||
"DataView",
|
||||
"DataView.prototype.getFloat32",
|
||||
"DataView.prototype.getFloat64",
|
||||
"DataView.prototype.getInt8",
|
||||
"DataView.prototype.getInt16",
|
||||
"DataView.prototype.getInt32",
|
||||
"DataView.prototype.getUint16",
|
||||
"DataView.prototype.getUint32",
|
||||
"DataView.prototype.setUint8",
|
||||
"default-parameters",
|
||||
"destructuring-assignment",
|
||||
"destructuring-binding",
|
||||
"Float32Array",
|
||||
"Float64Array",
|
||||
"for-of",
|
||||
"generators",
|
||||
"globalThis",
|
||||
"hashbang",
|
||||
"Int8Array",
|
||||
"Int32Array",
|
||||
"Intl.ListFormat",
|
||||
"Intl.Locale",
|
||||
"Intl.NumberFormat-unified",
|
||||
"Intl.RelativeTimeFormat",
|
||||
"Intl.Segmenter",
|
||||
"IsHTMLDDA",
|
||||
"json-superset",
|
||||
"let",
|
||||
"Map",
|
||||
"new.target",
|
||||
"Object.fromEntries",
|
||||
"Object.is",
|
||||
"object-rest",
|
||||
"object-spread",
|
||||
"optional-catch-binding",
|
||||
"Promise.prototype.finally",
|
||||
"Proxy",
|
||||
"Reflect",
|
||||
"Reflect.construct",
|
||||
"Reflect.set",
|
||||
"Reflect.setPrototypeOf",
|
||||
"regexp-dotall",
|
||||
"regexp-lookbehind",
|
||||
"regexp-named-groups",
|
||||
"regexp-unicode-property-escapes",
|
||||
"SharedArrayBuffer",
|
||||
"Set",
|
||||
"String.fromCodePoint",
|
||||
"String.prototype.endsWith",
|
||||
"String.prototype.includes",
|
||||
"String.prototype.matchAll",
|
||||
"String.prototype.trimEnd",
|
||||
"String.prototype.trimStart",
|
||||
"string-trimming",
|
||||
"super",
|
||||
"Symbol",
|
||||
"Symbol.asyncIterator",
|
||||
"Symbol.hasInstance",
|
||||
"Symbol.isConcatSpreadable",
|
||||
"Symbol.iterator",
|
||||
"Symbol.match",
|
||||
"Symbol.matchAll",
|
||||
"Symbol.prototype.description",
|
||||
"Symbol.replace",
|
||||
"Symbol.search",
|
||||
"Symbol.split",
|
||||
"Symbol.species",
|
||||
"Symbol.toPrimitive",
|
||||
"Symbol.toStringTag",
|
||||
"Symbol.unscopables",
|
||||
"tail-call-optimization",
|
||||
"template",
|
||||
"TypedArray",
|
||||
"u180e",
|
||||
"Uint8Array",
|
||||
"Uint8ClampedArray",
|
||||
"Uint16Array",
|
||||
"WeakMap",
|
||||
"WeakSet",
|
||||
"well-formed-json-stringify",
|
||||
];
|
||||
|
||||
const ignoredTests = ["built-ins/RegExp/", "language/literals/regexp/"];
|
||||
|
||||
const featuresToPlugins = {
|
||||
BigInt: "bigInt",
|
||||
"class-fields-private": "classPrivateProperties",
|
||||
"class-fields-public": "classProperties",
|
||||
"class-methods-private": "classPrivateMethods",
|
||||
"class-static-fields-public": "classProperties",
|
||||
"class-static-fields-private": "classPrivateProperties",
|
||||
"class-static-methods-private": "classPrivateMethods",
|
||||
"dynamic-import": "dynamicImport",
|
||||
"export-star-as-namespace-from-module": "exportNamespaceFrom",
|
||||
"import.meta": "importMeta",
|
||||
"numeric-separator-literal": "numericSeparator",
|
||||
"optional-chaining": "optionalChaining",
|
||||
"top-level-await": "topLevelAwait",
|
||||
};
|
||||
|
||||
function getPlugins(features) {
|
||||
return (
|
||||
features &&
|
||||
features
|
||||
.map(f => {
|
||||
if (!featuresToPlugins[f] && !ignoredFeatures.includes(f)) {
|
||||
unmappedFeatures.add(f);
|
||||
}
|
||||
return featuresToPlugins[f];
|
||||
})
|
||||
.filter(Boolean)
|
||||
);
|
||||
}
|
||||
|
||||
const unmappedFeatures = new Set();
|
||||
|
||||
exports.getUnmappedFeatures = function() {
|
||||
return unmappedFeatures;
|
||||
};
|
||||
|
||||
exports.getTests = function(testDir) {
|
||||
const stream = new TestStream(testDir, {
|
||||
omitRuntime: true,
|
||||
});
|
||||
const tests = [];
|
||||
|
||||
stream.on("data", test => {
|
||||
// strip test/
|
||||
const fileName = test.file.substr(5);
|
||||
|
||||
if (ignoredTests.some(start => fileName.startsWith(start))) return;
|
||||
|
||||
tests.push({
|
||||
contents: test.contents,
|
||||
fileName,
|
||||
id: `${fileName}(${test.scenario})`,
|
||||
sourceType: test.attrs.flags.module ? "module" : "script",
|
||||
plugins: getPlugins(test.attrs.features),
|
||||
expectedError:
|
||||
!!test.attrs.negative &&
|
||||
(test.attrs.negative.phase === "parse" ||
|
||||
test.attrs.negative.phase === "early"),
|
||||
});
|
||||
});
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on("end", () => resolve(tests));
|
||||
stream.on("error", reject);
|
||||
});
|
||||
};
|
||||
|
||||
exports.runTest = function(test) {
|
||||
try {
|
||||
parse(test.contents, {
|
||||
sourceType: test.sourceType,
|
||||
plugins: test.plugins,
|
||||
});
|
||||
test.actualError = false;
|
||||
} catch (err) {
|
||||
test.actualError = true;
|
||||
}
|
||||
|
||||
test.result = test.expectedError !== test.actualError ? "fail" : "pass";
|
||||
|
||||
return test;
|
||||
};
|
||||
|
||||
exports.getWhitelist = function(filename) {
|
||||
return pfs.readFile(filename, "utf-8").then(function(contents) {
|
||||
return contents
|
||||
.split("\n")
|
||||
.map(function(line) {
|
||||
return line.replace(/#.*$/, "").trim();
|
||||
})
|
||||
.filter(function(line) {
|
||||
return line.length > 0;
|
||||
})
|
||||
.reduce(function(table, filename) {
|
||||
table[filename] = true;
|
||||
return table;
|
||||
}, Object.create(null));
|
||||
});
|
||||
};
|
||||
|
||||
exports.updateWhitelist = function(filename, summary) {
|
||||
return pfs.readFile(filename, "utf-8").then(function(contents) {
|
||||
const toRemove = summary.disallowed.success
|
||||
.concat(summary.disallowed.failure)
|
||||
.map(function(test) {
|
||||
return test.id;
|
||||
})
|
||||
.concat(summary.unrecognized);
|
||||
const toAdd = summary.disallowed.falsePositive
|
||||
.concat(summary.disallowed.falseNegative)
|
||||
.map(function(test) {
|
||||
return test.id;
|
||||
});
|
||||
const newContents = contents
|
||||
.split("\n")
|
||||
.map(function(line) {
|
||||
const testId = line.replace(/#.*$/, "").trim();
|
||||
|
||||
if (toRemove.indexOf(testId) > -1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return line;
|
||||
})
|
||||
.filter(function(line) {
|
||||
return line !== null && line !== "";
|
||||
})
|
||||
.concat(toAdd)
|
||||
.sort()
|
||||
.join("\n");
|
||||
|
||||
return pfs.writeFile(filename, newContents, "utf-8");
|
||||
});
|
||||
};
|
||||
|
||||
exports.interpret = function(results, whitelist) {
|
||||
const summary = {
|
||||
passed: true,
|
||||
allowed: {
|
||||
success: [],
|
||||
failure: [],
|
||||
falsePositive: [],
|
||||
falseNegative: [],
|
||||
},
|
||||
disallowed: {
|
||||
success: [],
|
||||
failure: [],
|
||||
falsePositive: [],
|
||||
falseNegative: [],
|
||||
},
|
||||
unrecognized: null,
|
||||
};
|
||||
|
||||
results.forEach(function(result) {
|
||||
let classification, isAllowed;
|
||||
const inWhitelist = result.id in whitelist;
|
||||
delete whitelist[result.id];
|
||||
|
||||
if (!result.expectedError) {
|
||||
if (!result.actualError) {
|
||||
classification = "success";
|
||||
isAllowed = !inWhitelist;
|
||||
} else {
|
||||
classification = "falseNegative";
|
||||
isAllowed = inWhitelist;
|
||||
}
|
||||
} else {
|
||||
if (!result.actualError) {
|
||||
classification = "falsePositive";
|
||||
isAllowed = inWhitelist;
|
||||
} else {
|
||||
classification = "failure";
|
||||
isAllowed = !inWhitelist;
|
||||
}
|
||||
}
|
||||
|
||||
summary.passed &= isAllowed;
|
||||
summary[isAllowed ? "allowed" : "disallowed"][classification].push(result);
|
||||
});
|
||||
|
||||
summary.unrecognized = Object.keys(whitelist);
|
||||
summary.passed = !!summary.passed && summary.unrecognized.length === 0;
|
||||
|
||||
return summary;
|
||||
};
|
||||
@@ -191,4 +191,4 @@ language/statements/class/elements/syntax/early-errors/invalid-names/method-outt
|
||||
language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-bad-reference.js(default)
|
||||
language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-bad-reference.js(strict mode)
|
||||
language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(default)
|
||||
language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(strict mode)
|
||||
language/statements/class/elements/syntax/early-errors/invalid-names/method-outter-member-expression-this.js(strict mode)
|
||||
Reference in New Issue
Block a user