diff --git a/scripts/flow_tests_whitelist.txt b/scripts/flow_tests_whitelist.txt new file mode 100644 index 0000000000..aa0d9bafd4 --- /dev/null +++ b/scripts/flow_tests_whitelist.txt @@ -0,0 +1,56 @@ +# This file lists tests that are known to produce incorrect results when parsed +# with Babylon: +# +# - Tests that are expected to parse successfully but for which Babylon reports +# a syntax error +# - Tests that contain invalid syntax but for which Babylon reports no syntax +# error +# +# Entries should be removed incrementally as Babylon is improved. + +ES6/binding-pattern/object-pattern/await-prop-in-async-function.js +ES6/computed_properties/migrated_0000.js +JSX_invalid/migrated_0000.js +arrow_function/param-dflt-yield-expr.js +arrow_function_invalid/migrated_0002.js +async_await/async_generic_method.js +async_await/migrated_0020.js +async_await/migrated_0024.js +async_await/migrated_0027.js +async_generators/migrated_0007.js +class_properties/migrated_0000.js +class_properties/migrated_0005.js +class_properties/migrated_0011.js +class_properties/migrated_0016.js +class_properties/migrated_0021.js +class_properties/migrated_0026.js +decorators/migrated_0003.js +decorators/migrated_0007.js +dynamic_import/migrated_0000.js +dynamic_import/migrated_0001.js +dynamic_import/migrated_0002.js +dynamic_import/migrated_0003.js +dynamic_import/migrated_0004.js +invalid_syntax/migrated_0000.js +invalid_syntax/migrated_0001.js +invalid_syntax/migrated_0002.js +invalid_syntax/migrated_0003.js +private_class_properties/valid.js +types/annotations/migrated_0001.js +types/annotations_in_comments_invalid/migrated_0000.js +types/annotations_in_comments_invalid/migrated_0001.js +types/annotations_in_comments_invalid/migrated_0002.js +types/annotations_in_comments_invalid/migrated_0003.js +types/annotations_in_comments_invalid/migrated_0004.js +types/declare_export_invalid/migrated_0013.js +types/declare_statements_invalid/migrated_0001.js +types/number_literal_invalid/migrated_0000.js +types/parameter_defaults/migrated_0023.js +types/parameter_defaults/migrated_0026.js +types/parameter_defaults/migrated_0028.js +types/parameter_defaults/migrated_0029.js +types/parameter_defaults/migrated_0030.js +types/parameter_defaults/migrated_0031.js +types/parameter_defaults/migrated_0032.js +types/string_literal_invalid/migrated_0000.js +types/typecasts_invalid/migrated_0001.js \ No newline at end of file diff --git a/scripts/run_flow_tests.js b/scripts/run_flow_tests.js index f74df55c62..a8ba4f2def 100644 --- a/scripts/run_flow_tests.js +++ b/scripts/run_flow_tests.js @@ -6,6 +6,7 @@ const chalk = require("chalk"); const parse = require("..").parse; const TESTS_FOLDER = path.join(__dirname, "../build/flow/src/parser/test/flow"); +const WHITELIST_PATH = path.join(__dirname, "./flow_tests_whitelist.txt"); function map_get_default(map, key, defaultConstructor) { if (map.has(key)) { @@ -16,6 +17,14 @@ function map_get_default(map, key, defaultConstructor) { return value; } +function get_whitelist(filename) { + return fs + .readFileSync(filename, "utf8") + .split("\n") + .map(line => line.replace(/#.*$/, "").trim()) + .filter(Boolean); +} + function list_files(root, dir) { const files = fs.readdirSync(dir ? path.join(root, dir) : root); let result = []; @@ -48,15 +57,13 @@ function get_tests(root_dir) { const cases = map_get_default(tests, test_name, Map); const case_ = map_get_default(cases, case_name, Object); - const content = fs.readFileSync(path.join(root_dir, file), { - encoding: "utf8", - }); + const content = fs.readFileSync(path.join(root_dir, file), "utf8"); const ext = case_parts[case_parts.length - 1]; const kind = case_parts.length > 2 ? case_parts[case_parts.length - 2] : null; if (ext === "js") { - case_.name = case_name; + case_.file = file; case_.content = content; } else if (ext === "json" && kind === "tree") { case_.expected_ast = JSON.parse(content); @@ -80,17 +87,31 @@ const flowOptionsMapping = { esproposal_decorators: "decorators", }; -let failedTests = 0; -let successTests = 0; +const summary = { + passed: true, + allowed: { + success: [], + failure: [], + }, + disallowed: { + success: [], + failure: [], + }, + unrecognized: [] +}; + const tests = get_tests(TESTS_FOLDER); -tests.forEach((section, sectionName) => { - console.log(""); - console.log(`### ${sectionName} ###`); +const whitelist = get_whitelist(WHITELIST_PATH); + +const unrecognized = new Set(whitelist); + +tests.forEach(section => { section.forEach(test => { const shouldSuccess = test.expected_ast && (!Array.isArray(test.expected_ast.errors) || test.expected_ast.errors.length === 0); + const inWhitelist = whitelist.indexOf(test.file) > -1; const babylonOptions = Object.assign({}, options); babylonOptions.plugins = babylonOptions.plugins.slice(); @@ -122,53 +143,97 @@ tests.forEach((section, sectionName) => { } catch (e) {} } - if ((!failed && shouldSuccess) || (failed && !shouldSuccess)) { - successTests++; - console.log(chalk.green(`✔ ${test.name}`)); - } else { - failedTests++; - printErrorMessage( - test.name, - exception, - shouldSuccess, - test.expected_ast, - babylonOptions - ); - } + const isSuccess = shouldSuccess !== failed; + const isAllowed = isSuccess !== inWhitelist; + + summary[isAllowed ? "allowed" : "disallowed"][ + isSuccess ? "success" : "failure" + ].push({ test, exception, shouldSuccess, babylonOptions }); + summary.passed &= isAllowed; + + unrecognized.delete(test.file); + + process.stdout.write(chalk.gray(".")); }); }); -function printErrorMessage( - code, - exception, - shouldSuccess, - expectedAst, - babylonOptions -) { - console.log(chalk.red(`✘ ${code}`)); - console.log( - chalk.yellow( - ` Should ${shouldSuccess - ? "parse successfully" - : `fail parsing`}, but did not` - ) - ); - if (exception) - console.log(chalk.yellow(` Failed with: \`${exception.message}\``)); - if (Array.isArray(expectedAst.errors) && expectedAst.errors.length > 0) { - console.log(chalk.yellow(" Expected error message similar to:")); - expectedAst.errors.forEach(error => { - console.log(` • ${error.message}`); - }); - } - - console.log( - chalk.yellow(` Active plugins: ${JSON.stringify(babylonOptions.plugins)}`) - ); -} +summary.unrecognized = Array.from(unrecognized); +summary.passed &= summary.unrecognized.length === 0; +// This is needed because, after the dots written using +// `process.stdout.write(".")` there is no final newline console.log(); -console.log(chalk.green(`✔ ${successTests} tests passed`)); -console.log(chalk.red(`✘ ${failedTests} tests failed`)); -process.exit(failedTests > 0 ? 1 : 0); +console.log("\n-- FAILED TESTS --"); +summary.disallowed.failure.forEach( + ({ test, shouldSuccess, exception, babylonOptions }) => { + console.log(chalk.red(`✘ ${test.file}`)); + if (shouldSuccess) { + console.log(chalk.yellow(" Should parse successfully, but did not")); + console.log(chalk.yellow(` Failed with: \`${exception.message}\``)); + } else { + console.log(chalk.yellow(" Should fail parsing, but did not")); + } + console.log( + chalk.yellow( + ` Active plugins: ${JSON.stringify(babylonOptions.plugins)}` + ) + ); + } +); +summary.disallowed.success.forEach( + ({ test, shouldSuccess, babylonOptions }) => { + console.log(chalk.red(`✘ ${test.file}`)); + if (shouldSuccess) { + console.log( + chalk.yellow( + " Correctly parsed successfully, but" + + " was disallowed by the whitelist" + ) + ); + } else { + console.log( + chalk.yellow( + " Correctly failed parsing, but" + + " was disallowed by the whitelist" + ) + ); + } + console.log( + chalk.yellow( + ` Active plugins: ${JSON.stringify(babylonOptions.plugins)}` + ) + ); + } +); + +console.log("-- SUMMARY --"); +console.log( + chalk.green("✔ " + summary.allowed.success.length + " tests passed") +); +console.log( + chalk.green( + "✔ " + + summary.allowed.failure.length + + " tests failed but were allowed in the whitelist" + ) +); +console.log( + chalk.red("✘ " + summary.disallowed.failure.length + " tests failed") +); +console.log( + chalk.red( + "✘ " + + summary.disallowed.success.length + + " tests passed but were disallowed in the whitelist" + ) +); +console.log( + chalk.red( + "✘ " + + summary.unrecognized.length + + " tests specified in the whitelist were not found" + ) +); + +process.exit(summary.passed ? 0 : 1);