Fix flow test runner (#647)

* Fix flow test runner

* Retry parsing in scriptmode

scripts/run_flow_tests.js

@@ -1,56 +1,115 @@
 "use strict";
 
 const path = require("path");
+const fs = require("fs");
 const chalk = require("chalk");
 const parse = require("..").parse;
 
-const flowDirectory = path.join(__dirname, "../build/flow");
-const hardcodedTestFile = path.join(flowDirectory, "src/parser/test/hardcoded_tests.js");
+function list_files(root, dir) {
+  const files = fs.readdirSync(dir ? path.join(root, dir) : root);
+  let result = [];
+  for (let i = 0; i < files.length; i++) {
+    const file = dir ? path.join(dir, files[i]) : files[i];
+    const stats = fs.statSync(path.join(root, file));
+    if (stats.isDirectory()) {
+      result = result.concat(list_files(root, file));
+    } else {
+      result.push(file);
+    }
+  }
+  return result.sort();
+}
+
+function get_tests(root_dir) {
+  const files = list_files(root_dir);
+  const tests = {};
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const test_name = path.dirname(file);
+    const case_parts = path.basename(file).split(".");
+    const case_name = case_parts[0];
+
+    // Hack to ignore hidden files.
+    if (case_name === "") {
+      continue;
+    }
+
+    const cases = (tests[test_name] = tests[test_name] || {});
+    const case_ = (cases[case_name] = cases[case_name] || {});
+
+    const content = fs.readFileSync(path.join(root_dir, file), {
+      encoding: "utf8",
+    });
+    const ext = case_parts[case_parts.length - 1];
+    const kind =
+      case_parts.length > 2 ? case_parts[case_parts.length - 2] : null;
+
+    if (ext === "js") {
+      case_.name = case_name;
+      case_.content = content;
+    } else if (ext === "json" && kind === "tree") {
+      case_.expected_ast = JSON.parse(content);
+    } else if (ext === "json" && kind === "options") {
+      case_.options = JSON.parse(content);
+    }
+  }
+  return tests;
+}
+
+function get_hardcoded_tests() {
+  const tests = get_tests(
+    path.join(__dirname, "../build/flow/src/parser/test/flow")
+  );
+  const result = {};
+  for (const section in tests) {
+    if (tests.hasOwnProperty(section)) {
+      const test = tests[section];
+      const cases = [];
+      // TODO: use Object.values if we require new enough node
+      for (const case_ in test) {
+        if (test.hasOwnProperty(case_)) {
+          cases.push(test[case_]);
+        }
+      }
+      result[section] = { tests: cases };
+    }
+  }
+  return result;
+}
 
-const hardcodedTests = require(hardcodedTestFile).sections;
 const options = {
-  plugins: [
-    "jsx",
-    "flow",
-    "asyncGenerators",
-    "objectRestSpread"
-  ],
+  plugins: ["jsx", "flow", "asyncGenerators", "objectRestSpread"],
   sourceType: "module",
+  ranges: true,
 };
 
 const flowOptionsMapping = {
-  "esproposal_class_instance_fields": "classProperties",
-  "esproposal_class_static_fields": "classProperties",
-  "esproposal_export_star_as": "exportExtensions",
-  "esproposal_decorators": "decorators",
+  esproposal_class_instance_fields: "classProperties",
+  esproposal_class_static_fields: "classProperties",
+  esproposal_export_star_as: "exportExtensions",
+  esproposal_decorators: "decorators",
 };
 
 let failedTests = 0;
 let successTests = 0;
-Object.keys(hardcodedTests).forEach((sectionName) => {
+const hardcodedTests = get_hardcoded_tests();
+Object.keys(hardcodedTests).forEach(sectionName => {
   console.log("");
   console.log(`### ${sectionName} ###`);
-  Object.keys(hardcodedTests[sectionName]).forEach((code) => {
-    const shouldSuccess = !Object.keys(hardcodedTests[sectionName][code])
-      .some(
-        (key) => {
-          const value = hardcodedTests[sectionName][code][key];
-          if (key === "errors.length" && value === 0) return false;
-          if (key === "errors" && Array.isArray(value) && value.length === 0) return false;
-          if (!key.startsWith("errors")) return false;
+  hardcodedTests[sectionName].tests.forEach(test => {
+    const shouldSuccess =
+      test.expected_ast &&
+      (!Array.isArray(test.expected_ast.errors) ||
+        test.expected_ast.errors.length === 0);
 
-          return true;
-        }
-      );
-
-    const parserOptions = hardcodedTests[sectionName][code]["%parse_options%"];
-    const babylonOptions = Object.assign({}, options );
+    const babylonOptions = Object.assign({}, options);
     babylonOptions.plugins = babylonOptions.plugins.slice();
 
-    if (parserOptions) {
-      Object.keys(parserOptions).forEach((option) => {
-        if (!parserOptions[option]) return;
-        if (!flowOptionsMapping[option]) throw new Error("Parser options not mapped " + option);
+    if (test.options) {
+      Object.keys(test.options).forEach(option => {
+        if (!test.options[option]) return;
+        if (!flowOptionsMapping[option])
+          throw new Error("Parser options not mapped " + option);
         babylonOptions.plugins.push(flowOptionsMapping[option]);
       });
     }
@@ -58,34 +117,68 @@ Object.keys(hardcodedTests).forEach((sectionName) => {
     let failed = false;
     let exception = null;
     try {
-      parse(code, babylonOptions);
+      parse(test.content, babylonOptions);
     } catch (e) {
       exception = e;
       failed = true;
+      // lets retry in script mode
+      try {
+        parse(
+          test.content,
+          Object.assign({}, babylonOptions, { sourceType: "script" })
+        );
+        exception = null;
+        failed = false;
+      } catch (e) {}
     }
 
-    if (!failed && shouldSuccess || failed && !shouldSuccess) {
+    if ((!failed && shouldSuccess) || (failed && !shouldSuccess)) {
       successTests++;
-      console.log(chalk.green(`✔ ${code}`));
+      console.log(chalk.green(`✔ ${test.name}`));
     } else {
       failedTests++;
-      printErrorMessage(code, exception, shouldSuccess, hardcodedTests[sectionName][code], babylonOptions)
+      printErrorMessage(
+        test.name,
+        exception,
+        shouldSuccess,
+        test.expected_ast,
+        babylonOptions
+      );
     }
   });
 });
 
-function printErrorMessage(code, exception, shouldSuccess, data, babylonOptions) {
+function printErrorMessage(
+  code,
+  exception,
+  shouldSuccess,
+  expectedAst,
+  babylonOptions
+) {
   console.log(chalk.red(`✘ ${code}`));
-  console.log(chalk.yellow(`  Should ${shouldSuccess ? "parse successfully" : `fail parsing`}, but did not`));
-  if (exception) console.log(chalk.yellow(`  Failed with: \`${exception.message}\``));
-  if (data["errors.0.message"]) console.log(chalk.yellow(`  Expected error message similar to: \`${data["errors.0.message"]}\``));
-  console.log(chalk.yellow(`  Active plugins: ${JSON.stringify(babylonOptions.plugins)}`));
+  console.log(
+    chalk.yellow(
+      `  Should ${shouldSuccess
+        ? "parse successfully"
+        : `fail parsing`}, but did not`
+    )
+  );
+  if (exception)
+    console.log(chalk.yellow(`  Failed with: \`${exception.message}\``));
+  if (Array.isArray(expectedAst.errors) && expectedAst.errors.length > 0) {
+    console.log(chalk.yellow("  Expected error message similar to:"));
+    expectedAst.errors.forEach(error => {
+      console.log(`    • ${error.message}`);
+    });
+  }
 
+  console.log(
+    chalk.yellow(`  Active plugins: ${JSON.stringify(babylonOptions.plugins)}`)
+  );
 }
 
 console.log();
 console.log(chalk.green(`✔ ${successTests} tests passed`));
-console.log(chalk.red(`✔ ${failedTests} tests failed`));
+console.log(chalk.red(`✘ ${failedTests} tests failed`));
 
 process.exit(failedTests > 0 ? 1 : 0);
-

yarn.lock

@@ -2050,7 +2050,7 @@ flat-cache@^1.2.1:
     graceful-fs "^4.1.2"
     write "^0.2.1"
 
-flow-bin@0.51.0:
+flow-bin@^0.51.0:
   version "0.51.0"
   resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.51.0.tgz#97a3c511a17caa25d8fad58fc17aaabcb86319fe"