Run tests of flow with babylon (#225)

* Run tests of flow with babylon

* Fix travis

* Fix typo

* Again...

* Brtter hint

* proper exit code

* Fix some flase-positives and better reporting

* Enable some plugins, that flow supports by default

.travis.yml

@@ -15,6 +15,7 @@ before_install:
 
 before_script:
   - 'if [ -n "${BABEL-}" ]; then make bootstrap-babel ; fi'
+  - 'if [ -n "${FLOWTESTS-}" ]; then make bootstrap-flow ; fi'
   - 'BABEL_ENV=test npm run build'
     # Switch back to node version currently being tested prior to test run
   - 'nvm use $TRAVIS_NODE_VERSION;'
@@ -22,8 +23,9 @@ before_script:
 script:
   - 'if [ -n "${LINT-}" ]; then npm run lint ; fi'
   - 'if [ -n "${FLOW-}" ]; then npm run flow ; fi'
+  - 'if [ -n "${FLOWTESTS-}" ]; then make test-flow ; fi'
   - 'if [ -n "${BABEL-}" ]; then make test-babel ; fi'
-  - 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ] && [ -z "${BABEL-}" ]; then npm run test-ci ; fi'
+  - 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ] && [ -z "${BABEL-}" ] && [ -z "${FLOWTESTS-}" ]; then npm run test-ci ; fi'
 
 matrix:
   fast_finish: true
@@ -34,8 +36,13 @@ matrix:
       env: FLOW=true
     - node_js: "node"
       env: BABEL=true
+    - node_js: "node"
+      env: FLOWTESTS=true
+  allow_failures:
+    - node_js: "node"
+      env: FLOWTESTS=true
 
-after_success: 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ]; then npm run coverage ; fi'
+after_success: 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" && [ -z "${FLOWTESTS-}" ]; then npm run coverage ; fi'
 
 notifications:
   slack: babeljs:5Wy4QX13KVkGy9CnU0rmvgeK

Makefile

@@ -21,3 +21,10 @@ test-babel:
 	mv -f package.nonyc.json package.json; \
 	../../node_modules/.bin/nyc --no-instrument --no-source-map --report-dir ../../coverage node_modules/mocha/bin/_mocha `scripts/_get-test-directories.sh` --opts test/mocha.opts; \
 	mv .nyc_output ../../.nyc_output
+
+bootstrap-flow: clean
+	mkdir ./build
+	git clone --depth=1 --branch=master https://github.com/facebook/flow.git ./build/flow
+
+test-flow:
+	node scripts/run_flow_tests.js

package.json

@@ -20,6 +20,7 @@
     "babel-plugin-transform-flow-strip-types": "^6.14.0",
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-stage-0": "^6.5.0",
+    "chalk": "^1.1.3",
     "codecov": "^1.0.1",
     "cross-env": "^2.0.1",
     "eslint": "^3.7.1",

scripts/run_flow_tests.js

@@ -0,0 +1,91 @@
+"use strict";
+
+const path = require("path");
+const chalk = require("chalk");
+const parse = require("..").parse;
+
+const flowDirectory = path.join(__dirname, "../build/flow");
+const hardcodedTestFile = path.join(flowDirectory, "src/parser/test/hardcoded_tests.js");
+
+const hardcodedTests = require(hardcodedTestFile).sections;
+const options = {
+  plugins: [
+    "jsx",
+    "flow",
+    "asyncGenerators",
+    "objectRestSpread"
+  ],
+  sourceType: "module",
+};
+
+const flowOptionsMapping = {
+  "esproposal_class_instance_fields": "classProperties",
+  "esproposal_class_static_fields": "classProperties",
+  "esproposal_export_star_as": "exportExtensions",
+  "esproposal_decorators": "decorators",
+};
+
+let failedTests = 0;
+let successTests = 0;
+Object.keys(hardcodedTests).forEach((sectionName) => {
+  console.log("");
+  console.log(`### ${sectionName} ###`);
+  Object.keys(hardcodedTests[sectionName]).forEach((code) => {
+    const shouldSuccess = !Object.keys(hardcodedTests[sectionName][code])
+      .some(
+        (key) => {
+          const value = hardcodedTests[sectionName][code][key];
+          if (key === "errors.length" && value === 0) return false;
+          if (key === "errors" && Array.isArray(value) && value.length === 0) return false;
+          if (!key.startsWith("errors")) return false;
+
+          return true;
+        }
+      );
+
+    const parserOptions = hardcodedTests[sectionName][code]["%parse_options%"];
+    const babylonOptions = Object.assign({}, options );
+    babylonOptions.plugins = babylonOptions.plugins.slice();
+
+    if (parserOptions) {
+      Object.keys(parserOptions).forEach((option) => {
+        if (!parserOptions[option]) return;
+        if (!flowOptionsMapping[option]) throw new Error("Parser options not mapped " + option);
+        babylonOptions.plugins.push(flowOptionsMapping[option]);
+      });
+    }
+
+    let failed = false;
+    let exception = null;
+    try {
+      parse(code, babylonOptions);
+    } catch (e) {
+      exception = e;
+      failed = true;
+    }
+
+    if (!failed && shouldSuccess || failed && !shouldSuccess) {
+      successTests++;
+      console.log(chalk.green(`✔ ${code}`));
+    } else {
+      failedTests++;
+      printErrorMessage(code, exception, shouldSuccess, hardcodedTests[sectionName][code], babylonOptions)
+    }
+  });
+});
+
+function printErrorMessage(code, exception, shouldSuccess, data, babylonOptions) {
+  console.log(chalk.red(`✘ ${code}`));
+  console.log(chalk.yellow(`  Should ${shouldSuccess ? "parse successfully" : `fail parsing`}, but did not`));
+  if (exception) console.log(chalk.yellow(`  Failed with: \`${exception.message}\``));
+  if (data["errors.0.message"]) console.log(chalk.yellow(`  Expected error message similar to: \`${data["errors.0.message"]}\``));
+  console.log(chalk.yellow(`  Active plugins: ${JSON.stringify(babylonOptions.plugins)}`));
+
+}
+
+console.log();
+console.log(chalk.green(`✔ ${successTests} tests passed`));
+console.log(chalk.red(`✔ ${failedTests} tests failed`));
+
+process.exit(failedTests > 0 ? 1 : 0);
+