From 58887ed14e911c988d20eac8e174bf48e166da9d Mon Sep 17 00:00:00 2001 From: Daniel Tschinder Date: Mon, 14 Nov 2016 18:16:36 +0100 Subject: [PATCH] Run tests of flow with babylon (#225) * Run tests of flow with babylon * Fix travis * Fix typo * Again... * Brtter hint * proper exit code * Fix some flase-positives and better reporting * Enable some plugins, that flow supports by default --- .travis.yml | 11 ++++- Makefile | 7 +++ package.json | 1 + scripts/run_flow_tests.js | 91 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 scripts/run_flow_tests.js diff --git a/.travis.yml b/.travis.yml index 8ec6ff48d0..11fee98596 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,7 @@ before_install: before_script: - 'if [ -n "${BABEL-}" ]; then make bootstrap-babel ; fi' + - 'if [ -n "${FLOWTESTS-}" ]; then make bootstrap-flow ; fi' - 'BABEL_ENV=test npm run build' # Switch back to node version currently being tested prior to test run - 'nvm use $TRAVIS_NODE_VERSION;' @@ -22,8 +23,9 @@ before_script: script: - 'if [ -n "${LINT-}" ]; then npm run lint ; fi' - 'if [ -n "${FLOW-}" ]; then npm run flow ; fi' + - 'if [ -n "${FLOWTESTS-}" ]; then make test-flow ; fi' - 'if [ -n "${BABEL-}" ]; then make test-babel ; fi' - - 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ] && [ -z "${BABEL-}" ]; then npm run test-ci ; fi' + - 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ] && [ -z "${BABEL-}" ] && [ -z "${FLOWTESTS-}" ]; then npm run test-ci ; fi' matrix: fast_finish: true @@ -34,8 +36,13 @@ matrix: env: FLOW=true - node_js: "node" env: BABEL=true + - node_js: "node" + env: FLOWTESTS=true + allow_failures: + - node_js: "node" + env: FLOWTESTS=true -after_success: 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" ]; then npm run coverage ; fi' +after_success: 'if [ -z "${LINT-}" ] && [ -z "${FLOW-}" && [ -z "${FLOWTESTS-}" ]; then npm run coverage ; fi' notifications: slack: babeljs:5Wy4QX13KVkGy9CnU0rmvgeK diff --git a/Makefile b/Makefile index a34e1fa2b4..22e088f90a 100644 --- a/Makefile +++ b/Makefile @@ -21,3 +21,10 @@ test-babel: mv -f package.nonyc.json package.json; \ ../../node_modules/.bin/nyc --no-instrument --no-source-map --report-dir ../../coverage node_modules/mocha/bin/_mocha `scripts/_get-test-directories.sh` --opts test/mocha.opts; \ mv .nyc_output ../../.nyc_output + +bootstrap-flow: clean + mkdir ./build + git clone --depth=1 --branch=master https://github.com/facebook/flow.git ./build/flow + +test-flow: + node scripts/run_flow_tests.js diff --git a/package.json b/package.json index 53879db31f..05b4db18b9 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "babel-plugin-transform-flow-strip-types": "^6.14.0", "babel-preset-es2015": "^6.14.0", "babel-preset-stage-0": "^6.5.0", + "chalk": "^1.1.3", "codecov": "^1.0.1", "cross-env": "^2.0.1", "eslint": "^3.7.1", diff --git a/scripts/run_flow_tests.js b/scripts/run_flow_tests.js new file mode 100644 index 0000000000..e7c2830f57 --- /dev/null +++ b/scripts/run_flow_tests.js @@ -0,0 +1,91 @@ +"use strict"; + +const path = require("path"); +const chalk = require("chalk"); +const parse = require("..").parse; + +const flowDirectory = path.join(__dirname, "../build/flow"); +const hardcodedTestFile = path.join(flowDirectory, "src/parser/test/hardcoded_tests.js"); + +const hardcodedTests = require(hardcodedTestFile).sections; +const options = { + plugins: [ + "jsx", + "flow", + "asyncGenerators", + "objectRestSpread" + ], + sourceType: "module", +}; + +const flowOptionsMapping = { + "esproposal_class_instance_fields": "classProperties", + "esproposal_class_static_fields": "classProperties", + "esproposal_export_star_as": "exportExtensions", + "esproposal_decorators": "decorators", +}; + +let failedTests = 0; +let successTests = 0; +Object.keys(hardcodedTests).forEach((sectionName) => { + console.log(""); + console.log(`### ${sectionName} ###`); + Object.keys(hardcodedTests[sectionName]).forEach((code) => { + const shouldSuccess = !Object.keys(hardcodedTests[sectionName][code]) + .some( + (key) => { + const value = hardcodedTests[sectionName][code][key]; + if (key === "errors.length" && value === 0) return false; + if (key === "errors" && Array.isArray(value) && value.length === 0) return false; + if (!key.startsWith("errors")) return false; + + return true; + } + ); + + const parserOptions = hardcodedTests[sectionName][code]["%parse_options%"]; + const babylonOptions = Object.assign({}, options ); + babylonOptions.plugins = babylonOptions.plugins.slice(); + + if (parserOptions) { + Object.keys(parserOptions).forEach((option) => { + if (!parserOptions[option]) return; + if (!flowOptionsMapping[option]) throw new Error("Parser options not mapped " + option); + babylonOptions.plugins.push(flowOptionsMapping[option]); + }); + } + + let failed = false; + let exception = null; + try { + parse(code, babylonOptions); + } catch (e) { + exception = e; + failed = true; + } + + if (!failed && shouldSuccess || failed && !shouldSuccess) { + successTests++; + console.log(chalk.green(`✔ ${code}`)); + } else { + failedTests++; + printErrorMessage(code, exception, shouldSuccess, hardcodedTests[sectionName][code], babylonOptions) + } + }); +}); + +function printErrorMessage(code, exception, shouldSuccess, data, babylonOptions) { + console.log(chalk.red(`✘ ${code}`)); + console.log(chalk.yellow(` Should ${shouldSuccess ? "parse successfully" : `fail parsing`}, but did not`)); + if (exception) console.log(chalk.yellow(` Failed with: \`${exception.message}\``)); + if (data["errors.0.message"]) console.log(chalk.yellow(` Expected error message similar to: \`${data["errors.0.message"]}\``)); + console.log(chalk.yellow(` Active plugins: ${JSON.stringify(babylonOptions.plugins)}`)); + +} + +console.log(); +console.log(chalk.green(`✔ ${successTests} tests passed`)); +console.log(chalk.red(`✔ ${failedTests} tests failed`)); + +process.exit(failedTests > 0 ? 1 : 0); +