Integrate Test262 (#654)

* Integrate Test262 Introduce a GNU Make target for retrieving TC-39's Test262 suite and validating parsing of the files it contains. Interpret each file as a parser test in accordance with that project's `INTERPRETING.md` document. Allow for the specification of allowed failures via a "whitelist" file so that the test suite may help prevent regressions in this project in situations where this project has known bugs. Initialize the "whitelist" file with a listing of all tests that are currently failing. Extend the continuous integration environment's configuration to automatically run these tests. * use graceful-fs and latest yarn on travis
2017-08-06 21:20:25 -04:00 · 2017-08-06 21:20:25 -04:00 · 0466504d7b
commit 0466504d7b
parent fb6d0491f6
8 changed files with 1669 additions and 2 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -6,15 +6,19 @@ node_js:
  - "4"

 env:
-  - JOB=test
+  global:
+    - PATH=$HOME/.yarn/bin:$PATH
+    - JOB=test

 before_install:
+  - curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 0.28.4
  - yarn global add greenkeeper-lockfile@1

 before_script:
  - greenkeeper-lockfile-update
  - 'if [ "$JOB" = "babel-test" ]; then make bootstrap-babel ; fi'
  - 'if [ "$JOB" = "flow-test" ]; then make bootstrap-flow ; fi'
+  - 'if [ "$JOB" = "test262-test" ]; then make bootstrap-test262 ; fi'
  - 'if [ "$JOB" = "test" ]; then yarn run build; fi'

 script:
@ -22,6 +26,7 @@ script:
  - 'if [ "$JOB" = "lint" ]; then yarn run lint && yarn run flow; fi'
  - 'if [ "$JOB" = "flow-test" ]; then make test-flow; fi'
  - 'if [ "$JOB" = "babel-test" ]; then make test-babel; fi'
+  - 'if [ "$JOB" = "test262-test" ]; then make test-test262; fi'
  - 'if [ "$JOB" = "test-coverage" ]; then yarn run test-coverage; fi'

 matrix:
@ -35,6 +40,8 @@ matrix:
      env: JOB=babel-test
    - node_js: "lts/*"
      env: JOB=flow-test
+    - node_js: node
+      env: JOB=test262-test
  allow_failures:
    - node_js: "lts/*"
      env: JOB=flow-test
--- a/.yarnrc
+++ b/.yarnrc
@ -0,0 +1 @@
+workspaces-experimental true
--- a/12
+++ b/12
@ -1,4 +1,5 @@
 MAKEFLAGS = -j1
+TEST262_COMMIT = 4ea2931f169d4a4b5a9a4a7c731cc92bf7b3e13c

 export NODE_ENV = test

@ -30,3 +31,14 @@ bootstrap-flow: clean

 test-flow:
 	node scripts/run_flow_tests.js
+
+bootstrap-test262: clean
+	mkdir ./build
+	git clone https://github.com/tc39/test262.git ./build/test262
+	cd build/test262 && git checkout $(TEST262_COMMIT)
+
+test-test262:
+	node scripts/run_test262.js
+
+test-test262-update-whitelist:
+	node scripts/run_test262.js --update-whitelist
--- a/package.json
+++ b/package.json
@ -36,6 +36,7 @@
    "eslint-plugin-flowtype": "^2.34.0",
    "eslint-plugin-prettier": "^2.1.2",
    "flow-bin": "^0.52.0",
+    "graceful-fs": "^4.1.11",
    "husky": "^0.14.1",
    "lint-staged": "^4.0.0",
    "nyc": "^11.0.3",
@ -45,7 +46,8 @@
    "rollup-plugin-babel": "3.0.0-alpha.15",
    "rollup-plugin-node-resolve": "^3.0.0",
    "rollup-watch": "^4.0.0",
-    "unicode-9.0.0": "~0.7.0"
+    "unicode-9.0.0": "~0.7.0",
+    "util.promisify": "^1.0.0"
  },
  "bin": {
    "babylon": "./bin/babylon.js"
--- a/scripts/run_test262.js
+++ b/scripts/run_test262.js
@ -0,0 +1,113 @@
+"use strict";
+
+const path = require("path");
+const chalk = require("chalk");
+const utils = require("./run_test262_utils");
+
+const testDir = path.join(__dirname, "..", "build", "test262", "test");
+const whitelistFile = path.join(__dirname, "test262_whitelist.txt");
+const plugins = ["asyncGenerators", "objectRestSpread"];
+const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1;
+
+Promise.all([utils.getTests(testDir), utils.getWhitelist(whitelistFile)])
+  .then(function([tests, whitelist]) {
+    const total = tests.length;
+    const reportInc = Math.floor(total / 20);
+
+    console.log(`Now running ${total} tests...`);
+
+    const results = tests.map(function(test, idx) {
+      if (idx % reportInc === 0) {
+        console.log(`> ${Math.round(100 * idx / total)}% complete`);
+      }
+
+      return utils.runTest(test, plugins);
+    });
+
+    return utils.interpret(results, whitelist);
+  })
+  .then(function(summary) {
+    const goodnews = [
+      summary.allowed.success.length + " valid programs parsed without error",
+      summary.allowed.failure.length +
+        " invalid programs produced a parsing error",
+      summary.allowed.falsePositive.length +
+        " invalid programs did not produce a parsing error" +
+        " (and allowed by the whitelist file)",
+      summary.allowed.falseNegative.length +
+        " valid programs produced a parsing error" +
+        " (and allowed by the whitelist file)",
+    ];
+    const badnews = [];
+    const badnewsDetails = [];
+
+    void [
+      {
+        tests: summary.disallowed.success,
+        label:
+          "valid programs parsed without error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.failure,
+        label:
+          "invalid programs produced a parsing error" +
+          " (in violation of the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falsePositive,
+        label:
+          "invalid programs did not produce a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.disallowed.falseNegative,
+        label:
+          "valid programs produced a parsing error" +
+          " (without a corresponding entry in the whitelist file)",
+      },
+      {
+        tests: summary.unrecognized,
+        label: "non-existent programs specified in the whitelist file",
+      },
+    ].forEach(function({ tests, label }) {
+      if (!tests.length) {
+        return;
+      }
+
+      const desc = tests.length + " " + label;
+
+      badnews.push(desc);
+      badnewsDetails.push(desc + ":");
+      badnewsDetails.push(
+        ...tests.map(function(test) {
+          return test.id;
+        })
+      );
+    });
+
+    console.log("Testing complete.");
+    console.log("Summary:");
+    console.log(chalk.green(goodnews.join("\n").replace(/^/gm, " ✔ ")));
+
+    if (!summary.passed) {
+      console.log("");
+      console.log(chalk.red(badnews.join("\n").replace(/^/gm, " ✘ ")));
+      console.log("");
+      console.log("Details:");
+      console.log(badnewsDetails.join("\n").replace(/^/gm, "   "));
+    }
+
+    if (shouldUpdate) {
+      return utils.updateWhitelist(whitelistFile, summary).then(function() {
+        console.log("");
+        console.log("Whitelist file updated.");
+      });
+    } else {
+      process.exitCode = summary.passed ? 0 : 1;
+    }
+  })
+  .catch(function(err) {
+    console.error(err);
+    process.exitCode = 1;
+  });
--- a/scripts/run_test262_utils.js
+++ b/scripts/run_test262_utils.js
@ -0,0 +1,233 @@
+"use strict";
+
+const fs = require("graceful-fs");
+const path = require("path");
+const promisify = require("util.promisify");
+const pfs = {
+  readFile: promisify(fs.readFile),
+  writeFile: promisify(fs.writeFile),
+  readdir: promisify(fs.readdir),
+  stat: promisify(fs.stat)
+};
+
+const parse = require("..").parse;
+
+const modulePattern = /^\s*-\s*module\s*$|^\s*flags\s*:.*\bmodule\b/m;
+const noStrictPattern = /^\s*-\s*noStrict\s*$|^\s*flags\s*:.*\bnoStrict\b/m;
+const onlyStrictPattern = /^\s*-\s*onlyStrict\s*$|^\s*flags\s*:.*\bonlyStrict\b/m;
+const rawPattern = /^\s*-\s*raw\s*$|^\s*flags\s*:.*\braw\b/m;
+const testNamePattern = /^(?!.*_FIXTURE).*\.[jJ][sS]$/;
+
+function flatten(array) {
+  const flattened = [];
+  array.forEach(function(element) {
+    if (Array.isArray(element)) {
+      flattened.push.apply(flattened, element);
+    } else {
+      flattened.push(element);
+    }
+  });
+  return flattened;
+}
+
+function hasEarlyError(src) {
+  return !!(
+    src.match(/^\s*negative:\s*$/m) && src.match(/^\s+phase:\s*early\s*$/m)
+  );
+}
+
+function readDirDeep(dirName) {
+  return pfs.readdir(dirName).then(function(contents) {
+    return Promise.all(
+      contents.map(function(name) {
+        return findTests(path.join(dirName, name));
+      })
+    ).then(flatten)
+  });
+}
+
+function findTests(name) {
+  return pfs.stat(name).then(function(stat) {
+    if (stat.isDirectory()) {
+      return readDirDeep(name);
+    }
+
+    return name;
+  });
+}
+
+function readTest(fileName, testDir) {
+  if (!testNamePattern.test(fileName)) {
+    return Promise.resolve([]);
+  }
+
+  return pfs.readFile(fileName, "utf-8").then(function(contents) {
+    return makeScenarios(path.relative(testDir, fileName), contents);
+  });
+}
+
+function makeScenarios(fileName, testContent) {
+  const scenarios = [];
+  const base = {
+    fileName: fileName,
+    isModule: modulePattern.test(testContent),
+    expectedError: hasEarlyError(testContent),
+  };
+  const isNoStrict = noStrictPattern.test(testContent);
+  const isOnlyStrict = onlyStrictPattern.test(testContent);
+  const isRaw = rawPattern.test(testContent);
+
+  if (!isOnlyStrict) {
+    scenarios.push(
+      Object.assign(
+        {
+          id: fileName + "(default)",
+          content: testContent,
+        },
+        base
+      )
+    );
+  }
+
+  if (!isNoStrict && !isRaw) {
+    scenarios.push(
+      Object.assign(
+        {
+          id: fileName + "(strict mode)",
+          content: "'use strict';\n" + testContent,
+        },
+        base
+      )
+    );
+  }
+
+  return scenarios;
+}
+
+exports.getTests = function(testDir) {
+  return findTests(testDir)
+    .then(function(testPaths) {
+      return Promise.all(
+        testPaths.map(function(path) {
+          return readTest(path, testDir);
+        })
+      );
+    })
+    .then(flatten);
+};
+
+exports.runTest = function(test, plugins) {
+  const sourceType = test.isModule ? "module" : "script";
+
+  try {
+    parse(test.content, { sourceType: sourceType, plugins: plugins });
+    test.actualError = false;
+  } catch (err) {
+    test.actualError = true;
+  }
+
+  test.result = test.expectedError !== test.actualError ? "fail" : "pass";
+
+  return test;
+};
+
+exports.getWhitelist = function(filename) {
+  return pfs.readFile(filename, "utf-8").then(function(contents) {
+    return contents
+      .split("\n")
+      .map(function(line) {
+        return line.replace(/#.*$/, "").trim();
+      })
+      .filter(function(line) {
+        return line.length > 0;
+      })
+      .reduce(function(table, filename) {
+        table[filename] = true;
+        return table;
+      }, Object.create(null));
+  });
+};
+
+exports.updateWhitelist = function(filename, summary) {
+  return pfs.readFile(filename, "utf-8").then(function(contents) {
+    const toRemove = summary.disallowed.success
+      .concat(summary.disallowed.failure)
+      .map(function(test) {
+        return test.id;
+      });
+    const toAdd = summary.disallowed.falsePositive
+      .concat(summary.disallowed.falseNegative)
+      .map(function(test) {
+        return test.id;
+      });
+    const newContents = contents
+      .split("\n")
+      .map(function(line) {
+        const testId = line.replace(/#.*$/, "").trim();
+
+        if (toRemove.indexOf(testId) > -1) {
+          return null;
+        }
+
+        return line;
+      })
+      .filter(function(line) {
+        return line !== null;
+      })
+      .concat(toAdd)
+      .join("\n");
+
+    return pfs.writeFile(filename, newContents, "utf-8");
+  });
+};
+
+exports.interpret = function(results, whitelist) {
+  const summary = {
+    passed: true,
+    allowed: {
+      success: [],
+      failure: [],
+      falsePositive: [],
+      falseNegative: [],
+    },
+    disallowed: {
+      success: [],
+      failure: [],
+      falsePositive: [],
+      falseNegative: [],
+    },
+    unrecognized: null,
+  };
+
+  results.forEach(function(result) {
+    let classification, isAllowed;
+    const inWhitelist = result.id in whitelist;
+    delete whitelist[result.id];
+
+    if (!result.expectedError) {
+      if (!result.actualError) {
+        classification = "success";
+        isAllowed = !inWhitelist;
+      } else {
+        classification = "falseNegative";
+        isAllowed = inWhitelist;
+      }
+    } else {
+      if (!result.actualError) {
+        classification = "falsePositive";
+        isAllowed = inWhitelist;
+      } else {
+        classification = "failure";
+        isAllowed = !inWhitelist;
+      }
+    }
+
+    summary.passed &= isAllowed;
+    summary[isAllowed ? "allowed" : "disallowed"][classification].push(result);
+  });
+
+  summary.unrecognized = Object.keys(whitelist);
+  summary.passed = !!summary.passed && summary.unrecognized.length === 0;
+
+  return summary;
+};
--- a/scripts/test262_whitelist.txt
+++ b/scripts/test262_whitelist.txt
--- a/yarn.lock
+++ b/yarn.lock
@ -1836,6 +1836,13 @@ default-require-extensions@^1.0.0:
  dependencies:
    strip-bom "^2.0.0"

+define-properties@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94"
+  dependencies:
+    foreach "^2.0.5"
+    object-keys "^1.0.8"
+
 del@^2.0.2:
  version "2.2.2"
  resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8"
@ -1914,6 +1921,24 @@ error-ex@^1.2.0:
  dependencies:
    is-arrayish "^0.2.1"

+es-abstract@^1.5.1:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.8.0.tgz#3b00385e85729932beffa9163bbea1234e932914"
+  dependencies:
+    es-to-primitive "^1.1.1"
+    function-bind "^1.1.0"
+    has "^1.0.1"
+    is-callable "^1.1.3"
+    is-regex "^1.0.4"
+
+es-to-primitive@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d"
+  dependencies:
+    is-callable "^1.1.1"
+    is-date-object "^1.0.1"
+    is-symbol "^1.0.1"
+
 es6-error@^4.0.1, es6-error@^4.0.2:
  version "4.0.2"
  resolved "https://registry.yarnpkg.com/es6-error/-/es6-error-4.0.2.tgz#eec5c726eacef51b7f6b73c20db6e1b13b069c98"
@ -2194,6 +2219,10 @@ for-own@^0.1.4:
  dependencies:
    for-in "^1.0.1"

+foreach@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
+
 foreground-child@^1.5.3, foreground-child@^1.5.6:
  version "1.5.6"
  resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-1.5.6.tgz#4fd71ad2dfde96789b980a5c0a295937cb2f5ce9"
@ -2245,6 +2274,10 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:
    mkdirp ">=0.5 0"
    rimraf "2"

+function-bind@^1.0.2, function-bind@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771"
+
 function-name-support@^0.2.0:
  version "0.2.0"
  resolved "https://registry.yarnpkg.com/function-name-support/-/function-name-support-0.2.0.tgz#55d3bfaa6eafd505a50f9bc81fdf57564a0bb071"
@ -2408,6 +2441,12 @@ has-yarn@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/has-yarn/-/has-yarn-1.0.0.tgz#89e25db604b725c8f5976fff0addc921b828a5a7"

+has@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/has/-/has-1.0.1.tgz#8461733f538b0837c9361e39a9ab9e9704dc2f28"
+  dependencies:
+    function-bind "^1.0.2"
+
 hawk@~3.1.3:
  version "3.1.3"
  resolved "https://registry.yarnpkg.com/hawk/-/hawk-3.1.3.tgz#078444bd7c1640b0fe540d2c9b73d59678e8e1c4"
@ -2576,12 +2615,20 @@ is-builtin-module@^1.0.0:
  dependencies:
    builtin-modules "^1.0.0"

+is-callable@^1.1.1, is-callable@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.3.tgz#86eb75392805ddc33af71c92a0eedf74ee7604b2"
+
 is-ci@^1.0.10, is-ci@^1.0.7:
  version "1.0.10"
  resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-1.0.10.tgz#f739336b2632365061a9d48270cd56ae3369318e"
  dependencies:
    ci-info "^1.0.0"

+is-date-object@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16"
+
 is-dotfile@^1.0.0:
  version "1.0.3"
  resolved "https://registry.yarnpkg.com/is-dotfile/-/is-dotfile-1.0.3.tgz#a6a2f32ffd2dfb04f5ca25ecd0f6b83cf798a1e1"
@ -2696,6 +2743,12 @@ is-redirect@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24"

+is-regex@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491"
+  dependencies:
+    has "^1.0.1"
+
 is-resolvable@^1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/is-resolvable/-/is-resolvable-1.0.0.tgz#8df57c61ea2e3c501408d100fb013cf8d6e0cc62"
@ -2710,6 +2763,10 @@ is-stream@^1.0.0, is-stream@^1.1.0:
  version "1.1.0"
  resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"

+is-symbol@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572"
+
 is-typedarray@~1.0.0:
  version "1.0.0"
  resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
@ -3331,6 +3388,17 @@ object-assign@^4.0.1, object-assign@^4.1.0:
  version "4.1.1"
  resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"

+object-keys@^1.0.8:
+  version "1.0.11"
+  resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
+
+object.getownpropertydescriptors@^2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.0.3.tgz#8758c846f5b407adab0f236e0986f14b051caa16"
+  dependencies:
+    define-properties "^1.1.2"
+    es-abstract "^1.5.1"
+
 object.omit@^2.0.0:
  version "2.0.1"
  resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@ -4474,6 +4542,13 @@ util-deprecate@~1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"

+util.promisify@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/util.promisify/-/util.promisify-1.0.0.tgz#440f7165a459c9a16dc145eb8e72f35687097030"
+  dependencies:
+    define-properties "^1.1.2"
+    object.getownpropertydescriptors "^2.0.3"
+
 uuid@^3.0.0:
  version "3.1.0"
  resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"