Update test262 test script and a few keyword escape fixes (#7503)

* Update test262 and whitelist

* Use test262-stream

* Check escapes in contextual keywords

* Check escapes when parsing new.target

* Check escapes for getters/setters

* Check escapes for static class methods

* Check escapes on async arrow and functions

Makefile

@@ -1,6 +1,6 @@
 MAKEFLAGS = -j1
 FLOW_COMMIT = 622bbc4f07acb77eb1109830c70815f827401d90
-TEST262_COMMIT = 1282e842febf418ca27df13fa4b32f7e5021b470
+TEST262_COMMIT = 52f70e2f637731aae92a9c9a2d831310c3ab2e1e
 
 export NODE_ENV = test
 

package.json

@@ -58,6 +58,7 @@
     "rollup-plugin-babel": "^4.0.0-beta.0",
     "rollup-plugin-node-resolve": "^3.0.2",
     "rollup-stream": "^1.24.1",
+    "test262-stream": "^1.1.0",
     "through2": "^2.0.0",
     "uglify-js": "^2.4.16",
     "vinyl-buffer": "^1.0.1",

packages/babylon/src/parser/expression.js

@@ -572,6 +572,7 @@ export default class ExpressionParser extends LValParser {
 
   atPossibleAsync(base: N.Expression): boolean {
     return (
+      !this.state.containsEsc &&
       this.state.potentialArrowAt === base.start &&
       base.type === "Identifier" &&
       base.name === "async" &&
@@ -739,6 +740,7 @@ export default class ExpressionParser extends LValParser {
       case tt.name: {
         node = this.startNode();
         const allowAwait = this.state.value === "await" && this.state.inAsync;
+        const containsEsc = this.state.containsEsc;
         const allowYield = this.shouldAllowYieldIdentifier();
         const id = this.parseIdentifier(allowAwait || allowYield);
 
@@ -747,6 +749,7 @@ export default class ExpressionParser extends LValParser {
             return this.parseAwait(node);
           }
         } else if (
+          !containsEsc &&
           id.name === "async" &&
           this.match(tt._function) &&
           !this.canInsertSemicolon()
@@ -915,9 +918,11 @@ export default class ExpressionParser extends LValParser {
       }
     }
 
+    const containsEsc = this.state.containsEsc;
+
     node.property = this.parseIdentifier(true);
 
-    if (node.property.name !== propertyName) {
+    if (node.property.name !== propertyName || containsEsc) {
       this.raise(
         node.property.start,
         `The only valid meta property for ${meta.name} is ${
@@ -1304,6 +1309,8 @@ export default class ExpressionParser extends LValParser {
         isGenerator = this.eat(tt.star);
       }
 
+      const containsEsc = this.state.containsEsc;
+
       if (!isPattern && this.isContextual("async")) {
         if (isGenerator) this.unexpected();
 
@@ -1338,6 +1345,7 @@ export default class ExpressionParser extends LValParser {
         isAsync,
         isPattern,
         refShorthandDefaultPos,
+        containsEsc,
       );
       this.checkPropClash(prop, propHash);
 
@@ -1408,6 +1416,7 @@ export default class ExpressionParser extends LValParser {
     isGenerator: boolean,
     isAsync: boolean,
     isPattern: boolean,
+    containsEsc: boolean,
   ): ?N.ObjectMethod {
     if (isAsync || isGenerator || this.match(tt.parenL)) {
       if (isPattern) this.unexpected();
@@ -1422,7 +1431,7 @@ export default class ExpressionParser extends LValParser {
       );
     }
 
-    if (this.isGetterOrSetterMethod(prop, isPattern)) {
+    if (!containsEsc && this.isGetterOrSetterMethod(prop, isPattern)) {
       if (isGenerator || isAsync) this.unexpected();
       prop.kind = prop.key.name;
       this.parsePropertyName(prop);
@@ -1490,9 +1499,16 @@ export default class ExpressionParser extends LValParser {
     isAsync: boolean,
     isPattern: boolean,
     refShorthandDefaultPos: ?Pos,
+    containsEsc: boolean,
   ): void {
     const node =
-      this.parseObjectMethod(prop, isGenerator, isAsync, isPattern) ||
+      this.parseObjectMethod(
+        prop,
+        isGenerator,
+        isAsync,
+        isPattern,
+        containsEsc,
+      ) ||
       this.parseObjectProperty(
         prop,
         startPos,

packages/babylon/src/parser/statement.js

@@ -155,7 +155,7 @@ export default class StatementParser extends ExpressionParser {
         return result;
       }
       case tt.name:
-        if (this.state.value === "async") {
+        if (this.isContextual("async")) {
           // peek ahead and see if next token is a function
           const state = this.state.clone();
           this.next();
@@ -970,8 +970,11 @@ export default class StatementParser extends ExpressionParser {
     state: { hadConstructor: boolean },
   ): void {
     let isStatic = false;
+    const containsEsc = this.state.containsEsc;
+
     if (this.match(tt.name) && this.state.value === "static") {
       const key = this.parseIdentifier(true); // eats 'static'
+
       if (this.isClassMethod()) {
         const method: N.ClassMethod = (member: any);
 
@@ -997,7 +1000,10 @@ export default class StatementParser extends ExpressionParser {
         prop.static = false;
         classBody.body.push(this.parseClassProperty(prop));
         return;
+      } else if (containsEsc) {
+        throw this.unexpected();
       }
+
       // otherwise something static
       isStatic = true;
     }

packages/babylon/src/parser/util.js

@@ -46,7 +46,11 @@ export default class UtilParser extends Tokenizer {
   // Tests whether parsed token is a contextual keyword.
 
   isContextual(name: string): boolean {
-    return this.match(tt.name) && this.state.value === name;
+    return (
+      this.match(tt.name) &&
+      this.state.value === name &&
+      !this.state.containsEsc
+    );
   }
 
   isLookaheadContextual(name: string): boolean {
@@ -57,7 +61,7 @@ export default class UtilParser extends Tokenizer {
   // Consumes contextual keyword if possible.
 
   eatContextual(name: string): boolean {
-    return this.state.value === name && this.eat(tt.name);
+    return this.isContextual(name) && this.eat(tt.name);
   }
 
   // Asserts that following token is given contextual keyword.

scripts/tests/test262/run_babylon_test262.js

@@ -4,7 +4,7 @@ const path = require("path");
 const chalk = require("chalk");
 const utils = require("./run_babylon_test262_utils");
 
-const testDir = path.join(__dirname, "../../../build/test262/test");
+const testDir = path.join(__dirname, "../../../build/test262");
 const whitelistFile = path.join(__dirname, "test262_whitelist.txt");
 const plugins = ["asyncGenerators", "objectRestSpread", "optionalCatchBinding"];
 const shouldUpdate = process.argv.indexOf("--update-whitelist") > -1;

scripts/tests/test262/run_babylon_test262_utils.js

@@ -1,8 +1,8 @@
 "use strict";
 
 const fs = require("graceful-fs");
-const path = require("path");
 const promisify = require("util").promisify;
+const TestStream = require("test262-stream");
 const pfs = {
   readFile: promisify(fs.readFile),
   writeFile: promisify(fs.writeFile),
@@ -12,115 +12,35 @@ const pfs = {
 
 const parse = require("../../../packages/babylon").parse;
 
-const modulePattern = /^\s*-\s*module\s*$|^\s*flags\s*:.*\bmodule\b/m;
-const noStrictPattern = /^\s*-\s*noStrict\s*$|^\s*flags\s*:.*\bnoStrict\b/m;
-const onlyStrictPattern = /^\s*-\s*onlyStrict\s*$|^\s*flags\s*:.*\bonlyStrict\b/m;
-const rawPattern = /^\s*-\s*raw\s*$|^\s*flags\s*:.*\braw\b/m;
-const testNamePattern = /^(?!.*_FIXTURE).*\.[jJ][sS]$/;
-
-function flatten(array) {
-  const flattened = [];
-  array.forEach(function(element) {
-    if (Array.isArray(element)) {
-      flattened.push.apply(flattened, element);
-    } else {
-      flattened.push(element);
-    }
-  });
-  return flattened;
-}
-
-function hasEarlyError(src) {
-  return !!(
-    src.match(/^\s*negative:\s*$/m) && src.match(/^\s+phase:\s*early\s*$/m)
-  );
-}
-
-function readDirDeep(dirName) {
-  return pfs.readdir(dirName).then(function(contents) {
-    return Promise.all(
-      contents.map(function(name) {
-        return findTests(path.join(dirName, name));
-      })
-    ).then(flatten);
-  });
-}
-
-function findTests(name) {
-  return pfs.stat(name).then(function(stat) {
-    if (stat.isDirectory()) {
-      return readDirDeep(name);
-    }
-
-    return name;
-  });
-}
-
-function readTest(fileName, testDir) {
-  if (!testNamePattern.test(fileName)) {
-    return Promise.resolve([]);
-  }
-
-  return pfs.readFile(fileName, "utf-8").then(function(contents) {
-    return makeScenarios(path.relative(testDir, fileName), contents);
-  });
-}
-
-function makeScenarios(fileName, testContent) {
-  const scenarios = [];
-  const base = {
-    fileName: fileName,
-    isModule: modulePattern.test(testContent),
-    expectedError: hasEarlyError(testContent),
-  };
-  const isNoStrict = noStrictPattern.test(testContent);
-  const isOnlyStrict = onlyStrictPattern.test(testContent);
-  const isRaw = rawPattern.test(testContent);
-
-  if (!isOnlyStrict) {
-    scenarios.push(
-      Object.assign(
-        {
-          id: fileName + "(default)",
-          content: testContent,
-        },
-        base
-      )
-    );
-  }
-
-  if (!isNoStrict && !isRaw) {
-    scenarios.push(
-      Object.assign(
-        {
-          id: fileName + "(strict mode)",
-          content: "'use strict';\n" + testContent,
-        },
-        base
-      )
-    );
-  }
-
-  return scenarios;
-}
-
 exports.getTests = function(testDir) {
-  return findTests(testDir)
+  const stream = new TestStream(testDir, { omitRuntime: true });
-    .then(function(testPaths) {
+  const tests = [];
-      return Promise.all(
+
-        testPaths.map(function(path) {
+  stream.on("data", test => {
-          return readTest(path, testDir);
+    // strip test/
-        })
+    const fileName = test.file.substr(5);
-      );
+
-    })
+    tests.push({
-    .then(flatten);
+      contents: test.contents,
+      fileName,
+      id: `${fileName}(${test.scenario})`,
+      sourceType: test.attrs.flags.module ? "module" : "script",
+      expectedError:
+        !!test.attrs.negative &&
+        (test.attrs.negative.phase === "parse" ||
+          test.attrs.negative.phase === "early"),
+    });
+  });
+
+  return new Promise((resolve, reject) => {
+    stream.on("end", () => resolve(tests));
+    stream.on("error", reject);
+  });
 };
 
 exports.runTest = function(test, plugins) {
-  const sourceType = test.isModule ? "module" : "script";
-
   try {
-    parse(test.content, { sourceType: sourceType, plugins: plugins });
+    parse(test.contents, { sourceType: test.sourceType, plugins: plugins });
     test.actualError = false;
   } catch (err) {
     test.actualError = true;

yarn.lock

@@ -3523,7 +3523,7 @@ graceful-fs@^3.0.0:
   dependencies:
     natives "^1.1.0"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6:
+graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9:
   version "4.1.11"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
 
@@ -4663,6 +4663,13 @@ js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"
 
+js-yaml@^3.2.1:
+  version "3.11.0"
+  resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.11.0.tgz#597c1a8bd57152f26d622ce4117851a51f5ebaef"
+  dependencies:
+    argparse "^1.0.7"
+    esprima "^4.0.0"
+
 js-yaml@^3.7.0, js-yaml@^3.9.0, js-yaml@^3.9.1:
   version "3.11.0"
   resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.11.0.tgz#597c1a8bd57152f26d622ce4117851a51f5ebaef"
@@ -4808,6 +4815,12 @@ kind-of@^6.0.0, kind-of@^6.0.2:
   version "6.0.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
 
+klaw@^2.1.0:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/klaw/-/klaw-2.1.1.tgz#42b76894701169cc910fd0d19ce677b5fb378af1"
+  dependencies:
+    graceful-fs "^4.1.9"
+
 labeled-stream-splicer@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/labeled-stream-splicer/-/labeled-stream-splicer-2.0.0.tgz#a52e1d138024c00b86b1c0c91f677918b8ae0a59"
@@ -7261,6 +7274,20 @@ test-exclude@^4.1.1:
     read-pkg-up "^1.0.1"
     require-main-filename "^1.0.1"
 
+test262-parser@^2.0.7:
+  version "2.0.7"
+  resolved "https://registry.yarnpkg.com/test262-parser/-/test262-parser-2.0.7.tgz#733b46bf7759e747eae34b5b14d6a3c8d2082add"
+  dependencies:
+    js-yaml "^3.2.1"
+    through "^2.3.4"
+
+test262-stream@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/test262-stream/-/test262-stream-1.1.0.tgz#18265217b8ca800d9ec432e37e6b8d192e12c6fd"
+  dependencies:
+    klaw "^2.1.0"
+    test262-parser "^2.0.7"
+
 text-extensions@^1.0.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/text-extensions/-/text-extensions-1.7.0.tgz#faaaba2625ed746d568a23e4d0aacd9bf08a8b39"