diff --git a/src/tokenizer/index.js b/src/tokenizer/index.js index b41067c43b..743c3286a5 100644 --- a/src/tokenizer/index.js +++ b/src/tokenizer/index.js @@ -17,18 +17,24 @@ import State from "./state"; // The following character codes are forbidden from being // an immediate sibling of NumericLiteralSeparator _ -const forbiddenNumericLiteralSeparatorSiblings = [ - 46, // . - 66, // B - 69, // E - 79, // O - 88, // X - 95, // _ (multiple separators are not allowed) - 98, // b - 101, // e - 111, // o - 120, // x -]; +const forbiddenNumericSeparatorSiblings = { + decBinOct: [ + 46, // . + 66, // B + 69, // E + 79, // O + 95, // _ (multiple separators are not allowed) + 98, // b + 101, // e + 111, // o + ], + hex: [ + 46, // . + 88, // X + 95, // _ (multiple separators are not allowed) + 120, // x + ], +}; // Object type used to represent tokens. Note that normally, tokens // simply exist as properties on the parser object. This is only @@ -567,6 +573,9 @@ export default class Tokenizer extends LocationParser { readInt(radix: number, len?: number): number | null { const start = this.state.pos; + const forbiddenSiblings = radix === 16 ? + forbiddenNumericSeparatorSiblings.hex : + forbiddenNumericSeparatorSiblings.decBinOct; let total = 0; for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) { @@ -577,8 +586,8 @@ export default class Tokenizer extends LocationParser { const prev = this.input.charCodeAt(this.state.pos - 1); const next = this.input.charCodeAt(this.state.pos + 1); if (code === 95) { - if ((forbiddenNumericLiteralSeparatorSiblings.indexOf(prev) > -1) || - (forbiddenNumericLiteralSeparatorSiblings.indexOf(next) > -1) || + if ((forbiddenSiblings.indexOf(prev) > -1) || + (forbiddenSiblings.indexOf(next) > -1) || Number.isNaN(next)) { this.raise(this.state.pos, "Invalid NumericLiteralSeparator"); } diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-12/actual.js b/test/fixtures/experimental/numeric-literal-separator/valid-12/actual.js new file mode 100644 index 0000000000..4115d0bd97 --- /dev/null +++ b/test/fixtures/experimental/numeric-literal-separator/valid-12/actual.js @@ -0,0 +1 @@ +0xBE_be_EB_eb; diff --git a/test/fixtures/experimental/numeric-literal-separator/valid-12/expected.json b/test/fixtures/experimental/numeric-literal-separator/valid-12/expected.json new file mode 100644 index 0000000000..8719155cde --- /dev/null +++ b/test/fixtures/experimental/numeric-literal-separator/valid-12/expected.json @@ -0,0 +1,69 @@ +{ + "type": "File", + "start": 0, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "sourceType": "script", + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "expression": { + "type": "NumericLiteral", + "start": 0, + "end": 13, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 13 + } + }, + "extra": { + "rawValue": 3200183275, + "raw": "0xBE_be_EB_eb" + }, + "value": 3200183275 + } + } + ], + "directives": [] + } +} \ No newline at end of file