NumberLiteralSeparator: Stage 1 feature plugin. Closes gh-538 (#541)

* NumberLiteralSeparator: Stage 1 feature plugin

Signed-off-by: Rick Waldron <waldron.rick@gmail.com>

* fix-up per review

Signed-off-by: Rick Waldron <waldron.rick@gmail.com>

* nit: forbiddenNumericLiteralSeparatorSibling -> forbiddenNumericLiteralSeparatorSiblings

Signed-off-by: Rick Waldron <waldron.rick@gmail.com>

* fix-up to change includes -> indexOf

Signed-off-by: Rick Waldron <waldron.rick@gmail.com>
This commit is contained in:
Rick Waldron 2017-05-26 16:37:05 -04:00 committed by Henry Zhu
parent 589ceb4ee7
commit b344f62056
83 changed files with 935 additions and 1 deletions

View File

@ -138,6 +138,7 @@ require("babylon").parse("code", {
- `functionBind`
- `functionSent`
- `dynamicImport`
- `numericSeparator` ([proposal](https://github.com/samuelgoto/proposal-numeric-separator))
### FAQ

View File

@ -13,6 +13,23 @@ import { SourceLocation } from "../util/location";
import { lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace } from "../util/whitespace";
import State from "./state";
// The following character codes are forbidden from being
// an immediate sibling of NumericLiteralSeparator _
const forbiddenNumericLiteralSeparatorSiblings = [
46, // .
66, // B
69, // E
79, // O
88, // X
95, // _ (multiple separators are not allowed)
98, // b
101, // e
111, // o
120, // x
];
// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
@ -555,6 +572,23 @@ export default class Tokenizer extends LocationParser {
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
const code = this.input.charCodeAt(this.state.pos);
let val;
if (this.hasPlugin("numericSeparator")) {
const prev = this.input.charCodeAt(this.state.pos - 1);
const next = this.input.charCodeAt(this.state.pos + 1);
if (code === 95) {
if ((forbiddenNumericLiteralSeparatorSiblings.indexOf(prev) > -1) ||
(forbiddenNumericLiteralSeparatorSiblings.indexOf(next) > -1) ||
Number.isNaN(next)) {
this.raise(this.state.pos, "Invalid NumericLiteralSeparator");
}
// Ignore this _ character
++this.state.pos;
continue;
}
}
if (code >= 97) {
val = code - 97 + 10; // a
} else if (code >= 65) {
@ -608,7 +642,7 @@ export default class Tokenizer extends LocationParser {
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
const str = this.input.slice(start, this.state.pos);
const str = this.input.slice(start, this.state.pos).replace(/_/g, "");
let val;
if (isFloat) {
val = parseFloat(str);

View File

@ -0,0 +1 @@
1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:1)" }

View File

@ -0,0 +1 @@
1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }

View File

@ -0,0 +1 @@
0x1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }

View File

@ -0,0 +1 @@
0xa_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }

View File

@ -0,0 +1 @@
0x_a_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0x__1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0x_1__1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0x_1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0o_1_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0o_11

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0o_01_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0b_0_1_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
1_1__

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }

View File

@ -0,0 +1 @@
0b_01_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
0b01_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }

View File

@ -0,0 +1 @@
0o1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }

View File

@ -0,0 +1 @@
0o_1_1_

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:2)" }

View File

@ -0,0 +1 @@
._1_1

View File

@ -0,0 +1 @@
{ "throws": "Unexpected token (1:0)" }

View File

@ -0,0 +1 @@
0o01_8

View File

@ -0,0 +1 @@
{ "throws": "Unexpected token, expected ; (1:5)" }

View File

@ -0,0 +1 @@
0b2_1

View File

@ -0,0 +1 @@
{ "throws": "Expected number in radix 2 (1:2)" }

View File

@ -0,0 +1 @@
0xZ_1

View File

@ -0,0 +1 @@
{ "throws": "Expected number in radix 16 (1:2)" }

View File

@ -0,0 +1 @@
1__1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:1)" }

View File

@ -0,0 +1 @@
1_1_.1_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:3)" }

View File

@ -0,0 +1 @@
1_1._1_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:4)" }

View File

@ -0,0 +1 @@
1_1.1_e1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }

View File

@ -0,0 +1 @@
1_1.1_E1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:5)" }

View File

@ -0,0 +1 @@
1_1.1e_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }

View File

@ -0,0 +1 @@
1_1.1E_1

View File

@ -0,0 +1 @@
{ "throws": "Invalid NumericLiteralSeparator (1:6)" }

View File

@ -0,0 +1,3 @@
{
"plugins": ["numericSeparator"]
}

View File

@ -0,0 +1 @@
1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"program": {
"type": "Program",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
},
"extra": {
"rawValue": 11,
"raw": "1_1"
},
"value": 11
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
1_1.1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"program": {
"type": "Program",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"extra": {
"rawValue": 11.11,
"raw": "1_1.1_1"
},
"value": 11.11
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0o1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"program": {
"type": "Program",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"extra": {
"rawValue": 9,
"raw": "0o1_1"
},
"value": 9
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0o0_11

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"program": {
"type": "Program",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"extra": {
"rawValue": 9,
"raw": "0o0_11"
},
"value": 9
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
1.1_0_1e1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"program": {
"type": "Program",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"extra": {
"rawValue": 11.01,
"raw": "1.1_0_1e1"
},
"value": 11.01
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
1.1_0_1E1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"program": {
"type": "Program",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 9
}
},
"extra": {
"rawValue": 11.01,
"raw": "1.1_0_1E1"
},
"value": 11.01
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
.1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 4,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 4
}
},
"program": {
"type": "Program",
"start": 0,
"end": 4,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 4
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 4,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 4
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 4,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 4
}
},
"extra": {
"rawValue": 0.11,
"raw": ".1_1"
},
"value": 0.11
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0x1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"program": {
"type": "Program",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"extra": {
"rawValue": 17,
"raw": "0x1_1"
},
"value": 17
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0xa_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"program": {
"type": "Program",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"extra": {
"rawValue": 161,
"raw": "0xa_1"
},
"value": 161
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0xA_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"program": {
"type": "Program",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 5
}
},
"extra": {
"rawValue": 161,
"raw": "0xA_1"
},
"value": 161
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0b01_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"program": {
"type": "Program",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 6,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 6
}
},
"extra": {
"rawValue": 3,
"raw": "0b01_1"
},
"value": 3
}
}
],
"directives": []
}
}

View File

@ -0,0 +1 @@
0b0_1_1

View File

@ -0,0 +1,69 @@
{
"type": "File",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"program": {
"type": "Program",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"expression": {
"type": "NumericLiteral",
"start": 0,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 7
}
},
"extra": {
"rawValue": 3,
"raw": "0b0_1_1"
},
"value": 3
}
}
],
"directives": []
}
}