From 459e289d63d720c4ae9e78bbaef3a9afe2274de2 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Thu, 2 Nov 2017 09:10:26 +0100 Subject: [PATCH 01/18] feat: setup constants --- packages/babylon/src/tokenizer/index.js | 25 ++++++++++++------------ packages/babylon/src/util/charCodes.js | 26 +++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 packages/babylon/src/util/charCodes.js diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index fa484ed017..8c0cf7bf15 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -4,6 +4,7 @@ import type { Options } from "../options"; import type { Position } from "../util/location"; +import charCodes from "../util/charCodes"; import { isIdentifierStart, isIdentifierChar, @@ -328,31 +329,31 @@ export default class Tokenizer extends LocationParser { loop: while (this.state.pos < this.input.length) { const ch = this.input.charCodeAt(this.state.pos); switch (ch) { - case 32: // space - case 160: // non-breaking space + case charCodes.space: + case charCodes.nonBreakingSpace: ++this.state.pos; break; - case 13: // '\r' carriage return + case charCodes.carriageReturn: if (this.input.charCodeAt(this.state.pos + 1) === 10) { ++this.state.pos; } - case 10: // '\n' line feed - case 8232: // line separator - case 8233: // paragraph separator + case charCodes.lineFeed: + case charCodes.lineSeparator: + case charCodes.paragraphSeparator: ++this.state.pos; ++this.state.curLine; this.state.lineStart = this.state.pos; break; - case 47: // '/' + case charCodes.slash: // '/' switch (this.input.charCodeAt(this.state.pos + 1)) { - case 42: // '*' + case charCodes.asterisk: // '*' this.skipBlockComment(); break; - case 47: + case charCodes.slash: this.skipLineComment(2); break; @@ -873,7 +874,7 @@ export default class Tokenizer extends LocationParser { val = code - 97 + 10; // a } else if (code >= 65) { val = code - 65 + 10; // A - } else if (code >= 48 && code <= 57) { + } else if (charCodes.isIn09(code)) { val = code - 48; // 0-9 } else { val = Infinity; @@ -901,7 +902,7 @@ export default class Tokenizer extends LocationParser { this.raise(this.state.start + 2, "Expected number in radix " + radix); if (this.hasPlugin("bigInt")) { - if (this.input.charCodeAt(this.state.pos) === 0x6e) { + if (this.input.charCodeAt(this.state.pos) === charCodes.letterN) { // 'n' ++this.state.pos; isBigInt = true; @@ -924,7 +925,7 @@ export default class Tokenizer extends LocationParser { readNumber(startsWithDot: boolean): void { const start = this.state.pos; - let octal = this.input.charCodeAt(start) === 0x30; // '0' + let octal = this.input.charCodeAt(start) === charCodes.digit0; // '0' let isFloat = false; let isBigInt = false; diff --git a/packages/babylon/src/util/charCodes.js b/packages/babylon/src/util/charCodes.js new file mode 100644 index 0000000000..b91ec1ebd9 --- /dev/null +++ b/packages/babylon/src/util/charCodes.js @@ -0,0 +1,26 @@ +// @flow + +type Char = number; + +const charCodes = { + space: 32, + nonBreakingSpace: 160, + carriageReturn: 13, // '\r' + lineFeed: 10, // '\n' + lineSeparator: 8232, + paragraphSeparator: 8233, + + slash: 47, // '/' + asterisk: 42, // '*' + + letterN: 110, // 'n' + + digit0: 48, // '0' + digit9: 57, // '9' +}; + +export function isIn09(code: Char): boolean { + return code >= charCodes.digit0 && code <= charCodes.digit9; +} + +export default charCodes; From abb48507092d40ff56d5ed174744e9c5729f134a Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Thu, 2 Nov 2017 16:40:08 +0100 Subject: [PATCH 02/18] refactor: use charCodes --- packages/babylon/src/tokenizer/index.js | 33 ++++++++++++++----------- packages/babylon/src/util/charCodes.js | 16 ++++++++++-- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index 8c0cf7bf15..e1a6927593 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -364,7 +364,7 @@ export default class Tokenizer extends LocationParser { default: if ( - (ch > 8 && ch < 14) || + (ch > charCodes.backSpace && ch < charCodes.shiftOut) || (ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) ) { ++this.state.pos; @@ -528,8 +528,11 @@ export default class Tokenizer extends LocationParser { if (next === code) { size = - code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2; - if (this.input.charCodeAt(this.state.pos + size) === 61) { + code === charCodes.greaterThan && + this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan + ? 3 + : 2; + if (this.input.charCodeAt(this.state.pos + size) === charCodes.equalsTo) { this.finishOp(tt.assign, size + 1); return; } @@ -538,11 +541,11 @@ export default class Tokenizer extends LocationParser { } if ( - next === 33 && - code === 60 && + next === charCodes.exclamationMark && + code === charCodes.lessThan && !this.inModule && - this.input.charCodeAt(this.state.pos + 2) === 45 && - this.input.charCodeAt(this.state.pos + 3) === 45 + this.input.charCodeAt(this.state.pos + 2) === charCodes.dash && + this.input.charCodeAt(this.state.pos + 3) === charCodes.dash ) { // `` line comment @@ -513,7 +519,7 @@ export default class Tokenizer extends LocationParser { return; } - if (next === 61) { + if (next === charCodes.equalsTo) { this.finishOp(tt.assign, 2); } else { this.finishOp(tt.plusMin, 1); @@ -564,7 +570,7 @@ export default class Tokenizer extends LocationParser { readToken_eq_excl(code: number): void { // '=!' const next = this.input.charCodeAt(this.state.pos + 1); - if (next === 61) { + if (next === charCodes.equalsTo) { this.finishOp( tt.equality, this.input.charCodeAt(this.state.pos + 2) === charCodes.equalsTo @@ -653,10 +659,10 @@ export default class Tokenizer extends LocationParser { this.finishToken(tt.bracketR); return; - case 123: + case charCodes.leftCurlyBrace: if ( this.hasPlugin("flow") && - this.input.charCodeAt(this.state.pos + 1) === 124 + this.input.charCodeAt(this.state.pos + 1) === charCodes.verticalBar ) { this.finishOp(tt.braceBarL, 2); } else { @@ -690,7 +696,7 @@ export default class Tokenizer extends LocationParser { this.finishToken(tt.at); return; - case 96: // '`' + case charCodes.graveAccent: ++this.state.pos; this.finishToken(tt.backQuote); return; @@ -716,63 +722,63 @@ export default class Tokenizer extends LocationParser { } // Anything else beginning with a digit is an integer, octal // number, or float. - case 49: - case 50: - case 51: - case 52: - case 53: - case 54: - case 55: - case 56: - case charCodes.digit9: // 1-9 + case charCodes.digit1: + case charCodes.digit2: + case charCodes.digit3: + case charCodes.digit4: + case charCodes.digit5: + case charCodes.digit6: + case charCodes.digit7: + case charCodes.digit8: + case charCodes.digit9: this.readNumber(false); return; // Quotes produce strings. - case 34: - case 39: // '"', "'" + case charCodes.questionMark: + case charCodes.apostrophe: this.readString(code); return; - // Operators are parsed inline in tiny state machines. '=' (61) is + // Operators are parsed inline in tiny state machines. '=' (charCodes.equalsTo) is // often referred to. `finishOp` simply skips the amount of // characters it is given as second argument, and returns a token // of the type given by its first argument. - case 47: // '/' + case charCodes.slash: this.readToken_slash(); return; - case 37: - case 42: // '%*' + case charCodes.percentSign: + case charCodes.asterisk: this.readToken_mult_modulo(code); return; - case 124: - case 38: // '|&' + case charCodes.verticalBar: + case charCodes.ampersand: this.readToken_pipe_amp(code); return; - case 94: // '^' + case charCodes.caret: this.readToken_caret(); return; - case 43: - case 45: // '+-' + case charCodes.plusSign: + case charCodes.dash: // '+-' this.readToken_plus_min(code); return; - case 60: - case 62: // '<>' + case charCodes.lessThan: + case charCodes.greaterThan: this.readToken_lt_gt(code); return; - case 61: - case 33: // '=!' + case charCodes.equalsTo: + case charCodes.exclamationMark: this.readToken_eq_excl(code); return; - case 126: // '~' + case charCodes.tilde: this.finishOp(tt.tilde, 1); return; } @@ -877,10 +883,10 @@ export default class Tokenizer extends LocationParser { } } - if (code >= 97) { - val = code - 97 + 10; // a - } else if (code >= 65) { - val = code - 65 + 10; // A + if (code >= charCodes.lowercaseA) { + val = code - charCodes.lowercaseA + 10; // a + } else if (code >= charCodes.uppercaseA) { + val = code - charCodes.uppercaseA + 10; // A } else if (charCodes.isDigit(code)) { val = code - charCodes.digit0; // 0-9 } else { @@ -941,26 +947,27 @@ export default class Tokenizer extends LocationParser { if (octal && this.state.pos == start + 1) octal = false; // number === 0 let next = this.input.charCodeAt(this.state.pos); - if (next === 0x2e && !octal) { - // '.' + if (next === charCodes.dot && !octal) { ++this.state.pos; this.readInt(10); isFloat = true; next = this.input.charCodeAt(this.state.pos); } - if ((next === 0x45 || next === 0x65) && !octal) { - // 'Ee' + if ( + (next === charCodes.uppercaseE || next === charCodes.lowercaseE) && + !octal + ) { next = this.input.charCodeAt(++this.state.pos); - if (next === 0x2b || next === 0x2d) ++this.state.pos; // '+-' + if (next === charCodes.plusSign || next === charCodes.dash) + ++this.state.pos; if (this.readInt(10) === null) this.raise(start, "Invalid number"); isFloat = true; next = this.input.charCodeAt(this.state.pos); } if (this.hasPlugin("bigInt")) { - if (next === 0x6e) { - // 'n' + if (next === charCodes.lowercaseN) { // disallow floats and legacy octal syntax, new style octal ("0o") is handled in this.readRadixNumber if (isFloat || octal) this.raise(start, "Invalid BigIntLiteral"); ++this.state.pos; @@ -1000,7 +1007,7 @@ export default class Tokenizer extends LocationParser { const ch = this.input.charCodeAt(this.state.pos); let code; - if (ch === 123) { + if (ch === charCodes.leftCurlyBrace) { // '{' const codePos = ++this.state.pos; code = this.readHexChar( @@ -1033,7 +1040,7 @@ export default class Tokenizer extends LocationParser { this.raise(this.state.start, "Unterminated string constant"); const ch = this.input.charCodeAt(this.state.pos); if (ch === quote) break; - if (ch === 92) { + if (ch === charCodes.backslash) { // '\' out += this.input.slice(chunkStart, this.state.pos); // $FlowFixMe @@ -1060,12 +1067,13 @@ export default class Tokenizer extends LocationParser { this.raise(this.state.start, "Unterminated template"); const ch = this.input.charCodeAt(this.state.pos); if ( - ch === 96 || - (ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) + ch === charCodes.graveAccent || + (ch === charCodes.dollarSign && + this.input.charCodeAt(this.state.pos + 1) === + charCodes.leftCurlyBrace) ) { - // '`', '${' if (this.state.pos === this.state.start && this.match(tt.template)) { - if (ch === 36) { + if (ch === charCodes.dollarSign) { this.state.pos += 2; this.finishToken(tt.dollarBraceL); return; @@ -1079,7 +1087,7 @@ export default class Tokenizer extends LocationParser { this.finishToken(tt.template, containsInvalid ? null : out); return; } - if (ch === 92) { + if (ch === charCodes.backslash) { // '\' out += this.input.slice(chunkStart, this.state.pos); const escaped = this.readEscapedChar(true); @@ -1146,14 +1154,14 @@ export default class Tokenizer extends LocationParser { ++this.state.curLine; return ""; default: - if (ch >= charCodes.digit0 && ch <= 55) { + if (ch >= charCodes.digit0 && ch <= charCodes.digit7) { const codePos = this.state.pos - 1; // $FlowFixMe let octalStr = this.input .substr(this.state.pos - 1, 3) .match(/^[0-7]+/)[0]; let octal = parseInt(octalStr, 8); - if (octal > 255) { + if (octal > 2557) { octalStr = octalStr.slice(0, -1); octal = parseInt(octalStr, 8); } @@ -1208,8 +1216,7 @@ export default class Tokenizer extends LocationParser { const ch = this.fullCharCodeAtPos(); if (isIdentifierChar(ch)) { this.state.pos += ch <= 0xffff ? 1 : 2; - } else if (ch === 92) { - // "\" + } else if (ch === charCodes.backslash) { this.state.containsEsc = true; word += this.input.slice(chunkStart, this.state.pos); From b93800d3cbbad0ecc58dda7866446c32d1e29038 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Fri, 3 Nov 2017 09:52:19 +0100 Subject: [PATCH 08/18] fix: use charcode package --- packages/babylon/package.json | 3 +- packages/babylon/src/tokenizer/index.js | 5 +- packages/babylon/src/util/charCodes.js | 110 ------------------------ 3 files changed, 4 insertions(+), 114 deletions(-) delete mode 100644 packages/babylon/src/util/charCodes.js diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 0289526ca5..bc6d49946e 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -28,7 +28,8 @@ "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0", "rollup-watch": "^4.0.0", - "unicode-9.0.0": "~0.7.0" + "unicode-9.0.0": "~0.7.0", + "charcodes": "^0.0.2" }, "bin": { "babylon": "./bin/babylon.js" diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index b16420574a..fa4ead52c4 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -4,7 +4,7 @@ import type { Options } from "../options"; import type { Position } from "../util/location"; -import * as charCodes from "../util/charCodes"; +import * as charCodes from "charcodes"; import { isIdentifierStart, isIdentifierChar, @@ -916,8 +916,7 @@ export default class Tokenizer extends LocationParser { this.raise(this.state.start + 2, "Expected number in radix " + radix); if (this.hasPlugin("bigInt")) { - if (this.input.charCodeAt(this.state.pos) === charCodes.letterLowerN) { - // 'n' + if (this.input.charCodeAt(this.state.pos) === charCodes.lowercaseN) { ++this.state.pos; isBigInt = true; } diff --git a/packages/babylon/src/util/charCodes.js b/packages/babylon/src/util/charCodes.js deleted file mode 100644 index f74c404939..0000000000 --- a/packages/babylon/src/util/charCodes.js +++ /dev/null @@ -1,110 +0,0 @@ -// @flow - -type Char = number; - -export const backSpace = 8; -export const lineFeed = 10; // '\n' -export const carriageReturn = 13; // '\r' -export const shiftOut = 14; -export const space = 32; -export const exclamationMark = 33; // '!' -export const quotationMark = 34; // '"' -export const numberSign = 35; // '#' -export const dollarSign = 36; // '$' -export const percentSign = 37; // '%' -export const ampersand = 38; // '&' -export const apostrophe = 39; // ''' -export const leftParenthesis = 40; // '(' -export const rightParenthesis = 41; // ')' -export const asterisk = 42; // '*' -export const plusSign = 43; // '+' -export const comma = 44; // ',' -export const dash = 45; // '-' -export const dot = 46; // '.' -export const slash = 47; // '/' -export const digit0 = 48; // '0' -export const digit1 = 49; // '1' -export const digit2 = 50; // '2' -export const digit3 = 51; // '3' -export const digit4 = 52; // '4' -export const digit5 = 53; // '5' -export const digit6 = 54; // '6' -export const digit7 = 55; // '7' -export const digit8 = 56; // '8' -export const digit9 = 57; // '9' -export const colon = 58; // ':' -export const semicolon = 59; // ';' -export const lessThan = 60; // '<' -export const equalsTo = 61; // '=' -export const greaterThan = 62; // '>' -export const questionMark = 63; // '?' -export const atSign = 64; // '@' -export const uppercaseA = 65; // 'A' -export const uppercaseB = 66; // 'B' -export const uppercaseC = 67; // 'C' -export const uppercaseD = 68; // 'D' -export const uppercaseE = 69; // 'E' -export const uppercaseF = 70; // 'F' -export const uppercaseG = 71; // 'G' -export const uppercaseH = 72; // 'H' -export const uppercaseI = 73; // 'I' -export const uppercaseJ = 74; // 'J' -export const uppercaseK = 75; // 'K' -export const uppercaseL = 76; // 'L' -export const uppercaseM = 77; // 'M' -export const uppercaseN = 78; // 'N' -export const uppercaseO = 79; // 'O' -export const uppercaseP = 80; // 'P' -export const uppercaseQ = 81; // 'Q' -export const uppercaseR = 82; // 'R' -export const uppercaseS = 83; // 'S' -export const uppercaseT = 84; // 'T' -export const uppercaseU = 85; // 'U' -export const uppercaseV = 86; // 'V' -export const uppercaseW = 87; // 'W' -export const uppercaseX = 88; // 'X' -export const uppercaseY = 89; // 'Y' -export const uppercaseZ = 90; // 'Z' -export const leftSquareBracket = 91; // '[' -export const backslash = 92; // '\ ' -export const rightSquareBracket = 93; // ']' -export const caret = 94; // '^' -export const underscore = 95; // '_' -export const graveAccent = 96; // '`' -export const lowercaseA = 97; // 'a' -export const lowercaseB = 98; // 'b' -export const lowercaseC = 99; // 'c' -export const lowercaseD = 100; // 'd' -export const lowercaseE = 101; // 'e' -export const lowercaseF = 102; // 'f' -export const lowercaseG = 103; // 'g' -export const lowercaseH = 104; // 'h' -export const lowercaseI = 105; // 'i' -export const lowercaseJ = 106; // 'j' -export const lowercaseK = 107; // 'k' -export const lowercaseL = 108; // 'l' -export const lowercaseM = 109; // 'm' -export const lowercaseN = 110; // 'n' -export const lowercaseO = 111; // 'o' -export const lowercaseP = 112; // 'p' -export const lowercaseQ = 113; // 'q' -export const lowercaseR = 114; // 'r' -export const lowercaseS = 115; // 's' -export const lowercaseT = 116; // 't' -export const lowercaseU = 117; // 'u' -export const lowercaseV = 118; // 'v' -export const lowercaseW = 119; // 'w' -export const lowercaseX = 120; // 'x' -export const lowercaseY = 121; // 'y' -export const lowercaseZ = 122; // 'z' -export const leftCurlyBrace = 123; // '{' -export const verticalBar = 124; // '|' -export const rightCurlyBrace = 125; // '}' -export const tilde = 126; // '~' -export const nonBreakingSpace = 160; -export const lineSeparator = 8232; -export const paragraphSeparator = 8233; - -export function isDigit(code: Char): boolean { - return code >= digit0 && code <= digit9; -} From 3e34162092e12b271b629527ada7f02b81b5495a Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Tue, 7 Nov 2017 18:07:38 +0100 Subject: [PATCH 09/18] chore: bump charcode --- packages/babylon/package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index bc6d49946e..80eb768f45 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -24,12 +24,13 @@ }, "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.5", + "babel-plugin-transform-charcodes": "0.0.3-0", + "charcodes": "0.0.3-0", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0", "rollup-watch": "^4.0.0", - "unicode-9.0.0": "~0.7.0", - "charcodes": "^0.0.2" + "unicode-9.0.0": "~0.7.0" }, "bin": { "babylon": "./bin/babylon.js" From f3f005c67e2cbe2947174d9ba5b2a874824232ef Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Tue, 7 Nov 2017 19:16:04 +0100 Subject: [PATCH 10/18] feat: use charcode AOT transform --- packages/babylon/package.json | 4 ++-- packages/babylon/rollup.config.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 87f6cb26fe..8ce960d302 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -25,8 +25,8 @@ "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.31", "babel-plugin-transform-for-of-as-array": "1.0.4", - "babel-plugin-transform-charcodes": "0.0.3-0", - "charcodes": "0.0.3-0", + "babel-plugin-transform-charcodes": "0.0.3", + "charcodes": "0.0.3", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0", diff --git a/packages/babylon/rollup.config.js b/packages/babylon/rollup.config.js index 7ed7a136b4..f627c6a860 100644 --- a/packages/babylon/rollup.config.js +++ b/packages/babylon/rollup.config.js @@ -24,7 +24,7 @@ export default { ], "@babel/flow", ], - plugins: ["transform-for-of-as-array"], + plugins: ["transform-charcodes", "transform-for-of-as-array"], }), nodeResolve(), ], From cd8a869a189541e947cbd503194343ed859372cc Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Wed, 8 Nov 2017 13:20:11 +0100 Subject: [PATCH 11/18] fix: remove unwanted change --- packages/babylon/src/tokenizer/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index 252855641d..579c682b15 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -1172,7 +1172,7 @@ export default class Tokenizer extends LocationParser { .substr(this.state.pos - 1, 3) .match(/^[0-7]+/)[0]; let octal = parseInt(octalStr, 8); - if (octal > 2557) { + if (octal > 255) { octalStr = octalStr.slice(0, -1); octal = parseInt(octalStr, 8); } From 6b417a8ecb047ea479891a158cd4f2a9586382c9 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Wed, 8 Nov 2017 16:56:25 +0100 Subject: [PATCH 12/18] chore: bump charcode plugin --- packages/babylon/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 8ce960d302..1663b9bd12 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -25,7 +25,7 @@ "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.31", "babel-plugin-transform-for-of-as-array": "1.0.4", - "babel-plugin-transform-charcodes": "0.0.3", + "babel-plugin-transform-charcodes": "0.0.5", "charcodes": "0.0.3", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", From 88c1b4ef1ce6adead64acb865d7a0bab119962ec Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Thu, 9 Nov 2017 17:13:36 +0100 Subject: [PATCH 13/18] feat: more charcodes --- packages/babylon/package.json | 4 +- packages/babylon/src/tokenizer/index.js | 120 ++++++++++++------------ 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 1663b9bd12..6df0578357 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -25,8 +25,8 @@ "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.31", "babel-plugin-transform-for-of-as-array": "1.0.4", - "babel-plugin-transform-charcodes": "0.0.5", - "charcodes": "0.0.3", + "babel-plugin-transform-charcodes": "0.0.7", + "charcodes": "0.0.6", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0", diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index 579c682b15..7a747d6b3c 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -53,6 +53,7 @@ allowedNumericSeparatorSiblings.bin = [ allowedNumericSeparatorSiblings.oct = [ // 0 - 7 ...allowedNumericSeparatorSiblings.bin, + charCodes.digit2, charCodes.digit3, charCodes.digit4, @@ -63,6 +64,7 @@ allowedNumericSeparatorSiblings.oct = [ allowedNumericSeparatorSiblings.dec = [ // 0 - 9 ...allowedNumericSeparatorSiblings.oct, + charCodes.digit8, charCodes.digit9, ]; @@ -70,20 +72,20 @@ allowedNumericSeparatorSiblings.dec = [ allowedNumericSeparatorSiblings.hex = [ // 0 - 9, A - F, a - f, ...allowedNumericSeparatorSiblings.dec, - // A - F + charCodes.uppercaseA, charCodes.uppercaseB, - 67, - 68, + charCodes.uppercaseC, + charCodes.uppercaseD, charCodes.uppercaseE, - 70, - // a - f + charCodes.uppercaseF, + charCodes.lowercaseA, charCodes.lowercaseB, - 99, - 100, + charCodes.lowercaseC, + charCodes.lowercaseD, charCodes.lowercaseE, - 102, + charCodes.lowercaseF, ]; // Object type used to represent tokens. Note that normally, tokens @@ -115,7 +117,7 @@ function codePointToString(code: number): string { } else { return String.fromCharCode( ((code - 0x10000) >> 10) + 0xd800, - ((code - 0x10000) & 1023) + 0xdc00, + ((code - 0x10000) & charCodes.lowercaseF) + 0xdc00, ); } } @@ -302,10 +304,10 @@ export default class Tokenizer extends LocationParser { let ch = this.input.charCodeAt((this.state.pos += startSkip)); if (this.state.pos < this.input.length) { while ( - ch !== 10 && - ch !== 13 && - ch !== 8232 && - ch !== 8233 && + ch !== charCodes.lineFeed && + ch !== charCodes.carriageReturn && + ch !== charCodes.lineSeparator && + ch !== charCodes.paragraphSeparator && ++this.state.pos < this.input.length ) { ch = this.input.charCodeAt(this.state.pos); @@ -335,7 +337,9 @@ export default class Tokenizer extends LocationParser { break; case charCodes.carriageReturn: - if (this.input.charCodeAt(this.state.pos + 1) === 10) { + if ( + this.input.charCodeAt(this.state.pos + 1) === charCodes.lineFeed + ) { ++this.state.pos; } @@ -347,9 +351,9 @@ export default class Tokenizer extends LocationParser { this.state.lineStart = this.state.pos; break; - case charCodes.slash: // '/' + case charCodes.slash: switch (this.input.charCodeAt(this.state.pos + 1)) { - case charCodes.asterisk: // '*' + case charCodes.asterisk: this.skipBlockComment(); break; @@ -471,7 +475,7 @@ export default class Tokenizer extends LocationParser { if (next === charCodes.greaterThan) { this.finishOp(tt.pipeline, 2); return; - } else if (next === 125 && this.hasPlugin("flow")) { + } else if (next === charCodes.rightCurlyBrace && this.hasPlugin("flow")) { // '|}' this.finishOp(tt.braceBarR, 2); return; @@ -611,7 +615,7 @@ export default class Tokenizer extends LocationParser { getTokenFromCode(code: number): void { switch (code) { - case 35: // '#' + case charCodes.numberSign: if ( (this.hasPlugin("classPrivateProperties") || this.hasPlugin("classPrivateMethods")) && @@ -635,27 +639,27 @@ export default class Tokenizer extends LocationParser { return; // Punctuation tokens. - case 40: + case charCodes.leftParenthesis: ++this.state.pos; this.finishToken(tt.parenL); return; - case 41: + case charCodes.rightParenthesis: ++this.state.pos; this.finishToken(tt.parenR); return; - case 59: + case charCodes.semicolon: ++this.state.pos; this.finishToken(tt.semi); return; - case 44: + case charCodes.comma: ++this.state.pos; this.finishToken(tt.comma); return; - case 91: + case charCodes.leftSquareBracket: ++this.state.pos; this.finishToken(tt.bracketL); return; - case 93: + case charCodes.rightSquareBracket: ++this.state.pos; this.finishToken(tt.bracketR); return; @@ -672,15 +676,15 @@ export default class Tokenizer extends LocationParser { } return; - case 125: + case charCodes.rightCurlyBrace: ++this.state.pos; this.finishToken(tt.braceR); return; - case 58: + case charCodes.colon: if ( this.hasPlugin("functionBind") && - this.input.charCodeAt(this.state.pos + 1) === 58 + this.input.charCodeAt(this.state.pos + 1) === charCodes.colon ) { this.finishOp(tt.doubleColon, 2); } else { @@ -689,10 +693,10 @@ export default class Tokenizer extends LocationParser { } return; - case 63: + case charCodes.questionMark: this.readToken_question(); return; - case 64: + case charCodes.atSign: ++this.state.pos; this.finishToken(tt.at); return; @@ -703,7 +707,6 @@ export default class Tokenizer extends LocationParser { return; case charCodes.digit0: { - // '0' const next = this.input.charCodeAt(this.state.pos + 1); // '0x', '0X' - hex number if (next === charCodes.lowercaseX || next === charCodes.uppercaseX) { @@ -736,7 +739,7 @@ export default class Tokenizer extends LocationParser { return; // Quotes produce strings. - case charCodes.questionMark: + case charCodes.quotationMark: case charCodes.apostrophe: this.readString(code); return; @@ -765,7 +768,7 @@ export default class Tokenizer extends LocationParser { return; case charCodes.plusSign: - case charCodes.dash: // '+-' + case charCodes.dash: this.readToken_plus_min(code); return; @@ -887,9 +890,9 @@ export default class Tokenizer extends LocationParser { } if (code >= charCodes.lowercaseA) { - val = code - charCodes.lowercaseA + 10; // a + val = code - charCodes.lowercaseA + charCodes.lineFeed; } else if (code >= charCodes.uppercaseA) { - val = code - charCodes.uppercaseA + 10; // A + val = code - charCodes.uppercaseA + charCodes.lineFeed; } else if (charCodes.isDigit(code)) { val = code - charCodes.digit0; // 0-9 } else { @@ -943,7 +946,7 @@ export default class Tokenizer extends LocationParser { readNumber(startsWithDot: boolean): void { const start = this.state.pos; - let octal = this.input.charCodeAt(start) === charCodes.digit0; // '0' + let octal = this.input.charCodeAt(start) === charCodes.digit0; let isFloat = false; let isBigInt = false; @@ -1016,7 +1019,6 @@ export default class Tokenizer extends LocationParser { let code; if (ch === charCodes.leftCurlyBrace) { - // '{' const codePos = ++this.state.pos; code = this.readHexChar( this.input.indexOf("}", this.state.pos) - this.state.pos, @@ -1050,7 +1052,6 @@ export default class Tokenizer extends LocationParser { const ch = this.input.charCodeAt(this.state.pos); if (ch === quote) break; if (ch === charCodes.backslash) { - // '\' out += this.input.slice(chunkStart, this.state.pos); // $FlowFixMe out += this.readEscapedChar(false); @@ -1099,7 +1100,6 @@ export default class Tokenizer extends LocationParser { return; } if (ch === charCodes.backslash) { - // '\' out += this.input.slice(chunkStart, this.state.pos); const escaped = this.readEscapedChar(true); if (escaped === null) { @@ -1112,9 +1112,11 @@ export default class Tokenizer extends LocationParser { out += this.input.slice(chunkStart, this.state.pos); ++this.state.pos; switch (ch) { - case 13: - if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; - case 10: + case charCodes.carriageReturn: + if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) { + ++this.state.pos; + } + case charCodes.lineFeed: out += "\n"; break; default: @@ -1137,30 +1139,31 @@ export default class Tokenizer extends LocationParser { const ch = this.input.charCodeAt(++this.state.pos); ++this.state.pos; switch (ch) { - case 110: - return "\n"; // 'n' -> '\n' - case 114: - return "\r"; // 'r' -> '\r' + case charCodes.lowercaseN: + return "\n"; + case charCodes.lowercaseR: + return "\r"; case charCodes.lowercaseX: { const code = this.readHexChar(2, throwOnInvalid); return code === null ? null : String.fromCharCode(code); } - case 117: { - // 'u' + case charCodes.lowercaseU: { const code = this.readCodePoint(throwOnInvalid); return code === null ? null : codePointToString(code); } - case 116: - return "\t"; // 't' -> '\t' + case charCodes.lowercaseT: + return "\t"; case charCodes.lowercaseB: - return "\b"; // 'b' -> '\b' - case 118: - return "\u000b"; // 'v' -> '\u000b' - case 102: - return "\f"; // 'f' -> '\f' - case 13: - if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; // '\r\n' - case 10: // ' \n' + return "\b"; + case charCodes.lowercaseV: + return "\u000b"; + case charCodes.lowercaseF: + return "\f"; + case charCodes.carriageReturn: + if (this.input.charCodeAt(this.state.pos) === charCodes.lineFeed) { + ++this.state.pos; + } + case charCodes.lineFeed: this.state.lineStart = this.state.pos; ++this.state.curLine; return ""; @@ -1233,8 +1236,7 @@ export default class Tokenizer extends LocationParser { word += this.input.slice(chunkStart, this.state.pos); const escStart = this.state.pos; - if (this.input.charCodeAt(++this.state.pos) !== 117) { - // "u" + if (this.input.charCodeAt(++this.state.pos) !== charCodes.lowercaseU) { this.raise( this.state.pos, "Expecting Unicode escape sequence \\uXXXX", From b79e3c0e99865c5078249d09e3351cbaac503cc0 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Thu, 9 Nov 2017 18:07:10 +0100 Subject: [PATCH 14/18] fix: minor changes --- packages/babylon/src/tokenizer/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index 7a747d6b3c..1f45a76414 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -117,7 +117,7 @@ function codePointToString(code: number): string { } else { return String.fromCharCode( ((code - 0x10000) >> 10) + 0xd800, - ((code - 0x10000) & charCodes.lowercaseF) + 0xdc00, + ((code - 0x10000) & 1023) + 0xdc00, ); } } @@ -233,7 +233,7 @@ export default class Tokenizer extends LocationParser { readToken(code: number): void { // Identifier or keyword. '\uXXXX' sequences are allowed in // identifiers, so '\' also dispatches to that. - if (isIdentifierStart(code) || code === charCodes.backslash /* '\' */) { + if (isIdentifierStart(code) || code === charCodes.backslash) { this.readWord(); } else { this.getTokenFromCode(code); From 097ff400018d237fb1bd7f04825487c43493c1de Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Sat, 11 Nov 2017 18:08:43 +0100 Subject: [PATCH 15/18] feat: more charCodes --- packages/babylon/src/tokenizer/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/babylon/src/tokenizer/index.js b/packages/babylon/src/tokenizer/index.js index 1f45a76414..608a85bf79 100644 --- a/packages/babylon/src/tokenizer/index.js +++ b/packages/babylon/src/tokenizer/index.js @@ -369,7 +369,8 @@ export default class Tokenizer extends LocationParser { default: if ( (ch > charCodes.backSpace && ch < charCodes.shiftOut) || - (ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) + (ch >= charCodes.oghamSpaceMark && + nonASCIIwhitespace.test(String.fromCharCode(ch))) ) { ++this.state.pos; } else { From 186f04a3e996e68c02aa749c41849083686f84b5 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Mon, 13 Nov 2017 13:15:32 +0100 Subject: [PATCH 16/18] feat: use charcodes in JSX plugin --- packages/babylon/src/plugins/jsx/index.js | 29 ++++++++++++++--------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/babylon/src/plugins/jsx/index.js b/packages/babylon/src/plugins/jsx/index.js index 9478bec17c..15444d8738 100644 --- a/packages/babylon/src/plugins/jsx/index.js +++ b/packages/babylon/src/plugins/jsx/index.js @@ -1,5 +1,7 @@ // @flow +import * as charCodes from "charcodes"; + import XHTMLEntities from "./xhtml"; import type Parser from "../../parser"; import { TokenType, types as tt } from "../../tokenizer/types"; @@ -84,10 +86,10 @@ export default (superClass: Class): Class => const ch = this.input.charCodeAt(this.state.pos); switch (ch) { - case 60: // "<" - case 123: // "{" + case charCodes.lessThan: + case charCodes.leftCurlyBrace: if (this.state.pos === this.state.start) { - if (ch === 60 && this.state.exprAllowed) { + if (ch === charCodes.lessThan && this.state.exprAllowed) { ++this.state.pos; return this.finishToken(tt.jsxTagStart); } @@ -96,7 +98,7 @@ export default (superClass: Class): Class => out += this.input.slice(chunkStart, this.state.pos); return this.finishToken(tt.jsxText, out); - case 38: // "&" + case charCodes.ampersand: out += this.input.slice(chunkStart, this.state.pos); out += this.jsxReadEntity(); chunkStart = this.state.pos; @@ -118,7 +120,10 @@ export default (superClass: Class): Class => const ch = this.input.charCodeAt(this.state.pos); let out; ++this.state.pos; - if (ch === 13 && this.input.charCodeAt(this.state.pos) === 10) { + if ( + ch === charCodes.carriageReturn && + this.input.charCodeAt(this.state.pos) === charCodes.lineFeed + ) { ++this.state.pos; out = normalizeCRLF ? "\n" : "\r\n"; } else { @@ -140,8 +145,7 @@ export default (superClass: Class): Class => const ch = this.input.charCodeAt(this.state.pos); if (ch === quote) break; - if (ch === 38) { - // "&" + if (ch === charCodes.ampersand) { out += this.input.slice(chunkStart, this.state.pos); out += this.jsxReadEntity(); chunkStart = this.state.pos; @@ -205,7 +209,7 @@ export default (superClass: Class): Class => const start = this.state.pos; do { ch = this.input.charCodeAt(++this.state.pos); - } while (isIdentifierChar(ch) || ch === 45); // "-" + } while (isIdentifierChar(ch) || ch === charCodes.dash); return this.finishToken( tt.jsxName, this.input.slice(start, this.state.pos), @@ -513,17 +517,20 @@ export default (superClass: Class): Class => return this.jsxReadWord(); } - if (code === 62) { + if (code === charCodes.greaterThan) { ++this.state.pos; return this.finishToken(tt.jsxTagEnd); } - if ((code === 34 || code === 39) && context === tc.j_oTag) { + if ( + (code === charCodes.quotationMark || code === charCodes.apostrophe) && + context === tc.j_oTag + ) { return this.jsxReadString(code); } } - if (code === 60 && this.state.exprAllowed) { + if (code === charCodes.lessThan && this.state.exprAllowed) { ++this.state.pos; return this.finishToken(tt.jsxTagStart); } From 4c584ae341564ff8719e78018ec06f02ab93f4ad Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Wed, 15 Nov 2017 19:53:30 +0100 Subject: [PATCH 17/18] chore: upgrade and fix charcodes --- packages/babylon/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 6df0578357..7cff5cd493 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -25,8 +25,8 @@ "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.31", "babel-plugin-transform-for-of-as-array": "1.0.4", - "babel-plugin-transform-charcodes": "0.0.7", - "charcodes": "0.0.6", + "babel-plugin-transform-charcodes": "0.0.9", + "charcodes": "0.0.9", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0", From 57e2c45cbeb66e494d2caf6aa05a87d4aede90ec Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Thu, 16 Nov 2017 09:57:20 +0100 Subject: [PATCH 18/18] chore: upgrade charcode --- packages/babylon/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/babylon/package.json b/packages/babylon/package.json index 7cff5cd493..d80eff69ae 100644 --- a/packages/babylon/package.json +++ b/packages/babylon/package.json @@ -25,8 +25,8 @@ "devDependencies": { "@babel/helper-fixtures": "7.0.0-beta.31", "babel-plugin-transform-for-of-as-array": "1.0.4", - "babel-plugin-transform-charcodes": "0.0.9", - "charcodes": "0.0.9", + "babel-plugin-transform-charcodes": "0.0.10", + "charcodes": "0.0.10", "rollup": "^0.50.0", "rollup-plugin-babel": "^4.0.0-beta.0", "rollup-plugin-node-resolve": "^3.0.0",