From ed13a4ab0541266e5afbc954db3933d8aa4480f6 Mon Sep 17 00:00:00 2001 From: Sven SAULEAU Date: Tue, 10 Jan 2017 20:22:56 +0100 Subject: [PATCH] Refactor tokenizer types file (#263) * refactor(tokenizer): use class for keywork tokens * refactor(tokenizer): re-use constacts in types * refactor(tokenizer): binop token type * feat(tokenizer): use beforeAndStartExpr for shortcut * fix(tokenizer): typo in keywordTokenType * refactor(tokenizer): don't use spread operator * refactor(tokenizer): constant for isLoop, isAssign, prefix, postfix * fix(tokenizer): remove constant beforeAndStartExpr * style(tokenizer): space after comma --- src/tokenizer/types.js | 179 ++++++++++++++++++++++------------------- 1 file changed, 96 insertions(+), 83 deletions(-) diff --git a/src/tokenizer/types.js b/src/tokenizer/types.js index 86d49e10cb..b82dfe0902 100644 --- a/src/tokenizer/types.js +++ b/src/tokenizer/types.js @@ -16,6 +16,13 @@ // to know when parsing a label, in order to allow or disallow // continue jumps to that label. +const beforeExpr = true; +const startsExpr = true; +const isLoop = true; +const isAssign = true; +const prefix = true; +const postfix = true; + export class TokenType { constructor(label, conf = {}) { this.label = label; @@ -32,38 +39,47 @@ export class TokenType { } } -function binop(name, prec) { - return new TokenType(name, {beforeExpr: true, binop: prec}); +class KeywordTokenType extends TokenType { + constructor(name, options = {}) { + options.keyword = name; + + super(name, options); + } +} + +export class BinopTokenType extends TokenType { + constructor(name, prec) { + super(name, { beforeExpr, binop: prec }); + } } -const beforeExpr = {beforeExpr: true}, startsExpr = {startsExpr: true}; export const types = { - num: new TokenType("num", startsExpr), - regexp: new TokenType("regexp", startsExpr), - string: new TokenType("string", startsExpr), - name: new TokenType("name", startsExpr), + num: new TokenType("num", { startsExpr }), + regexp: new TokenType("regexp", { startsExpr }), + string: new TokenType("string", { startsExpr }), + name: new TokenType("name", { startsExpr }), eof: new TokenType("eof"), // Punctuation token types. - bracketL: new TokenType("[", {beforeExpr: true, startsExpr: true}), + bracketL: new TokenType("[", { beforeExpr, startsExpr }), bracketR: new TokenType("]"), - braceL: new TokenType("{", {beforeExpr: true, startsExpr: true}), - braceBarL: new TokenType("{|", {beforeExpr: true, startsExpr: true}), + braceL: new TokenType("{", { beforeExpr, startsExpr }), + braceBarL: new TokenType("{|", { beforeExpr, startsExpr }), braceR: new TokenType("}"), braceBarR: new TokenType("|}"), - parenL: new TokenType("(", {beforeExpr: true, startsExpr: true}), + parenL: new TokenType("(", { beforeExpr, startsExpr }), parenR: new TokenType(")"), - comma: new TokenType(",", beforeExpr), - semi: new TokenType(";", beforeExpr), - colon: new TokenType(":", beforeExpr), - doubleColon: new TokenType("::", beforeExpr), + comma: new TokenType(",", { beforeExpr }), + semi: new TokenType(";", { beforeExpr }), + colon: new TokenType(":", { beforeExpr }), + doubleColon: new TokenType("::", { beforeExpr }), dot: new TokenType("."), - question: new TokenType("?", beforeExpr), - arrow: new TokenType("=>", beforeExpr), + question: new TokenType("?", { beforeExpr }), + arrow: new TokenType("=>", { beforeExpr }), template: new TokenType("template"), - ellipsis: new TokenType("...", beforeExpr), - backQuote: new TokenType("`", startsExpr), - dollarBraceL: new TokenType("${", {beforeExpr: true, startsExpr: true}), + ellipsis: new TokenType("...", { beforeExpr }), + backQuote: new TokenType("`", { startsExpr }), + dollarBraceL: new TokenType("${", { beforeExpr, startsExpr }), at: new TokenType("@"), // Operators. These carry several kinds of properties to help the @@ -80,69 +96,66 @@ export const types = { // binary operators with a very low precedence, that should result // in AssignmentExpression nodes. - eq: new TokenType("=", {beforeExpr: true, isAssign: true}), - assign: new TokenType("_=", {beforeExpr: true, isAssign: true}), - incDec: new TokenType("++/--", {prefix: true, postfix: true, startsExpr: true}), - prefix: new TokenType("prefix", {beforeExpr: true, prefix: true, startsExpr: true}), - logicalOR: binop("||", 1), - logicalAND: binop("&&", 2), - bitwiseOR: binop("|", 3), - bitwiseXOR: binop("^", 4), - bitwiseAND: binop("&", 5), - equality: binop("==/!=", 6), - relational: binop("", 7), - bitShift: binop("<>", 8), - plusMin: new TokenType("+/-", {beforeExpr: true, binop: 9, prefix: true, startsExpr: true}), - modulo: binop("%", 10), - star: binop("*", 10), - slash: binop("/", 10), - exponent: new TokenType("**", {beforeExpr: true, binop: 11, rightAssociative: true}) + eq: new TokenType("=", { beforeExpr, isAssign }), + assign: new TokenType("_=", { beforeExpr, isAssign }), + incDec: new TokenType("++/--", { prefix, postfix, startsExpr }), + prefix: new TokenType("prefix", { beforeExpr, prefix, startsExpr }), + logicalOR: new BinopTokenType("||", 1), + logicalAND: new BinopTokenType("&&", 2), + bitwiseOR: new BinopTokenType("|", 3), + bitwiseXOR: new BinopTokenType("^", 4), + bitwiseAND: new BinopTokenType("&", 5), + equality: new BinopTokenType("==/!=", 6), + relational: new BinopTokenType("", 7), + bitShift: new BinopTokenType("<>", 8), + plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }), + modulo: new BinopTokenType("%", 10), + star: new BinopTokenType("*", 10), + slash: new BinopTokenType("/", 10), + exponent: new TokenType("**", { beforeExpr, binop: 11, rightAssociative: true }) +}; + +export const keywords = { + "break": new KeywordTokenType("break"), + "case": new KeywordTokenType("case", { beforeExpr }), + "catch": new KeywordTokenType("catch"), + "continue": new KeywordTokenType("continue"), + "debugger": new KeywordTokenType("debugger"), + "default": new KeywordTokenType("default", { beforeExpr }), + "do": new KeywordTokenType("do", { isLoop, beforeExpr }), + "else": new KeywordTokenType("else", { beforeExpr }), + "finally": new KeywordTokenType("finally"), + "for": new KeywordTokenType("for", { isLoop }), + "function": new KeywordTokenType("function", { startsExpr }), + "if": new KeywordTokenType("if"), + "return": new KeywordTokenType("return", { beforeExpr }), + "switch": new KeywordTokenType("switch"), + "throw": new KeywordTokenType("throw", { beforeExpr }), + "try": new KeywordTokenType("try"), + "var": new KeywordTokenType("var"), + "let": new KeywordTokenType("let"), + "const": new KeywordTokenType("const"), + "while": new KeywordTokenType("while", { isLoop }), + "with": new KeywordTokenType("with"), + "new": new KeywordTokenType("new", { beforeExpr, startsExpr }), + "this": new KeywordTokenType("this", { startsExpr }), + "super": new KeywordTokenType("super", { startsExpr }), + "class": new KeywordTokenType("class"), + "extends": new KeywordTokenType("extends", { beforeExpr }), + "export": new KeywordTokenType("export"), + "import": new KeywordTokenType("import"), + "yield": new KeywordTokenType("yield", { beforeExpr, startsExpr }), + "null": new KeywordTokenType("null", { startsExpr }), + "true": new KeywordTokenType("true", { startsExpr }), + "false": new KeywordTokenType("false", { startsExpr }), + "in": new KeywordTokenType("in", { beforeExpr, binop: 7 }), + "instanceof": new KeywordTokenType("instanceof", { beforeExpr, binop: 7 }), + "typeof": new KeywordTokenType("typeof", { beforeExpr, prefix, startsExpr }), + "void": new KeywordTokenType("void", { beforeExpr, prefix, startsExpr }), + "delete": new KeywordTokenType("delete", { beforeExpr, prefix, startsExpr }) }; // Map keyword names to token types. - -export const keywords = {}; - -// Succinct definitions of keyword token types -function kw(name, options = {}) { - options.keyword = name; - keywords[name] = types["_" + name] = new TokenType(name, options); -} - -kw("break"); -kw("case", beforeExpr); -kw("catch"); -kw("continue"); -kw("debugger"); -kw("default", beforeExpr); -kw("do", {isLoop: true, beforeExpr: true}); -kw("else", beforeExpr); -kw("finally"); -kw("for", {isLoop: true}); -kw("function", startsExpr); -kw("if"); -kw("return", beforeExpr); -kw("switch"); -kw("throw", beforeExpr); -kw("try"); -kw("var"); -kw("let"); -kw("const"); -kw("while", {isLoop: true}); -kw("with"); -kw("new", {beforeExpr: true, startsExpr: true}); -kw("this", startsExpr); -kw("super", startsExpr); -kw("class"); -kw("extends", beforeExpr); -kw("export"); -kw("import"); -kw("yield", {beforeExpr: true, startsExpr: true}); -kw("null", startsExpr); -kw("true", startsExpr); -kw("false", startsExpr); -kw("in", {beforeExpr: true, binop: 7}); -kw("instanceof", {beforeExpr: true, binop: 7}); -kw("typeof", {beforeExpr: true, prefix: true, startsExpr: true}); -kw("void", {beforeExpr: true, prefix: true, startsExpr: true}); -kw("delete", {beforeExpr: true, prefix: true, startsExpr: true}); +Object.keys(keywords).forEach((name) => { + types["_" + name] = keywords[name]; +});