diff --git a/README.md b/README.md index cc5395c1c9..fb14b5a8ce 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,8 @@ mind. When in doubt, use `.parse()`. - **ranges**: Adds a `ranges` property to each node: `[node.start, node.end]` +- **tokens**: Adds all parsed tokens to a `tokens` property on the `File` node + ### Output Babylon generates AST according to [Babel AST format][]. diff --git a/src/options.js b/src/options.js index 72d65ed0df..f789cae388 100755 --- a/src/options.js +++ b/src/options.js @@ -13,6 +13,7 @@ export type Options = { plugins: $ReadOnlyArray; strictMode: ?boolean; ranges: boolean; + tokens: boolean; }; export const defaultOptions: Options = { @@ -44,6 +45,8 @@ export const defaultOptions: Options = { // // [range]: https://bugzilla.mozilla.org/show_bug.cgi?id=745678 ranges: false, + // Adds all parsed tokens to a `tokens` property on the `File` node + tokens: false, }; // Interpret and default an options object diff --git a/src/parser/base.js b/src/parser/base.js index 3795b71668..972d60c800 100644 --- a/src/parser/base.js +++ b/src/parser/base.js @@ -16,7 +16,6 @@ export default class BaseParser { state: State; input: string; - isReservedWord(word: string): boolean { if (word === "await") { return this.inModule; diff --git a/src/parser/statement.js b/src/parser/statement.js index 0c21effd7e..bb9b4bf18b 100644 --- a/src/parser/statement.js +++ b/src/parser/statement.js @@ -28,9 +28,10 @@ export default class StatementParser extends ExpressionParser { this.parseBlockBody(program, true, true, tt.eof); - file.program = this.finishNode(program, "Program"); + file.program = this.finishNode(program, "Program"); file.comments = this.state.comments; - file.tokens = this.state.tokens; + + if (this.options.tokens) file.tokens = this.state.tokens; return this.finishNode(file, "File"); } diff --git a/src/tokenizer/index.js b/src/tokenizer/index.js index 9549a21e31..dca36f2175 100644 --- a/src/tokenizer/index.js +++ b/src/tokenizer/index.js @@ -84,7 +84,7 @@ export default class Tokenizer extends LocationParser { // Move to the next token next(): void { - if (!this.isLookahead) { + if (this.options.tokens && !this.isLookahead) { this.state.tokens.push(new Token(this.state)); } @@ -199,7 +199,7 @@ export default class Tokenizer extends LocationParser { }; if (!this.isLookahead) { - this.state.tokens.push(comment); + if (this.options.tokens) this.state.tokens.push(comment); this.state.comments.push(comment); this.addComment(comment); } diff --git a/test/fixtures/core/opts/tokens-false/actual.js b/test/fixtures/core/opts/tokens-false/actual.js new file mode 100644 index 0000000000..a5433428e9 --- /dev/null +++ b/test/fixtures/core/opts/tokens-false/actual.js @@ -0,0 +1,3 @@ +var a = 1; + +var b = a + 1; diff --git a/test/fixtures/core/opts/tokens-false/expected.json b/test/fixtures/core/opts/tokens-false/expected.json new file mode 100644 index 0000000000..e8ab354792 --- /dev/null +++ b/test/fixtures/core/opts/tokens-false/expected.json @@ -0,0 +1,207 @@ +{ + "type": "File", + "start": 0, + "end": 26, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 26, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "sourceType": "script", + "body": [ + { + "type": "VariableDeclaration", + "start": 0, + "end": 10, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 10 + } + }, + "declarations": [ + { + "type": "VariableDeclarator", + "start": 4, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 4 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "id": { + "type": "Identifier", + "start": 4, + "end": 5, + "loc": { + "start": { + "line": 1, + "column": 4 + }, + "end": { + "line": 1, + "column": 5 + }, + "identifierName": "a" + }, + "name": "a" + }, + "init": { + "type": "NumericLiteral", + "start": 8, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 8 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + } + ], + "kind": "var" + }, + { + "type": "VariableDeclaration", + "start": 12, + "end": 26, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "declarations": [ + { + "type": "VariableDeclarator", + "start": 16, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "id": { + "type": "Identifier", + "start": 16, + "end": 17, + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 5 + }, + "identifierName": "b" + }, + "name": "b" + }, + "init": { + "type": "BinaryExpression", + "start": 20, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "left": { + "type": "Identifier", + "start": 20, + "end": 21, + "loc": { + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 9 + }, + "identifierName": "a" + }, + "name": "a" + }, + "operator": "+", + "right": { + "type": "NumericLiteral", + "start": 24, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 12 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + } + } + ], + "kind": "var" + } + ], + "directives": [] + } +} \ No newline at end of file diff --git a/test/fixtures/core/opts/tokens-false/options.json b/test/fixtures/core/opts/tokens-false/options.json new file mode 100644 index 0000000000..43b8c57d2f --- /dev/null +++ b/test/fixtures/core/opts/tokens-false/options.json @@ -0,0 +1,3 @@ +{ + "tokens": false +} diff --git a/test/fixtures/core/opts/tokens-true/actual.js b/test/fixtures/core/opts/tokens-true/actual.js new file mode 100644 index 0000000000..a5433428e9 --- /dev/null +++ b/test/fixtures/core/opts/tokens-true/actual.js @@ -0,0 +1,3 @@ +var a = 1; + +var b = a + 1; diff --git a/test/fixtures/core/opts/tokens-true/expected.json b/test/fixtures/core/opts/tokens-true/expected.json new file mode 100644 index 0000000000..1af20e4259 --- /dev/null +++ b/test/fixtures/core/opts/tokens-true/expected.json @@ -0,0 +1,556 @@ +{ + "type": "File", + "start": 0, + "end": 26, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 26, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "sourceType": "script", + "body": [ + { + "type": "VariableDeclaration", + "start": 0, + "end": 10, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 10 + } + }, + "declarations": [ + { + "type": "VariableDeclarator", + "start": 4, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 4 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "id": { + "type": "Identifier", + "start": 4, + "end": 5, + "loc": { + "start": { + "line": 1, + "column": 4 + }, + "end": { + "line": 1, + "column": 5 + }, + "identifierName": "a" + }, + "name": "a" + }, + "init": { + "type": "NumericLiteral", + "start": 8, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 8 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + } + ], + "kind": "var" + }, + { + "type": "VariableDeclaration", + "start": 12, + "end": 26, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 14 + } + }, + "declarations": [ + { + "type": "VariableDeclarator", + "start": 16, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "id": { + "type": "Identifier", + "start": 16, + "end": 17, + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 5 + }, + "identifierName": "b" + }, + "name": "b" + }, + "init": { + "type": "BinaryExpression", + "start": 20, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "left": { + "type": "Identifier", + "start": 20, + "end": 21, + "loc": { + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 9 + }, + "identifierName": "a" + }, + "name": "a" + }, + "operator": "+", + "right": { + "type": "NumericLiteral", + "start": 24, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 12 + }, + "end": { + "line": 3, + "column": 13 + } + }, + "extra": { + "rawValue": 1, + "raw": "1" + }, + "value": 1 + } + } + } + ], + "kind": "var" + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "var", + "keyword": "var", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "var", + "start": 0, + "end": 3, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 3 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "a", + "start": 4, + "end": 5, + "loc": { + "start": { + "line": 1, + "column": 4 + }, + "end": { + "line": 1, + "column": 5 + } + } + }, + { + "type": { + "label": "=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": true, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "=", + "start": 6, + "end": 7, + "loc": { + "start": { + "line": 1, + "column": 6 + }, + "end": { + "line": 1, + "column": 7 + } + } + }, + { + "type": { + "label": "num", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": 1, + "start": 8, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 8 + }, + "end": { + "line": 1, + "column": 9 + } + } + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 9, + "end": 10, + "loc": { + "start": { + "line": 1, + "column": 9 + }, + "end": { + "line": 1, + "column": 10 + } + } + }, + { + "type": { + "label": "var", + "keyword": "var", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "var", + "start": 12, + "end": 15, + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 3 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "b", + "start": 16, + "end": 17, + "loc": { + "start": { + "line": 3, + "column": 4 + }, + "end": { + "line": 3, + "column": 5 + } + } + }, + { + "type": { + "label": "=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": true, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "=", + "start": 18, + "end": 19, + "loc": { + "start": { + "line": 3, + "column": 6 + }, + "end": { + "line": 3, + "column": 7 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "a", + "start": 20, + "end": 21, + "loc": { + "start": { + "line": 3, + "column": 8 + }, + "end": { + "line": 3, + "column": 9 + } + } + }, + { + "type": { + "label": "+/-", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": true, + "postfix": false, + "binop": 9, + "updateContext": null + }, + "value": "+", + "start": 22, + "end": 23, + "loc": { + "start": { + "line": 3, + "column": 10 + }, + "end": { + "line": 3, + "column": 11 + } + } + }, + { + "type": { + "label": "num", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": 1, + "start": 24, + "end": 25, + "loc": { + "start": { + "line": 3, + "column": 12 + }, + "end": { + "line": 3, + "column": 13 + } + } + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 25, + "end": 26, + "loc": { + "start": { + "line": 3, + "column": 13 + }, + "end": { + "line": 3, + "column": 14 + } + } + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 26, + "end": 26, + "loc": { + "start": { + "line": 3, + "column": 14 + }, + "end": { + "line": 3, + "column": 14 + } + } + } + ] +} \ No newline at end of file diff --git a/test/fixtures/core/opts/tokens-true/options.json b/test/fixtures/core/opts/tokens-true/options.json new file mode 100644 index 0000000000..ea1a5da64d --- /dev/null +++ b/test/fixtures/core/opts/tokens-true/options.json @@ -0,0 +1,3 @@ +{ + "tokens": true +} diff --git a/test/helpers/runFixtureTests.js b/test/helpers/runFixtureTests.js index c37edcff04..f6f78aed3c 100644 --- a/test/helpers/runFixtureTests.js +++ b/test/helpers/runFixtureTests.js @@ -80,7 +80,6 @@ function runTest(test, parseFunction) { throw err; } - delete ast.tokens; if (ast.comments && !ast.comments.length) delete ast.comments; if (!test.expect.code && !opts.throws && !process.env.CI) { @@ -133,7 +132,7 @@ function misMatch(exp, act) { } for (var prop in act) { - if (prop === "__clone") { + if (typeof act[prop] === "function") { continue; }