Make tokens optional (#563)

Adding tokens to the ast is significant slower and most tools
don't ever use them anyway

src/options.js

@@ -13,6 +13,7 @@ export type Options = {
   plugins: $ReadOnlyArray<string>;
   strictMode: ?boolean;
   ranges: boolean;
+  tokens: boolean;
 };
 
 export const defaultOptions: Options = {
@@ -44,6 +45,8 @@ export const defaultOptions: Options = {
   //
   // [range]: https://bugzilla.mozilla.org/show_bug.cgi?id=745678
   ranges: false,
+  // Adds all parsed tokens to a `tokens` property on the `File` node
+  tokens: false,
 };
 
 // Interpret and default an options object

src/parser/base.js

@@ -16,7 +16,6 @@ export default class BaseParser {
   state: State;
   input: string;
 
-
   isReservedWord(word: string): boolean {
     if (word === "await") {
       return this.inModule;

src/parser/statement.js

@@ -28,9 +28,10 @@ export default class StatementParser extends ExpressionParser {
 
     this.parseBlockBody(program, true, true, tt.eof);
 
-    file.program  = this.finishNode(program, "Program");
+    file.program = this.finishNode(program, "Program");
     file.comments = this.state.comments;
-    file.tokens   = this.state.tokens;
+
+    if (this.options.tokens) file.tokens = this.state.tokens;
 
     return this.finishNode(file, "File");
   }

src/tokenizer/index.js

@@ -84,7 +84,7 @@ export default class Tokenizer extends LocationParser {
   // Move to the next token
 
   next(): void {
-    if (!this.isLookahead) {
+    if (this.options.tokens && !this.isLookahead) {
       this.state.tokens.push(new Token(this.state));
     }
 
@@ -199,7 +199,7 @@ export default class Tokenizer extends LocationParser {
     };
 
     if (!this.isLookahead) {
-      this.state.tokens.push(comment);
+      if (this.options.tokens) this.state.tokens.push(comment);
       this.state.comments.push(comment);
       this.addComment(comment);
     }