Merge pull request #3565 from loganfsmyth/codegen-append-only

Make the code generator write-only to avoid exponential time generation
This commit is contained in:
Logan Smyth 2016-07-06 18:33:00 -07:00 committed by GitHub
commit 193b9b5797
9 changed files with 392 additions and 354 deletions

View File

@ -1,267 +1,113 @@
import type Position from "./position";
import repeat from "lodash/repeat";
import Position from "./position";
import type SourceMap from "./source-map";
import trimEnd from "lodash/trimEnd";
const SPACES_RE = /^[ \t]+$/;
/**
* Buffer for collecting generated output.
* The Buffer class exists to manage the queue of tokens being pushed onto the output string
* in such a way that the final string buffer is treated as write-only until the final .get()
* call. This allows V8 to optimize the output efficiently by not requiring it to store the
* string in contiguous memory.
*/
export default class Buffer {
constructor(position: Position, format: Object) {
this.printedCommentStarts = {};
this.parenPushNewlineState = null;
this.position = position;
this._indent = format.indent.base;
this.format = format;
this.buf = "";
// Maintaining a reference to the last char in the buffer is an optimization
// to make sure that v8 doesn't "flatten" the string more often than needed
// see https://github.com/babel/babel/pull/3283 for details.
this.last = "";
this.map = null;
this._sourcePosition = {
line: null,
column: null,
filename: null,
};
this._endsWithWord = false;
constructor(map: ?SourceMap) {
this._map = map;
}
printedCommentStarts: Object;
parenPushNewlineState: ?Object;
position: Position;
_indent: number;
format: Object;
buf: string;
last: string;
_map: SourceMap = null;
_buf: string = "";
_last: string = "";
_queue: Array = [];
_position: Position = new Position;
_sourcePosition: Object = {
line: null,
column: null,
filename: null,
};
/**
* Description
* Get the final string output from the buffer, along with the sourcemap if one exists.
*/
catchUp(node: Object) {
// catch up to this nodes newline if we're behind
if (node.loc && this.format.retainLines && this.buf) {
while (this.position.line < node.loc.start.line) {
this.push("\n");
}
}
}
get(): Object {
this._flush();
/**
* Get the current trimmed buffer.
*/
get(): string {
return trimEnd(this.buf);
}
/**
* Get the current indent.
*/
getIndent(): string {
if (this.format.compact || this.format.concise) {
return "";
} else {
return repeat(this.format.indent.style, this._indent);
}
}
/**
* Get the current indent size.
*/
indentSize(): number {
return this.getIndent().length;
}
/**
* Increment indent size.
*/
indent() {
this._indent++;
}
/**
* Decrement indent size.
*/
dedent() {
this._indent--;
}
/**
* Add a semicolon to the buffer.
*/
semicolon() {
this.token(";");
}
/**
* Add a right brace to the buffer.
*/
rightBrace() {
this.newline(true);
if (this.format.minified && !this._lastPrintedIsEmptyStatement) {
this._removeLast(";");
}
this.token("}");
}
/**
* Add a keyword to the buffer.
*/
keyword(name: string) {
this.word(name);
this.space();
}
/**
* Add a space to the buffer unless it is compact.
*/
space() {
if (this.format.compact) return;
if (this.buf && !this.endsWith(" ") && !this.endsWith("\n")) {
this.push(" ");
}
}
/**
* Writes a token that can't be safely parsed without taking whitespace into account.
*/
word(str: string) {
if (this._endsWithWord) this.push(" ");
this.push(str);
this._endsWithWord = true;
}
/**
* Writes a simple token.
*/
token(str: string) {
// space is mandatory to avoid outputting <!--
// http://javascript.spec.whatwg.org/#comment-syntax
if ((str === "--" && this.last === "!") ||
// Need spaces for operators of the same kind to avoid: `a+++b`
(str[0] === "+" && this.last === "+") ||
(str[0] === "-" && this.last === "-")) {
this.push(" ");
}
this.push(str);
}
/**
* Remove the last character.
*/
removeLast(cha: string) {
if (this.format.compact) return;
return this._removeLast(cha);
}
_removeLast(cha: string) {
if (!this.endsWith(cha)) return;
this.buf = this.buf.slice(0, -1);
this.last = this.buf[this.buf.length - 1];
this.position.unshift(cha);
}
/**
* Set some state that will be modified if a newline has been inserted before any
* non-space characters.
*
* This is to prevent breaking semantics for terminatorless separator nodes. eg:
*
* return foo;
*
* returns `foo`. But if we do:
*
* return
* foo;
*
* `undefined` will be returned and not `foo` due to the terminator.
*/
startTerminatorless(): Object {
return this.parenPushNewlineState = {
printed: false
return {
code: trimEnd(this._buf),
map: this._map ? this._map.get() : null,
};
}
/**
* Print an ending parentheses if a starting one has been printed.
* Add a string to the buffer that cannot be reverted.
*/
endTerminatorless(state: Object) {
if (state.printed) {
this.dedent();
this.newline();
this.token(")");
}
append(str: string): void {
this._flush();
this._append(str, this._sourcePosition.line, this._sourcePosition.column,
this._sourcePosition.filename);
}
/**
* Add a newline (or many newlines), maintaining formatting.
* Strips multiple newlines if removeLast is true.
* Add a string to the buffer than can be reverted.
*/
newline(i?: boolean | number, removeLast?: boolean) {
if (this.format.retainLines || this.format.compact) return;
if (this.format.concise) {
this.space();
return;
}
// never allow more than two lines
if (this.endsWith("\n\n")) return;
if (typeof i === "boolean") removeLast = i;
if (typeof i !== "number") i = 1;
i = Math.min(2, i);
if (this.endsWith("{\n") || this.endsWith(":\n")) i--;
if (i <= 0) return;
// remove the last newline
if (removeLast) {
this.removeLast("\n");
}
this.removeLast(" ");
this._removeSpacesAfterLastNewline();
for (let j = 0; j < i; j++) {
this.push("\n");
}
queue(str: string): void {
this._queue.unshift([str, this._sourcePosition.line, this._sourcePosition.column,
this._sourcePosition.filename]);
}
/**
* If buffer ends with a newline and some spaces after it, trim those spaces.
*/
_flush(): void {
let item;
while (item = this._queue.pop()) this._append(...item);
}
_removeSpacesAfterLastNewline() {
let lastNewlineIndex = this.buf.lastIndexOf("\n");
if (lastNewlineIndex >= 0 && this.get().length <= lastNewlineIndex) {
let toRemove = this.buf.slice(lastNewlineIndex + 1);
this.buf = this.buf.substring(0, lastNewlineIndex + 1);
this.last = "\n";
this.position.unshift(toRemove);
_append(str: string, line: number, column: number, filename: ?string): void {
// If there the line is ending, adding a new mapping marker is redundant
if (this._map && str[0] !== "\n") this._map.mark(this._position, line, column, filename);
this._buf += str;
this._last = str[str.length - 1];
this._position.push(str);
}
removeTrailingSpaces(): void {
while (this._queue.length > 0 && SPACES_RE.test(this._queue[0][0])) this._queue.shift();
}
removeTrailingNewline(): void {
if (this._queue.length > 0 && this._queue[0][0] === "\n") this._queue.shift();
}
removeLastSemicolon(): void {
if (this._queue.length > 0 && this._queue[0][0] === ";") this._queue.shift();
}
endsWith(str: string): boolean {
const end = this._last + this._queue.reduce((acc, item) => item[0] + acc, "");
if (str.length <= end.length) {
return end.slice(-str.length) === str;
}
// We assume that everything being matched is at most a single token plus some whitespace,
// which everything currently is, but otherwise we'd have to expand _last or check _buf.
return false;
}
getLast(): string {
if (this._queue.length > 0) {
const last = this._queue[this._queue.length - 1][0];
return last[last.length - 1];
}
return this._last;
}
hasContent(): boolean {
return this._queue.length > 0 || !!this._last;
}
/**
@ -269,7 +115,7 @@ export default class Buffer {
* will be given this position in the sourcemap.
*/
source(prop: string, loc: Location) {
source(prop: string, loc: Location): void {
if (prop && !loc) return;
let pos = loc ? loc[prop] : null;
@ -283,8 +129,8 @@ export default class Buffer {
* Call a callback with a specific source location and restore on completion.
*/
withSource(prop: string, loc: Location, cb: () => void) {
if (!this.opts.sourceMaps) return cb();
withSource(prop: string, loc: Location, cb: () => void): void {
if (!this._map) return cb();
// Use the call stack to manage a stack of "source location" data.
let originalLine = this._sourcePosition.line;
@ -300,62 +146,21 @@ export default class Buffer {
this._sourcePosition.filename = originalFilename;
}
/**
* Push a string to the buffer, maintaining indentation and newlines.
*/
getCurrentColumn(): number {
const extra = this._queue.reduce((acc, item) => item[0] + acc, "");
const lastIndex = extra.lastIndexOf("\n");
push(str: string) {
if (!this.format.compact && this._indent && str[0] !== "\n") {
// we've got a newline before us so prepend on the indentation
if (this.endsWith("\n")) str = this.getIndent() + str;
}
// see startTerminatorless() instance method
let parenPushNewlineState = this.parenPushNewlineState;
if (parenPushNewlineState) {
for (let i = 0; i < str.length; i++) {
let cha = str[i];
// we can ignore spaces since they wont interupt a terminatorless separator
if (cha === " ") continue;
this.parenPushNewlineState = null;
if (cha === "\n" || cha === "/") {
// we're going to break this terminator expression so we need to add a parentheses
str = "(" + str;
this.indent();
parenPushNewlineState.printed = true;
}
break;
}
}
// If there the line is ending, adding a new mapping marker is redundant
if (this.opts.sourceMaps && str[0] !== "\n") this.map.mark(this._sourcePosition);
//
this.position.push(str);
this.buf += str;
this.last = str[str.length - 1];
// Clear any state-tracking flags that may have been set.
this._endsWithWord = false;
return lastIndex === -1 ? this._position.column + extra.length : (extra.length - 1 - lastIndex);
}
/**
* Test if the buffer ends with a string.
*/
getCurrentLine(): number {
const extra = this._queue.reduce((acc, item) => item[0] + acc, "");
endsWith(str: string): boolean {
if (Array.isArray(str)) return str.some((s) => this.endsWith(s));
if (str.length === 1) {
return this.last === str;
} else {
return this.buf.slice(-str.length) === str;
let count = 0;
for (let i = 0; i < extra.length; i++) {
if (extra[i] === "\n") count++;
}
}
return this._position.line + count;
}
}

View File

@ -21,7 +21,7 @@ export function BlockStatement(node: Object) {
if (node.directives && node.directives.length) this.newline();
this.printSequence(node.body, node, { indent: true });
if (!this.format.retainLines && !this.format.concise) this.removeLast("\n");
if (!this.format.retainLines && !this.format.concise) this.removeTrailingNewline();
this.source("end", node.loc);
this.rightBrace();

View File

@ -88,7 +88,7 @@ export function Decorator(node: Object) {
function commaSeparatorNewline() {
this.token(",");
this.push("\n");
this.newline();
}
export function CallExpression(node: Object) {

View File

@ -208,15 +208,15 @@ export function DebuggerStatement() {
function variableDeclarationIdent() {
// "let " or "var " indentation.
this.token(",");
this.push("\n");
for (let i = 0; i < 4; i++) this.push(" ");
this.newline();
for (let i = 0; i < 4; i++) this.space(true);
}
function constDeclarationIdent() {
// "const " indentation.
this.token(",");
this.push("\n");
for (let i = 0; i < 6; i++) this.push(" ");
this.newline();
for (let i = 0; i < 6; i++) this.space(true);
}
export function VariableDeclaration(node: Object, parent: Object) {

View File

@ -1,7 +1,6 @@
import detectIndent from "detect-indent";
import Whitespace from "./whitespace";
import SourceMap from "./source-map";
import Position from "./position";
import * as messages from "babel-messages";
import Printer from "./printer";
@ -18,12 +17,11 @@ class Generator extends Printer {
let tokens = ast.tokens || [];
let format = Generator.normalizeOptions(code, opts, tokens);
let position = new Position;
let map = opts.sourceMaps ? new SourceMap(opts, code) : null;
super(position, format);
super(format, map);
this.comments = comments;
this.position = position;
this.tokens = tokens;
this.format = format;
this.opts = opts;
@ -31,7 +29,6 @@ class Generator extends Printer {
this._inForStatementInitCounter = 0;
this.whitespace = new Whitespace(tokens);
this.map = new SourceMap(position, opts, code);
}
format: {
@ -54,8 +51,6 @@ class Generator extends Printer {
auxiliaryCommentBefore: string;
auxiliaryCommentAfter: string;
whitespace: Whitespace;
position: Position;
map: SourceMap;
comments: Array<Object>;
tokens: Array<Object>;
opts: Object;
@ -153,10 +148,7 @@ class Generator extends Printer {
this.print(this.ast);
this.printAuxAfterComment();
return {
map: this.map.get(),
code: this.get()
};
return this._buf.get();
}
}

View File

@ -5,12 +5,257 @@ import Buffer from "./buffer";
import * as n from "./node";
import * as t from "babel-types";
export default class Printer extends Buffer {
constructor(...args) {
super(...args);
export default class Printer {
constructor(format, map) {
this._format = format || {};
this._buf = new Buffer(map);
this.insideAux = false;
this.printAuxAfterOnNextUserNode = false;
this._printStack = [];
this.printedCommentStarts = {};
this.parenPushNewlineState = null;
this._indent = 0;
}
printedCommentStarts: Object;
parenPushNewlineState: ?Object;
/**
* Get the current indent.
*/
_getIndent(): string {
if (this._format.compact || this._format.concise) {
return "";
} else {
return repeat(this._format.indent.style, this._indent);
}
}
/**
* Increment indent size.
*/
indent(): void {
this._indent++;
}
/**
* Decrement indent size.
*/
dedent(): void {
this._indent--;
}
/**
* Add a semicolon to the buffer.
*/
semicolon(): void {
this._append(";", true /* queue */);
}
/**
* Add a right brace to the buffer.
*/
rightBrace(): void {
if (!this.endsWith("\n")) this.newline();
if (this._format.minified && !this._lastPrintedIsEmptyStatement) {
this._buf.removeLastSemicolon();
}
this.token("}");
}
/**
* Add a keyword to the buffer.
*/
keyword(name: string): void {
this.word(name);
this.space();
}
/**
* Add a space to the buffer unless it is compact.
*/
space(force: boolean = false): void {
if (this._format.compact) return;
if ((this._buf.hasContent() && !this.endsWith(" ") && !this.endsWith("\n")) || force) {
this._space();
}
}
/**
* Writes a token that can't be safely parsed without taking whitespace into account.
*/
word(str: string): void {
if (this._endsWithWord) this._space();
this._append(str);
this._endsWithWord = true;
}
/**
* Writes a simple token.
*/
token(str: string): void {
const last = this._buf.getLast();
// space is mandatory to avoid outputting <!--
// http://javascript.spec.whatwg.org/#comment-syntax
if ((str === "--" && last === "!") ||
// Need spaces for operators of the same kind to avoid: `a+++b`
(str[0] === "+" && last === "+") ||
(str[0] === "-" && last === "-")) {
this._space();
}
this._append(str);
}
/**
* Add a newline (or many newlines), maintaining formatting.
*/
newline(i?: number): void {
if (this._format.retainLines || this._format.compact) return;
if (this._format.concise) {
this.space();
return;
}
// never allow more than two lines
if (this.endsWith("\n\n")) return;
if (typeof i !== "number") i = 1;
i = Math.min(2, i);
if (this.endsWith("{\n") || this.endsWith(":\n")) i--;
if (i <= 0) return;
this._buf.removeTrailingSpaces();
for (let j = 0; j < i; j++) {
this._newline();
}
}
endsWith(str: string): boolean {
return this._buf.endsWith(str);
}
removeTrailingNewline(): void {
this._buf.removeTrailingNewline();
}
source(prop: string, loc: Object): void {
this._catchUp(prop, loc);
this._buf.source(prop, loc);
}
withSource(prop: string, loc: Object, cb: () => void): void {
this._catchUp(prop, loc);
this._buf.withSource(prop, loc, cb);
}
_space(): void {
this._append(" ", true /* queue */);
}
_newline(): void {
this._append("\n", true /* queue */);
}
_append(str: string, queue: boolean = false) {
this._maybeAddParen(str);
this._maybeIndent(str);
if (queue) this._buf.queue(str);
else this._buf.append(str);
this._endsWithWord = false;
}
_maybeIndent(str: string): void {
// we've got a newline before us so prepend on the indentation
if (!this._format.compact && this._indent && this.endsWith("\n") && str[0] !== "\n") {
this._buf.queue(this._getIndent());
}
}
_maybeAddParen(str: string): void {
// see startTerminatorless() instance method
let parenPushNewlineState = this.parenPushNewlineState;
if (!parenPushNewlineState) return;
this.parenPushNewlineState = null;
let i;
for (i = 0; i < str.length && str[i] === " "; i++) continue;
if (i === str.length) return;
const cha = str[i];
if (cha === "\n" || cha === "/") {
// we're going to break this terminator expression so we need to add a parentheses
this.token("(");
this.indent();
parenPushNewlineState.printed = true;
}
}
_catchUp(prop: string, loc: Object) {
if (!this._format.retainLines) return;
// catch up to this nodes newline if we're behind
const pos = loc ? loc[prop] : null;
if (pos && pos.line !== null) {
while (this._buf.getCurrentLine() < pos.line) {
this._newline();
}
}
}
/**
* Set some state that will be modified if a newline has been inserted before any
* non-space characters.
*
* This is to prevent breaking semantics for terminatorless separator nodes. eg:
*
* return foo;
*
* returns `foo`. But if we do:
*
* return
* foo;
*
* `undefined` will be returned and not `foo` due to the terminator.
*/
startTerminatorless(): Object {
return this.parenPushNewlineState = {
printed: false
};
}
/**
* Print an ending parentheses if a starting one has been printed.
*/
endTerminatorless(state: Object) {
if (state.printed) {
this.dedent();
this.newline();
this.token(")");
}
}
print(node, parent, opts = {}) {
@ -25,9 +270,9 @@ export default class Printer extends Buffer {
let oldInAux = this.insideAux;
this.insideAux = !node.loc;
let oldConcise = this.format.concise;
let oldConcise = this._format.concise;
if (node._compact) {
this.format.concise = true;
this._format.concise = true;
}
let printMethod = this[node.type];
@ -45,8 +290,6 @@ export default class Printer extends Buffer {
this.printLeadingComments(node, parent);
this.catchUp(node);
this._printNewline(true, node, parent, opts);
if (opts.before) opts.before();
@ -67,14 +310,14 @@ export default class Printer extends Buffer {
this._printStack.pop();
if (opts.after) opts.after();
this.format.concise = oldConcise;
this._format.concise = oldConcise;
this.insideAux = oldInAux;
this._printNewline(false, node, parent, opts);
}
printAuxBeforeComment(wasInAux) {
let comment = this.format.auxiliaryCommentBefore;
let comment = this._format.auxiliaryCommentBefore;
if (!wasInAux && this.insideAux && !this.printAuxAfterOnNextUserNode) {
this.printAuxAfterOnNextUserNode = true;
if (comment) this.printComment({
@ -87,7 +330,7 @@ export default class Printer extends Buffer {
printAuxAfterComment() {
if (this.printAuxAfterOnNextUserNode) {
this.printAuxAfterOnNextUserNode = false;
let comment = this.format.auxiliaryCommentAfter;
let comment = this._format.auxiliaryCommentAfter;
if (comment) this.printComment({
type: "CommentBlock",
value: comment
@ -96,7 +339,7 @@ export default class Printer extends Buffer {
}
getPossibleRaw(node) {
if (this.format.minified) return;
if (this._format.minified) return;
let extra = node.extra;
if (extra && extra.raw != null && extra.rawValue != null && node.value === extra.rawValue) {
@ -195,7 +438,7 @@ export default class Printer extends Buffer {
_printNewline(leading, node, parent, opts) {
// Fast path since 'this.newline' does nothing when not tracking lines.
if (this.format.retainLines || this.format.compact) return;
if (this._format.retainLines || this._format.compact) return;
if (!opts.statement && !n.isUserWhitespacable(node, parent)) {
return;
@ -203,7 +446,7 @@ export default class Printer extends Buffer {
// Fast path for concise since 'this.newline' just inserts a space when
// concise formatting is in use.
if (this.format.concise) {
if (this._format.concise) {
this.space();
return;
}
@ -227,7 +470,7 @@ export default class Printer extends Buffer {
if (needs(node, parent)) lines++;
// generated nodes can't add starting file whitespace
if (!this.buf) lines = 0;
if (!this._buf.hasContent()) lines = 0;
}
this.newline(lines);
@ -240,14 +483,14 @@ export default class Printer extends Buffer {
}
shouldPrintComment(comment) {
if (this.format.shouldPrintComment) {
return this.format.shouldPrintComment(comment.value);
if (this._format.shouldPrintComment) {
return this._format.shouldPrintComment(comment.value);
} else {
if (!this.format.minified &&
if (!this._format.minified &&
(comment.value.indexOf("@license") >= 0 || comment.value.indexOf("@preserve") >= 0)) {
return true;
} else {
return this.format.comments;
return this._format.comments;
}
}
}
@ -264,38 +507,35 @@ export default class Printer extends Buffer {
}
// Exclude comments from source mappings since they will only clutter things.
this.withSource(null, null, () => {
this.catchUp(comment);
this.withSource("start", comment.loc, () => {
// whitespace before
this.newline(this.whitespace.getNewlinesBefore(comment));
if (!this.endsWith(["[", "{"])) this.space();
if (!this.endsWith("[") && !this.endsWith("{")) this.space();
let column = this.position.column;
let val = this.generateComment(comment);
//
if (comment.type === "CommentBlock" && this.format.indent.adjustMultilineComment) {
if (comment.type === "CommentBlock" && this._format.indent.adjustMultilineComment) {
let offset = comment.loc && comment.loc.start.column;
if (offset) {
let newlineRegex = new RegExp("\\n\\s{1," + offset + "}", "g");
val = val.replace(newlineRegex, "\n");
}
let indent = Math.max(this.indentSize(), column);
val = val.replace(/\n/g, `\n${repeat(" ", indent)}`);
let indentSize = Math.max(this._getIndent().length, this._buf.getCurrentColumn());
val = val.replace(/\n(?!$)/g, `\n${repeat(" ", indentSize)}`);
}
// force a newline for line comments when retainLines is set in case the next printed node
// doesn't catch up
if ((this.format.compact || this.format.concise || this.format.retainLines) &&
if ((this._format.compact || this._format.concise || this._format.retainLines) &&
comment.type === "CommentLine") {
val += "\n";
}
//
this.push(val);
this.token(val);
// whitespace after
this.newline(this.whitespace.getNewlinesAfter(comment));

View File

@ -1,12 +1,12 @@
import sourceMap from "source-map";
import type Position from "./position";
/**
* Build a sourcemap.
*/
export default class SourceMap {
constructor(position, opts, code) {
this.position = position;
constructor(opts, code) {
this.opts = opts;
this.last = {generated: {}, original: {}};
@ -46,35 +46,33 @@ export default class SourceMap {
* values to insert a mapping to nothing.
*/
mark(sourcePos: Object) {
mark(position: Position, line: number, column: number, filename: ?string) {
let map = this.map;
if (!map) return; // no source map
let position = this.position;
// Adding an empty mapping at the start of a generated line just clutters the map.
if (this._lastGenLine !== position.line && sourcePos.line === null) return;
if (this._lastGenLine !== position.line && line === null) return;
// If this mapping points to the same source location as the last one, we can ignore it since
// the previous one covers it.
if (this._lastGenLine === position.line && this._lastSourceLine === sourcePos.line &&
this._lastSourceColumn === sourcePos.column) {
if (this._lastGenLine === position.line && this._lastSourceLine === line &&
this._lastSourceColumn === column) {
return;
}
this._lastGenLine = position.line;
this._lastSourceLine = sourcePos.line;
this._lastSourceColumn = sourcePos.column;
this._lastSourceLine = line;
this._lastSourceColumn = column;
map.addMapping({
generated: {
line: position.line,
column: position.column
},
source: sourcePos.line == null ? null : sourcePos.filename || this.opts.sourceFileName,
original: sourcePos.line == null ? null : {
line: sourcePos.line,
column: sourcePos.column,
source: line == null ? null : filename || this.opts.sourceFileName,
original: line == null ? null : {
line: line,
column: column,
},
});
}

View File

@ -1,4 +1,6 @@
function foo(l){
return(
l);}
l);
}

View File

@ -1,10 +1,11 @@
function foo(l) {
return (
l);}
l);
}
function foo() {
return (
1 && 2 ||
3);}
3);
}