Faster tokenizer lookahead (#13341)

* refactor: simplify token context structure

* add benchmark

* perf: return a sub-state on tokenizer lookahead

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* remove irrelevant comment

* fix: guard curPosition with isLookahead

* add test cases

Co-authored-by: Brian Ng <bng412@gmail.com>
This commit is contained in:
Huáng Jùnliàng
2021-05-25 21:12:38 -04:00
committed by GitHub
parent b1f57e5fb5
commit acf2a10899
10 changed files with 684 additions and 29 deletions

View File

@@ -15,6 +15,10 @@ import { isIdentifierChar, isIdentifierStart } from "../../util/identifier";
import type { Position } from "../../util/location";
import { isNewLine } from "../../util/whitespace";
import { Errors, makeErrorTemplates, ErrorCodes } from "../../parser/error";
import type { LookaheadState } from "../../tokenizer/state";
import State from "../../tokenizer/state";
type JSXLookaheadState = LookaheadState & { inPropertyName: boolean };
const HEX_NUMBER = /^[\da-fA-F]+$/;
const DECIMAL_NUMBER = /^\d+$/;
@@ -573,6 +577,14 @@ export default (superClass: Class<Parser>): Class<Parser> =>
}
}
createLookaheadState(state: State): JSXLookaheadState {
const lookaheadState = ((super.createLookaheadState(
state,
): any): JSXLookaheadState);
lookaheadState.inPropertyName = state.inPropertyName;
return lookaheadState;
}
getTokenFromCode(code: number): void {
if (this.state.inPropertyName) return super.getTokenFromCode(code);