Faster tokenizer lookahead (#13341)

* refactor: simplify token context structure

* add benchmark

* perf: return a sub-state on tokenizer lookahead

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* remove irrelevant comment

* fix: guard curPosition with isLookahead

* add test cases

Co-authored-by: Brian Ng <bng412@gmail.com>
This commit is contained in:
Huáng Jùnliàng
2021-05-25 21:12:38 -04:00
committed by GitHub
parent b1f57e5fb5
commit acf2a10899
10 changed files with 684 additions and 29 deletions

View File

@@ -178,3 +178,14 @@ export default class State {
return state;
}
}
export type LookaheadState = {
pos: number,
value: any,
type: TokenType,
start: number,
end: number,
/* Used only in readSlashToken */
exprAllowed: boolean,
inType: boolean,
};