Faster tokenizer lookahead (#13341)

* refactor: simplify token context structure

* add benchmark

* perf: return a sub-state on tokenizer lookahead

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* Update packages/babel-parser/src/tokenizer/index.js

Co-authored-by: Brian Ng <bng412@gmail.com>

* remove irrelevant comment

* fix: guard curPosition with isLookahead

* add test cases

Co-authored-by: Brian Ng <bng412@gmail.com>
This commit is contained in:
Huáng Jùnliàng
2021-05-25 21:12:38 -04:00
committed by GitHub
parent b1f57e5fb5
commit acf2a10899
10 changed files with 684 additions and 29 deletions

View File

@@ -0,0 +1,2 @@
/*1*/ export /*2*/ { /*3*/ A /*4*/, /*5*/ B /*6*/ as /*7*/ C /*8*/ } /*9*/ from /*10*/ "foo";
/*1*/ export /*2*/ * /*3*/ from /*4*/ "foo"

View File

@@ -0,0 +1,242 @@
{
"type": "File",
"start":0,"end":137,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":43}},
"program": {
"type": "Program",
"start":0,"end":137,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":43}},
"sourceType": "module",
"interpreter": null,
"body": [
{
"type": "ExportNamedDeclaration",
"start":6,"end":93,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":93}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":94,"end":99,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
}
],
"exportKind": "value",
"specifiers": [
{
"type": "ExportSpecifier",
"start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "2",
"start":13,"end":18,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "4",
"start":29,"end":34,"loc":{"start":{"line":1,"column":29},"end":{"line":1,"column":34}}
},
{
"type": "CommentBlock",
"value": "5",
"start":36,"end":41,"loc":{"start":{"line":1,"column":36},"end":{"line":1,"column":41}}
}
],
"local": {
"type": "Identifier",
"start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28},"identifierName":"A"},
"name": "A"
},
"exported": {
"type": "Identifier",
"start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28},"identifierName":"A"},
"name": "A"
}
},
{
"type": "ExportSpecifier",
"start":42,"end":60,"loc":{"start":{"line":1,"column":42},"end":{"line":1,"column":60}},
"trailingComments": [
{
"type": "CommentBlock",
"value": "8",
"start":61,"end":66,"loc":{"start":{"line":1,"column":61},"end":{"line":1,"column":66}}
},
{
"type": "CommentBlock",
"value": "9",
"start":69,"end":74,"loc":{"start":{"line":1,"column":69},"end":{"line":1,"column":74}}
},
{
"type": "CommentBlock",
"value": "10",
"start":80,"end":86,"loc":{"start":{"line":1,"column":80},"end":{"line":1,"column":86}}
}
],
"local": {
"type": "Identifier",
"start":42,"end":43,"loc":{"start":{"line":1,"column":42},"end":{"line":1,"column":43},"identifierName":"B"},
"trailingComments": [
{
"type": "CommentBlock",
"value": "6",
"start":44,"end":49,"loc":{"start":{"line":1,"column":44},"end":{"line":1,"column":49}}
}
],
"name": "B"
},
"exported": {
"type": "Identifier",
"start":59,"end":60,"loc":{"start":{"line":1,"column":59},"end":{"line":1,"column":60},"identifierName":"C"},
"leadingComments": [
{
"type": "CommentBlock",
"value": "6",
"start":44,"end":49,"loc":{"start":{"line":1,"column":44},"end":{"line":1,"column":49}}
},
{
"type": "CommentBlock",
"value": "7",
"start":53,"end":58,"loc":{"start":{"line":1,"column":53},"end":{"line":1,"column":58}}
}
],
"name": "C"
}
}
],
"source": {
"type": "StringLiteral",
"start":87,"end":92,"loc":{"start":{"line":1,"column":87},"end":{"line":1,"column":92}},
"extra": {
"rawValue": "foo",
"raw": "\"foo\""
},
"value": "foo"
},
"declaration": null
},
{
"type": "ExportAllDeclaration",
"start":100,"end":137,"loc":{"start":{"line":2,"column":6},"end":{"line":2,"column":43}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":94,"end":99,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
}
],
"exportKind": "value",
"source": {
"type": "StringLiteral",
"start":132,"end":137,"loc":{"start":{"line":2,"column":38},"end":{"line":2,"column":43}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "2",
"start":107,"end":112,"loc":{"start":{"line":2,"column":13},"end":{"line":2,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":115,"end":120,"loc":{"start":{"line":2,"column":21},"end":{"line":2,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":126,"end":131,"loc":{"start":{"line":2,"column":32},"end":{"line":2,"column":37}}
}
],
"extra": {
"rawValue": "foo",
"raw": "\"foo\""
},
"value": "foo"
}
}
],
"directives": []
},
"comments": [
{
"type": "CommentBlock",
"value": "1",
"start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
},
{
"type": "CommentBlock",
"value": "2",
"start":13,"end":18,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":29,"end":34,"loc":{"start":{"line":1,"column":29},"end":{"line":1,"column":34}}
},
{
"type": "CommentBlock",
"value": "5",
"start":36,"end":41,"loc":{"start":{"line":1,"column":36},"end":{"line":1,"column":41}}
},
{
"type": "CommentBlock",
"value": "6",
"start":44,"end":49,"loc":{"start":{"line":1,"column":44},"end":{"line":1,"column":49}}
},
{
"type": "CommentBlock",
"value": "7",
"start":53,"end":58,"loc":{"start":{"line":1,"column":53},"end":{"line":1,"column":58}}
},
{
"type": "CommentBlock",
"value": "8",
"start":61,"end":66,"loc":{"start":{"line":1,"column":61},"end":{"line":1,"column":66}}
},
{
"type": "CommentBlock",
"value": "9",
"start":69,"end":74,"loc":{"start":{"line":1,"column":69},"end":{"line":1,"column":74}}
},
{
"type": "CommentBlock",
"value": "10",
"start":80,"end":86,"loc":{"start":{"line":1,"column":80},"end":{"line":1,"column":86}}
},
{
"type": "CommentBlock",
"value": "1",
"start":94,"end":99,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
},
{
"type": "CommentBlock",
"value": "2",
"start":107,"end":112,"loc":{"start":{"line":2,"column":13},"end":{"line":2,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":115,"end":120,"loc":{"start":{"line":2,"column":21},"end":{"line":2,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":126,"end":131,"loc":{"start":{"line":2,"column":32},"end":{"line":2,"column":37}}
}
]
}

View File

@@ -0,0 +1,2 @@
/*1*/ import /*2*/ D /*3*/, /*4*/ { /*5*/ A /*6*/, /*7*/ B /*8*/ as /*9*/ C /*10*/ } /*11*/ from /*12*/ "foo";
/*1*/ import /*2*/ * /*3*/ as /*4*/ foo /*5*/ from /*6*/ "foo";

View File

@@ -0,0 +1,331 @@
{
"type": "File",
"start":0,"end":174,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":63}},
"program": {
"type": "Program",
"start":0,"end":174,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":63}},
"sourceType": "module",
"interpreter": null,
"body": [
{
"type": "ImportDeclaration",
"start":6,"end":110,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":110}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":111,"end":116,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
}
],
"importKind": "value",
"specifiers": [
{
"type": "ImportDefaultSpecifier",
"start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "2",
"start":13,"end":18,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":18}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "3",
"start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
}
],
"local": {
"type": "Identifier",
"start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"D"},
"name": "D"
}
},
{
"type": "ImportSpecifier",
"start":42,"end":43,"loc":{"start":{"line":1,"column":42},"end":{"line":1,"column":43}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
},
{
"type": "CommentBlock",
"value": "2",
"start":13,"end":18,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":28,"end":33,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":33}}
},
{
"type": "CommentBlock",
"value": "5",
"start":36,"end":41,"loc":{"start":{"line":1,"column":36},"end":{"line":1,"column":41}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "6",
"start":44,"end":49,"loc":{"start":{"line":1,"column":44},"end":{"line":1,"column":49}}
},
{
"type": "CommentBlock",
"value": "7",
"start":51,"end":56,"loc":{"start":{"line":1,"column":51},"end":{"line":1,"column":56}}
}
],
"imported": {
"type": "Identifier",
"start":42,"end":43,"loc":{"start":{"line":1,"column":42},"end":{"line":1,"column":43},"identifierName":"A"},
"name": "A"
},
"local": {
"type": "Identifier",
"start":42,"end":43,"loc":{"start":{"line":1,"column":42},"end":{"line":1,"column":43},"identifierName":"A"},
"name": "A"
}
},
{
"type": "ImportSpecifier",
"start":57,"end":75,"loc":{"start":{"line":1,"column":57},"end":{"line":1,"column":75}},
"trailingComments": [
{
"type": "CommentBlock",
"value": "10",
"start":76,"end":82,"loc":{"start":{"line":1,"column":76},"end":{"line":1,"column":82}}
},
{
"type": "CommentBlock",
"value": "11",
"start":85,"end":91,"loc":{"start":{"line":1,"column":85},"end":{"line":1,"column":91}}
},
{
"type": "CommentBlock",
"value": "12",
"start":97,"end":103,"loc":{"start":{"line":1,"column":97},"end":{"line":1,"column":103}}
}
],
"imported": {
"type": "Identifier",
"start":57,"end":58,"loc":{"start":{"line":1,"column":57},"end":{"line":1,"column":58},"identifierName":"B"},
"trailingComments": [
{
"type": "CommentBlock",
"value": "8",
"start":59,"end":64,"loc":{"start":{"line":1,"column":59},"end":{"line":1,"column":64}}
}
],
"name": "B"
},
"local": {
"type": "Identifier",
"start":74,"end":75,"loc":{"start":{"line":1,"column":74},"end":{"line":1,"column":75},"identifierName":"C"},
"leadingComments": [
{
"type": "CommentBlock",
"value": "8",
"start":59,"end":64,"loc":{"start":{"line":1,"column":59},"end":{"line":1,"column":64}}
},
{
"type": "CommentBlock",
"value": "9",
"start":68,"end":73,"loc":{"start":{"line":1,"column":68},"end":{"line":1,"column":73}}
}
],
"name": "C"
}
}
],
"source": {
"type": "StringLiteral",
"start":104,"end":109,"loc":{"start":{"line":1,"column":104},"end":{"line":1,"column":109}},
"extra": {
"rawValue": "foo",
"raw": "\"foo\""
},
"value": "foo"
}
},
{
"type": "ImportDeclaration",
"start":117,"end":174,"loc":{"start":{"line":2,"column":6},"end":{"line":2,"column":63}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "1",
"start":111,"end":116,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
}
],
"importKind": "value",
"specifiers": [
{
"type": "ImportNamespaceSpecifier",
"start":130,"end":150,"loc":{"start":{"line":2,"column":19},"end":{"line":2,"column":39}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "2",
"start":124,"end":129,"loc":{"start":{"line":2,"column":13},"end":{"line":2,"column":18}}
}
],
"trailingComments": [
{
"type": "CommentBlock",
"value": "5",
"start":151,"end":156,"loc":{"start":{"line":2,"column":40},"end":{"line":2,"column":45}}
}
],
"local": {
"type": "Identifier",
"start":147,"end":150,"loc":{"start":{"line":2,"column":36},"end":{"line":2,"column":39},"identifierName":"foo"},
"leadingComments": [
{
"type": "CommentBlock",
"value": "3",
"start":132,"end":137,"loc":{"start":{"line":2,"column":21},"end":{"line":2,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":141,"end":146,"loc":{"start":{"line":2,"column":30},"end":{"line":2,"column":35}}
}
],
"name": "foo"
}
}
],
"source": {
"type": "StringLiteral",
"start":168,"end":173,"loc":{"start":{"line":2,"column":57},"end":{"line":2,"column":62}},
"leadingComments": [
{
"type": "CommentBlock",
"value": "5",
"start":151,"end":156,"loc":{"start":{"line":2,"column":40},"end":{"line":2,"column":45}}
},
{
"type": "CommentBlock",
"value": "6",
"start":162,"end":167,"loc":{"start":{"line":2,"column":51},"end":{"line":2,"column":56}}
}
],
"extra": {
"rawValue": "foo",
"raw": "\"foo\""
},
"value": "foo"
}
}
],
"directives": []
},
"comments": [
{
"type": "CommentBlock",
"value": "1",
"start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
},
{
"type": "CommentBlock",
"value": "2",
"start":13,"end":18,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":28,"end":33,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":33}}
},
{
"type": "CommentBlock",
"value": "5",
"start":36,"end":41,"loc":{"start":{"line":1,"column":36},"end":{"line":1,"column":41}}
},
{
"type": "CommentBlock",
"value": "6",
"start":44,"end":49,"loc":{"start":{"line":1,"column":44},"end":{"line":1,"column":49}}
},
{
"type": "CommentBlock",
"value": "7",
"start":51,"end":56,"loc":{"start":{"line":1,"column":51},"end":{"line":1,"column":56}}
},
{
"type": "CommentBlock",
"value": "8",
"start":59,"end":64,"loc":{"start":{"line":1,"column":59},"end":{"line":1,"column":64}}
},
{
"type": "CommentBlock",
"value": "9",
"start":68,"end":73,"loc":{"start":{"line":1,"column":68},"end":{"line":1,"column":73}}
},
{
"type": "CommentBlock",
"value": "10",
"start":76,"end":82,"loc":{"start":{"line":1,"column":76},"end":{"line":1,"column":82}}
},
{
"type": "CommentBlock",
"value": "11",
"start":85,"end":91,"loc":{"start":{"line":1,"column":85},"end":{"line":1,"column":91}}
},
{
"type": "CommentBlock",
"value": "12",
"start":97,"end":103,"loc":{"start":{"line":1,"column":97},"end":{"line":1,"column":103}}
},
{
"type": "CommentBlock",
"value": "1",
"start":111,"end":116,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":5}}
},
{
"type": "CommentBlock",
"value": "2",
"start":124,"end":129,"loc":{"start":{"line":2,"column":13},"end":{"line":2,"column":18}}
},
{
"type": "CommentBlock",
"value": "3",
"start":132,"end":137,"loc":{"start":{"line":2,"column":21},"end":{"line":2,"column":26}}
},
{
"type": "CommentBlock",
"value": "4",
"start":141,"end":146,"loc":{"start":{"line":2,"column":30},"end":{"line":2,"column":35}}
},
{
"type": "CommentBlock",
"value": "5",
"start":151,"end":156,"loc":{"start":{"line":2,"column":40},"end":{"line":2,"column":45}}
},
{
"type": "CommentBlock",
"value": "6",
"start":162,"end":167,"loc":{"start":{"line":2,"column":51},"end":{"line":2,"column":56}}
}
]
}