Make tokens optional (#563)

Adding tokens to the ast is significant slower and most tools
don't ever use them anyway
This commit is contained in:
Daniel Tschinder 2017-06-27 20:26:24 -07:00 committed by Henry Zhu
parent 3d03414c05
commit fecdb6feeb
12 changed files with 786 additions and 7 deletions

View File

@ -63,6 +63,8 @@ mind. When in doubt, use `.parse()`.
- **ranges**: Adds a `ranges` property to each node: `[node.start, node.end]`
- **tokens**: Adds all parsed tokens to a `tokens` property on the `File` node
### Output
Babylon generates AST according to [Babel AST format][].

View File

@ -13,6 +13,7 @@ export type Options = {
plugins: $ReadOnlyArray<string>;
strictMode: ?boolean;
ranges: boolean;
tokens: boolean;
};
export const defaultOptions: Options = {
@ -44,6 +45,8 @@ export const defaultOptions: Options = {
//
// [range]: https://bugzilla.mozilla.org/show_bug.cgi?id=745678
ranges: false,
// Adds all parsed tokens to a `tokens` property on the `File` node
tokens: false,
};
// Interpret and default an options object

View File

@ -16,7 +16,6 @@ export default class BaseParser {
state: State;
input: string;
isReservedWord(word: string): boolean {
if (word === "await") {
return this.inModule;

View File

@ -28,9 +28,10 @@ export default class StatementParser extends ExpressionParser {
this.parseBlockBody(program, true, true, tt.eof);
file.program = this.finishNode(program, "Program");
file.program = this.finishNode(program, "Program");
file.comments = this.state.comments;
file.tokens = this.state.tokens;
if (this.options.tokens) file.tokens = this.state.tokens;
return this.finishNode(file, "File");
}

View File

@ -84,7 +84,7 @@ export default class Tokenizer extends LocationParser {
// Move to the next token
next(): void {
if (!this.isLookahead) {
if (this.options.tokens && !this.isLookahead) {
this.state.tokens.push(new Token(this.state));
}
@ -199,7 +199,7 @@ export default class Tokenizer extends LocationParser {
};
if (!this.isLookahead) {
this.state.tokens.push(comment);
if (this.options.tokens) this.state.tokens.push(comment);
this.state.comments.push(comment);
this.addComment(comment);
}

View File

@ -0,0 +1,3 @@
var a = 1;
var b = a + 1;

View File

@ -0,0 +1,207 @@
{
"type": "File",
"start": 0,
"end": 26,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"program": {
"type": "Program",
"start": 0,
"end": 26,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"sourceType": "script",
"body": [
{
"type": "VariableDeclaration",
"start": 0,
"end": 10,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 10
}
},
"declarations": [
{
"type": "VariableDeclarator",
"start": 4,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 9
}
},
"id": {
"type": "Identifier",
"start": 4,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 5
},
"identifierName": "a"
},
"name": "a"
},
"init": {
"type": "NumericLiteral",
"start": 8,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 8
},
"end": {
"line": 1,
"column": 9
}
},
"extra": {
"rawValue": 1,
"raw": "1"
},
"value": 1
}
}
],
"kind": "var"
},
{
"type": "VariableDeclaration",
"start": 12,
"end": 26,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"declarations": [
{
"type": "VariableDeclarator",
"start": 16,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 4
},
"end": {
"line": 3,
"column": 13
}
},
"id": {
"type": "Identifier",
"start": 16,
"end": 17,
"loc": {
"start": {
"line": 3,
"column": 4
},
"end": {
"line": 3,
"column": 5
},
"identifierName": "b"
},
"name": "b"
},
"init": {
"type": "BinaryExpression",
"start": 20,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 8
},
"end": {
"line": 3,
"column": 13
}
},
"left": {
"type": "Identifier",
"start": 20,
"end": 21,
"loc": {
"start": {
"line": 3,
"column": 8
},
"end": {
"line": 3,
"column": 9
},
"identifierName": "a"
},
"name": "a"
},
"operator": "+",
"right": {
"type": "NumericLiteral",
"start": 24,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 12
},
"end": {
"line": 3,
"column": 13
}
},
"extra": {
"rawValue": 1,
"raw": "1"
},
"value": 1
}
}
}
],
"kind": "var"
}
],
"directives": []
}
}

View File

@ -0,0 +1,3 @@
{
"tokens": false
}

View File

@ -0,0 +1,3 @@
var a = 1;
var b = a + 1;

View File

@ -0,0 +1,556 @@
{
"type": "File",
"start": 0,
"end": 26,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"program": {
"type": "Program",
"start": 0,
"end": 26,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"sourceType": "script",
"body": [
{
"type": "VariableDeclaration",
"start": 0,
"end": 10,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 10
}
},
"declarations": [
{
"type": "VariableDeclarator",
"start": 4,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 9
}
},
"id": {
"type": "Identifier",
"start": 4,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 5
},
"identifierName": "a"
},
"name": "a"
},
"init": {
"type": "NumericLiteral",
"start": 8,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 8
},
"end": {
"line": 1,
"column": 9
}
},
"extra": {
"rawValue": 1,
"raw": "1"
},
"value": 1
}
}
],
"kind": "var"
},
{
"type": "VariableDeclaration",
"start": 12,
"end": 26,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 14
}
},
"declarations": [
{
"type": "VariableDeclarator",
"start": 16,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 4
},
"end": {
"line": 3,
"column": 13
}
},
"id": {
"type": "Identifier",
"start": 16,
"end": 17,
"loc": {
"start": {
"line": 3,
"column": 4
},
"end": {
"line": 3,
"column": 5
},
"identifierName": "b"
},
"name": "b"
},
"init": {
"type": "BinaryExpression",
"start": 20,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 8
},
"end": {
"line": 3,
"column": 13
}
},
"left": {
"type": "Identifier",
"start": 20,
"end": 21,
"loc": {
"start": {
"line": 3,
"column": 8
},
"end": {
"line": 3,
"column": 9
},
"identifierName": "a"
},
"name": "a"
},
"operator": "+",
"right": {
"type": "NumericLiteral",
"start": 24,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 12
},
"end": {
"line": 3,
"column": 13
}
},
"extra": {
"rawValue": 1,
"raw": "1"
},
"value": 1
}
}
}
],
"kind": "var"
}
],
"directives": []
},
"tokens": [
{
"type": {
"label": "var",
"keyword": "var",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": "var",
"start": 0,
"end": 3,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 3
}
}
},
{
"type": {
"label": "name",
"beforeExpr": false,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null
},
"value": "a",
"start": 4,
"end": 5,
"loc": {
"start": {
"line": 1,
"column": 4
},
"end": {
"line": 1,
"column": 5
}
}
},
{
"type": {
"label": "=",
"beforeExpr": true,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": true,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": "=",
"start": 6,
"end": 7,
"loc": {
"start": {
"line": 1,
"column": 6
},
"end": {
"line": 1,
"column": 7
}
}
},
{
"type": {
"label": "num",
"beforeExpr": false,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": 1,
"start": 8,
"end": 9,
"loc": {
"start": {
"line": 1,
"column": 8
},
"end": {
"line": 1,
"column": 9
}
}
},
{
"type": {
"label": ";",
"beforeExpr": true,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start": 9,
"end": 10,
"loc": {
"start": {
"line": 1,
"column": 9
},
"end": {
"line": 1,
"column": 10
}
}
},
{
"type": {
"label": "var",
"keyword": "var",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": "var",
"start": 12,
"end": 15,
"loc": {
"start": {
"line": 3,
"column": 0
},
"end": {
"line": 3,
"column": 3
}
}
},
{
"type": {
"label": "name",
"beforeExpr": false,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null
},
"value": "b",
"start": 16,
"end": 17,
"loc": {
"start": {
"line": 3,
"column": 4
},
"end": {
"line": 3,
"column": 5
}
}
},
{
"type": {
"label": "=",
"beforeExpr": true,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": true,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": "=",
"start": 18,
"end": 19,
"loc": {
"start": {
"line": 3,
"column": 6
},
"end": {
"line": 3,
"column": 7
}
}
},
{
"type": {
"label": "name",
"beforeExpr": false,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null
},
"value": "a",
"start": 20,
"end": 21,
"loc": {
"start": {
"line": 3,
"column": 8
},
"end": {
"line": 3,
"column": 9
}
}
},
{
"type": {
"label": "+/-",
"beforeExpr": true,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": true,
"postfix": false,
"binop": 9,
"updateContext": null
},
"value": "+",
"start": 22,
"end": 23,
"loc": {
"start": {
"line": 3,
"column": 10
},
"end": {
"line": 3,
"column": 11
}
}
},
{
"type": {
"label": "num",
"beforeExpr": false,
"startsExpr": true,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"value": 1,
"start": 24,
"end": 25,
"loc": {
"start": {
"line": 3,
"column": 12
},
"end": {
"line": 3,
"column": 13
}
}
},
{
"type": {
"label": ";",
"beforeExpr": true,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start": 25,
"end": 26,
"loc": {
"start": {
"line": 3,
"column": 13
},
"end": {
"line": 3,
"column": 14
}
}
},
{
"type": {
"label": "eof",
"beforeExpr": false,
"startsExpr": false,
"rightAssociative": false,
"isLoop": false,
"isAssign": false,
"prefix": false,
"postfix": false,
"binop": null,
"updateContext": null
},
"start": 26,
"end": 26,
"loc": {
"start": {
"line": 3,
"column": 14
},
"end": {
"line": 3,
"column": 14
}
}
}
]
}

View File

@ -0,0 +1,3 @@
{
"tokens": true
}

View File

@ -80,7 +80,6 @@ function runTest(test, parseFunction) {
throw err;
}
delete ast.tokens;
if (ast.comments && !ast.comments.length) delete ast.comments;
if (!test.expect.code && !opts.throws && !process.env.CI) {
@ -133,7 +132,7 @@ function misMatch(exp, act) {
}
for (var prop in act) {
if (prop === "__clone") {
if (typeof act[prop] === "function") {
continue;
}