cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

finish reorganisation of babylon

Browse Source
This commit is contained in:
Sebastian McKenzie
2015-07-25 08:03:39 +01:00
parent 7e7ad89724
commit 2598299e64
20 changed files with 748 additions and 776 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
packages/babylon/src/index.js
View File

@@ -1,18 +1,18 @@
import { Parser, plugins } from "./state";
import "./parseutil";
import "./statement";
import "./lval";
import "./expression";
import "./node";
import "./location";
import "./lookahead";
import Parser, { plugins } from "./parser";
import "./parser/util";
import "./parser/statement";
import "./parser/lval";
import "./parser/expression";
import "./parser/node";
import "./parser/location";
import "./parser/comments";
import { types as tokTypes } from "./tokenizer/types";
import "./tokenizer";
import "./tokenizer/context";
import "./comments";
import flowPlugin from "./plugins/flow";
import jsxPlugin from "./plugins/jsx";
plugins.flow = flowPlugin;
plugins.jsx = jsxPlugin;

59
packages/babylon/src/lookahead.js
View File

@@ -1,59 +0,0 @@
import { Parser } from "./state";
const pp = Parser.prototype;
var STATE_KEYS = [
"lastTokStartLoc",
"lastTokEndLoc",
"lastTokStart",
"lastTokEnd",
"lineStart",
"startLoc",
"curLine",
"endLoc",
"start",
"pos",
"end",
"type",
"value",
"exprAllowed",
"potentialArrowAt",
"currLine",
"input",
"inType",
"inFunction",
"inGenerator",
"labels",
"tokens",
"comments"
];
pp.getState = function () {
var state = {};
for (var key of (STATE_KEYS: Array)) {
state[key] = this[key];
}
state.comments = this.comments.slice();
state.context = this.context.slice();
state.tokens = this.tokens.slice();
state.labels = this.labels.slice();
return state;
};
pp.setState = function (state) {
for (var key in state) {
this[key] = state[key];
}
};
pp.lookahead = function () {
var old = this.getState();
this.isLookahead = true;
this.next();
this.isLookahead = false;
var curr = this.getState();
this.setState(old);
return curr;
};

34
packages/babylon/src/comments.js → packages/babylon/src/parser/comments.js
View File

@@ -22,7 +22,7 @@
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
import { Parser } from "./state";
import Parser from "./index";
function last(stack) {
return stack[stack.length - 1];
@@ -31,25 +31,25 @@ function last(stack) {
const pp = Parser.prototype;
pp.addComment = function (comment) {
this.trailingComments.push(comment);
this.leadingComments.push(comment);
this.state.trailingComments.push(comment);
this.state.leadingComments.push(comment);
};
pp.processComment = function (node) {
if (node.type === "Program" && node.body.length > 0) return;
var stack = this.bottomRightStack;
var stack = this.state.bottomRightStack;
var lastChild, trailingComments, i;
if (this.trailingComments.length > 0) {
if (this.state.trailingComments.length > 0) {
// If the first comment in trailingComments comes after the
// current node, then we're good - all comments in the array will
// come after the node and so it's safe to add then as official
// trailingComments.
if (this.trailingComments[0].start >= node.end) {
trailingComments = this.trailingComments;
this.trailingComments = [];
if (this.state.trailingComments[0].start >= node.end) {
trailingComments = this.state.trailingComments;
this.state.trailingComments = [];
} else {
// Otherwise, if the first comment doesn't come after the
// current node, that means we have a mix of leading and trailing
@@ -57,7 +57,7 @@ pp.processComment = function (node) {
// same items as trailingComments. Reset trailingComments to
// zero items and we'll handle this by evaluating leadingComments
// later.
this.trailingComments.length = 0;
this.state.trailingComments.length = 0;
}
} else {
var lastInStack = last(stack);
@@ -89,10 +89,10 @@ pp.processComment = function (node) {
}
}
}
} else if (this.leadingComments.length > 0) {
if (last(this.leadingComments).end <= node.start) {
node.leadingComments = this.leadingComments;
this.leadingComments = [];
} else if (this.state.leadingComments.length > 0) {
if (last(this.state.leadingComments).end <= node.start) {
node.leadingComments = this.state.leadingComments;
this.state.leadingComments = [];
} else {
// https://github.com/eslint/espree/issues/2
//
@@ -105,8 +105,8 @@ pp.processComment = function (node) {
// This loop figures out the stopping point between the actual
// leading and trailing comments by finding the location of the
// first comment that comes after the given node.
for (i = 0; i < this.leadingComments.length; i++) {
if (this.leadingComments[i].end > node.start) {
for (i = 0; i < this.state.leadingComments.length; i++) {
if (this.state.leadingComments[i].end > node.start) {
break;
}
}
@@ -115,14 +115,14 @@ pp.processComment = function (node) {
// that comes after the node. Keep in mind that this could
// result in an empty array, and if so, the array must be
// deleted.
node.leadingComments = this.leadingComments.slice(0, i);
node.leadingComments = this.state.leadingComments.slice(0, i);
if (node.leadingComments.length === 0) {
node.leadingComments = null;
}
// Similarly, trailing comments are attached later. The variable
// must be reset to null if there are no trailing comments.
trailingComments = this.leadingComments.slice(i);
trailingComments = this.state.leadingComments.slice(i);
if (trailingComments.length === 0) {
trailingComments = null;
}

190
packages/babylon/src/expression.js → packages/babylon/src/parser/expression.js Executable file → Normal file
View File

@@ -16,9 +16,9 @@
//
// [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser
import { types as tt } from "./tokenizer/types";
import { Parser } from "./state";
import { reservedWords } from "./util/identifier";
import { types as tt } from "../tokenizer/types";
import Parser from "./index";
import { reservedWords } from "../util/identifier";
const pp = Parser.prototype;
@@ -60,9 +60,9 @@ pp.checkPropClash = function (prop, propHash) {
// delayed syntax error at correct position).
pp.parseExpression = function (noIn, refShorthandDefaultPos) {
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
let expr = this.parseMaybeAssign(noIn, refShorthandDefaultPos);
if (this.type === tt.comma) {
if (this.state.type === tt.comma) {
let node = this.startNodeAt(startPos, startLoc);
node.expressions = [expr];
while (this.eat(tt.comma)) {
@@ -78,7 +78,7 @@ pp.parseExpression = function (noIn, refShorthandDefaultPos) {
// operators like `+=`.
pp.parseMaybeAssign = function (noIn, refShorthandDefaultPos, afterLeftParse) {
if (this.type === tt._yield && this.inGenerator) {
if (this.state.type === tt._yield && this.state.inGenerator) {
return this.parseYield();
}
@@ -89,16 +89,16 @@ pp.parseMaybeAssign = function (noIn, refShorthandDefaultPos, afterLeftParse) {
} else {
failOnShorthandAssign = false;
}
let startPos = this.start, startLoc = this.startLoc;
if (this.type === tt.parenL || this.type === tt.name) {
this.potentialArrowAt = this.start;
let startPos = this.state.start, startLoc = this.state.startLoc;
if (this.state.type === tt.parenL || this.state.type === tt.name) {
this.state.potentialArrowAt = this.state.start;
}
let left = this.parseMaybeConditional(noIn, refShorthandDefaultPos);
if (afterLeftParse) left = afterLeftParse.call(this, left, startPos, startLoc);
if (this.type.isAssign) {
if (this.state.type.isAssign) {
let node = this.startNodeAt(startPos, startLoc);
node.operator = this.value;
node.left = this.type === tt.eq ? this.toAssignable(left) : left;
node.operator = this.state.value;
node.left = this.state.type === tt.eq ? this.toAssignable(left) : left;
refShorthandDefaultPos.start = 0; // reset because shorthand default was used correctly
this.checkLVal(left);
if (left.parenthesizedExpression) {
@@ -124,7 +124,7 @@ pp.parseMaybeAssign = function (noIn, refShorthandDefaultPos, afterLeftParse) {
// Parse a ternary conditional (`?:`) operator.
pp.parseMaybeConditional = function (noIn, refShorthandDefaultPos) {
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
let expr = this.parseExprOps(noIn, refShorthandDefaultPos);
if (refShorthandDefaultPos && refShorthandDefaultPos.start) return expr;
if (this.eat(tt.question)) {
@@ -141,7 +141,7 @@ pp.parseMaybeConditional = function (noIn, refShorthandDefaultPos) {
// Start the precedence parser.
pp.parseExprOps = function (noIn, refShorthandDefaultPos) {
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
let expr = this.parseMaybeUnary(refShorthandDefaultPos);
if (refShorthandDefaultPos && refShorthandDefaultPos.start) return expr;
return this.parseExprOp(expr, startPos, startLoc, -1, noIn);
@@ -154,15 +154,15 @@ pp.parseExprOps = function (noIn, refShorthandDefaultPos) {
// operator that has a lower precedence than the set it is parsing.
pp.parseExprOp = function(left, leftStartPos, leftStartLoc, minPrec, noIn) {
let prec = this.type.binop;
if (prec != null && (!noIn || this.type !== tt._in)) {
let prec = this.state.type.binop;
if (prec != null && (!noIn || this.state.type !== tt._in)) {
if (prec > minPrec) {
let node = this.startNodeAt(leftStartPos, leftStartLoc);
node.left = left;
node.operator = this.value;
let op = this.type;
node.operator = this.state.value;
let op = this.state.type;
this.next();
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
node.right = this.parseExprOp(this.parseMaybeUnary(), startPos, startLoc, op.rightAssociative ? prec - 1 : prec, noIn);
this.finishNode(node, (op === tt.logicalOR || op === tt.logicalAND) ? "LogicalExpression" : "BinaryExpression");
return this.parseExprOp(node, leftStartPos, leftStartLoc, minPrec, noIn);
@@ -174,9 +174,9 @@ pp.parseExprOp = function(left, leftStartPos, leftStartLoc, minPrec, noIn) {
// Parse unary operators, both prefix and postfix.
pp.parseMaybeUnary = function (refShorthandDefaultPos) {
if (this.type.prefix) {
let node = this.startNode(), update = this.type === tt.incDec;
node.operator = this.value;
if (this.state.type.prefix) {
let node = this.startNode(), update = this.state.type === tt.incDec;
node.operator = this.state.value;
node.prefix = true;
this.next();
node.argument = this.parseMaybeUnary();
@@ -188,12 +188,12 @@ pp.parseMaybeUnary = function (refShorthandDefaultPos) {
}
return this.finishNode(node, update ? "UpdateExpression" : "UnaryExpression");
}
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
let expr = this.parseExprSubscripts(refShorthandDefaultPos);
if (refShorthandDefaultPos && refShorthandDefaultPos.start) return expr;
while (this.type.postfix && !this.canInsertSemicolon()) {
while (this.state.type.postfix && !this.canInsertSemicolon()) {
let node = this.startNodeAt(startPos, startLoc);
node.operator = this.value;
node.operator = this.state.value;
node.prefix = false;
node.argument = expr;
this.checkLVal(expr);
@@ -206,7 +206,7 @@ pp.parseMaybeUnary = function (refShorthandDefaultPos) {
// Parse call, dot, and `[]`-subscript expressions.
pp.parseExprSubscripts = function (refShorthandDefaultPos) {
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
let expr = this.parseExprAtom(refShorthandDefaultPos);
if (refShorthandDefaultPos && refShorthandDefaultPos.start) {
return expr;
@@ -235,7 +235,7 @@ pp.parseSubscripts = function(base, startPos, startLoc, noCalls) {
node.computed = true;
this.expect(tt.bracketR);
base = this.finishNode(node, "MemberExpression");
} else if (!noCalls && this.type === tt.parenL) {
} else if (!noCalls && this.state.type === tt.parenL) {
let possibleAsync = false;
if (base.type === "Identifier" && base.name === "async" && !this.canInsertSemicolon()) possibleAsync = true;
this.next();
@@ -245,12 +245,12 @@ pp.parseSubscripts = function(base, startPos, startLoc, noCalls) {
node.arguments = this.parseExprList(tt.parenR, this.options.features["es7.trailingFunctionCommas"]);
base = this.finishNode(node, "CallExpression");
if (possibleAsync && (this.type === tt.colon || this.type === tt.arrow)) {
if (possibleAsync && (this.state.type === tt.colon || this.state.type === tt.arrow)) {
base = this.parseAsyncArrowFromCallExpression(this.startNodeAt(startPos, startLoc), node);
} else {
this.toReferencedList(node.arguments);
}
} else if (this.type === tt.backQuote) {
} else if (this.state.type === tt.backQuote) {
let node = this.startNodeAt(startPos, startLoc);
node.tag = base;
node.quasi = this.parseTemplate();
@@ -270,7 +270,7 @@ pp.parseAsyncArrowFromCallExpression = function (node, call) {
// Parse a no-call expression (like argument of `new` or `::` operators).
pp.parseNoCallExpr = function () {
let startPos = this.start, startLoc = this.startLoc;
let startPos = this.state.start, startLoc = this.state.startLoc;
return this.parseSubscripts(this.parseExprAtom(), startPos, startLoc, true);
};
@@ -280,31 +280,31 @@ pp.parseNoCallExpr = function () {
// or `{}`.
pp.parseExprAtom = function (refShorthandDefaultPos) {
let node, canBeArrow = this.potentialArrowAt === this.start;
switch (this.type) {
let node, canBeArrow = this.state.potentialArrowAt === this.state.start;
switch (this.state.type) {
case tt._super:
if (!this.inFunction)
this.raise(this.start, "'super' outside of function or class");
if (!this.state.inFunction)
this.raise(this.state.start, "'super' outside of function or class");
case tt._this:
let type = this.type === tt._this ? "ThisExpression" : "Super";
let type = this.state.type === tt._this ? "ThisExpression" : "Super";
node = this.startNode();
this.next();
return this.finishNode(node, type);
case tt._yield:
if (this.inGenerator) this.unexpected();
if (this.state.inGenerator) this.unexpected();
case tt._do:
if (this.options.features["es7.doExpressions"]) {
let node = this.startNode();
this.next();
var oldInFunction = this.inFunction;
var oldLabels = this.labels;
this.labels = [];
this.inFunction = false;
var oldInFunction = this.state.inFunction;
var oldLabels = this.state.labels;
this.state.labels = [];
this.state.inFunction = false;
node.body = this.parseBlock();
this.inFunction = oldInFunction;
this.labels = oldLabels;
this.state.inFunction = oldInFunction;
this.state.labels = oldLabels;
return this.finishNode(node, "DoExpression");
}
@@ -316,10 +316,10 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
if (this.options.features["es7.asyncFunctions"]) {
if (id.name === "await") {
if (this.inAsync) return this.parseAwait(node);
} else if (id.name === "async" && this.type === tt._function && !this.canInsertSemicolon()) {
} else if (id.name === "async" && this.state.type === tt._function && !this.canInsertSemicolon()) {
this.next();
return this.parseFunction(node, false, false, true);
} else if (canBeArrow && id.name === "async" && this.type === tt.name) {
} else if (canBeArrow && id.name === "async" && this.state.type === tt.name) {
var params = [this.parseIdent()];
this.expect(tt.arrow);
// var foo = bar => {};
@@ -334,18 +334,18 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
return id;
case tt.regexp:
let value = this.value;
let value = this.state.value;
node = this.parseLiteral(value.value);
node.regex = {pattern: value.pattern, flags: value.flags};
return node;
case tt.num: case tt.string:
return this.parseLiteral(this.value);
return this.parseLiteral(this.state.value);
case tt._null: case tt._true: case tt._false:
node = this.startNode();
node.rawValue = node.value = this.type === tt._null ? null : this.type === tt._true;
node.raw = this.type.keyword;
node.rawValue = node.value = this.state.type === tt._null ? null : this.state.type === tt._true;
node.raw = this.state.type.keyword;
this.next();
return this.finishNode(node, "Literal");
@@ -356,7 +356,7 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
node = this.startNode();
this.next();
// check whether this is array comprehension or regular array
if (this.options.features["es7.comprehensions"] && this.type === tt._for) {
if (this.options.features["es7.comprehensions"] && this.state.type === tt._for) {
return this.parseComprehension(node, false);
}
node.elements = this.parseExprList(tt.bracketR, true, true, refShorthandDefaultPos);
@@ -404,7 +404,7 @@ pp.parseExprAtom = function (refShorthandDefaultPos) {
pp.parseLiteral = function (value) {
let node = this.startNode();
node.rawValue = node.value = value;
node.raw = this.input.slice(this.start, this.end);
node.raw = this.input.slice(this.state.start, this.state.end);
this.next();
return this.finishNode(node, "Literal");
};
@@ -417,43 +417,43 @@ pp.parseParenExpression = function () {
};
pp.parseParenAndDistinguishExpression = function (startPos, startLoc, canBeArrow, isAsync) {
startPos = startPos || this.start;
startLoc = startLoc || this.startLoc;
startPos = startPos || this.state.start;
startLoc = startLoc || this.state.startLoc;
let val;
this.next();
if (this.options.features["es7.comprehensions"] && this.type === tt._for) {
if (this.options.features["es7.comprehensions"] && this.state.type === tt._for) {
return this.parseComprehension(this.startNodeAt(startPos, startLoc), true);
}
let innerStartPos = this.start, innerStartLoc = this.startLoc;
let innerStartPos = this.state.start, innerStartLoc = this.state.startLoc;
let exprList = [], first = true;
let refShorthandDefaultPos = {start: 0}, spreadStart, innerParenStart, optionalCommaStart;
while (this.type !== tt.parenR) {
while (this.state.type !== tt.parenR) {
if (first) {
first = false;
} else {
this.expect(tt.comma);
if (this.type === tt.parenR && this.options.features["es7.trailingFunctionCommas"]) {
optionalCommaStart = this.start;
if (this.state.type === tt.parenR && this.options.features["es7.trailingFunctionCommas"]) {
optionalCommaStart = this.state.start;
break;
}
}
if (this.type === tt.ellipsis) {
let spreadNodeStartPos = this.start, spreadNodeStartLoc = this.startLoc;
spreadStart = this.start;
if (this.state.type === tt.ellipsis) {
let spreadNodeStartPos = this.state.start, spreadNodeStartLoc = this.state.startLoc;
spreadStart = this.state.start;
exprList.push(this.parseParenItem(this.parseRest(), spreadNodeStartLoc, spreadNodeStartPos));
break;
} else {
if (this.type === tt.parenL && !innerParenStart) {
innerParenStart = this.start;
if (this.state.type === tt.parenL && !innerParenStart) {
innerParenStart = this.state.start;
}
exprList.push(this.parseMaybeAssign(false, refShorthandDefaultPos, this.parseParenItem));
}
}
let innerEndPos = this.start;
let innerEndLoc = this.startLoc;
let innerEndPos = this.state.start;
let innerEndLoc = this.state.startLoc;
this.expect(tt.parenR);
if (canBeArrow && !this.canInsertSemicolon() && this.eat(tt.arrow)) {
@@ -465,7 +465,7 @@ pp.parseParenAndDistinguishExpression = function (startPos, startLoc, canBeArrow
if (isAsync) {
return;
} else {
this.unexpected(this.lastTokStart);
this.unexpected(this.state.lastTokStart);
}
}
if (optionalCommaStart) this.unexpected(optionalCommaStart);
@@ -525,11 +525,11 @@ pp.parseNew = function () {
pp.parseTemplateElement = function () {
let elem = this.startNode();
elem.value = {
raw: this.input.slice(this.start, this.end).replace(/\r\n?/g, "\n"),
cooked: this.value
raw: this.input.slice(this.state.start, this.state.end).replace(/\r\n?/g, "\n"),
cooked: this.state.value
};
this.next();
elem.tail = this.type === tt.backQuote;
elem.tail = this.state.type === tt.backQuote;
return this.finishNode(elem, "TemplateElement");
};
@@ -564,7 +564,7 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
if (this.afterTrailingComma(tt.braceR)) break;
}
while (this.type === tt.at) {
while (this.state.type === tt.at) {
decorators.push(this.parseDecorator());
}
@@ -573,7 +573,7 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
prop.decorators = decorators;
decorators = [];
}
if (this.options.features["es7.objectRestSpread"] && this.type === tt.ellipsis) {
if (this.options.features["es7.objectRestSpread"] && this.state.type === tt.ellipsis) {
prop = this.parseSpread();
prop.type = "SpreadProperty";
node.properties.push(prop);
@@ -582,15 +582,15 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
prop.method = false;
prop.shorthand = false;
if (isPattern || refShorthandDefaultPos) {
startPos = this.start;
startLoc = this.startLoc;
startPos = this.state.start;
startLoc = this.state.startLoc;
}
if (!isPattern)
isGenerator = this.eat(tt.star);
if (this.options.features["es7.asyncFunctions"] && this.isContextual("async")) {
if (isGenerator || isPattern) this.unexpected();
var asyncId = this.parseIdent();
if (this.type === tt.colon || this.type === tt.parenL) {
if (this.state.type === tt.colon || this.state.type === tt.parenL) {
prop.key = asyncId;
} else {
isAsync = true;
@@ -604,21 +604,21 @@ pp.parseObj = function (isPattern, refShorthandDefaultPos) {
node.properties.push(this.finishNode(prop, "Property"));
}
if (decorators.length) {
this.raise(this.start, "You have trailing decorators with no property");
this.raise(this.state.start, "You have trailing decorators with no property");
}
return this.finishNode(node, isPattern ? "ObjectPattern" : "ObjectExpression");
};
pp.parseObjPropValue = function (prop, startPos, startLoc, isGenerator, isAsync, isPattern, refShorthandDefaultPos) {
if (this.eat(tt.colon)) {
prop.value = isPattern ? this.parseMaybeDefault(this.start, this.startLoc) : this.parseMaybeAssign(false, refShorthandDefaultPos);
prop.value = isPattern ? this.parseMaybeDefault(this.state.start, this.state.startLoc) : this.parseMaybeAssign(false, refShorthandDefaultPos);
prop.kind = "init";
} else if (this.type === tt.parenL) {
} else if (this.state.type === tt.parenL) {
if (isPattern) this.unexpected();
prop.kind = "init";
prop.method = true;
prop.value = this.parseMethod(isGenerator, isAsync);
} else if (!prop.computed && prop.key.type === "Identifier" && (prop.key.name === "get" || prop.key.name === "set") && (this.type !== tt.comma && this.type !== tt.braceR)) {
} else if (!prop.computed && prop.key.type === "Identifier" && (prop.key.name === "get" || prop.key.name === "set") && (this.state.type !== tt.comma && this.state.type !== tt.braceR)) {
if (isGenerator || isAsync || isPattern) this.unexpected();
prop.kind = prop.key.name;
this.parsePropertyName(prop);
@@ -639,9 +639,9 @@ pp.parseObjPropValue = function (prop, startPos, startLoc, isGenerator, isAsync,
(!this.options.allowReserved && this.isReservedWord(prop.key.name)))
this.raise(prop.key.start, "Binding " + prop.key.name);
prop.value = this.parseMaybeDefault(startPos, startLoc, prop.key.__clone());
} else if (this.type === tt.eq && refShorthandDefaultPos) {
} else if (this.state.type === tt.eq && refShorthandDefaultPos) {
if (!refShorthandDefaultPos.start)
refShorthandDefaultPos.start = this.start;
refShorthandDefaultPos.start = this.state.start;
prop.value = this.parseMaybeDefault(startPos, startLoc, prop.key.__clone());
} else {
prop.value = prop.key.__clone();
@@ -660,7 +660,7 @@ pp.parsePropertyName = function (prop) {
return prop.key;
} else {
prop.computed = false;
return prop.key = (this.type === tt.num || this.type === tt.string) ? this.parseExprAtom() : this.parseIdent(true);
return prop.key = (this.state.type === tt.num || this.state.type === tt.string) ? this.parseExprAtom() : this.parseIdent(true);
}
};
@@ -699,7 +699,7 @@ pp.parseArrowExpression = function (node, params, isAsync) {
// Parse function body and check parameters.
pp.parseFunctionBody = function (node, allowExpression) {
let isExpression = allowExpression && this.type !== tt.braceL;
let isExpression = allowExpression && this.state.type !== tt.braceL;
var oldInAsync = this.inAsync;
this.inAsync = node.async;
@@ -709,11 +709,11 @@ pp.parseFunctionBody = function (node, allowExpression) {
} else {
// Start a new scope with regard to labels and the `inFunction`
// flag (restore them to their old value afterwards).
let oldInFunc = this.inFunction, oldInGen = this.inGenerator, oldLabels = this.labels;
this.inFunction = true; this.inGenerator = node.generator; this.labels = [];
let oldInFunc = this.state.inFunction, oldInGen = this.state.inGenerator, oldLabels = this.state.labels;
this.state.inFunction = true; this.state.inGenerator = node.generator; this.state.labels = [];
node.body = this.parseBlock(true);
node.expression = false;
this.inFunction = oldInFunc; this.inGenerator = oldInGen; this.labels = oldLabels;
this.state.inFunction = oldInFunc; this.state.inGenerator = oldInGen; this.state.labels = oldLabels;
}
this.inAsync = oldInAsync;
@@ -756,9 +756,9 @@ pp.parseExprList = function (close, allowTrailingComma, allowEmpty, refShorthand
pp.parseExprListItem = function (allowEmpty, refShorthandDefaultPos) {
let elt;
if (allowEmpty && this.type === tt.comma) {
if (allowEmpty && this.state.type === tt.comma) {
elt = null;
} else if (this.type === tt.ellipsis) {
} else if (this.state.type === tt.ellipsis) {
elt = this.parseSpread(refShorthandDefaultPos);
} else {
elt = this.parseMaybeAssign(false, refShorthandDefaultPos);
@@ -772,14 +772,14 @@ pp.parseExprListItem = function (allowEmpty, refShorthandDefaultPos) {
pp.parseIdent = function (liberal) {
let node = this.startNode();
if (this.type === tt.name) {
if (this.state.type === tt.name) {
if (!liberal &&
((!this.options.allowReserved && this.isReservedWord(this.value)) ||
(this.strict && reservedWords.strict(this.value))))
this.raise(this.start, "The keyword '" + this.value + "' is reserved");
node.name = this.value;
} else if (liberal && this.type.keyword) {
node.name = this.type.keyword;
((!this.options.allowReserved && this.isReservedWord(this.state.value)) ||
(this.strict && reservedWords.strict(this.state.value))))
this.raise(this.state.start, "The keyword '" + this.state.value + "' is reserved");
node.name = this.state.value;
} else if (liberal && this.state.type.keyword) {
node.name = this.state.type.keyword;
} else {
this.unexpected();
}
@@ -803,7 +803,7 @@ pp.parseAwait = function (node) {
pp.parseYield = function () {
let node = this.startNode();
this.next();
if (this.type === tt.semi || this.canInsertSemicolon() || (this.type !== tt.star && !this.type.startsExpr)) {
if (this.state.type === tt.semi || this.canInsertSemicolon() || (this.state.type !== tt.star && !this.state.type.startsExpr)) {
node.delegate = false;
node.argument = null;
} else {
@@ -817,7 +817,7 @@ pp.parseYield = function () {
pp.parseComprehension = function (node, isGenerator) {
node.blocks = [];
while (this.type === tt._for) {
while (this.state.type === tt._for) {
let block = this.startNode();
this.next();
this.expect(tt.parenL);

34
packages/babylon/src/state.js → packages/babylon/src/parser/index.js Executable file → Normal file
View File

@@ -1,14 +1,14 @@
import { reservedWords, isKeyword } from "./util/identifier";
import { getOptions } from "./options";
import Tokenizer from "./tokenizer";
import { reservedWords, isKeyword } from "../util/identifier";
import { getOptions } from "../options";
import Tokenizer from "../tokenizer";
// Registered plugins
export const plugins = {};
export class Parser extends Tokenizer {
export default class Parser extends Tokenizer {
constructor(options, input) {
super();
super(input);
this.options = getOptions(options);
this.isKeyword = isKeyword;
@@ -20,30 +20,8 @@ export class Parser extends Tokenizer {
this.inModule = this.options.sourceType === "module";
this.strict = this.options.strictMode === false ? false : this.inModule;
// Used to signify the start of a potential arrow function
this.potentialArrowAt = -1;
// Flags to track whether we are in a function, a generator.
this.inFunction = this.inGenerator = false;
// Labels in scope.
this.labels = [];
// Leading decorators.
this.decorators = [];
// Token store.
this.tokens = [];
// Comment store.
this.comments = [];
// Comment attachment store
this.trailingComments = [];
this.leadingComments = [];
this.bottomRightStack = [];
// If enabled, skip leading hashbang line.
if (this.pos === 0 && this.input[0] === "#" && this.input[1] === "!") {
if (this.state.pos === 0 && this.input[0] === "#" && this.input[1] === "!") {
this.skipLineComment(2);
}
}

9
packages/babylon/src/location.js → packages/babylon/src/parser/location.js Executable file → Normal file
View File

@@ -1,5 +1,5 @@
import {Position, getLineInfo} from "./locutil";
import { Parser } from "./state";
import { getLineInfo } from "../util/location";
import Parser from "./index";
const pp = Parser.prototype;
@@ -15,10 +15,5 @@ pp.raise = function (pos, message) {
let err = new SyntaxError(message);
err.pos = pos;
err.loc = loc;
err.raisedAt = this.pos;
throw err;
};
pp.curPosition = function () {
return new Position(this.curLine, this.pos - this.lineStart);
};

18
packages/babylon/src/lval.js → packages/babylon/src/parser/lval.js Executable file → Normal file
View File

@@ -1,6 +1,6 @@
import { types as tt } from "./tokenizer/types";
import { Parser } from "./state";
import { reservedWords } from "./util/identifier";
import { types as tt } from "../tokenizer/types";
import Parser from "./index";
import { reservedWords } from "../util/identifier";
const pp = Parser.prototype;
@@ -92,14 +92,14 @@ pp.parseSpread = function (refShorthandDefaultPos) {
pp.parseRest = function () {
let node = this.startNode();
this.next();
node.argument = this.type === tt.name || this.type === tt.bracketL ? this.parseBindingAtom() : this.unexpected();
node.argument = this.state.type === tt.name || this.state.type === tt.bracketL ? this.parseBindingAtom() : this.unexpected();
return this.finishNode(node, "RestElement");
};
// Parses lvalue (assignable) atom.
pp.parseBindingAtom = function () {
switch (this.type) {
switch (this.state.type) {
case tt.name:
return this.parseIdent();
@@ -122,11 +122,11 @@ pp.parseBindingList = function (close, allowEmpty, allowTrailingComma) {
while (!this.eat(close)) {
if (first) first = false;
else this.expect(tt.comma);
if (allowEmpty && this.type === tt.comma) {
if (allowEmpty && this.state.type === tt.comma) {
elts.push(null);
} else if (allowTrailingComma && this.afterTrailingComma(close)) {
break;
} else if (this.type === tt.ellipsis) {
} else if (this.state.type === tt.ellipsis) {
elts.push(this.parseAssignableListItemTypes(this.parseRest()));
this.expect(close);
break;
@@ -146,8 +146,8 @@ pp.parseAssignableListItemTypes = function (param) {
// Parses assignment pattern around given atom if possible.
pp.parseMaybeDefault = function (startPos, startLoc, left) {
startLoc = startLoc || this.startLoc;
startPos = startPos || this.start;
startLoc = startLoc || this.state.startLoc;
startPos = startPos || this.state.start;
left = left || this.parseBindingAtom();
if (!this.eat(tt.eq)) return left;

8
packages/babylon/src/node.js → packages/babylon/src/parser/node.js Executable file → Normal file
View File

@@ -1,5 +1,5 @@
import { Parser } from "./state";
import { SourceLocation } from "./locutil";
import Parser from "./index";
import { SourceLocation } from "../util/location";
// Start an AST node, attaching a start offset.
@@ -24,7 +24,7 @@ export class Node {
}
pp.startNode = function () {
return new Node(this, this.start, this.startLoc);
return new Node(this, this.state.start, this.state.startLoc);
};
pp.startNodeAt = function (pos, loc) {
@@ -42,7 +42,7 @@ function finishNodeAt(node, type, pos, loc) {
// Finish an AST node, adding `type` and `end` properties.
pp.finishNode = function (node, type) {
return finishNodeAt.call(this, node, type, this.lastTokEnd, this.lastTokEndLoc);
return finishNodeAt.call(this, node, type, this.state.lastTokEnd, this.state.lastTokEndLoc);
};
// Finish node at given position

271
packages/babylon/src/statement.js → packages/babylon/src/parser/statement.js Executable file → Normal file
View File

@@ -1,6 +1,6 @@
import { types as tt } from "./tokenizer/types";
import { Parser } from "./state";
import { lineBreak } from "./util/whitespace";
import { types as tt } from "../tokenizer/types";
import Parser from "./index";
import { lineBreak } from "../util/whitespace";
const pp = Parser.prototype;
@@ -16,7 +16,7 @@ pp.parseTopLevel = function (file, program) {
program.body = [];
let first = true;
while (this.type !== tt.eof) {
while (this.state.type !== tt.eof) {
let stmt = this.parseStatement(true, true);
program.body.push(stmt);
if (first) {
@@ -27,8 +27,8 @@ pp.parseTopLevel = function (file, program) {
this.next();
file.program = this.finishNode(program, "Program");
file.comments = this.comments;
file.tokens = this.tokens;
file.comments = this.state.comments;
file.tokens = this.state.tokens;
return this.finishNode(file, "File");
};
@@ -43,96 +43,99 @@ const loopLabel = {kind: "loop"}, switchLabel = {kind: "switch"};
// does not help.
pp.parseStatement = function (declaration, topLevel) {
if (this.type === tt.at) {
if (this.state.type === tt.at) {
this.parseDecorators(true);
}
let starttype = this.type, node = this.startNode();
let starttype = this.state.type, node = this.startNode();
// Most types of statements are recognized by the keyword they
// start with. Many are trivial to parse, some require a bit of
// complexity.
switch (starttype) {
case tt._break: case tt._continue: return this.parseBreakContinueStatement(node, starttype.keyword);
case tt._debugger: return this.parseDebuggerStatement(node);
case tt._do: return this.parseDoStatement(node);
case tt._for: return this.parseForStatement(node);
case tt._function:
if (!declaration) this.unexpected();
return this.parseFunctionStatement(node);
case tt._break: case tt._continue: return this.parseBreakContinueStatement(node, starttype.keyword);
case tt._debugger: return this.parseDebuggerStatement(node);
case tt._do: return this.parseDoStatement(node);
case tt._for: return this.parseForStatement(node);
case tt._function:
if (!declaration) this.unexpected();
return this.parseFunctionStatement(node);
case tt._class:
if (!declaration) this.unexpected();
this.takeDecorators(node);
return this.parseClass(node, true);
case tt._class:
if (!declaration) this.unexpected();
this.takeDecorators(node);
return this.parseClass(node, true);
case tt._if: return this.parseIfStatement(node);
case tt._return: return this.parseReturnStatement(node);
case tt._switch: return this.parseSwitchStatement(node);
case tt._throw: return this.parseThrowStatement(node);
case tt._try: return this.parseTryStatement(node);
case tt._let: case tt._const: if (!declaration) this.unexpected(); // NOTE: falls through to _var
case tt._var: return this.parseVarStatement(node, starttype);
case tt._while: return this.parseWhileStatement(node);
case tt._with: return this.parseWithStatement(node);
case tt.braceL: return this.parseBlock();
case tt.semi: return this.parseEmptyStatement(node);
case tt._export:
case tt._import:
if (!this.options.allowImportExportEverywhere) {
if (!topLevel)
this.raise(this.start, "'import' and 'export' may only appear at the top level");
case tt._if: return this.parseIfStatement(node);
case tt._return: return this.parseReturnStatement(node);
case tt._switch: return this.parseSwitchStatement(node);
case tt._throw: return this.parseThrowStatement(node);
case tt._try: return this.parseTryStatement(node);
case tt._let: case tt._const: if (!declaration) this.unexpected(); // NOTE: falls through to _var
case tt._var: return this.parseVarStatement(node, starttype);
case tt._while: return this.parseWhileStatement(node);
case tt._with: return this.parseWithStatement(node);
case tt.braceL: return this.parseBlock();
case tt.semi: return this.parseEmptyStatement(node);
case tt._export:
case tt._import:
if (!this.options.allowImportExportEverywhere) {
if (!topLevel)
this.raise(this.state.start, "'import' and 'export' may only appear at the top level");
if (!this.inModule)
this.raise(this.start, "'import' and 'export' may appear only with 'sourceType: module'");
}
return starttype === tt._import ? this.parseImport(node) : this.parseExport(node);
case tt.name:
if (this.options.features["es7.asyncFunctions"] && this.value === "async") {
var lookahead = this.lookahead();
if (lookahead.type === tt._function && !this.canInsertSemicolon.call(lookahead)) {
this.next();
this.expect(tt._function);
return this.parseFunction(node, true, false, true);
if (!this.inModule)
this.raise(this.state.start, "'import' and 'export' may appear only with 'sourceType: module'");
}
}
return starttype === tt._import ? this.parseImport(node) : this.parseExport(node);
// If the statement does not start with a statement keyword or a
// brace, it's an ExpressionStatement or LabeledStatement. We
// simply start parsing an expression, and afterwards, if the
// next token is a colon and the expression was a simple
// Identifier node, we switch to interpreting it as a label.
default:
let maybeName = this.value, expr = this.parseExpression();
case tt.name:
if (this.options.features["es7.asyncFunctions"] && this.state.value === "async") {
// peek ahead and see if next token is a function
var state = this.state.clone();
this.next();
if (this.state.type === tt._function && !this.canInsertSemicolon()) {
this.expect(tt._function);
return this.parseFunction(node, true, false, true);
} else {
this.state = state;
}
}
if (starttype === tt.name && expr.type === "Identifier" && this.eat(tt.colon)) {
return this.parseLabeledStatement(node, maybeName, expr);
} else {
return this.parseExpressionStatement(node, expr);
}
// If the statement does not start with a statement keyword or a
// brace, it's an ExpressionStatement or LabeledStatement. We
// simply start parsing an expression, and afterwards, if the
// next token is a colon and the expression was a simple
// Identifier node, we switch to interpreting it as a label.
default:
let maybeName = this.state.value, expr = this.parseExpression();
if (starttype === tt.name && expr.type === "Identifier" && this.eat(tt.colon)) {
return this.parseLabeledStatement(node, maybeName, expr);
} else {
return this.parseExpressionStatement(node, expr);
}
}
};
pp.takeDecorators = function (node) {
if (this.decorators.length) {
node.decorators = this.decorators;
this.decorators = [];
if (this.state.decorators.length) {
node.decorators = this.state.decorators;
this.state.decorators = [];
}
};
pp.parseDecorators = function (allowExport) {
while (this.type === tt.at) {
this.decorators.push(this.parseDecorator());
while (this.state.type === tt.at) {
this.state.decorators.push(this.parseDecorator());
}
if (allowExport && this.type === tt._export) {
if (allowExport && this.state.type === tt._export) {
return;
}
if (this.type !== tt._class) {
this.raise(this.start, "Leading decorators must be attached to a class declaration");
if (this.state.type !== tt._class) {
this.raise(this.state.start, "Leading decorators must be attached to a class declaration");
}
};
@@ -152,7 +155,7 @@ pp.parseBreakContinueStatement = function (node, keyword) {
if (this.eat(tt.semi) || this.insertSemicolon()) {
node.label = null;
} else if (this.type !== tt.name) {
} else if (this.state.type !== tt.name) {
this.unexpected();
} else {
node.label = this.parseIdent();
@@ -161,14 +164,14 @@ pp.parseBreakContinueStatement = function (node, keyword) {
// Verify that there is an actual destination to break or
// continue to.
for (var i = 0; i < this.labels.length; ++i) {
let lab = this.labels[i];
for (var i = 0; i < this.state.labels.length; ++i) {
let lab = this.state.labels[i];
if (node.label == null || lab.name === node.label.name) {
if (lab.kind != null && (isBreak || lab.kind === "loop")) break;
if (node.label && isBreak) break;
}
}
if (i === this.labels.length) this.raise(node.start, "Unsyntactic " + keyword);
if (i === this.state.labels.length) this.raise(node.start, "Unsyntactic " + keyword);
return this.finishNode(node, isBreak ? "BreakStatement" : "ContinueStatement");
};
@@ -180,9 +183,9 @@ pp.parseDebuggerStatement = function (node) {
pp.parseDoStatement = function (node) {
this.next();
this.labels.push(loopLabel);
this.state.labels.push(loopLabel);
node.body = this.parseStatement(false);
this.labels.pop();
this.state.labels.pop();
this.expect(tt._while);
node.test = this.parseParenExpression();
this.eat(tt.semi);
@@ -199,19 +202,19 @@ pp.parseDoStatement = function (node) {
pp.parseForStatement = function (node) {
this.next();
this.labels.push(loopLabel);
this.state.labels.push(loopLabel);
this.expect(tt.parenL);
if (this.type === tt.semi) {
if (this.state.type === tt.semi) {
return this.parseFor(node, null);
}
if (this.type === tt._var || this.type === tt._let || this.type === tt._const) {
let init = this.startNode(), varKind = this.type;
if (this.state.type === tt._var || this.state.type === tt._let || this.state.type === tt._const) {
let init = this.startNode(), varKind = this.state.type;
this.next();
this.parseVar(init, true, varKind);
this.finishNode(init, "VariableDeclaration");
if ((this.type === tt._in || this.isContextual("of")) && init.declarations.length === 1 &&
if ((this.state.type === tt._in || this.isContextual("of")) && init.declarations.length === 1 &&
!(varKind !== tt._var && init.declarations[0].init))
return this.parseForIn(node, init);
return this.parseFor(node, init);
@@ -219,7 +222,7 @@ pp.parseForStatement = function (node) {
let refShorthandDefaultPos = {start: 0};
let init = this.parseExpression(true, refShorthandDefaultPos);
if (this.type === tt._in || this.isContextual("of")) {
if (this.state.type === tt._in || this.isContextual("of")) {
this.toAssignable(init);
this.checkLVal(init);
return this.parseForIn(node, init);
@@ -243,8 +246,8 @@ pp.parseIfStatement = function (node) {
};
pp.parseReturnStatement = function (node) {
if (!this.inFunction && !this.options.allowReturnOutsideFunction) {
this.raise(this.start, "'return' outside of function");
if (!this.state.inFunction && !this.options.allowReturnOutsideFunction) {
this.raise(this.state.start, "'return' outside of function");
}
this.next();
@@ -268,15 +271,15 @@ pp.parseSwitchStatement = function (node) {
node.discriminant = this.parseParenExpression();
node.cases = [];
this.expect(tt.braceL);
this.labels.push(switchLabel);
this.state.labels.push(switchLabel);
// Statements under must be grouped (by label) in SwitchCase
// nodes. `cur` is used to keep the node that we are currently
// adding statements to.
for (var cur, sawDefault; this.type !== tt.braceR; ) {
if (this.type === tt._case || this.type === tt._default) {
let isCase = this.type === tt._case;
for (var cur, sawDefault; this.state.type !== tt.braceR; ) {
if (this.state.type === tt._case || this.state.type === tt._default) {
let isCase = this.state.type === tt._case;
if (cur) this.finishNode(cur, "SwitchCase");
node.cases.push(cur = this.startNode());
cur.consequent = [];
@@ -284,7 +287,7 @@ pp.parseSwitchStatement = function (node) {
if (isCase) {
cur.test = this.parseExpression();
} else {
if (sawDefault) this.raise(this.lastTokStart, "Multiple default clauses");
if (sawDefault) this.raise(this.state.lastTokStart, "Multiple default clauses");
sawDefault = true;
cur.test = null;
}
@@ -296,14 +299,14 @@ pp.parseSwitchStatement = function (node) {
}
if (cur) this.finishNode(cur, "SwitchCase");
this.next(); // Closing brace
this.labels.pop();
this.state.labels.pop();
return this.finishNode(node, "SwitchStatement");
};
pp.parseThrowStatement = function (node) {
this.next();
if (lineBreak.test(this.input.slice(this.lastTokEnd, this.start)))
this.raise(this.lastTokEnd, "Illegal newline after throw");
if (lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start)))
this.raise(this.state.lastTokEnd, "Illegal newline after throw");
node.argument = this.parseExpression();
this.semicolon();
return this.finishNode(node, "ThrowStatement");
@@ -317,7 +320,7 @@ pp.parseTryStatement = function (node) {
this.next();
node.block = this.parseBlock();
node.handler = null;
if (this.type === tt._catch) {
if (this.state.type === tt._catch) {
let clause = this.startNode();
this.next();
this.expect(tt.parenL);
@@ -348,14 +351,14 @@ pp.parseVarStatement = function (node, kind) {
pp.parseWhileStatement = function (node) {
this.next();
node.test = this.parseParenExpression();
this.labels.push(loopLabel);
this.state.labels.push(loopLabel);
node.body = this.parseStatement(false);
this.labels.pop();
this.state.labels.pop();
return this.finishNode(node, "WhileStatement");
};
pp.parseWithStatement = function (node) {
if (this.strict) this.raise(this.start, "'with' in strict mode");
if (this.strict) this.raise(this.state.start, "'with' in strict mode");
this.next();
node.object = this.parseParenExpression();
node.body = this.parseStatement(false);
@@ -368,26 +371,26 @@ pp.parseEmptyStatement = function (node) {
};
pp.parseLabeledStatement = function (node, maybeName, expr) {
for (let label of (this.labels: Array)){
for (let label of (this.state.labels: Array)){
if (label.name === maybeName) {
this.raise(expr.start, `Label '${maybeName}' is already declared`);
}
}
let kind = this.type.isLoop ? "loop" : this.type === tt._switch ? "switch" : null;
for (let i = this.labels.length - 1; i >= 0; i--) {
let label = this.labels[i];
let kind = this.state.type.isLoop ? "loop" : this.state.type === tt._switch ? "switch" : null;
for (let i = this.state.labels.length - 1; i >= 0; i--) {
let label = this.state.labels[i];
if (label.statementStart === node.start) {
label.statementStart = this.start;
label.statementStart = this.state.start;
label.kind = kind;
} else {
break;
}
}
this.labels.push({name: maybeName, kind: kind, statementStart: this.start});
this.state.labels.push({name: maybeName, kind: kind, statementStart: this.state.start});
node.body = this.parseStatement(true);
this.labels.pop();
this.state.labels.pop();
node.label = expr;
return this.finishNode(node, "LabeledStatement");
};
@@ -426,12 +429,12 @@ pp.parseBlock = function (allowStrict) {
pp.parseFor = function (node, init) {
node.init = init;
this.expect(tt.semi);
node.test = this.type === tt.semi ? null : this.parseExpression();
node.test = this.state.type === tt.semi ? null : this.parseExpression();
this.expect(tt.semi);
node.update = this.type === tt.parenR ? null : this.parseExpression();
node.update = this.state.type === tt.parenR ? null : this.parseExpression();
this.expect(tt.parenR);
node.body = this.parseStatement(false);
this.labels.pop();
this.state.labels.pop();
return this.finishNode(node, "ForStatement");
};
@@ -439,13 +442,13 @@ pp.parseFor = function (node, init) {
// same from parser's perspective.
pp.parseForIn = function (node, init) {
let type = this.type === tt._in ? "ForInStatement" : "ForOfStatement";
let type = this.state.type === tt._in ? "ForInStatement" : "ForOfStatement";
this.next();
node.left = init;
node.right = this.parseExpression();
this.expect(tt.parenR);
node.body = this.parseStatement(false);
this.labels.pop();
this.state.labels.pop();
return this.finishNode(node, type);
};
@@ -459,10 +462,10 @@ pp.parseVar = function (node, isFor, kind) {
this.parseVarHead(decl);
if (this.eat(tt.eq)) {
decl.init = this.parseMaybeAssign(isFor);
} else if (kind === tt._const && !(this.type === tt._in || this.isContextual("of"))) {
} else if (kind === tt._const && !(this.state.type === tt._in || this.isContextual("of"))) {
this.unexpected();
} else if (decl.id.type !== "Identifier" && !(isFor && (this.type === tt._in || this.isContextual("of")))) {
this.raise(this.lastTokEnd, "Complex binding patterns require an initialization value");
} else if (decl.id.type !== "Identifier" && !(isFor && (this.state.type === tt._in || this.isContextual("of")))) {
this.raise(this.state.lastTokEnd, "Complex binding patterns require an initialization value");
} else {
decl.init = null;
}
@@ -484,7 +487,7 @@ pp.parseFunction = function (node, isStatement, allowExpressionBody, isAsync) {
this.initFunction(node, isAsync);
node.generator = this.eat(tt.star);
if (isStatement || this.type === tt.name) {
if (isStatement || this.state.type === tt.name) {
node.id = this.parseIdent();
}
@@ -512,7 +515,7 @@ pp.parseClass = function (node, isStatement) {
let decorators = [];
while (!this.eat(tt.braceR)) {
if (this.eat(tt.semi)) continue;
if (this.type === tt.at) {
if (this.state.type === tt.at) {
decorators.push(this.parseDecorator());
continue;
}
@@ -521,10 +524,10 @@ pp.parseClass = function (node, isStatement) {
method.decorators = decorators;
decorators = [];
}
let isMaybeStatic = this.type === tt.name && this.value === "static";
let isMaybeStatic = this.state.type === tt.name && this.state.value === "static";
var isGenerator = this.eat(tt.star), isAsync = false;
this.parsePropertyName(method);
method.static = isMaybeStatic && this.type !== tt.parenL;
method.static = isMaybeStatic && this.state.type !== tt.parenL;
if (method.static) {
if (isGenerator) this.unexpected();
isGenerator = this.eat(tt.star);
@@ -534,7 +537,7 @@ pp.parseClass = function (node, isStatement) {
classBody.body.push(this.parseClassProperty(method));
continue;
}
if (this.options.features["es7.asyncFunctions"] && this.type !== tt.parenL &&
if (this.options.features["es7.asyncFunctions"] && this.state.type !== tt.parenL &&
!method.computed && method.key.type === "Identifier" && method.key.name === "async") {
isAsync = true;
this.parsePropertyName(method);
@@ -543,7 +546,7 @@ pp.parseClass = function (node, isStatement) {
method.kind = "method";
if (!method.computed) {
let {key} = method;
if (!isAsync && !isGenerator && key.type === "Identifier" && this.type !== tt.parenL && (key.name === "get" || key.name === "set")) {
if (!isAsync && !isGenerator && key.type === "Identifier" && this.state.type !== tt.parenL && (key.name === "get" || key.name === "set")) {
isGetSet = true;
method.kind = key.name;
key = this.parsePropertyName(method);
@@ -576,7 +579,7 @@ pp.parseClass = function (node, isStatement) {
}
if (decorators.length) {
this.raise(this.start, "You have trailing decorators with no method");
this.raise(this.state.start, "You have trailing decorators with no method");
}
node.body = this.finishNode(classBody, "ClassBody");
@@ -584,11 +587,11 @@ pp.parseClass = function (node, isStatement) {
};
pp.isClassProperty = function () {
return this.type === tt.eq || (this.type === tt.semi || this.canInsertSemicolon());
return this.state.type === tt.eq || (this.state.type === tt.semi || this.canInsertSemicolon());
};
pp.parseClassProperty = function (node) {
if (this.type === tt.eq) {
if (this.state.type === tt.eq) {
if (!this.options.features["es7.classProperties"]) this.unexpected();
this.next();
node.value = this.parseMaybeAssign();
@@ -605,7 +608,7 @@ pp.parseClassMethod = function (classBody, method, isGenerator, isAsync) {
};
pp.parseClassId = function (node, isStatement) {
node.id = this.type === tt.name ? this.parseIdent() : isStatement ? this.unexpected() : null;
node.id = this.state.type === tt.name ? this.parseIdent() : isStatement ? this.unexpected() : null;
};
pp.parseClassSuper = function (node) {
@@ -617,7 +620,7 @@ pp.parseClassSuper = function (node) {
pp.parseExport = function (node) {
this.next();
// export * from '...'
if (this.type === tt.star) {
if (this.state.type === tt.star) {
let specifier = this.startNode();
this.next();
if (this.options.features["es7.exportExtensions"] && this.eatContextual("as")) {
@@ -633,7 +636,7 @@ pp.parseExport = function (node) {
let specifier = this.startNode();
specifier.exported = this.parseIdent(true);
node.specifiers = [this.finishNode(specifier, "ExportDefaultSpecifier")];
if (this.type === tt.comma && this.lookahead().type === tt.star) {
if (this.state.type === tt.comma && this.lookahead().type === tt.star) {
this.expect(tt.comma);
let specifier = this.startNode();
this.expect(tt.star);
@@ -657,7 +660,7 @@ pp.parseExport = function (node) {
if (needsSemi) this.semicolon();
this.checkExport(node);
return this.finishNode(node, "ExportDefaultDeclaration");
} else if (this.type.keyword || this.shouldParseExportDeclaration()) {
} else if (this.state.type.keyword || this.shouldParseExportDeclaration()) {
node.declaration = this.parseStatement(true);
node.specifiers = [];
node.source = null;
@@ -665,7 +668,7 @@ pp.parseExport = function (node) {
node.declaration = null;
node.specifiers = this.parseExportSpecifiers();
if (this.eatContextual("from")) {
node.source = this.type === tt.string ? this.parseExprAtom() : this.unexpected();
node.source = this.state.type === tt.string ? this.parseExprAtom() : this.unexpected();
} else {
node.source = null;
}
@@ -676,11 +679,11 @@ pp.parseExport = function (node) {
};
pp.isExportDefaultSpecifier = function () {
if (this.type === tt.name) {
return this.value !== "type" && this.value !== "async";
if (this.state.type === tt.name) {
return this.state.value !== "type" && this.state.value !== "async";
}
if (this.type !== tt._default) {
if (this.state.type !== tt._default) {
return false;
}
@@ -696,7 +699,7 @@ pp.parseExportSpecifiersMaybe = function (node) {
pp.parseExportFrom = function (node) {
this.expectContextual("from");
node.source = this.type === tt.string ? this.parseExprAtom() : this.unexpected();
node.source = this.state.type === tt.string ? this.parseExprAtom() : this.unexpected();
this.semicolon();
this.checkExport(node);
};
@@ -706,7 +709,7 @@ pp.shouldParseExportDeclaration = function () {
};
pp.checkExport = function (node) {
if (this.decorators.length) {
if (this.state.decorators.length) {
var isClass = node.declaration && (node.declaration.type === "ClassDeclaration" || node.declaration.type === "ClassExpression");
if (!node.declaration || !isClass) {
this.raise(node.start, "You can only use decorators on an export when exporting a class");
@@ -731,7 +734,7 @@ pp.parseExportSpecifiers = function () {
}
let node = this.startNode();
node.local = this.parseIdent(this.type === tt._default);
node.local = this.parseIdent(this.state.type === tt._default);
node.exported = this.eatContextual("as") ? this.parseIdent(true) : node.local.__clone();
nodes.push(this.finishNode(node, "ExportSpecifier"));
}
@@ -745,14 +748,14 @@ pp.parseImport = function (node) {
this.next();
// import '...'
if (this.type === tt.string) {
if (this.state.type === tt.string) {
node.specifiers = [];
node.source = this.parseExprAtom();
} else {
node.specifiers = [];
this.parseImportSpecifiers(node);
this.expectContextual("from");
node.source = this.type === tt.string ? this.parseExprAtom() : this.unexpected();
node.source = this.state.type === tt.string ? this.parseExprAtom() : this.unexpected();
}
this.semicolon();
return this.finishNode(node, "ImportDeclaration");
@@ -762,14 +765,14 @@ pp.parseImport = function (node) {
pp.parseImportSpecifiers = function (node) {
var first = true;
if (this.type === tt.name) {
if (this.state.type === tt.name) {
// import defaultObj, { x, y as z } from '...'
var startPos = this.start, startLoc = this.startLoc;
var startPos = this.state.start, startLoc = this.state.startLoc;
node.specifiers.push(this.parseImportSpecifierDefault(this.parseIdent(), startPos, startLoc));
if (!this.eat(tt.comma)) return;
}
if (this.type === tt.star) {
if (this.state.type === tt.star) {
let specifier = this.startNode();
this.next();
this.expectContextual("as");

24
packages/babylon/src/parseutil.js → packages/babylon/src/parser/util.js Executable file → Normal file
View File

@@ -1,6 +1,6 @@
import { types as tt } from "./tokenizer/types";
import { Parser } from "./state";
import { lineBreak } from "./util/whitespace";
import { types as tt } from "../tokenizer/types";
import Parser from "./index";
import { lineBreak } from "../util/whitespace";
const pp = Parser.prototype;
@@ -16,7 +16,7 @@ pp.isUseStrict = function (stmt) {
// type, and if yes, consumes it as a side effect.
pp.eat = function (type) {
if (this.type === type) {
if (this.state.type === type) {
this.next();
return true;
} else {
@@ -27,7 +27,7 @@ pp.eat = function (type) {
// TODO
pp.isRelational = function (op) {
return this.type === tt.relational && this.value === op;
return this.state.type === tt.relational && this.state.value === op;
};
// TODO
@@ -43,13 +43,13 @@ pp.expectRelational = function (op) {
// Tests whether parsed token is a contextual keyword.
pp.isContextual = function (name) {
return this.type === tt.name && this.value === name;
return this.state.type === tt.name && this.state.value === name;
};
// Consumes contextual keyword if possible.
pp.eatContextual = function (name) {
return this.value === name && this.eat(tt.name);
return this.state.value === name && this.eat(tt.name);
};
// Asserts that following token is given contextual keyword.
@@ -61,9 +61,9 @@ pp.expectContextual = function (name) {
// Test whether a semicolon can be inserted at the current position.
pp.canInsertSemicolon = function () {
return this.type === tt.eof ||
this.type === tt.braceR ||
lineBreak.test(this.input.slice(this.lastTokEnd, this.start));
return this.state.type === tt.eof ||
this.state.type === tt.braceR ||
lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start));
};
pp.insertSemicolon = function () {
@@ -80,7 +80,7 @@ pp.semicolon = function () {
};
pp.afterTrailingComma = function (tokType) {
if (this.type === tokType) {
if (this.state.type === tokType) {
this.next();
return true;
}
@@ -96,5 +96,5 @@ pp.expect = function (type) {
// Raise an unexpected token error.
pp.unexpected = function (pos) {
this.raise(pos != null ? pos : this.start, "Unexpected token");
this.raise(pos != null ? pos : this.state.start, "Unexpected token");
};

104
packages/babylon/src/plugins/flow.js
View File

@@ -1,5 +1,5 @@
import { types as tt } from "../tokenizer/types";
import { Parser } from "../state";
import Parser from "../parser";
var pp = Parser.prototype;
@@ -50,11 +50,11 @@ pp.flowParseDeclareFunction = function (node) {
};
pp.flowParseDeclare = function (node) {
if (this.type === tt._class) {
if (this.state.type === tt._class) {
return this.flowParseDeclareClass(node);
} else if (this.type === tt._function) {
} else if (this.state.type === tt._function) {
return this.flowParseDeclareFunction(node);
} else if (this.type === tt._var) {
} else if (this.state.type === tt._var) {
return this.flowParseDeclareVariable(node);
} else if (this.isContextual("module")) {
return this.flowParseDeclareModule(node);
@@ -73,7 +73,7 @@ pp.flowParseDeclareVariable = function (node) {
pp.flowParseDeclareModule = function (node) {
this.next();
if (this.type === tt.string) {
if (this.state.type === tt.string) {
node.id = this.parseExprAtom();
} else {
node.id = this.parseIdent();
@@ -82,7 +82,7 @@ pp.flowParseDeclareModule = function (node) {
var bodyNode = node.body = this.startNode();
var body = bodyNode.body = [];
this.expect(tt.braceL);
while (this.type !== tt.braceR) {
while (this.state.type !== tt.braceR) {
var node2 = this.startNode();
// todo: declare check
@@ -193,7 +193,7 @@ pp.flowParseTypeParameterInstantiation = function () {
};
pp.flowParseObjectPropertyKey = function () {
return (this.type === tt.num || this.type === tt.string) ? this.parseExprAtom() : this.parseIdent(true);
return (this.state.type === tt.num || this.state.type === tt.string) ? this.parseExprAtom() : this.parseIdent(true);
};
pp.flowParseObjectTypeIndexer = function (node, isStatic) {
@@ -219,9 +219,9 @@ pp.flowParseObjectTypeMethodish = function (node) {
}
this.expect(tt.parenL);
while (this.type === tt.name) {
while (this.state.type === tt.name) {
node.params.push(this.flowParseFunctionTypeParam());
if (this.type !== tt.parenR) {
if (this.state.type !== tt.parenR) {
this.expect(tt.comma);
}
}
@@ -266,25 +266,25 @@ pp.flowParseObjectType = function (allowStatic) {
this.expect(tt.braceL);
while (this.type !== tt.braceR) {
var startPos = this.start, startLoc = this.startLoc;
while (this.state.type !== tt.braceR) {
var startPos = this.state.start, startLoc = this.state.startLoc;
node = this.startNode();
if (allowStatic && this.isContextual("static")) {
this.next();
isStatic = true;
}
if (this.type === tt.bracketL) {
if (this.state.type === tt.bracketL) {
nodeStart.indexers.push(this.flowParseObjectTypeIndexer(node, isStatic));
} else if (this.type === tt.parenL || this.isRelational("<")) {
} else if (this.state.type === tt.parenL || this.isRelational("<")) {
nodeStart.callProperties.push(this.flowParseObjectTypeCallProperty(node, allowStatic));
} else {
if (isStatic && this.type === tt.colon) {
if (isStatic && this.state.type === tt.colon) {
propertyKey = this.parseIdent();
} else {
propertyKey = this.flowParseObjectPropertyKey();
}
if (this.isRelational("<") || this.type === tt.parenL) {
if (this.isRelational("<") || this.state.type === tt.parenL) {
// This is a method property
nodeStart.properties.push(this.flowParseObjectTypeMethod(startPos, startLoc, isStatic, propertyKey));
} else {
@@ -307,7 +307,7 @@ pp.flowParseObjectType = function (allowStatic) {
};
pp.flowObjectTypeSemicolon = function () {
if (!this.eat(tt.semi) && !this.eat(tt.comma) && this.type !== tt.braceR) {
if (!this.eat(tt.semi) && !this.eat(tt.comma) && this.state.type !== tt.braceR) {
this.unexpected();
}
};
@@ -344,9 +344,9 @@ pp.flowParseTupleType = function () {
node.types = [];
this.expect(tt.bracketL);
// We allow trailing commas
while (this.pos < this.input.length && this.type !== tt.bracketR) {
while (this.state.pos < this.input.length && this.state.type !== tt.bracketR) {
node.types.push(this.flowParseType());
if (this.type === tt.bracketR) break;
if (this.state.type === tt.bracketR) break;
this.expect(tt.comma);
}
this.expect(tt.bracketR);
@@ -367,9 +367,9 @@ pp.flowParseFunctionTypeParam = function () {
pp.flowParseFunctionTypeParams = function () {
var ret = { params: [], rest: null };
while (this.type === tt.name) {
while (this.state.type === tt.name) {
ret.params.push(this.flowParseFunctionTypeParam());
if (this.type !== tt.parenR) {
if (this.state.type !== tt.parenR) {
this.expect(tt.comma);
}
}
@@ -409,13 +409,13 @@ pp.flowIdentToTypeAnnotation = function (startPos, startLoc, node, id) {
// primary types are kind of like primary expressions...they're the
// primitives with which other types are constructed.
pp.flowParsePrimaryType = function () {
var startPos = this.start, startLoc = this.startLoc;
var startPos = this.state.start, startLoc = this.state.startLoc;
var node = this.startNode();
var tmp;
var type;
var isGroupedType = false;
switch (this.type) {
switch (this.state.type) {
case tt.name:
return this.flowIdentToTypeAnnotation(startPos, startLoc, node, this.parseIdent());
@@ -426,7 +426,7 @@ pp.flowParsePrimaryType = function () {
return this.flowParseTupleType();
case tt.relational:
if (this.value === "<") {
if (this.state.value === "<") {
node.typeParameters = this.flowParseTypeParameterDeclaration();
this.expect(tt.parenL);
tmp = this.flowParseFunctionTypeParams();
@@ -445,8 +445,8 @@ pp.flowParsePrimaryType = function () {
this.next();
// Check to see if this is actually a grouped type
if (this.type !== tt.parenR && this.type !== tt.ellipsis) {
if (this.type === tt.name) {
if (this.state.type !== tt.parenR && this.state.type !== tt.ellipsis) {
if (this.state.type === tt.name) {
var token = this.lookahead().type;
isGroupedType = token !== tt.question && token !== tt.colon;
} else {
@@ -488,19 +488,19 @@ pp.flowParsePrimaryType = function () {
return this.finishNode(node, "FunctionTypeAnnotation");
case tt.string:
node.rawValue = node.value = this.value;
node.raw = this.input.slice(this.start, this.end);
node.rawValue = node.value = this.state.value;
node.raw = this.input.slice(this.state.start, this.state.end);
this.next();
return this.finishNode(node, "StringLiteralTypeAnnotation");
case tt.num:
node.rawValue = node.value = this.value;
node.raw = this.input.slice(this.start, this.end);
node.rawValue = node.value = this.state.value;
node.raw = this.input.slice(this.state.start, this.state.end);
this.next();
return this.finishNode(node, "NumberLiteralTypeAnnotation");
default:
if (this.type.keyword === "typeof") {
if (this.state.type.keyword === "typeof") {
return this.flowParseTypeofType();
}
}
@@ -511,7 +511,7 @@ pp.flowParsePrimaryType = function () {
pp.flowParsePostfixType = function () {
var node = this.startNode();
var type = node.elementType = this.flowParsePrimaryType();
if (this.type === tt.bracketL) {
if (this.state.type === tt.bracketL) {
this.expect(tt.bracketL);
this.expect(tt.bracketR);
return this.finishNode(node, "ArrayTypeAnnotation");
@@ -573,7 +573,7 @@ pp.flowParseTypeAnnotatableIdentifier = function (requireTypeAnnotation, canBeOp
isOptionalParam = true;
}
if (requireTypeAnnotation || this.type === tt.colon) {
if (requireTypeAnnotation || this.state.type === tt.colon) {
ident.typeAnnotation = this.flowParseTypeAnnotation();
this.finishNode(ident, ident.type);
}
@@ -590,7 +590,7 @@ export default function (instance) {
// function name(): string {}
instance.extend("parseFunctionBody", function (inner) {
return function (node, allowExpression) {
if (this.type === tt.colon && !allowExpression) {
if (this.state.type === tt.colon && !allowExpression) {
// if allowExpression is true then we're parsing an arrow function and if
// there's a return type then it's been handled elsewhere
node.returnType = this.flowParseTypeAnnotation();
@@ -603,7 +603,7 @@ export default function (instance) {
instance.extend("parseStatement", function (inner) {
return function (declaration, topLevel) {
// strict mode handling of `interface` since it's a reserved word
if (this.strict && this.type === tt.name && this.value === "interface") {
if (this.strict && this.state.type === tt.name && this.state.value === "interface") {
var node = this.startNode();
this.next();
return this.flowParseInterface(node);
@@ -617,10 +617,10 @@ export default function (instance) {
return function (node, expr) {
if (expr.type === "Identifier") {
if (expr.name === "declare") {
if (this.type === tt._class || this.type === tt.name || this.type === tt._function || this.type === tt._var) {
if (this.state.type === tt._class || this.state.type === tt.name || this.state.type === tt._function || this.state.type === tt._var) {
return this.flowParseDeclare(node);
}
} else if (this.type === tt.name) {
} else if (this.state.type === tt.name) {
if (expr.name === "interface") {
return this.flowParseInterface(node);
} else if (expr.name === "type") {
@@ -641,12 +641,12 @@ export default function (instance) {
instance.extend("parseParenItem", function () {
return function (node, startLoc, startPos, forceArrow?) {
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
var typeCastNode = this.startNodeAt(startLoc, startPos);
typeCastNode.expression = node;
typeCastNode.typeAnnotation = this.flowParseTypeAnnotation();
if (forceArrow && this.type !== tt.arrow) {
if (forceArrow && this.state.type !== tt.arrow) {
this.unexpected();
}
@@ -735,7 +735,7 @@ export default function (instance) {
return function (allowEmpty, refShorthandDefaultPos) {
var container = this.startNode();
var node = inner.call(this, allowEmpty, refShorthandDefaultPos);
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
container._exprListItem = true;
container.expression = node;
container.typeAnnotation = this.flowParseTypeAnnotation();
@@ -748,7 +748,7 @@ export default function (instance) {
instance.extend("parseClassProperty", function (inner) {
return function (node) {
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
node.typeAnnotation = this.flowParseTypeAnnotation();
}
return inner.call(this, node);
@@ -757,7 +757,7 @@ export default function (instance) {
instance.extend("isClassProperty", function (inner) {
return function () {
return this.type === tt.colon || inner.call(this);
return this.state.type === tt.colon || inner.call(this);
};
});
@@ -801,7 +801,7 @@ export default function (instance) {
var typeParameters;
if (this.isRelational("<")) {
typeParameters = this.flowParseTypeParameterDeclaration();
if (this.type !== tt.parenL) this.unexpected();
if (this.state.type !== tt.parenL) this.unexpected();
}
inner.apply(this, arguments);
prop.value.typeParameters = typeParameters;
@@ -813,7 +813,7 @@ export default function (instance) {
if (this.eat(tt.question)) {
param.optional = true;
}
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
param.typeAnnotation = this.flowParseTypeAnnotation();
}
this.finishNode(param, param.type);
@@ -825,7 +825,7 @@ export default function (instance) {
return function (node) {
node.importKind = "value";
var kind = (this.type === tt._typeof ? "typeof" : (this.isContextual("type") ? "type" : null));
var kind = (this.state.type === tt._typeof ? "typeof" : (this.isContextual("type") ? "type" : null));
if (kind) {
var lh = this.lookahead();
if ((lh.type === tt.name && lh.value !== "from") || lh.type === tt.braceL || lh.type === tt.star) {
@@ -852,7 +852,7 @@ export default function (instance) {
instance.extend("parseVarHead", function (inner) {
return function (decl) {
inner.call(this, decl);
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
decl.id.typeAnnotation = this.flowParseTypeAnnotation();
this.finishNode(decl.id, decl.id.type);
}
@@ -862,7 +862,7 @@ export default function (instance) {
// var foo = (async (): number => {});
instance.extend("parseAsyncArrowFromCallExpression", function (inner) {
return function (node, call) {
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
node.returnType = this.flowParseTypeAnnotation();
}
@@ -872,8 +872,8 @@ export default function (instance) {
instance.extend("parseParenAndDistinguishExpression", function (inner) {
return function (startPos, startLoc, canBeArrow, isAsync) {
startPos = startPos || this.start;
startLoc = startLoc || this.startLoc;
startPos = startPos || this.state.start;
startLoc = startLoc || this.state.startLoc;
if (this.lookahead().type === tt.parenR) {
// var foo = (): number => {};
@@ -881,21 +881,21 @@ export default function (instance) {
this.expect(tt.parenR);
let node = this.startNodeAt(startPos, startLoc);
if (this.type === tt.colon) node.returnType = this.flowParseTypeAnnotation();
if (this.state.type === tt.colon) node.returnType = this.flowParseTypeAnnotation();
this.expect(tt.arrow);
return this.parseArrowExpression(node, [], isAsync);
} else {
// var foo = (foo): number => {};
let node = inner.call(this, startPos, startLoc, canBeArrow, isAsync);
var state = this.getState();
var state = this.state.clone();
if (this.type === tt.colon) {
if (this.state.type === tt.colon) {
try {
return this.parseParenItem(node, startPos, startLoc, true);
} catch (err) {
if (err instanceof SyntaxError) {
this.setState(state);
this.state = state;
return node;
} else {
throw err;

162
packages/babylon/src/plugins/jsx/index.js
View File

@@ -1,7 +1,7 @@
import XHTMLEntities from "./xhtml";
import { TokenType, types as tt } from "../../tokenizer/types";
import { TokContext, types as tc } from "../../tokenizer/context";
import { Parser } from "../../state";
import Parser from "../../parser";
import { isIdentifierChar, isIdentifierStart } from "../../util/identifier";
import { isNewLine } from "../../util/whitespace";
@@ -18,18 +18,18 @@ tt.jsxTagStart = new TokenType("jsxTagStart");
tt.jsxTagEnd = new TokenType("jsxTagEnd");
tt.jsxTagStart.updateContext = function() {
this.context.push(tc.j_expr); // treat as beginning of JSX expression
this.context.push(tc.j_oTag); // start opening tag context
this.exprAllowed = false;
this.state.context.push(tc.j_expr); // treat as beginning of JSX expression
this.state.context.push(tc.j_oTag); // start opening tag context
this.state.exprAllowed = false;
};
tt.jsxTagEnd.updateContext = function(prevType) {
var out = this.context.pop();
var out = this.state.context.pop();
if (out === tc.j_oTag && prevType === tt.slash || out === tc.j_cTag) {
this.context.pop();
this.exprAllowed = this.curContext() === tc.j_expr;
this.state.context.pop();
this.state.exprAllowed = this.curContext() === tc.j_expr;
} else {
this.exprAllowed = true;
this.state.exprAllowed = true;
}
};
@@ -38,94 +38,94 @@ var pp = Parser.prototype;
// Reads inline JSX contents token.
pp.jsxReadToken = function() {
var out = "", chunkStart = this.pos;
var out = "", chunkStart = this.state.pos;
for (;;) {
if (this.pos >= this.input.length) {
this.raise(this.start, "Unterminated JSX contents");
if (this.state.pos >= this.input.length) {
this.raise(this.state.start, "Unterminated JSX contents");
}
var ch = this.input.charCodeAt(this.pos);
var ch = this.input.charCodeAt(this.state.pos);
switch (ch) {
case 60: // "<"
case 123: // "{"
if (this.pos === this.start) {
if (ch === 60 && this.exprAllowed) {
++this.pos;
if (this.state.pos === this.state.start) {
if (ch === 60 && this.state.exprAllowed) {
++this.state.pos;
return this.finishToken(tt.jsxTagStart);
}
return this.getTokenFromCode(ch);
}
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
return this.finishToken(tt.jsxText, out);
case 38: // "&"
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity();
chunkStart = this.pos;
chunkStart = this.state.pos;
break;
default:
if (isNewLine(ch)) {
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadNewLine(true);
chunkStart = this.pos;
chunkStart = this.state.pos;
} else {
++this.pos;
++this.state.pos;
}
}
}
};
pp.jsxReadNewLine = function(normalizeCRLF) {
var ch = this.input.charCodeAt(this.pos);
var ch = this.input.charCodeAt(this.state.pos);
var out;
++this.pos;
if (ch === 13 && this.input.charCodeAt(this.pos) === 10) {
++this.pos;
++this.state.pos;
if (ch === 13 && this.input.charCodeAt(this.state.pos) === 10) {
++this.state.pos;
out = normalizeCRLF ? "\n" : "\r\n";
} else {
out = String.fromCharCode(ch);
}
++this.curLine;
this.lineStart = this.pos;
++this.state.curLine;
this.state.lineStart = this.state.pos;
return out;
};
pp.jsxReadString = function(quote) {
var out = "", chunkStart = ++this.pos;
var out = "", chunkStart = ++this.state.pos;
for (;;) {
if (this.pos >= this.input.length) {
this.raise(this.start, "Unterminated string constant");
if (this.state.pos >= this.input.length) {
this.raise(this.state.start, "Unterminated string constant");
}
var ch = this.input.charCodeAt(this.pos);
var ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break;
if (ch === 38) { // "&"
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadEntity();
chunkStart = this.pos;
chunkStart = this.state.pos;
} else if (isNewLine(ch)) {
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.jsxReadNewLine(false);
chunkStart = this.pos;
chunkStart = this.state.pos;
} else {
++this.pos;
++this.state.pos;
}
}
out += this.input.slice(chunkStart, this.pos++);
out += this.input.slice(chunkStart, this.state.pos++);
return this.finishToken(tt.string, out);
};
pp.jsxReadEntity = function() {
var str = "", count = 0, entity;
var ch = this.input[this.pos];
if (ch !== "&") this.raise(this.pos, "Entity must start with an ampersand");
var ch = this.input[this.state.pos];
if (ch !== "&") this.raise(this.state.pos, "Entity must start with an ampersand");
var startPos = ++this.pos;
while (this.pos < this.input.length && count++ < 10) {
ch = this.input[this.pos++];
var startPos = ++this.state.pos;
while (this.state.pos < this.input.length && count++ < 10) {
ch = this.input[this.state.pos++];
if (ch === ";") {
if (str[0] === "#") {
if (str[1] === "x") {
@@ -145,7 +145,7 @@ pp.jsxReadEntity = function() {
str += ch;
}
if (!entity) {
this.pos = startPos;
this.state.pos = startPos;
return "&";
}
return entity;
@@ -160,11 +160,11 @@ pp.jsxReadEntity = function() {
// by isIdentifierStart in readToken.
pp.jsxReadWord = function() {
var ch, start = this.pos;
var ch, start = this.state.pos;
do {
ch = this.input.charCodeAt(++this.pos);
ch = this.input.charCodeAt(++this.state.pos);
} while (isIdentifierChar(ch) || ch === 45); // "-"
return this.finishToken(tt.jsxName, this.input.slice(start, this.pos));
return this.finishToken(tt.jsxName, this.input.slice(start, this.state.pos));
};
// Transforms JSX element name to string.
@@ -187,10 +187,10 @@ function getQualifiedJSXName(object) {
pp.jsxParseIdentifier = function() {
var node = this.startNode();
if (this.type === tt.jsxName) {
node.name = this.value;
} else if (this.type.keyword) {
node.name = this.type.keyword;
if (this.state.type === tt.jsxName) {
node.name = this.state.value;
} else if (this.state.type.keyword) {
node.name = this.state.type.keyword;
} else {
this.unexpected();
}
@@ -201,7 +201,7 @@ pp.jsxParseIdentifier = function() {
// Parse namespaced identifier.
pp.jsxParseNamespacedName = function() {
var startPos = this.start, startLoc = this.startLoc;
var startPos = this.state.start, startLoc = this.state.startLoc;
var name = this.jsxParseIdentifier();
if (!this.eat(tt.colon)) return name;
@@ -215,7 +215,7 @@ pp.jsxParseNamespacedName = function() {
// or single identifier.
pp.jsxParseElementName = function() {
var startPos = this.start, startLoc = this.startLoc;
var startPos = this.state.start, startLoc = this.state.startLoc;
var node = this.jsxParseNamespacedName();
while (this.eat(tt.dot)) {
var newNode = this.startNodeAt(startPos, startLoc);
@@ -229,7 +229,7 @@ pp.jsxParseElementName = function() {
// Parses any type of JSX attribute value.
pp.jsxParseAttributeValue = function() {
switch (this.type) {
switch (this.state.type) {
case tt.braceL:
var node = this.jsxParseExpressionContainer();
if (node.expression.type === "JSXEmptyExpression") {
@@ -243,7 +243,7 @@ pp.jsxParseAttributeValue = function() {
return this.parseExprAtom();
default:
this.raise(this.start, "JSX value should be either an expression or a quoted JSX text");
this.raise(this.state.start, "JSX value should be either an expression or a quoted JSX text");
}
};
@@ -252,13 +252,13 @@ pp.jsxParseAttributeValue = function() {
// at the beginning of the next one (right brace).
pp.jsxParseEmptyExpression = function() {
var tmp = this.start;
this.start = this.lastTokEnd;
this.lastTokEnd = tmp;
var tmp = this.state.start;
this.state.start = this.state.lastTokEnd;
this.state.lastTokEnd = tmp;
tmp = this.startLoc;
this.startLoc = this.lastTokEndLoc;
this.lastTokEndLoc = tmp;
tmp = this.state.startLoc;
this.state.startLoc = this.state.lastTokEndLoc;
this.state.lastTokEndLoc = tmp;
return this.finishNode(this.startNode(), "JSXEmptyExpression");
};
@@ -269,7 +269,7 @@ pp.jsxParseEmptyExpression = function() {
pp.jsxParseExpressionContainer = function() {
var node = this.startNode();
this.next();
if (this.type === tt.braceR) {
if (this.state.type === tt.braceR) {
node.expression = this.jsxParseEmptyExpression();
} else {
node.expression = this.parseExpression();
@@ -299,7 +299,7 @@ pp.jsxParseOpeningElementAt = function(startPos, startLoc) {
var node = this.startNodeAt(startPos, startLoc);
node.attributes = [];
node.name = this.jsxParseElementName();
while (this.type !== tt.slash && this.type !== tt.jsxTagEnd) {
while (this.state.type !== tt.slash && this.state.type !== tt.jsxTagEnd) {
node.attributes.push(this.jsxParseAttribute());
}
node.selfClosing = this.eat(tt.slash);
@@ -327,9 +327,9 @@ pp.jsxParseElementAt = function(startPos, startLoc) {
if (!openingElement.selfClosing) {
contents: for (;;) {
switch (this.type) {
switch (this.state.type) {
case tt.jsxTagStart:
startPos = this.start; startLoc = this.startLoc;
startPos = this.state.start; startLoc = this.state.startLoc;
this.next();
if (this.eat(tt.slash)) {
closingElement = this.jsxParseClosingElementAt(startPos, startLoc);
@@ -361,8 +361,8 @@ pp.jsxParseElementAt = function(startPos, startLoc) {
node.openingElement = openingElement;
node.closingElement = closingElement;
node.children = children;
if (this.type === tt.relational && this.value === "<") {
this.raise(this.start, "Adjacent JSX elements must be wrapped in an enclosing tag");
if (this.state.type === tt.relational && this.state.value === "<") {
this.raise(this.state.start, "Adjacent JSX elements must be wrapped in an enclosing tag");
}
return this.finishNode(node, "JSXElement");
};
@@ -370,7 +370,7 @@ pp.jsxParseElementAt = function(startPos, startLoc) {
// Parses entire JSX element from current position.
pp.jsxParseElement = function() {
var startPos = this.start, startLoc = this.startLoc;
var startPos = this.state.start, startLoc = this.state.startLoc;
this.next();
return this.jsxParseElementAt(startPos, startLoc);
};
@@ -378,9 +378,9 @@ pp.jsxParseElement = function() {
export default function(instance) {
instance.extend("parseExprAtom", function(inner) {
return function(refShortHandDefaultPos) {
if (this.type === tt.jsxText)
return this.parseLiteral(this.value);
else if (this.type === tt.jsxTagStart)
if (this.state.type === tt.jsxText)
return this.parseLiteral(this.state.value);
else if (this.state.type === tt.jsxTagStart)
return this.jsxParseElement();
else
return inner.call(this, refShortHandDefaultPos);
@@ -397,7 +397,7 @@ export default function(instance) {
if (isIdentifierStart(code)) return this.jsxReadWord();
if (code === 62) {
++this.pos;
++this.state.pos;
return this.finishToken(tt.jsxTagEnd);
}
@@ -405,8 +405,8 @@ export default function(instance) {
return this.jsxReadString(code);
}
if (code === 60 && this.exprAllowed) {
++this.pos;
if (code === 60 && this.state.exprAllowed) {
++this.state.pos;
return this.finishToken(tt.jsxTagStart);
}
return inner.call(this, code);
@@ -415,16 +415,16 @@ export default function(instance) {
instance.extend("updateContext", function(inner) {
return function(prevType) {
if (this.type === tt.braceL) {
if (this.state.type === tt.braceL) {
var curContext = this.curContext();
if (curContext === tc.j_oTag) this.context.push(tc.b_expr);
else if (curContext === tc.j_expr) this.context.push(tc.b_tmpl);
if (curContext === tc.j_oTag) this.state.context.push(tc.b_expr);
else if (curContext === tc.j_expr) this.state.context.push(tc.b_tmpl);
else inner.call(this, prevType);
this.exprAllowed = true;
} else if (this.type === tt.slash && prevType === tt.jsxTagStart) {
this.context.length -= 2; // do not consider JSX expr -> JSX open tag -> ... anymore
this.context.push(tc.j_cTag); // reconsider as closing tag context
this.exprAllowed = false;
this.state.exprAllowed = true;
} else if (this.state.type === tt.slash && prevType === tt.jsxTagStart) {
this.state.context.length -= 2; // do not consider JSX expr -> JSX open tag -> ... anymore
this.state.context.push(tc.j_cTag); // reconsider as closing tag context
this.state.exprAllowed = false;
} else {
return inner.call(this, prevType);
}

74
packages/babylon/src/tokenizer/context.js
View File

@@ -2,9 +2,7 @@
// given point in the program is loosely based on sweet.js' approach.
// See https://github.com/mozilla/sweet.js/wiki/design
import { Parser } from "../state";
import { types as tt } from "./types";
import { lineBreak } from "../util/whitespace";
export class TokContext {
constructor(token, isExpr, preserveSpace, override) {
@@ -25,75 +23,39 @@ export const types = {
f_expr: new TokContext("function", true)
};
const pp = Parser.prototype;
pp.initialContext = function () {
return [types.b_stat];
};
pp.braceIsBlock = function (prevType) {
if (prevType === tt.colon) {
let parent = this.curContext();
if (parent === types.b_stat || parent === types.b_expr)
return !parent.isExpr;
}
if (prevType === tt._return)
return lineBreak.test(this.input.slice(this.lastTokEnd, this.start));
if (prevType === tt._else || prevType === tt.semi || prevType === tt.eof)
return true;
if (prevType === tt.braceL)
return this.curContext() === types.b_stat;
return !this.exprAllowed;
};
pp.updateContext = function (prevType) {
let update, type = this.type;
if (type.keyword && prevType === tt.dot) {
this.exprAllowed = false;
} else if (update = type.updateContext) {
update.call(this, prevType);
} else {
this.exprAllowed = type.beforeExpr;
}
};
// Token-specific context update code
tt.parenR.updateContext = tt.braceR.updateContext = function () {
if (this.context.length === 1) {
this.exprAllowed = true;
if (this.state.context.length === 1) {
this.state.exprAllowed = true;
return;
}
let out = this.context.pop();
let out = this.state.context.pop();
if (out === types.b_stat && this.curContext() === types.f_expr) {
this.context.pop();
this.exprAllowed = false;
this.state.context.pop();
this.state.exprAllowed = false;
} else if (out === types.b_tmpl) {
this.exprAllowed = true;
this.state.exprAllowed = true;
} else {
this.exprAllowed = !out.isExpr;
this.state.exprAllowed = !out.isExpr;
}
};
tt.braceL.updateContext = function (prevType) {
this.context.push(this.braceIsBlock(prevType) ? types.b_stat : types.b_expr);
this.exprAllowed = true;
this.state.context.push(this.braceIsBlock(prevType) ? types.b_stat : types.b_expr);
this.state.exprAllowed = true;
};
tt.dollarBraceL.updateContext = function () {
this.context.push(types.b_tmpl);
this.exprAllowed = true;
this.state.context.push(types.b_tmpl);
this.state.exprAllowed = true;
};
tt.parenL.updateContext = function (prevType) {
let statementParens = prevType === tt._if || prevType === tt._for || prevType === tt._with || prevType === tt._while;
this.context.push(statementParens ? types.p_stat : types.p_expr);
this.exprAllowed = true;
this.state.context.push(statementParens ? types.p_stat : types.p_expr);
this.state.exprAllowed = true;
};
tt.incDec.updateContext = function () {
@@ -102,17 +64,17 @@ tt.incDec.updateContext = function () {
tt._function.updateContext = function () {
if (this.curContext() !== types.b_stat) {
this.context.push(types.f_expr);
this.state.context.push(types.f_expr);
}
this.exprAllowed = false;
this.state.exprAllowed = false;
};
tt.backQuote.updateContext = function () {
if (this.curContext() === types.q_tmpl) {
this.context.pop();
this.state.context.pop();
} else {
this.context.push(types.q_tmpl);
this.state.context.push(types.q_tmpl);
}
this.exprAllowed = false;
this.state.exprAllowed = false;
};

403
packages/babylon/src/tokenizer/index.js
View File

@@ -1,20 +1,21 @@
import { isIdentifierStart, isIdentifierChar } from "../util/identifier";
import { types as tt, keywords as keywordTypes } from "./types";
import { SourceLocation } from "../locutil";
import { types as ct } from "./context";
import { SourceLocation } from "../util/location";
import { lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace } from "../util/whitespace";
import State from "./state";
// Object type used to represent tokens. Note that normally, tokens
// simply exist as properties on the parser object. This is only
// used for the onToken callback and the external tokenizer.
export class Token {
constructor(p) {
this.type = p.type;
this.value = p.value;
this.start = p.start;
this.end = p.end;
this.loc = new SourceLocation(p.startLoc, p.endLoc);
constructor(state) {
this.type = state.type;
this.value = state.value;
this.start = state.start;
this.end = state.end;
this.loc = new SourceLocation(state.startLoc, state.endLoc);
}
}
@@ -53,50 +54,34 @@ function codePointToString(code) {
var containsEsc;
export default class Tokenizer {
constructor() {
// The current position of the tokenizer in the input.
this.pos = this.lineStart = 0;
this.curLine = 1;
// Properties of the current token:
// Its type
this.type = tt.eof;
// For tokens that include more information than their type, the value
this.value = null;
// Its start and end offset
this.start = this.end = this.pos;
// And, if locations are used, the {line, column} object
// corresponding to those offsets
this.startLoc = this.endLoc = this.curPosition();
// Position information for the previous token
this.lastTokEndLoc = this.lastTokStartLoc = null;
this.lastTokStart = this.lastTokEnd = this.pos;
// The context stack is used to superficially track syntactic
// context to predict whether a regular expression is allowed in a
// given position.
this.context = this.initialContext();
this.exprAllowed = true;
constructor(input) {
this.state = new State;
this.state.init(input);
}
// Move to the next token
next() {
if (!this.isLookahead) {
this.tokens.push(new Token(this));
}
this.state.tokens.push(new Token(this.state));
this.lastTokEnd = this.end;
this.lastTokStart = this.start;
this.lastTokEndLoc = this.endLoc;
this.lastTokStartLoc = this.startLoc;
this.state.lastTokEnd = this.state.end;
this.state.lastTokStart = this.state.start;
this.state.lastTokEndLoc = this.state.endLoc;
this.state.lastTokStartLoc = this.state.startLoc;
this.nextToken();
};
}
lookahead() {
var old = this.state.clone();
this.next();
var curr = this.state.clone();
this.state = old;
return curr;
}
getToken() {
this.next();
return new Token(this);
return new Token(this.state);
}
// Toggle strict mode. Re-reads the next number or string to please
@@ -104,17 +89,17 @@ export default class Tokenizer {
setStrict(strict) {
this.strict = strict;
if (this.type !== tt.num && this.type !== tt.string) return;
this.pos = this.start;
while (this.pos < this.lineStart) {
this.lineStart = this.input.lastIndexOf("\n", this.lineStart - 2) + 1;
--this.curLine;
if (this.state.type !== tt.num && this.state.type !== tt.string) return;
this.state.pos = this.state.start;
while (this.state.pos < this.state.lineStart) {
this.state.lineStart = this.input.lastIndexOf("\n", this.state.lineStart - 2) + 1;
--this.state.curLine;
}
this.nextToken();
}
curContext() {
return this.context[this.context.length - 1];
return this.state.context[this.state.context.length - 1];
}
// Read a single token, updating the parser object's token-related
@@ -124,9 +109,9 @@ export default class Tokenizer {
let curContext = this.curContext();
if (!curContext || !curContext.preserveSpace) this.skipSpace();
this.start = this.pos;
this.startLoc = this.curPosition();
if (this.pos >= this.input.length) return this.finishToken(tt.eof);
this.state.start = this.state.pos;
this.state.startLoc = this.state.curPosition();
if (this.state.pos >= this.input.length) return this.finishToken(tt.eof);
if (curContext.override) {
return curContext.override(this);
@@ -145,10 +130,10 @@ export default class Tokenizer {
}
fullCharCodeAtPos() {
let code = this.input.charCodeAt(this.pos);
let code = this.input.charCodeAt(this.state.pos);
if (code <= 0xd7ff || code >= 0xe000) return code;
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
return (code << 10) + next - 0x35fdc00;
}
@@ -162,63 +147,63 @@ export default class Tokenizer {
range: [start, end]
};
this.tokens.push(comment);
this.comments.push(comment);
this.state.tokens.push(comment);
this.state.comments.push(comment);
this.addComment(comment);
}
skipBlockComment() {
let startLoc = this.curPosition();
let start = this.pos, end = this.input.indexOf("*/", this.pos += 2);
if (end === -1) this.raise(this.pos - 2, "Unterminated comment");
let startLoc = this.state.curPosition();
let start = this.state.pos, end = this.input.indexOf("*/", this.state.pos += 2);
if (end === -1) this.raise(this.state.pos - 2, "Unterminated comment");
this.pos = end + 2;
this.state.pos = end + 2;
lineBreakG.lastIndex = start;
let match;
while ((match = lineBreakG.exec(this.input)) && match.index < this.pos) {
++this.curLine;
this.lineStart = match.index + match[0].length;
while ((match = lineBreakG.exec(this.input)) && match.index < this.state.pos) {
++this.state.curLine;
this.state.lineStart = match.index + match[0].length;
}
this.pushComment(true, this.input.slice(start + 2, end), start, this.pos, startLoc, this.curPosition());
this.pushComment(true, this.input.slice(start + 2, end), start, this.state.pos, startLoc, this.state.curPosition());
}
skipLineComment(startSkip) {
let start = this.pos;
let startLoc = this.curPosition();
let ch = this.input.charCodeAt(this.pos += startSkip);
while (this.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) {
++this.pos;
ch = this.input.charCodeAt(this.pos);
let start = this.state.pos;
let startLoc = this.state.curPosition();
let ch = this.input.charCodeAt(this.state.pos += startSkip);
while (this.state.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) {
++this.state.pos;
ch = this.input.charCodeAt(this.state.pos);
}
this.pushComment(false, this.input.slice(start + startSkip, this.pos), start, this.pos, startLoc, this.curPosition());
this.pushComment(false, this.input.slice(start + startSkip, this.state.pos), start, this.state.pos, startLoc, this.state.curPosition());
}
// Called at the start of the parse and after every token. Skips
// whitespace and comments, and.
skipSpace() {
loop: while (this.pos < this.input.length) {
let ch = this.input.charCodeAt(this.pos);
loop: while (this.state.pos < this.input.length) {
let ch = this.input.charCodeAt(this.state.pos);
switch (ch) {
case 32: case 160: // ' '
++this.pos;
++this.state.pos;
break;
case 13:
if (this.input.charCodeAt(this.pos + 1) === 10) {
++this.pos;
if (this.input.charCodeAt(this.state.pos + 1) === 10) {
++this.state.pos;
}
case 10: case 8232: case 8233:
++this.pos;
++this.curLine;
this.lineStart = this.pos;
++this.state.pos;
++this.state.curLine;
this.state.lineStart = this.state.pos;
break;
case 47: // '/'
switch (this.input.charCodeAt(this.pos + 1)) {
switch (this.input.charCodeAt(this.state.pos + 1)) {
case 42: // '*'
this.skipBlockComment();
break;
@@ -234,7 +219,7 @@ export default class Tokenizer {
default:
if (ch > 8 && ch < 14 || ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) {
++this.pos;
++this.state.pos;
} else {
break loop;
}
@@ -248,11 +233,11 @@ export default class Tokenizer {
// right position.
finishToken(type, val) {
this.end = this.pos;
this.endLoc = this.curPosition();
let prevType = this.type;
this.type = type;
this.value = val;
this.state.end = this.state.pos;
this.state.endLoc = this.state.curPosition();
let prevType = this.state.type;
this.state.type = type;
this.state.value = val;
this.updateContext(prevType);
}
@@ -267,23 +252,23 @@ export default class Tokenizer {
// All in the name of speed.
//
readToken_dot() {
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next >= 48 && next <= 57) return this.readNumber(true);
let next2 = this.input.charCodeAt(this.pos + 2);
let next2 = this.input.charCodeAt(this.state.pos + 2);
if (next === 46 && next2 === 46) { // 46 = dot '.'
this.pos += 3;
this.state.pos += 3;
return this.finishToken(tt.ellipsis);
} else {
++this.pos;
++this.state.pos;
return this.finishToken(tt.dot);
}
}
readToken_slash() { // '/'
let next = this.input.charCodeAt(this.pos + 1);
if (this.exprAllowed) {
++this.pos;
let next = this.input.charCodeAt(this.state.pos + 1);
if (this.state.exprAllowed) {
++this.state.pos;
return this.readRegexp();
}
if (next === 61) return this.finishOp(tt.assign, 2);
@@ -293,11 +278,11 @@ export default class Tokenizer {
readToken_mult_modulo(code) { // '%*'
var type = code === 42 ? tt.star : tt.modulo;
var width = 1;
var next = this.input.charCodeAt(this.pos + 1);
var next = this.input.charCodeAt(this.state.pos + 1);
if (next === 42 && this.options.features["es7.exponentiationOperator"]) { // '*'
width++;
next = this.input.charCodeAt(this.pos + 2);
next = this.input.charCodeAt(this.state.pos + 2);
type = tt.exponent;
}
@@ -310,14 +295,14 @@ export default class Tokenizer {
}
readToken_pipe_amp(code) { // '|&'
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2);
if (next === 61) return this.finishOp(tt.assign, 2);
return this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1);
}
readToken_caret() { // '^'
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next === 61) {
return this.finishOp(tt.assign, 2);
} else {
@@ -326,10 +311,10 @@ export default class Tokenizer {
}
readToken_plus_min(code) { // '+-'
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next === code) {
if (next === 45 && this.input.charCodeAt(this.pos + 2) === 62 && lineBreak.test(this.input.slice(this.lastTokEnd, this.pos))) {
if (next === 45 && this.input.charCodeAt(this.state.pos + 2) === 62 && lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.pos))) {
// A `-->` line comment
this.skipLineComment(3);
this.skipSpace();
@@ -346,16 +331,16 @@ export default class Tokenizer {
}
readToken_lt_gt(code) { // '<>'
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
let size = 1;
if (next === code) {
size = code === 62 && this.input.charCodeAt(this.pos + 2) === 62 ? 3 : 2;
if (this.input.charCodeAt(this.pos + size) === 61) return this.finishOp(tt.assign, size + 1);
size = code === 62 && this.input.charCodeAt(this.state.pos + 2) === 62 ? 3 : 2;
if (this.input.charCodeAt(this.state.pos + size) === 61) return this.finishOp(tt.assign, size + 1);
return this.finishOp(tt.bitShift, size);
}
if (next === 33 && code === 60 && this.input.charCodeAt(this.pos + 2) === 45 && this.input.charCodeAt(this.pos + 3) === 45) {
if (next === 33 && code === 60 && this.input.charCodeAt(this.state.pos + 2) === 45 && this.input.charCodeAt(this.state.pos + 3) === 45) {
if (this.inModule) this.unexpected();
// `<!--`, an XML-style comment that should be interpreted as a line comment
this.skipLineComment(4);
@@ -364,17 +349,17 @@ export default class Tokenizer {
}
if (next === 61) {
size = this.input.charCodeAt(this.pos + 2) === 61 ? 3 : 2;
size = this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2;
}
return this.finishOp(tt.relational, size);
}
readToken_eq_excl(code) { // '=!'
let next = this.input.charCodeAt(this.pos + 1);
if (next === 61) return this.finishOp(tt.equality, this.input.charCodeAt(this.pos + 2) === 61 ? 3 : 2);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next === 61) return this.finishOp(tt.equality, this.input.charCodeAt(this.state.pos + 2) === 61 ? 3 : 2);
if (code === 61 && next === 62) { // '=>'
this.pos += 2;
this.state.pos += 2;
return this.finishToken(tt.arrow);
}
return this.finishOp(code === 61 ? tt.eq : tt.prefix, 1);
@@ -388,32 +373,32 @@ export default class Tokenizer {
return this.readToken_dot();
// Punctuation tokens.
case 40: ++this.pos; return this.finishToken(tt.parenL);
case 41: ++this.pos; return this.finishToken(tt.parenR);
case 59: ++this.pos; return this.finishToken(tt.semi);
case 44: ++this.pos; return this.finishToken(tt.comma);
case 91: ++this.pos; return this.finishToken(tt.bracketL);
case 93: ++this.pos; return this.finishToken(tt.bracketR);
case 123: ++this.pos; return this.finishToken(tt.braceL);
case 125: ++this.pos; return this.finishToken(tt.braceR);
case 40: ++this.state.pos; return this.finishToken(tt.parenL);
case 41: ++this.state.pos; return this.finishToken(tt.parenR);
case 59: ++this.state.pos; return this.finishToken(tt.semi);
case 44: ++this.state.pos; return this.finishToken(tt.comma);
case 91: ++this.state.pos; return this.finishToken(tt.bracketL);
case 93: ++this.state.pos; return this.finishToken(tt.bracketR);
case 123: ++this.state.pos; return this.finishToken(tt.braceL);
case 125: ++this.state.pos; return this.finishToken(tt.braceR);
case 58:
if (this.options.features["es7.functionBind"] && this.input.charCodeAt(this.pos + 1) === 58) {
if (this.options.features["es7.functionBind"] && this.input.charCodeAt(this.state.pos + 1) === 58) {
return this.finishOp(tt.doubleColon, 2);
} else {
++this.pos;
++this.state.pos;
return this.finishToken(tt.colon);
}
case 63: ++this.pos; return this.finishToken(tt.question);
case 64: ++this.pos; return this.finishToken(tt.at);
case 63: ++this.state.pos; return this.finishToken(tt.question);
case 64: ++this.state.pos; return this.finishToken(tt.at);
case 96: // '`'
++this.pos;
++this.state.pos;
return this.finishToken(tt.backQuote);
case 48: // '0'
let next = this.input.charCodeAt(this.pos + 1);
let next = this.input.charCodeAt(this.state.pos + 1);
if (next === 120 || next === 88) return this.readRadixNumber(16); // '0x', '0X' - hex number
if (next === 111 || next === 79) return this.readRadixNumber(8); // '0o', '0O' - octal number
if (next === 98 || next === 66) return this.readRadixNumber(2); // '0b', '0B' - binary number
@@ -456,20 +441,20 @@ export default class Tokenizer {
return this.finishOp(tt.prefix, 1);
}
this.raise(this.pos, `Unexpected character '${codePointToString(code)}'`);
this.raise(this.state.pos, `Unexpected character '${codePointToString(code)}'`);
}
finishOp(type, size) {
let str = this.input.slice(this.pos, this.pos + size);
this.pos += size;
let str = this.input.slice(this.state.pos, this.state.pos + size);
this.state.pos += size;
return this.finishToken(type, str);
}
readRegexp() {
let escaped, inClass, start = this.pos;
let escaped, inClass, start = this.state.pos;
for (;;) {
if (this.pos >= this.input.length) this.raise(start, "Unterminated regular expression");
let ch = this.input.charAt(this.pos);
if (this.state.pos >= this.input.length) this.raise(start, "Unterminated regular expression");
let ch = this.input.charAt(this.state.pos);
if (lineBreak.test(ch)) this.raise(start, "Unterminated regular expression");
if (escaped) {
escaped = false;
@@ -479,10 +464,10 @@ export default class Tokenizer {
else if (ch === "/" && !inClass) break;
escaped = ch === "\\";
}
++this.pos;
++this.state.pos;
}
let content = this.input.slice(start, this.pos);
++this.pos;
let content = this.input.slice(start, this.state.pos);
++this.state.pos;
// Need to use `readWord1` because '\uXXXX' sequences are allowed
// here (don't ask).
let mods = this.readWord1();
@@ -525,51 +510,51 @@ export default class Tokenizer {
// will return `null` unless the integer has exactly `len` digits.
readInt(radix, len) {
let start = this.pos, total = 0;
let start = this.state.pos, total = 0;
for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
let code = this.input.charCodeAt(this.pos), val;
let code = this.input.charCodeAt(this.state.pos), val;
if (code >= 97) val = code - 97 + 10; // a
else if (code >= 65) val = code - 65 + 10; // A
else if (code >= 48 && code <= 57) val = code - 48; // 0-9
else val = Infinity;
if (val >= radix) break;
++this.pos;
++this.state.pos;
total = total * radix + val;
}
if (this.pos === start || len != null && this.pos - start !== len) return null;
if (this.state.pos === start || len != null && this.state.pos - start !== len) return null;
return total;
}
readRadixNumber(radix) {
this.pos += 2; // 0x
this.state.pos += 2; // 0x
let val = this.readInt(radix);
if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix);
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number");
if (val == null) this.raise(this.state.start + 2, "Expected number in radix " + radix);
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
return this.finishToken(tt.num, val);
}
// Read an integer, octal integer, or floating-point number.
readNumber(startsWithDot) {
let start = this.pos, isFloat = false, octal = this.input.charCodeAt(this.pos) === 48;
let start = this.state.pos, isFloat = false, octal = this.input.charCodeAt(this.state.pos) === 48;
if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number");
let next = this.input.charCodeAt(this.pos);
let next = this.input.charCodeAt(this.state.pos);
if (next === 46) { // '.'
++this.pos;
++this.state.pos;
this.readInt(10);
isFloat = true;
next = this.input.charCodeAt(this.pos);
next = this.input.charCodeAt(this.state.pos);
}
if (next === 69 || next === 101) { // 'eE'
next = this.input.charCodeAt(++this.pos);
if (next === 43 || next === 45) ++this.pos; // '+-'
next = this.input.charCodeAt(++this.state.pos);
if (next === 43 || next === 45) ++this.state.pos; // '+-'
if (this.readInt(10) === null) this.raise(start, "Invalid number");
isFloat = true;
}
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number");
if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.state.pos, "Identifier directly after number");
let str = this.input.slice(start, this.pos), val;
let str = this.input.slice(start, this.state.pos), val;
if (isFloat) val = parseFloat(str);
else if (!octal || str.length === 1) val = parseInt(str, 10);
else if (/[89]/.test(str) || this.strict) this.raise(start, "Invalid number");
@@ -580,12 +565,12 @@ export default class Tokenizer {
// Read a string value, interpreting backslash-escapes.
readCodePoint() {
let ch = this.input.charCodeAt(this.pos), code;
let ch = this.input.charCodeAt(this.state.pos), code;
if (ch === 123) {
let codePos = ++this.pos;
code = this.readHexChar(this.input.indexOf("}", this.pos) - this.pos);
++this.pos;
let codePos = ++this.state.pos;
code = this.readHexChar(this.input.indexOf("}", this.state.pos) - this.state.pos);
++this.state.pos;
if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds");
} else {
code = this.readHexChar(4);
@@ -594,54 +579,54 @@ export default class Tokenizer {
}
readString(quote) {
let out = "", chunkStart = ++this.pos;
let out = "", chunkStart = ++this.state.pos;
for (;;) {
if (this.pos >= this.input.length) this.raise(this.start, "Unterminated string constant");
let ch = this.input.charCodeAt(this.pos);
if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated string constant");
let ch = this.input.charCodeAt(this.state.pos);
if (ch === quote) break;
if (ch === 92) { // '\'
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.readEscapedChar(false);
chunkStart = this.pos;
chunkStart = this.state.pos;
} else {
if (isNewLine(ch)) this.raise(this.start, "Unterminated string constant");
++this.pos;
if (isNewLine(ch)) this.raise(this.state.start, "Unterminated string constant");
++this.state.pos;
}
}
out += this.input.slice(chunkStart, this.pos++);
out += this.input.slice(chunkStart, this.state.pos++);
return this.finishToken(tt.string, out);
}
// Reads template string tokens.
readTmplToken() {
let out = "", chunkStart = this.pos;
let out = "", chunkStart = this.state.pos;
for (;;) {
if (this.pos >= this.input.length) this.raise(this.start, "Unterminated template");
let ch = this.input.charCodeAt(this.pos);
if (ch === 96 || ch === 36 && this.input.charCodeAt(this.pos + 1) === 123) { // '`', '${'
if (this.pos === this.start && this.type === tt.template) {
if (this.state.pos >= this.input.length) this.raise(this.state.start, "Unterminated template");
let ch = this.input.charCodeAt(this.state.pos);
if (ch === 96 || ch === 36 && this.input.charCodeAt(this.state.pos + 1) === 123) { // '`', '${'
if (this.state.pos === this.state.start && this.state.type === tt.template) {
if (ch === 36) {
this.pos += 2;
this.state.pos += 2;
return this.finishToken(tt.dollarBraceL);
} else {
++this.pos;
++this.state.pos;
return this.finishToken(tt.backQuote);
}
}
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
return this.finishToken(tt.template, out);
}
if (ch === 92) { // '\'
out += this.input.slice(chunkStart, this.pos);
out += this.input.slice(chunkStart, this.state.pos);
out += this.readEscapedChar(true);
chunkStart = this.pos;
chunkStart = this.state.pos;
} else if (isNewLine(ch)) {
out += this.input.slice(chunkStart, this.pos);
++this.pos;
out += this.input.slice(chunkStart, this.state.pos);
++this.state.pos;
switch (ch) {
case 13:
if (this.input.charCodeAt(this.pos) === 10) ++this.pos;
if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos;
case 10:
out += "\n";
break;
@@ -649,11 +634,11 @@ export default class Tokenizer {
out += String.fromCharCode(ch);
break;
}
++this.curLine;
this.lineStart = this.pos;
chunkStart = this.pos;
++this.state.curLine;
this.state.lineStart = this.state.pos;
chunkStart = this.state.pos;
} else {
++this.pos;
++this.state.pos;
}
}
}
@@ -661,8 +646,8 @@ export default class Tokenizer {
// Used to read escaped characters
readEscapedChar(inTemplate) {
let ch = this.input.charCodeAt(++this.pos);
++this.pos;
let ch = this.input.charCodeAt(++this.state.pos);
++this.state.pos;
switch (ch) {
case 110: return "\n"; // 'n' -> '\n'
case 114: return "\r"; // 'r' -> '\r'
@@ -672,23 +657,23 @@ export default class Tokenizer {
case 98: return "\b"; // 'b' -> '\b'
case 118: return "\u000b"; // 'v' -> '\u000b'
case 102: return "\f"; // 'f' -> '\f'
case 13: if (this.input.charCodeAt(this.pos) === 10) ++this.pos; // '\r\n'
case 13: if (this.input.charCodeAt(this.state.pos) === 10) ++this.state.pos; // '\r\n'
case 10: // ' \n'
this.lineStart = this.pos;
++this.curLine;
this.state.lineStart = this.state.pos;
++this.state.curLine;
return "";
default:
if (ch >= 48 && ch <= 55) {
let octalStr = this.input.substr(this.pos - 1, 3).match(/^[0-7]+/)[0];
let octalStr = this.input.substr(this.state.pos - 1, 3).match(/^[0-7]+/)[0];
let octal = parseInt(octalStr, 8);
if (octal > 255) {
octalStr = octalStr.slice(0, -1);
octal = parseInt(octalStr, 8);
}
if (octal > 0 && (this.strict || inTemplate)) {
this.raise(this.pos - 2, "Octal literal in strict mode");
this.raise(this.state.pos - 2, "Octal literal in strict mode");
}
this.pos += octalStr.length - 1;
this.state.pos += octalStr.length - 1;
return String.fromCharCode(octal);
}
return String.fromCharCode(ch);
@@ -698,7 +683,7 @@ export default class Tokenizer {
// Used to read character escape sequences ('\x', '\u', '\U').
readHexChar(len) {
let codePos = this.pos;
let codePos = this.state.pos;
let n = this.readInt(16, len);
if (n === null) this.raise(codePos, "Bad character escape sequence");
return n;
@@ -712,35 +697,35 @@ export default class Tokenizer {
readWord1() {
containsEsc = false;
let word = "", first = true, chunkStart = this.pos;
while (this.pos < this.input.length) {
let word = "", first = true, chunkStart = this.state.pos;
while (this.state.pos < this.input.length) {
let ch = this.fullCharCodeAtPos();
if (isIdentifierChar(ch, true)) {
this.pos += ch <= 0xffff ? 1 : 2;
this.state.pos += ch <= 0xffff ? 1 : 2;
} else if (ch === 92) { // "\"
containsEsc = true;
word += this.input.slice(chunkStart, this.pos);
let escStart = this.pos;
word += this.input.slice(chunkStart, this.state.pos);
let escStart = this.state.pos;
if (this.input.charCodeAt(++this.pos) !== 117) { // "u"
this.raise(this.pos, "Expecting Unicode escape sequence \\uXXXX");
if (this.input.charCodeAt(++this.state.pos) !== 117) { // "u"
this.raise(this.state.pos, "Expecting Unicode escape sequence \\uXXXX");
}
++this.pos;
++this.state.pos;
let esc = this.readCodePoint();
if (!(first ? isIdentifierStart : isIdentifierChar)(esc, true)) {
this.raise(escStart, "Invalid Unicode escape");
}
word += codePointToString(esc);
chunkStart = this.pos;
chunkStart = this.state.pos;
} else {
break;
}
first = false;
}
return word + this.input.slice(chunkStart, this.pos);
return word + this.input.slice(chunkStart, this.state.pos);
}
// Read an identifier or keyword token. Will check for reserved
@@ -753,4 +738,38 @@ export default class Tokenizer {
type = keywordTypes[word];
return this.finishToken(type, word);
}
braceIsBlock(prevType) {
if (prevType === tt.colon) {
let parent = this.curContext();
if (parent === ct.b_stat || parent === ct.b_expr) {
return !parent.isExpr;
}
}
if (prevType === tt._return) {
return lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start));
}
if (prevType === tt._else || prevType === tt.semi || prevType === tt.eof) {
return true;
}
if (prevType === tt.braceL) {
return this.curContext() === ct.b_stat;
}
return !this.state.exprAllowed;
}
updateContext(prevType) {
let update, type = this.state.type;
if (type.keyword && prevType === tt.dot) {
this.state.exprAllowed = false;
} else if (update = type.updateContext) {
update.call(this, prevType);
} else {
this.state.exprAllowed = type.beforeExpr;
}
}
}

73
packages/babylon/src/tokenizer/state.js Normal file
View File

@@ -0,0 +1,73 @@
import { Position } from "../util/location";
import { types as ct } from "./context";
import { types as tt } from "./types";
export default class State {
init(input) {
this.input = input;
// Used to signify the start of a potential arrow function
this.potentialArrowAt = -1;
// Flags to track whether we are in a function, a generator.
this.inFunction = this.inGenerator = false;
// Labels in scope.
this.labels = [];
// Leading decorators.
this.decorators = [];
// Token store.
this.tokens = [];
// Comment store.
this.comments = [];
// Comment attachment store
this.bottomRightStack = [];
this.trailingComments = [];
this.leadingComments = [];
// The current position of the tokenizer in the input.
this.pos = this.lineStart = 0;
this.curLine = 1;
// Properties of the current token:
// Its type
this.type = tt.eof;
// For tokens that include more information than their type, the value
this.value = null;
// Its start and end offset
this.start = this.end = this.pos;
// And, if locations are used, the {line, column} object
// corresponding to those offsets
this.startLoc = this.endLoc = this.curPosition();
// Position information for the previous token
this.lastTokEndLoc = this.lastTokStartLoc = null;
this.lastTokStart = this.lastTokEnd = this.pos;
// The context stack is used to superficially track syntactic
// context to predict whether a regular expression is allowed in a
// given position.
this.context = [ct.b_stat];
this.exprAllowed = true;
return this;
}
curPosition() {
return new Position(this.curLine, this.pos - this.lineStart);
}
clone() {
var state = new State;
for (var key in this) {
var val = this[key];
if (Array.isArray(val)) val = val.slice();
state[key] = val;
}
return state;
}
}

2
packages/babylon/src/locutil.js → packages/babylon/src/util/location.js
View File

@@ -1,4 +1,4 @@
import { lineBreakG } from "./util/whitespace";
import { lineBreakG } from "./whitespace";
// These are used when `options.locations` is on, for the
// `startLoc` and `endLoc` properties.

2
packages/babylon/test/fixtures/experimental/uncategorised/32/actual.js vendored
View File

@@ -1 +1 @@
export async function foo(){}
export async function foo() {}

26
packages/babylon/test/fixtures/experimental/uncategorised/32/expected.json vendored
View File

@@ -1,7 +1,7 @@
{
"type": "File",
"start": 0,
"end": 29,
"end": 30,
"loc": {
"start": {
"line": 1,
@@ -9,13 +9,13 @@
},
"end": {
"line": 1,
"column": 29
"column": 30
}
},
"program": {
"type": "Program",
"start": 0,
"end": 29,
"end": 30,
"loc": {
"start": {
"line": 1,
@@ -23,7 +23,7 @@
},
"end": {
"line": 1,
"column": 29
"column": 30
}
},
"sourceType": "module",
@@ -31,7 +31,7 @@
{
"type": "ExportNamedDeclaration",
"start": 0,
"end": 29,
"end": 30,
"loc": {
"start": {
"line": 1,
@@ -39,13 +39,13 @@
},
"end": {
"line": 1,
"column": 29
"column": 30
}
},
"declaration": {
"type": "FunctionDeclaration",
"start": 7,
"end": 29,
"end": 30,
"loc": {
"start": {
"line": 1,
@@ -53,7 +53,7 @@
},
"end": {
"line": 1,
"column": 29
"column": 30
}
},
"id": {
@@ -78,16 +78,16 @@
"params": [],
"body": {
"type": "BlockStatement",
"start": 27,
"end": 29,
"start": 28,
"end": 30,
"loc": {
"start": {
"line": 1,
"column": 27
"column": 28
},
"end": {
"line": 1,
"column": 29
"column": 30
}
},
"body": []
@@ -98,4 +98,4 @@
}
]
}
}
}