Store token type as number (#13768)

* refactor: abstract token metadata access

* refactor: move token-specific update context logic

* refactor: centralize token definitions

* refactor: abstract token type creation

* refactor: use number as token storage

* build: replace tt.* as number

* fix flow errors

* fix: build on Node 12

* Update packages/babel-parser/src/tokenizer/types.js

Co-authored-by: Nicolò Ribaudo <nicolo.ribaudo@gmail.com>

* refactor: rename token types exports to tt

* update unit test

* test: update Babel 8 test fixtures

* fix: centralize obsolete token type updateContext

* fix flow errors

Co-authored-by: Nicolò Ribaudo <nicolo.ribaudo@gmail.com>

babel.config.js

@@ -2,6 +2,7 @@
 
 const pathUtils = require("path");
 const fs = require("fs");
+const { parseSync } = require("@babel/core");
 
 function normalize(src) {
   return src.replace(/\//, pathUtils.sep);
@@ -176,7 +177,10 @@ module.exports = function (api) {
           "packages/babel-parser",
           "packages/babel-helper-validator-identifier",
         ].map(normalize),
-        plugins: ["babel-plugin-transform-charcodes"],
+        plugins: [
+          "babel-plugin-transform-charcodes",
+          pluginBabelParserTokenType,
+        ],
         assumptions: parserAssumptions,
       },
       {
@@ -583,3 +587,68 @@ function pluginImportMetaUrl({ types: t, template }) {
     },
   };
 }
+
+const tokenTypesMapping = new Map();
+const tokenTypeSourcePath = "./packages/babel-parser/src/tokenizer/types.js";
+
+function pluginBabelParserTokenType({
+  types: { isIdentifier, numericLiteral },
+}) {
+  return {
+    visitor: {
+      MemberExpression(path) {
+        const { node } = path;
+        if (
+          isIdentifier(node.object, { name: "tt" }) &&
+          isIdentifier(node.property) &&
+          !node.computed
+        ) {
+          const tokenName = node.property.name;
+          const tokenType = tokenTypesMapping.get(node.property.name);
+          if (tokenType === undefined) {
+            throw path.buildCodeFrameError(
+              `${tokenName} is not defined in ${tokenTypeSourcePath}`
+            );
+          }
+          path.replaceWith(numericLiteral(tokenType));
+        }
+      },
+    },
+  };
+}
+
+(function generateTokenTypesMapping() {
+  const tokenTypesAst = parseSync(
+    fs.readFileSync(tokenTypeSourcePath, {
+      encoding: "utf-8",
+    }),
+    {
+      configFile: false,
+      parserOpts: { attachComments: false, plugins: ["flow"] },
+    }
+  );
+
+  let typesDeclaration;
+  for (const n of tokenTypesAst.program.body) {
+    if (n.type === "ExportNamedDeclaration" && n.exportKind === "value") {
+      const declarations = n.declaration.declarations;
+      if (declarations !== undefined) typesDeclaration = declarations[0];
+      if (
+        typesDeclaration !== undefined &&
+        typesDeclaration.id.name === "types"
+      ) {
+        break;
+      }
+    }
+  }
+  if (typesDeclaration === undefined) {
+    throw new Error(
+      "The plugin can not find TokenType definition in " + tokenTypeSourcePath
+    );
+  }
+
+  const tokenTypesDefinition = typesDeclaration.init.properties;
+  for (let i = 0; i < tokenTypesDefinition.length; i++) {
+    tokenTypesMapping.set(tokenTypesDefinition[i].key.name, i);
+  }
+})();

packages/babel-parser/src/index.js

@@ -10,7 +10,7 @@ import {
 } from "./plugin-utils";
 import Parser from "./parser";
 
-import { types as tokTypes } from "./tokenizer/types";
+import { getExportedToken, tt as internalTokenTypes } from "./tokenizer/types";
 import "./tokenizer/context";
 
 import type { Expression, File } from "./types";
@@ -67,7 +67,15 @@ export function parseExpression(input: string, options?: Options): Expression {
   return parser.getExpression();
 }
 
-export { tokTypes };
+function generateExportedTokenTypes(internalTokenTypes) {
+  const tokenTypes = {};
+  for (const typeName of Object.keys(internalTokenTypes)) {
+    tokenTypes[typeName] = getExportedToken(internalTokenTypes[typeName]);
+  }
+  return tokenTypes;
+}
+
+export const tokTypes = generateExportedTokenTypes(internalTokenTypes);
 
 function getParser(options: ?Options, input: string): Parser {
   let cls = Parser;

packages/babel-parser/src/parser/expression.js

@@ -18,7 +18,19 @@
 //
 // [opp]: http://en.wikipedia.org/wiki/Operator-precedence_parser
 
-import { types as tt, type TokenType } from "../tokenizer/types";
+import {
+  tokenCanStartExpression,
+  tokenIsAssignment,
+  tokenIsKeyword,
+  tokenIsOperator,
+  tokenIsPostfix,
+  tokenIsPrefix,
+  tokenIsRightAssociative,
+  tokenLabelName,
+  tokenOperatorPrecedence,
+  tt,
+  type TokenType,
+} from "../tokenizer/types";
 import * as N from "../types";
 import LValParser from "./lval";
 import {
@@ -287,7 +299,7 @@ export default class ExpressionParser extends LValParser {
     if (afterLeftParse) {
       left = afterLeftParse.call(this, left, startPos, startLoc);
     }
-    if (this.state.type.isAssign) {
+    if (tokenIsAssignment(this.state.type)) {
       const node = this.startNodeAt(startPos, startLoc);
       const operator = this.state.value;
       node.operator = operator;
@@ -394,8 +406,7 @@ export default class ExpressionParser extends LValParser {
       const { start } = left;
 
       if (
-        // TODO: When migrating to TS, use tt._in.binop!
+        minPrec >= tokenOperatorPrecedence(tt._in) ||
-        minPrec >= ((tt._in.binop: any): number) ||
         !this.prodParam.hasIn ||
         !this.match(tt._in)
       ) {
@@ -405,10 +416,10 @@ export default class ExpressionParser extends LValParser {
       this.classScope.usePrivateName(value, start);
     }
 
-    let prec = this.state.type.binop;
-    if (prec != null && (this.prodParam.hasIn || !this.match(tt._in))) {
-      if (prec > minPrec) {
     const op = this.state.type;
+    if (tokenIsOperator(op) && (this.prodParam.hasIn || !this.match(tt._in))) {
+      let prec = tokenOperatorPrecedence(op);
+      if (prec > minPrec) {
         if (op === tt.pipeline) {
           this.expectPlugin("pipelineOperator");
           if (this.state.inFSharpPipelineDirectBody) {
@@ -426,7 +437,7 @@ export default class ExpressionParser extends LValParser {
         if (coalesce) {
           // Handle the precedence of `tt.coalesce` as equal to the range of logical expressions.
           // In other words, `node.right` shouldn't contain logical expressions in order to check the mixed error.
-          prec = ((tt.logicalAND: any): { binop: number }).binop;
+          prec = tokenOperatorPrecedence(tt.logicalAND);
         }
 
         this.next();
@@ -524,7 +535,7 @@ export default class ExpressionParser extends LValParser {
       this.parseMaybeUnaryOrPrivate(),
       startPos,
       startLoc,
-      op.rightAssociative ? prec - 1 : prec,
+      tokenIsRightAssociative(op) ? prec - 1 : prec,
     );
   }
 
@@ -576,7 +587,7 @@ export default class ExpressionParser extends LValParser {
     }
     const update = this.match(tt.incDec);
     const node = this.startNode();
-    if (this.state.type.prefix) {
+    if (tokenIsPrefix(this.state.type)) {
       node.operator = this.state.value;
       node.prefix = true;
 
@@ -609,9 +620,10 @@ export default class ExpressionParser extends LValParser {
     const expr = this.parseUpdate(node, update, refExpressionErrors);
 
     if (isAwait) {
+      const { type } = this.state;
       const startsExpr = this.hasPlugin("v8intrinsic")
-        ? this.state.type.startsExpr
+        ? tokenCanStartExpression(type)
-        : this.state.type.startsExpr && !this.match(tt.modulo);
+        : tokenCanStartExpression(type) && !this.match(tt.modulo);
       if (startsExpr && !this.isAmbiguousAwait()) {
         this.raiseOverwrite(startPos, Errors.AwaitNotInAsyncContext);
         return this.parseAwait(startPos, startLoc);
@@ -636,7 +648,7 @@ export default class ExpressionParser extends LValParser {
     const startLoc = this.state.startLoc;
     let expr = this.parseExprSubscripts(refExpressionErrors);
     if (this.checkExpressionErrors(refExpressionErrors, false)) return expr;
-    while (this.state.type.postfix && !this.canInsertSemicolon()) {
+    while (tokenIsPostfix(this.state.type) && !this.canInsertSemicolon()) {
       const node = this.startNodeAt(startPos, startLoc);
       node.operator = this.state.value;
       node.prefix = false;
@@ -1356,7 +1368,7 @@ export default class ExpressionParser extends LValParser {
       throw this.raise(
         start,
         Errors.PipeTopicUnconfiguredToken,
-        tokenType.label,
+        tokenLabelName(tokenType),
       );
     }
   }
@@ -1381,7 +1393,7 @@ export default class ExpressionParser extends LValParser {
           "pipelineOperator",
           "topicToken",
         );
-        return tokenType.label === pluginTopicToken;
+        return tokenLabelName(tokenType) === pluginTopicToken;
       }
       case "smart":
         return tokenType === tt.hash;
@@ -2527,8 +2539,8 @@ export default class ExpressionParser extends LValParser {
 
     if (type === tt.name) {
       name = this.state.value;
-    } else if (type.keyword) {
+    } else if (tokenIsKeyword(type)) {
-      name = type.keyword;
+      name = tokenLabelName(type);
     } else {
       throw this.unexpected();
     }
@@ -2538,7 +2550,7 @@ export default class ExpressionParser extends LValParser {
       // This will prevent this.next() from throwing about unexpected escapes.
       this.state.type = tt.name;
     } else {
-      this.checkReservedWord(name, start, !!type.keyword, false);
+      this.checkReservedWord(name, start, tokenIsKeyword(type), false);
     }
 
     this.next();

packages/babel-parser/src/parser/lval.js

@@ -2,7 +2,7 @@
 
 /*:: declare var invariant; */
 import * as charCodes from "charcodes";
-import { types as tt, type TokenType } from "../tokenizer/types";
+import { tt, type TokenType } from "../tokenizer/types";
 import type {
   TSParameterProperty,
   Decorator,

packages/babel-parser/src/parser/statement.js

@@ -1,7 +1,12 @@
 // @flow
 
 import * as N from "../types";
-import { types as tt, type TokenType } from "../tokenizer/types";
+import {
+  tokenIsLoop,
+  tt,
+  type TokenType,
+  getExportedToken,
+} from "../tokenizer/types";
 import ExpressionParser from "./expression";
 import { Errors, SourceTypeModuleErrors } from "./error";
 import { isIdentifierChar, isIdentifierStart } from "../util/identifier";
@@ -56,10 +61,11 @@ const keywordRelationalOperator = /in(?:stanceof)?/y;
  * @returns
  */
 function babel7CompatTokens(tokens) {
-  if (!process.env.BABEL_8_BREAKING) {
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
-      if (token.type === tt.privateName) {
+    const { type } = token;
+    if (type === tt.privateName) {
+      if (!process.env.BABEL_8_BREAKING) {
         const { loc, start, value, end } = token;
         const hashEndPos = start + 1;
         const hashEndLoc = new Position(loc.start.line, loc.start.column + 1);
@@ -68,7 +74,7 @@ function babel7CompatTokens(tokens) {
           1,
           // $FlowIgnore: hacky way to create token
           new Token({
-            type: tt.hash,
+            type: getExportedToken(tt.hash),
             value: "#",
             start: start,
             end: hashEndPos,
@@ -77,7 +83,7 @@ function babel7CompatTokens(tokens) {
           }),
           // $FlowIgnore: hacky way to create token
           new Token({
-            type: tt.name,
+            type: getExportedToken(tt.name),
             value: value,
             start: hashEndPos,
             end: end,
@@ -85,8 +91,14 @@ function babel7CompatTokens(tokens) {
             endLoc: loc.end,
           }),
         );
+        i++;
+        continue;
       }
     }
+    if (typeof type === "number") {
+      // $FlowIgnore: we manipulate `token` for performance reasons
+      token.type = getExportedToken(type);
+    }
   }
   return tokens;
 }
@@ -246,9 +258,9 @@ export default class StatementParser extends ExpressionParser {
 
     switch (starttype) {
       case tt._break:
+        return this.parseBreakContinueStatement(node, /* isBreak */ true);
       case tt._continue:
-        // $FlowFixMe
+        return this.parseBreakContinueStatement(node, /* isBreak */ false);
-        return this.parseBreakContinueStatement(node, starttype.keyword);
       case tt._debugger:
         return this.parseDebuggerStatement(node);
       case tt._do:
@@ -472,9 +484,8 @@ export default class StatementParser extends ExpressionParser {
 
   parseBreakContinueStatement(
     node: N.BreakStatement | N.ContinueStatement,
-    keyword: string,
+    isBreak: boolean,
   ): N.BreakStatement | N.ContinueStatement {
-    const isBreak = keyword === "break";
     this.next();
 
     if (this.isLineTerminator()) {
@@ -484,7 +495,7 @@ export default class StatementParser extends ExpressionParser {
       this.semicolon();
     }
 
-    this.verifyBreakContinue(node, keyword);
+    this.verifyBreakContinue(node, isBreak);
 
     return this.finishNode(
       node,
@@ -494,9 +505,8 @@ export default class StatementParser extends ExpressionParser {
 
   verifyBreakContinue(
     node: N.BreakStatement | N.ContinueStatement,
-    keyword: string,
+    isBreak: boolean,
   ) {
-    const isBreak = keyword === "break";
     let i;
     for (i = 0; i < this.state.labels.length; ++i) {
       const lab = this.state.labels[i];
@@ -506,7 +516,11 @@ export default class StatementParser extends ExpressionParser {
       }
     }
     if (i === this.state.labels.length) {
-      this.raise(node.start, Errors.IllegalBreakContinue, keyword);
+      this.raise(
+        node.start,
+        Errors.IllegalBreakContinue,
+        isBreak ? "break" : "continue",
+      );
     }
   }
 
@@ -850,7 +864,7 @@ export default class StatementParser extends ExpressionParser {
       }
     }
 
-    const kind = this.state.type.isLoop
+    const kind = tokenIsLoop(this.state.type)
       ? "loop"
       : this.match(tt._switch)
       ? "switch"
@@ -1994,7 +2008,8 @@ export default class StatementParser extends ExpressionParser {
   }
 
   shouldParseExportDeclaration(): boolean {
-    if (this.match(tt.at)) {
+    const { type } = this.state;
+    if (type === tt.at) {
       this.expectOnePlugin(["decorators", "decorators-legacy"]);
       if (this.hasPlugin("decorators")) {
         if (this.getPluginOption("decorators", "decoratorsBeforeExport")) {
@@ -2006,10 +2021,10 @@ export default class StatementParser extends ExpressionParser {
     }
 
     return (
-      this.state.type.keyword === "var" ||
+      type === tt._var ||
-      this.state.type.keyword === "const" ||
+      type === tt._const ||
-      this.state.type.keyword === "function" ||
+      type === tt._function ||
-      this.state.type.keyword === "class" ||
+      type === tt._class ||
       this.isLet() ||
       this.isAsyncFunction()
     );

packages/babel-parser/src/parser/util.js

@@ -1,6 +1,12 @@
 // @flow
 
-import { types as tt, TokenType } from "../tokenizer/types";
+import {
+  isTokenType,
+  tokenIsKeyword,
+  tokenLabelName,
+  tt,
+  type TokenType,
+} from "../tokenizer/types";
 import Tokenizer from "../tokenizer";
 import State from "../tokenizer/state";
 import type { Node } from "../types";
@@ -168,15 +174,19 @@ export default class UtilParser extends Tokenizer {
       template: "Unexpected token",
     },
   ): empty {
-    if (messageOrType instanceof TokenType) {
+    if (isTokenType(messageOrType)) {
       messageOrType = {
         code: ErrorCodes.SyntaxError,
         reasonCode: "UnexpectedToken",
-        template: `Unexpected token, expected "${messageOrType.label}"`,
+        template: `Unexpected token, expected "${tokenLabelName(
+          // $FlowIgnore: Flow does not support assertion signature and TokenType is opaque
+          messageOrType,
+        )}"`,
       };
     }
 
     /* eslint-disable @babel/development-internal/dry-error-messages */
+    // $FlowIgnore: Flow does not support assertion signature and TokenType is opaque
     throw this.raise(pos != null ? pos : this.state.start, messageOrType);
     /* eslint-enable @babel/development-internal/dry-error-messages */
   }
@@ -298,7 +308,7 @@ export default class UtilParser extends Tokenizer {
   isLiteralPropertyName(): boolean {
     return (
       this.match(tt.name) ||
-      !!this.state.type.keyword ||
+      tokenIsKeyword(this.state.type) ||
       this.match(tt.string) ||
       this.match(tt.num) ||
       this.match(tt.bigint) ||

packages/babel-parser/src/plugins/estree.js

@@ -1,6 +1,6 @@
 // @flow
 
-import { TokenType } from "../tokenizer/types";
+import { type TokenType } from "../tokenizer/types";
 import type Parser from "../parser";
 import type { ExpressionErrors } from "../parser/util";
 import * as N from "../types";

packages/babel-parser/src/plugins/flow/index.js

@@ -6,7 +6,12 @@
 /* eslint-disable @babel/development-internal/dry-error-messages */
 
 import type Parser from "../../parser";
-import { types as tt, type TokenType } from "../../tokenizer/types";
+import {
+  tokenIsKeyword,
+  tokenLabelName,
+  tt,
+  type TokenType,
+} from "../../tokenizer/types";
 import * as N from "../../types";
 import type { Position } from "../../util/location";
 import { types as tc } from "../../tokenizer/context";
@@ -157,7 +162,8 @@ function hasTypeImportKind(node: N.Node): boolean {
 
 function isMaybeDefaultImport(state: { type: TokenType, value: any }): boolean {
   return (
-    (state.type === tt.name || !!state.type.keyword) && state.value !== "from"
+    (state.type === tt.name || tokenIsKeyword(state.type)) &&
+    state.value !== "from"
   );
 }
 
@@ -1605,11 +1611,12 @@ export default (superClass: Class<Parser>): Class<Parser> =>
           this.next();
           return this.finishNode(node, "ExistsTypeAnnotation");
 
-        default:
+        case tt._typeof:
-          if (this.state.type.keyword === "typeof") {
           return this.flowParseTypeofType();
-          } else if (this.state.type.keyword) {
+
-            const label = this.state.type.label;
+        default:
+          if (tokenIsKeyword(this.state.type)) {
+            const label = tokenLabelName(this.state.type);
             this.next();
             return super.createIdentifier(node, label);
           }
@@ -2650,7 +2657,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         if (
           specifierTypeKind !== null &&
           !this.match(tt.name) &&
-          !this.state.type.keyword
+          !tokenIsKeyword(this.state.type)
         ) {
           // `import {type as ,` or `import {type as }`
           specifier.imported = as_ident;
@@ -2665,7 +2672,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       } else {
         if (
           specifierTypeKind !== null &&
-          (this.match(tt.name) || this.state.type.keyword)
+          (this.match(tt.name) || tokenIsKeyword(this.state.type))
         ) {
           // `import {type foo`
           specifier.imported = this.parseIdentifier(true);

packages/babel-parser/src/plugins/jsx/index.js

@@ -8,7 +8,13 @@ import * as charCodes from "charcodes";
 import XHTMLEntities from "./xhtml";
 import type Parser from "../../parser";
 import type { ExpressionErrors } from "../../parser/util";
-import { TokenType, types as tt } from "../../tokenizer/types";
+import {
+  tokenComesBeforeExpression,
+  tokenIsKeyword,
+  tokenLabelName,
+  type TokenType,
+  tt,
+} from "../../tokenizer/types";
 import { TokContext, types as tc } from "../../tokenizer/context";
 import * as N from "../../types";
 import { isIdentifierChar, isIdentifierStart } from "../../util/identifier";
@@ -45,23 +51,11 @@ const JsxErrors = makeErrorTemplates(
 /* eslint-disable sort-keys */
 
 // Be aware that this file is always executed and not only when the plugin is enabled.
-// Therefore this contexts and tokens do always exist.
+// Therefore the contexts do always exist.
 tc.j_oTag = new TokContext("<tag");
 tc.j_cTag = new TokContext("</tag");
 tc.j_expr = new TokContext("<tag>...</tag>", true);
 
-tt.jsxName = new TokenType("jsxName");
-tt.jsxText = new TokenType("jsxText", { beforeExpr: true });
-tt.jsxTagStart = new TokenType("jsxTagStart", { startsExpr: true });
-tt.jsxTagEnd = new TokenType("jsxTagEnd");
-
-tt.jsxTagStart.updateContext = context => {
-  context.push(
-    tc.j_expr, // treat as beginning of JSX expression
-    tc.j_oTag, // start opening tag context
-  );
-};
-
 function isFragment(object: ?N.JSXElement): boolean {
   return object
     ? object.type === "JSXOpeningFragment" ||
@@ -259,8 +253,8 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       const node = this.startNode();
       if (this.match(tt.jsxName)) {
         node.name = this.state.value;
-      } else if (this.state.type.keyword) {
+      } else if (tokenIsKeyword(this.state.type)) {
-        node.name = this.state.type.keyword;
+        node.name = tokenLabelName(this.state.type);
       } else {
         this.unexpected();
       }
@@ -624,6 +618,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         // reconsider as closing tag context
         context.splice(-2, 2, tc.j_cTag);
         this.state.exprAllowed = false;
+      } else if (type === tt.jsxTagStart) {
+        context.push(
+          tc.j_expr, // treat as beginning of JSX expression
+          tc.j_oTag, // start opening tag context
+        );
       } else if (type === tt.jsxTagEnd) {
         const out = context.pop();
         if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) {
@@ -633,12 +632,12 @@ export default (superClass: Class<Parser>): Class<Parser> =>
           this.state.exprAllowed = true;
         }
       } else if (
-        type.keyword &&
+        tokenIsKeyword(type) &&
         (prevType === tt.dot || prevType === tt.questionDot)
       ) {
         this.state.exprAllowed = false;
       } else {
-        this.state.exprAllowed = type.beforeExpr;
+        this.state.exprAllowed = tokenComesBeforeExpression(type);
       }
     }
   };

packages/babel-parser/src/plugins/placeholders.js

@@ -2,13 +2,11 @@
 
 import * as charCodes from "charcodes";
 
-import { types as tt, TokenType } from "../tokenizer/types";
+import { tokenLabelName, tt } from "../tokenizer/types";
 import type Parser from "../parser";
 import * as N from "../types";
 import { makeErrorTemplates, ErrorCodes } from "../parser/error";
 
-tt.placeholder = new TokenType("%%", { startsExpr: true });
-
 export type PlaceholderTypes =
   | "Identifier"
   | "StringLiteral"
@@ -288,7 +286,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         if (this.isUnparsedContextual(next, "from")) {
           if (
             this.input.startsWith(
-              tt.placeholder.label,
+              tokenLabelName(tt.placeholder),
               this.nextTokenStartSince(next + 4),
             )
           ) {

packages/babel-parser/src/plugins/typescript/index.js

@@ -7,7 +7,7 @@
 
 import type { TokenType } from "../../tokenizer/types";
 import type State from "../../tokenizer/state";
-import { types as tt } from "../../tokenizer/types";
+import { tokenOperatorPrecedence, tt } from "../../tokenizer/types";
 import { types as ct } from "../../tokenizer/context";
 import * as N from "../../types";
 import type { Position } from "../../util/location";
@@ -2195,7 +2195,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       minPrec: number,
     ) {
       if (
-        nonNull(tt._in.binop) > minPrec &&
+        tokenOperatorPrecedence(tt._in) > minPrec &&
         !this.hasPrecedingLineBreak() &&
         this.isContextual("as")
       ) {

packages/babel-parser/src/plugins/v8intrinsic.js

@@ -1,5 +1,5 @@
 import type Parser from "../parser";
-import { types as tt } from "../tokenizer/types";
+import { tt } from "../tokenizer/types";
 import * as N from "../types";
 
 export default (superClass: Class<Parser>): Class<Parser> =>

packages/babel-parser/src/tokenizer/context.js

@@ -3,8 +3,6 @@
 // The token context is used to track whether the apostrophe "`"
 // starts or ends a string template
 
-import { types as tt } from "./types";
-
 export class TokContext {
   constructor(token: string, preserveSpace?: boolean) {
     this.token = token;
@@ -21,39 +19,3 @@ export const types: {
   brace: new TokContext("{"),
   template: new TokContext("`", true),
 };
-
-// Token-specific context update code
-// Note that we should avoid accessing `this.prodParam` in context update,
-// because it is executed immediately when last token is consumed, which may be
-// before `this.prodParam` is updated. e.g.
-// ```
-// function *g() { () => yield / 2 }
-// ```
-// When `=>` is eaten, the context update of `yield` is executed, however,
-// `this.prodParam` still has `[Yield]` production because it is not yet updated
-
-tt.braceR.updateContext = context => {
-  context.pop();
-};
-
-// we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
-// ideally only dollarBraceL "${" needs a non-template context
-// in order to indicate that the last "`" in `${`" starts a new string template
-// inside a template element within outer string template.
-// but when we popped such context in `}`, we lost track of whether this
-// `}` matches a `${` or other tokens matching `}`, so we have to push
-// such context in every token that `}` will match.
-tt.braceL.updateContext =
-  tt.braceHashL.updateContext =
-  tt.dollarBraceL.updateContext =
-    context => {
-      context.push(types.brace);
-    };
-
-tt.backQuote.updateContext = context => {
-  if (context[context.length - 1] === types.template) {
-    context.pop();
-  } else {
-    context.push(types.template);
-  }
-};

packages/babel-parser/src/tokenizer/index.js

@@ -6,7 +6,13 @@ import type { Options } from "../options";
 import * as N from "../types";
 import * as charCodes from "charcodes";
 import { isIdentifierStart, isIdentifierChar } from "../util/identifier";
-import { types as tt, keywords as keywordTypes, type TokenType } from "./types";
+import {
+  tokenIsKeyword,
+  tokenLabelName,
+  tt,
+  keywords as keywordTypes,
+  type TokenType,
+} from "./types";
 import { type TokContext, types as ct } from "./context";
 import ParserErrors, { Errors, type ErrorTemplate } from "../parser/error";
 import { SourceLocation } from "../util/location";
@@ -1564,15 +1570,54 @@ export default class Tokenizer extends ParserErrors {
   }
 
   checkKeywordEscapes(): void {
-    const kw = this.state.type.keyword;
+    const { type } = this.state;
-    if (kw && this.state.containsEsc) {
+    if (tokenIsKeyword(type) && this.state.containsEsc) {
-      this.raise(this.state.start, Errors.InvalidEscapedReservedWord, kw);
+      this.raise(
+        this.state.start,
+        Errors.InvalidEscapedReservedWord,
+        tokenLabelName(type),
+      );
     }
   }
 
   // the prevType is required by the jsx plugin
   // eslint-disable-next-line no-unused-vars
   updateContext(prevType: TokenType): void {
-    this.state.type.updateContext?.(this.state.context);
+    // Token-specific context update code
+    // Note that we should avoid accessing `this.prodParam` in context update,
+    // because it is executed immediately when last token is consumed, which may be
+    // before `this.prodParam` is updated. e.g.
+    // ```
+    // function *g() { () => yield / 2 }
+    // ```
+    // When `=>` is eaten, the context update of `yield` is executed, however,
+    // `this.prodParam` still has `[Yield]` production because it is not yet updated
+    const { context, type } = this.state;
+    switch (type) {
+      case tt.braceR:
+        context.pop();
+        break;
+      // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
+      // ideally only dollarBraceL "${" needs a non-template context
+      // in order to indicate that the last "`" in `${`" starts a new string template
+      // inside a template element within outer string template.
+      // but when we popped such context in `}`, we lost track of whether this
+      // `}` matches a `${` or other tokens matching `}`, so we have to push
+      // such context in every token that `}` will match.
+      case tt.braceL:
+      case tt.braceHashL:
+      case tt.dollarBraceL:
+        context.push(ct.brace);
+        break;
+      case tt.backQuote:
+        if (context[context.length - 1] === ct.template) {
+          context.pop();
+        } else {
+          context.push(ct.template);
+        }
+        break;
+      default:
+        break;
+    }
   }
 }

packages/babel-parser/src/tokenizer/state.js

@@ -6,7 +6,7 @@ import type { CommentWhitespace } from "../parser/comments";
 import { Position } from "../util/location";
 
 import { types as ct, type TokContext } from "./context";
-import { types as tt, type TokenType } from "./types";
+import { tt, type TokenType } from "./types";
 import type { ParsingError, ErrorTemplate } from "../parser/error";
 
 type TopicContextState = {

packages/babel-parser/src/tokenizer/types.js

@@ -1,5 +1,5 @@
 // @flow
-import type { TokContext } from "./context";
+import { types as tc, type TokContext } from "./context";
 // ## Token types
 
 // The assignment of fine-grained, information-carrying type objects
@@ -41,7 +41,13 @@ type TokenOptions = {
   binop?: ?number,
 };
 
-export class TokenType {
+// Internally the tokenizer stores token as a number
+export opaque type TokenType = number;
+
+// The `ExportedTokenType` is exported via `tokTypes` and accessible
+// when `tokens: true` is enabled. Unlike internal token type, it provides
+// metadata of the tokens.
+export class ExportedTokenType {
   label: string;
   keyword: ?string;
   beforeExpr: boolean;
@@ -52,7 +58,8 @@ export class TokenType {
   prefix: boolean;
   postfix: boolean;
   binop: ?number;
-  updateContext: ?(context: Array<TokContext>) => void;
+  // todo(Babel 8): remove updateContext from exposed token layout
+  declare updateContext: ?(context: Array<TokContext>) => void;
 
   constructor(label: string, conf: TokenOptions = {}) {
     this.label = label;
@@ -65,63 +72,88 @@ export class TokenType {
     this.prefix = !!conf.prefix;
     this.postfix = !!conf.postfix;
     this.binop = conf.binop != null ? conf.binop : null;
+    if (!process.env.BABEL_8_BREAKING) {
       this.updateContext = null;
     }
   }
+}
 
 export const keywords = new Map<string, TokenType>();
 
 function createKeyword(name: string, options: TokenOptions = {}): TokenType {
   options.keyword = name;
-  const token = new TokenType(name, options);
+  const token = createToken(name, options);
   keywords.set(name, token);
   return token;
 }
 
 function createBinop(name: string, binop: number) {
-  return new TokenType(name, { beforeExpr, binop });
+  return createToken(name, { beforeExpr, binop });
 }
 
-export const types: { [name: string]: TokenType } = {
+let tokenTypeCounter = -1;
-  num: new TokenType("num", { startsExpr }),
+export const tokenTypes: ExportedTokenType[] = [];
-  bigint: new TokenType("bigint", { startsExpr }),
+const tokenLabels: string[] = [];
-  decimal: new TokenType("decimal", { startsExpr }),
+const tokenBinops: number[] = [];
-  regexp: new TokenType("regexp", { startsExpr }),
+const tokenBeforeExprs: boolean[] = [];
-  string: new TokenType("string", { startsExpr }),
+const tokenStartsExprs: boolean[] = [];
-  name: new TokenType("name", { startsExpr }),
+const tokenPrefixes: boolean[] = [];
-  privateName: new TokenType("#name", { startsExpr }),
+
-  eof: new TokenType("eof"),
+function createToken(name: string, options: TokenOptions = {}): TokenType {
+  ++tokenTypeCounter;
+  tokenLabels.push(name);
+  tokenBinops.push(options.binop ?? -1);
+  tokenBeforeExprs.push(options.beforeExpr ?? false);
+  tokenStartsExprs.push(options.startsExpr ?? false);
+  tokenPrefixes.push(options.prefix ?? false);
+  tokenTypes.push(new ExportedTokenType(name, options));
+
+  return tokenTypeCounter;
+}
+
+// For performance the token type helpers depend on the following declarations order.
+// When adding new token types, please also check if the token helpers need update.
+
+export const tt: { [name: string]: TokenType } = {
+  num: createToken("num", { startsExpr }),
+  bigint: createToken("bigint", { startsExpr }),
+  decimal: createToken("decimal", { startsExpr }),
+  regexp: createToken("regexp", { startsExpr }),
+  string: createToken("string", { startsExpr }),
+  name: createToken("name", { startsExpr }),
+  privateName: createToken("#name", { startsExpr }),
+  eof: createToken("eof"),
 
   // Punctuation token types.
-  bracketL: new TokenType("[", { beforeExpr, startsExpr }),
+  bracketL: createToken("[", { beforeExpr, startsExpr }),
-  bracketHashL: new TokenType("#[", { beforeExpr, startsExpr }),
+  bracketHashL: createToken("#[", { beforeExpr, startsExpr }),
-  bracketBarL: new TokenType("[|", { beforeExpr, startsExpr }),
+  bracketBarL: createToken("[|", { beforeExpr, startsExpr }),
-  bracketR: new TokenType("]"),
+  bracketR: createToken("]"),
-  bracketBarR: new TokenType("|]"),
+  bracketBarR: createToken("|]"),
-  braceL: new TokenType("{", { beforeExpr, startsExpr }),
+  braceL: createToken("{", { beforeExpr, startsExpr }),
-  braceBarL: new TokenType("{|", { beforeExpr, startsExpr }),
+  braceBarL: createToken("{|", { beforeExpr, startsExpr }),
-  braceHashL: new TokenType("#{", { beforeExpr, startsExpr }),
+  braceHashL: createToken("#{", { beforeExpr, startsExpr }),
-  braceR: new TokenType("}", { beforeExpr }),
+  braceR: createToken("}", { beforeExpr }),
-  braceBarR: new TokenType("|}"),
+  braceBarR: createToken("|}"),
-  parenL: new TokenType("(", { beforeExpr, startsExpr }),
+  parenL: createToken("(", { beforeExpr, startsExpr }),
-  parenR: new TokenType(")"),
+  parenR: createToken(")"),
-  comma: new TokenType(",", { beforeExpr }),
+  comma: createToken(",", { beforeExpr }),
-  semi: new TokenType(";", { beforeExpr }),
+  semi: createToken(";", { beforeExpr }),
-  colon: new TokenType(":", { beforeExpr }),
+  colon: createToken(":", { beforeExpr }),
-  doubleColon: new TokenType("::", { beforeExpr }),
+  doubleColon: createToken("::", { beforeExpr }),
-  dot: new TokenType("."),
+  dot: createToken("."),
-  question: new TokenType("?", { beforeExpr }),
+  question: createToken("?", { beforeExpr }),
-  questionDot: new TokenType("?."),
+  questionDot: createToken("?."),
-  arrow: new TokenType("=>", { beforeExpr }),
+  arrow: createToken("=>", { beforeExpr }),
-  template: new TokenType("template"),
+  template: createToken("template"),
-  ellipsis: new TokenType("...", { beforeExpr }),
+  ellipsis: createToken("...", { beforeExpr }),
-  backQuote: new TokenType("`", { startsExpr }),
+  backQuote: createToken("`", { startsExpr }),
-  dollarBraceL: new TokenType("${", { beforeExpr, startsExpr }),
+  dollarBraceL: createToken("${", { beforeExpr, startsExpr }),
-  at: new TokenType("@"),
+  at: createToken("@"),
-  hash: new TokenType("#", { startsExpr }),
+  hash: createToken("#", { startsExpr }),
 
   // Special hashbang token.
-  interpreterDirective: new TokenType("#!..."),
+  interpreterDirective: createToken("#!..."),
 
   // Operators. These carry several kinds of properties to help the
   // parser use them properly (the presence of these properties is
@@ -137,15 +169,19 @@ export const types: { [name: string]: TokenType } = {
   // binary operators with a very low precedence, that should result
   // in AssignmentExpression nodes.
 
-  eq: new TokenType("=", { beforeExpr, isAssign }),
+  // start: isAssign
-  assign: new TokenType("_=", { beforeExpr, isAssign }),
+  eq: createToken("=", { beforeExpr, isAssign }),
-  slashAssign: new TokenType("_=", { beforeExpr, isAssign }),
+  assign: createToken("_=", { beforeExpr, isAssign }),
+  slashAssign: createToken("_=", { beforeExpr, isAssign }),
   // This is only needed to support % as a Hack-pipe topic token. If the proposal
   // ends up choosing a different token, it can be merged with tt.assign.
-  moduloAssign: new TokenType("_=", { beforeExpr, isAssign }),
+  moduloAssign: createToken("_=", { beforeExpr, isAssign }),
-  incDec: new TokenType("++/--", { prefix, postfix, startsExpr }),
+  // end: isAssign
-  bang: new TokenType("!", { beforeExpr, prefix, startsExpr }),
+
-  tilde: new TokenType("~", { beforeExpr, prefix, startsExpr }),
+  incDec: createToken("++/--", { prefix, postfix, startsExpr }),
+  bang: createToken("!", { beforeExpr, prefix, startsExpr }),
+  tilde: createToken("~", { beforeExpr, prefix, startsExpr }),
+  // start: isBinop
   pipeline: createBinop("|>", 0),
   nullishCoalescing: createBinop("??", 1),
   logicalOR: createBinop("||", 1),
@@ -156,13 +192,13 @@ export const types: { [name: string]: TokenType } = {
   equality: createBinop("==/!=/===/!==", 6),
   relational: createBinop("</>/<=/>=", 7),
   bitShift: createBinop("<</>>/>>>", 8),
-  plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }),
+  plusMin: createToken("+/-", { beforeExpr, binop: 9, prefix, startsExpr }),
   // startsExpr: required by v8intrinsic plugin
-  modulo: new TokenType("%", { binop: 10, startsExpr }),
+  modulo: createToken("%", { binop: 10, startsExpr }),
   // unset `beforeExpr` as it can be `function *`
-  star: new TokenType("*", { binop: 10 }),
+  star: createToken("*", { binop: 10 }),
   slash: createBinop("/", 10),
-  exponent: new TokenType("**", {
+  exponent: createToken("**", {
     beforeExpr,
     binop: 11,
     rightAssociative: true,
@@ -171,16 +207,18 @@ export const types: { [name: string]: TokenType } = {
   // Keywords
   // Don't forget to update packages/babel-helper-validator-identifier/src/keyword.js
   // when new keywords are added
+  // start: isKeyword
+  _in: createKeyword("in", { beforeExpr, binop: 7 }),
+  _instanceof: createKeyword("instanceof", { beforeExpr, binop: 7 }),
+  // end: isBinop
   _break: createKeyword("break"),
   _case: createKeyword("case", { beforeExpr }),
   _catch: createKeyword("catch"),
   _continue: createKeyword("continue"),
   _debugger: createKeyword("debugger"),
   _default: createKeyword("default", { beforeExpr }),
-  _do: createKeyword("do", { isLoop, beforeExpr }),
   _else: createKeyword("else", { beforeExpr }),
   _finally: createKeyword("finally"),
-  _for: createKeyword("for", { isLoop }),
   _function: createKeyword("function", { startsExpr }),
   _if: createKeyword("if"),
   _return: createKeyword("return", { beforeExpr }),
@@ -189,7 +227,6 @@ export const types: { [name: string]: TokenType } = {
   _try: createKeyword("try"),
   _var: createKeyword("var"),
   _const: createKeyword("const"),
-  _while: createKeyword("while", { isLoop }),
   _with: createKeyword("with"),
   _new: createKeyword("new", { beforeExpr, startsExpr }),
   _this: createKeyword("this", { startsExpr }),
@@ -201,9 +238,99 @@ export const types: { [name: string]: TokenType } = {
   _null: createKeyword("null", { startsExpr }),
   _true: createKeyword("true", { startsExpr }),
   _false: createKeyword("false", { startsExpr }),
-  _in: createKeyword("in", { beforeExpr, binop: 7 }),
-  _instanceof: createKeyword("instanceof", { beforeExpr, binop: 7 }),
   _typeof: createKeyword("typeof", { beforeExpr, prefix, startsExpr }),
   _void: createKeyword("void", { beforeExpr, prefix, startsExpr }),
   _delete: createKeyword("delete", { beforeExpr, prefix, startsExpr }),
+  // start: isLoop
+  _do: createKeyword("do", { isLoop, beforeExpr }),
+  _for: createKeyword("for", { isLoop }),
+  _while: createKeyword("while", { isLoop }),
+  // end: isLoop
+  // end: isKeyword
+
+  // jsx plugin
+  jsxName: createToken("jsxName"),
+  jsxText: createToken("jsxText", { beforeExpr: true }),
+  jsxTagStart: createToken("jsxTagStart", { startsExpr: true }),
+  jsxTagEnd: createToken("jsxTagEnd"),
+
+  // placeholder plugin
+  placeholder: createToken("%%", { startsExpr: true }),
 };
+
+export function tokenComesBeforeExpression(token: TokenType): boolean {
+  return tokenBeforeExprs[token];
+}
+
+export function tokenCanStartExpression(token: TokenType): boolean {
+  return tokenStartsExprs[token];
+}
+
+export function tokenIsAssignment(token: TokenType): boolean {
+  return token >= tt.eq && token <= tt.moduloAssign;
+}
+
+export function tokenIsLoop(token: TokenType): boolean {
+  return token >= tt._do && token <= tt._while;
+}
+
+export function tokenIsKeyword(token: TokenType): boolean {
+  return token >= tt._in && token <= tt._while;
+}
+
+export function tokenIsOperator(token: TokenType): boolean {
+  return token >= tt.pipeline && token <= tt._instanceof;
+}
+
+export function tokenIsPostfix(token: TokenType): boolean {
+  return token === tt.incDec;
+}
+
+export function tokenIsPrefix(token: TokenType): boolean {
+  return tokenPrefixes[token];
+}
+
+export function tokenLabelName(token: TokenType): string {
+  return tokenLabels[token];
+}
+
+export function tokenOperatorPrecedence(token: TokenType): number {
+  return tokenBinops[token];
+}
+
+export function tokenIsRightAssociative(token: TokenType): boolean {
+  return token === tt.exponent;
+}
+
+export function getExportedToken(token: TokenType): ExportedTokenType {
+  return tokenTypes[token];
+}
+
+export function isTokenType(obj: any): boolean {
+  return typeof obj === "number";
+}
+
+if (!process.env.BABEL_8_BREAKING) {
+  tokenTypes[tt.braceR].updateContext = context => {
+    context.pop();
+  };
+
+  tokenTypes[tt.braceL].updateContext =
+    tokenTypes[tt.braceHashL].updateContext =
+    tokenTypes[tt.dollarBraceL].updateContext =
+      context => {
+        context.push(tc.brace);
+      };
+
+  tokenTypes[tt.backQuote].updateContext = context => {
+    if (context[context.length - 1] === tc.template) {
+      context.pop();
+    } else {
+      context.push(tc.template);
+    }
+  };
+
+  tokenTypes[tt.jsxTagStart].updateContext = context => {
+    context.push(tc.j_expr, tc.j_oTag);
+  };
+}

packages/babel-parser/test/fixtures/core/opts/private-name-tokens-true/output.json

@@ -53,8 +53,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "class",
       "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
@@ -69,8 +68,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "C",
       "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}}
@@ -99,8 +97,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "p",
       "start":12,"end":14,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":4}}
@@ -129,8 +126,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":16,"end":16,"loc":{"start":{"line":3,"column":1},"end":{"line":3,"column":1}}
     }

packages/babel-parser/test/fixtures/core/opts/tokens-true-babel-7/input.js

@@ -0,0 +1,3 @@
+var a = 1;
+
+var b = a + 1;

packages/babel-parser/test/fixtures/core/opts/tokens-true-babel-7/options.json

@@ -0,0 +1,4 @@
+{
+  "BABEL_8_BREAKING": false,
+  "tokens": true
+}

packages/babel-parser/test/fixtures/core/opts/tokens-true-babel-7/output.json

@@ -0,0 +1,282 @@
+{
+  "type": "File",
+  "start":0,"end":26,"loc":{"start":{"line":1,"column":0},"end":{"line":3,"column":14}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":26,"loc":{"start":{"line":1,"column":0},"end":{"line":3,"column":14}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "VariableDeclaration",
+        "start":0,"end":10,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":10}},
+        "declarations": [
+          {
+            "type": "VariableDeclarator",
+            "start":4,"end":9,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":9}},
+            "id": {
+              "type": "Identifier",
+              "start":4,"end":5,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5},"identifierName":"a"},
+              "name": "a"
+            },
+            "init": {
+              "type": "NumericLiteral",
+              "start":8,"end":9,"loc":{"start":{"line":1,"column":8},"end":{"line":1,"column":9}},
+              "extra": {
+                "rawValue": 1,
+                "raw": "1"
+              },
+              "value": 1
+            }
+          }
+        ],
+        "kind": "var"
+      },
+      {
+        "type": "VariableDeclaration",
+        "start":12,"end":26,"loc":{"start":{"line":3,"column":0},"end":{"line":3,"column":14}},
+        "declarations": [
+          {
+            "type": "VariableDeclarator",
+            "start":16,"end":25,"loc":{"start":{"line":3,"column":4},"end":{"line":3,"column":13}},
+            "id": {
+              "type": "Identifier",
+              "start":16,"end":17,"loc":{"start":{"line":3,"column":4},"end":{"line":3,"column":5},"identifierName":"b"},
+              "name": "b"
+            },
+            "init": {
+              "type": "BinaryExpression",
+              "start":20,"end":25,"loc":{"start":{"line":3,"column":8},"end":{"line":3,"column":13}},
+              "left": {
+                "type": "Identifier",
+                "start":20,"end":21,"loc":{"start":{"line":3,"column":8},"end":{"line":3,"column":9},"identifierName":"a"},
+                "name": "a"
+              },
+              "operator": "+",
+              "right": {
+                "type": "NumericLiteral",
+                "start":24,"end":25,"loc":{"start":{"line":3,"column":12},"end":{"line":3,"column":13}},
+                "extra": {
+                  "rawValue": 1,
+                  "raw": "1"
+                },
+                "value": 1
+              }
+            }
+          }
+        ],
+        "kind": "var"
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "var",
+        "keyword": "var",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "var",
+      "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "a",
+      "start":4,"end":5,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}
+    },
+    {
+      "type": {
+        "label": "=",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": true,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "=",
+      "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}}
+    },
+    {
+      "type": {
+        "label": "num",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": 1,
+      "start":8,"end":9,"loc":{"start":{"line":1,"column":8},"end":{"line":1,"column":9}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
+    },
+    {
+      "type": {
+        "label": "var",
+        "keyword": "var",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "var",
+      "start":12,"end":15,"loc":{"start":{"line":3,"column":0},"end":{"line":3,"column":3}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "b",
+      "start":16,"end":17,"loc":{"start":{"line":3,"column":4},"end":{"line":3,"column":5}}
+    },
+    {
+      "type": {
+        "label": "=",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": true,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "=",
+      "start":18,"end":19,"loc":{"start":{"line":3,"column":6},"end":{"line":3,"column":7}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "a",
+      "start":20,"end":21,"loc":{"start":{"line":3,"column":8},"end":{"line":3,"column":9}}
+    },
+    {
+      "type": {
+        "label": "+/-",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": true,
+        "postfix": false,
+        "binop": 9,
+        "updateContext": null
+      },
+      "value": "+",
+      "start":22,"end":23,"loc":{"start":{"line":3,"column":10},"end":{"line":3,"column":11}}
+    },
+    {
+      "type": {
+        "label": "num",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": 1,
+      "start":24,"end":25,"loc":{"start":{"line":3,"column":12},"end":{"line":3,"column":13}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":25,"end":26,"loc":{"start":{"line":3,"column":13},"end":{"line":3,"column":14}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":26,"end":26,"loc":{"start":{"line":3,"column":14},"end":{"line":3,"column":14}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/core/opts/tokens-true/options.json

@@ -1,3 +1,4 @@
 {
+  "BABEL_8_BREAKING": true,
   "tokens": true
 }

packages/babel-parser/test/fixtures/core/opts/tokens-true/output.json

@@ -82,8 +82,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "var",
       "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}
@@ -98,8 +97,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "a",
       "start":4,"end":5,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":5}}
@@ -114,8 +112,7 @@
         "isAssign": true,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "=",
       "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}}
@@ -130,8 +127,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": 1,
       "start":8,"end":9,"loc":{"start":{"line":1,"column":8},"end":{"line":1,"column":9}}
@@ -146,8 +142,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
     },
@@ -162,8 +157,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "var",
       "start":12,"end":15,"loc":{"start":{"line":3,"column":0},"end":{"line":3,"column":3}}
@@ -178,8 +172,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "b",
       "start":16,"end":17,"loc":{"start":{"line":3,"column":4},"end":{"line":3,"column":5}}
@@ -194,8 +187,7 @@
         "isAssign": true,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "=",
       "start":18,"end":19,"loc":{"start":{"line":3,"column":6},"end":{"line":3,"column":7}}
@@ -210,8 +202,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "a",
       "start":20,"end":21,"loc":{"start":{"line":3,"column":8},"end":{"line":3,"column":9}}
@@ -226,8 +217,7 @@
         "isAssign": false,
         "prefix": true,
         "postfix": false,
-        "binop": 9,
+        "binop": 9
-        "updateContext": null
       },
       "value": "+",
       "start":22,"end":23,"loc":{"start":{"line":3,"column":10},"end":{"line":3,"column":11}}
@@ -242,8 +232,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": 1,
       "start":24,"end":25,"loc":{"start":{"line":3,"column":12},"end":{"line":3,"column":13}}
@@ -258,8 +247,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":25,"end":26,"loc":{"start":{"line":3,"column":13},"end":{"line":3,"column":14}}
     },
@@ -273,8 +261,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":26,"end":26,"loc":{"start":{"line":3,"column":14},"end":{"line":3,"column":14}}
     }

packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true-babel-7/input.js

@@ -0,0 +1 @@
+<div propA={[key: value]} />

packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true-babel-7/options.json

@@ -0,0 +1,5 @@
+{
+  "BABEL_8_BREAKING": false,
+  "tokens": true,
+  "plugins": ["jsx", "flow"]
+}

packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true-babel-7/output.json

@@ -0,0 +1,295 @@
+{
+  "type": "File",
+  "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+  "errors": [
+    "SyntaxError: The type cast expression is expected to be wrapped with parenthesis. (1:16)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+    "sourceType": "module",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+        "expression": {
+          "type": "JSXElement",
+          "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+          "openingElement": {
+            "type": "JSXOpeningElement",
+            "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+            "name": {
+              "type": "JSXIdentifier",
+              "start":1,"end":4,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":4}},
+              "name": "div"
+            },
+            "attributes": [
+              {
+                "type": "JSXAttribute",
+                "start":5,"end":25,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":25}},
+                "name": {
+                  "type": "JSXIdentifier",
+                  "start":5,"end":10,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":10}},
+                  "name": "propA"
+                },
+                "value": {
+                  "type": "JSXExpressionContainer",
+                  "start":11,"end":25,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":25}},
+                  "expression": {
+                    "type": "ArrayExpression",
+                    "start":12,"end":24,"loc":{"start":{"line":1,"column":12},"end":{"line":1,"column":24}},
+                    "elements": [
+                      {
+                        "type": "TypeCastExpression",
+                        "start":13,"end":23,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":23}},
+                        "expression": {
+                          "type": "Identifier",
+                          "start":13,"end":16,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":16},"identifierName":"key"},
+                          "name": "key"
+                        },
+                        "typeAnnotation": {
+                          "type": "TypeAnnotation",
+                          "start":16,"end":23,"loc":{"start":{"line":1,"column":16},"end":{"line":1,"column":23}},
+                          "typeAnnotation": {
+                            "type": "GenericTypeAnnotation",
+                            "start":18,"end":23,"loc":{"start":{"line":1,"column":18},"end":{"line":1,"column":23}},
+                            "typeParameters": null,
+                            "id": {
+                              "type": "Identifier",
+                              "start":18,"end":23,"loc":{"start":{"line":1,"column":18},"end":{"line":1,"column":23},"identifierName":"value"},
+                              "name": "value"
+                            }
+                          }
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            ],
+            "selfClosing": true
+          },
+          "closingElement": null,
+          "children": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "jsxTagStart",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":0,"end":1,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":1}}
+    },
+    {
+      "type": {
+        "label": "jsxName",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "div",
+      "start":1,"end":4,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":4}}
+    },
+    {
+      "type": {
+        "label": "jsxName",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "propA",
+      "start":5,"end":10,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":10}}
+    },
+    {
+      "type": {
+        "label": "=",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": true,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "=",
+      "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}}
+    },
+    {
+      "type": {
+        "label": "{",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":11,"end":12,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":12}}
+    },
+    {
+      "type": {
+        "label": "[",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":12,"end":13,"loc":{"start":{"line":1,"column":12},"end":{"line":1,"column":13}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "key",
+      "start":13,"end":16,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":16}}
+    },
+    {
+      "type": {
+        "label": ":",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":16,"end":17,"loc":{"start":{"line":1,"column":16},"end":{"line":1,"column":17}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "value",
+      "start":18,"end":23,"loc":{"start":{"line":1,"column":18},"end":{"line":1,"column":23}}
+    },
+    {
+      "type": {
+        "label": "]",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":23,"end":24,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":24}}
+    },
+    {
+      "type": {
+        "label": "}",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":24,"end":25,"loc":{"start":{"line":1,"column":24},"end":{"line":1,"column":25}}
+    },
+    {
+      "type": {
+        "label": "/",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": 10,
+        "updateContext": null
+      },
+      "value": "/",
+      "start":26,"end":27,"loc":{"start":{"line":1,"column":26},"end":{"line":1,"column":27}}
+    },
+    {
+      "type": {
+        "label": "jsxTagEnd",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":28,"end":28,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":28}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/options.json

@@ -1,4 +1,5 @@
 {
+  "BABEL_8_BREAKING": true,
   "tokens": true,
   "plugins": ["jsx", "flow"]
 }

packages/babel-parser/test/fixtures/flow/typecasts/fail-without-parens-jsx-tokens-true/output.json

@@ -102,8 +102,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "div",
       "start":1,"end":4,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":4}}
@@ -118,8 +117,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "propA",
       "start":5,"end":10,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":10}}
@@ -134,8 +132,7 @@
         "isAssign": true,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "=",
       "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}}
@@ -164,8 +161,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":12,"end":13,"loc":{"start":{"line":1,"column":12},"end":{"line":1,"column":13}}
     },
@@ -179,8 +175,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "key",
       "start":13,"end":16,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":16}}
@@ -195,8 +190,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":16,"end":17,"loc":{"start":{"line":1,"column":16},"end":{"line":1,"column":17}}
     },
@@ -210,8 +204,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "value",
       "start":18,"end":23,"loc":{"start":{"line":1,"column":18},"end":{"line":1,"column":23}}
@@ -226,8 +219,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":23,"end":24,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":24}}
     },
@@ -255,8 +247,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 10,
+        "binop": 10
-        "updateContext": null
       },
       "value": "/",
       "start":26,"end":27,"loc":{"start":{"line":1,"column":26},"end":{"line":1,"column":27}}
@@ -271,8 +262,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
     },
@@ -286,8 +276,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":28,"end":28,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":28}}
     }

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-bar/input.js

@@ -0,0 +1 @@
+{||}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-bar/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
+  "tokens": true
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-bar/output.json

@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "RecordExpression",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "properties": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "{|",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "|}",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-hash/input.js

@@ -0,0 +1 @@
+#{}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-hash/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
+  "tokens": true
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/brace-hash/output.json

@@ -0,0 +1,67 @@
+{
+  "type": "File",
+  "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+        "expression": {
+          "type": "RecordExpression",
+          "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+          "properties": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "#{",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "}",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-bar/input.js

@@ -0,0 +1 @@
+[||]

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-bar/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
+  "tokens": true
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-bar/output.json

@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "TupleExpression",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "elements": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "[|",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "|]",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-hash/input.js

@@ -0,0 +1 @@
+#[]

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-hash/options.json

@@ -0,0 +1,4 @@
+{
+  "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
+  "tokens": true
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/bracket-hash/output.json

@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+        "expression": {
+          "type": "TupleExpression",
+          "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+          "elements": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "#[",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "]",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple-babel-7/options.json

@@ -0,0 +1,3 @@
+{
+  "BABEL_8_BREAKING": false
+}

packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json

@@ -30,8 +30,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
     },
@@ -45,8 +44,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
     },
@@ -60,8 +58,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
     }

packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json

@@ -58,8 +58,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
     }

packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json

@@ -30,8 +30,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
     },
@@ -45,8 +44,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
     },
@@ -60,8 +58,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
     }

packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json

@@ -30,8 +30,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
     },
@@ -45,8 +44,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
     },
@@ -60,8 +58,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
     }

packages/babel-parser/test/fixtures/tokens/record-and-tuple/options.json

@@ -0,0 +1,3 @@
+{
+  "BABEL_8_BREAKING": true
+}

packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json

@@ -85,8 +85,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "async",
       "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}}
@@ -101,8 +100,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": "<",
       "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}}
@@ -117,8 +115,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "T",
       "start":7,"end":8,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":8}}
@@ -133,8 +130,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": ">",
       "start":8,"end":9,"loc":{"start":{"line":1,"column":8},"end":{"line":1,"column":9}}
@@ -149,8 +145,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
     },
@@ -164,8 +159,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "a",
       "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}}
@@ -180,8 +174,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":11,"end":12,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":12}}
     },
@@ -195,8 +188,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "T",
       "start":13,"end":14,"loc":{"start":{"line":1,"column":13},"end":{"line":1,"column":14}}
@@ -211,8 +203,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":14,"end":15,"loc":{"start":{"line":1,"column":14},"end":{"line":1,"column":15}}
     },
@@ -226,8 +217,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":15,"end":16,"loc":{"start":{"line":1,"column":15},"end":{"line":1,"column":16}}
     },
@@ -241,8 +231,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "T",
       "start":17,"end":18,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":18}}
@@ -257,8 +246,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":19,"end":21,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":21}}
     },
@@ -272,8 +260,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "a",
       "start":22,"end":23,"loc":{"start":{"line":1,"column":22},"end":{"line":1,"column":23}}
@@ -288,8 +275,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":23,"end":24,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":24}}
     },
@@ -303,8 +289,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":24,"end":24,"loc":{"start":{"line":1,"column":24},"end":{"line":1,"column":24}}
     }

packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json

@@ -97,8 +97,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "type",
       "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}
@@ -113,8 +112,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "T",
       "start":5,"end":6,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":6}}
@@ -129,8 +127,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": "<",
       "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7}}
@@ -145,8 +142,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "U",
       "start":7,"end":8,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":8}}
@@ -162,8 +158,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "extends",
       "start":9,"end":16,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":16}}
@@ -178,8 +173,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "object",
       "start":17,"end":23,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":23}}
@@ -194,8 +188,7 @@
         "isAssign": true,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "=",
       "start":24,"end":25,"loc":{"start":{"line":1,"column":24},"end":{"line":1,"column":25}}
@@ -224,8 +217,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "x",
       "start":28,"end":29,"loc":{"start":{"line":1,"column":28},"end":{"line":1,"column":29}}
@@ -240,8 +232,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":29,"end":30,"loc":{"start":{"line":1,"column":29},"end":{"line":1,"column":30}}
     },
@@ -255,8 +246,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "number",
       "start":31,"end":37,"loc":{"start":{"line":1,"column":31},"end":{"line":1,"column":37}}
@@ -285,8 +275,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": ">",
       "start":39,"end":40,"loc":{"start":{"line":1,"column":39},"end":{"line":1,"column":40}}
@@ -301,8 +290,7 @@
         "isAssign": true,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "=",
       "start":41,"end":42,"loc":{"start":{"line":1,"column":41},"end":{"line":1,"column":42}}
@@ -317,8 +305,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "Array",
       "start":43,"end":48,"loc":{"start":{"line":1,"column":43},"end":{"line":1,"column":48}}
@@ -333,8 +320,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": "<",
       "start":48,"end":49,"loc":{"start":{"line":1,"column":48},"end":{"line":1,"column":49}}
@@ -349,8 +335,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "value": "U",
       "start":49,"end":50,"loc":{"start":{"line":1,"column":49},"end":{"line":1,"column":50}}
@@ -365,8 +350,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": 7,
+        "binop": 7
-        "updateContext": null
       },
       "value": ">",
       "start":50,"end":51,"loc":{"start":{"line":1,"column":50},"end":{"line":1,"column":51}}
@@ -381,8 +365,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":51,"end":52,"loc":{"start":{"line":1,"column":51},"end":{"line":1,"column":52}}
     },
@@ -396,8 +379,7 @@
         "isAssign": false,
         "prefix": false,
         "postfix": false,
-        "binop": null,
+        "binop": null
-        "updateContext": null
       },
       "start":52,"end":52,"loc":{"start":{"line":1,"column":52},"end":{"line":1,"column":52}}
     }

packages/babel-parser/test/unit/tokenizer/types.js

@@ -1,15 +1,21 @@
-import { types } from "../../../src/tokenizer/types";
+import { tt, tokenOperatorPrecedence } from "../../../src/tokenizer/types";
 
 describe("token types", () => {
   it("should check if the binOp for relational === in", () => {
-    expect(types.relational.binop).toEqual(types._in.binop);
+    expect(tokenOperatorPrecedence(tt.relational)).toEqual(
+      tokenOperatorPrecedence(tt._in),
+    );
   });
 
   it("should check if the binOp for relational === instanceOf", () => {
-    expect(types.relational.binop).toEqual(types._instanceof.binop);
+    expect(tokenOperatorPrecedence(tt.relational)).toEqual(
+      tokenOperatorPrecedence(tt._instanceof),
+    );
   });
 
   it("should check if the binOp for in === instanceOf", () => {
-    expect(types._in.binop).toEqual(types._instanceof.binop);
+    expect(tokenOperatorPrecedence(tt._in)).toEqual(
+      tokenOperatorPrecedence(tt._instanceof),
+    );
   });
 });