Improve template tokenizing (#13919)

* add benchmarks

* refactor: tokenize template as middle + tail

* perf: avoid push tc.brace

* refactor: overwrite skipSpace in jsx plugin

* transform tl.templateMiddle/Tail

* refactor: simplify JSX context tracking

* fix flow error

* refactor: move JSX context to context.js

* fix: ensure comment stack is correctly handled

* rename createPositionFromPosition

* rename token type and methods

* add tokenIsTemplate

* refactor: merge babel 7 logic in babel7CompatTokens

* fix flow error

benchmark/babel-parser/many-nested-block-elements/bench.mjs

@@ -0,0 +1,22 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+  return "{".repeat(length) + "0" + "}".repeat(length);
+}
+function benchCases(name, implementation, options) {
+  for (const length of [128, 256, 512, 1024]) {
+    const input = createInput(length);
+    suite.add(`${name} ${length} nested template elements`, () => {
+      implementation.parse(input, options);
+    });
+  }
+}
+
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();

benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs

@@ -0,0 +1,25 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+  return "<t a={x}>{y}".repeat(length) + "</t>".repeat(length);
+}
+function benchCases(name, implementation, options) {
+  for (const length of [128, 256, 512, 1024]) {
+    const input = createInput(length);
+    suite.add(
+      `${name} ${length} nested jsx elements with one attribute and text`,
+      () => {
+        implementation.parse(input, options);
+      }
+    );
+  }
+}
+
+benchCases("baseline", baseline, { plugins: ["jsx"] });
+benchCases("current", current, { plugins: ["jsx"] });
+
+suite.on("cycle", report).run();

benchmark/babel-parser/many-nested-template-elements/bench.mjs

@@ -0,0 +1,22 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+  return "` ${".repeat(length) + "0" + "}`".repeat(length);
+}
+function benchCases(name, implementation, options) {
+  for (const length of [128, 256, 512, 1024]) {
+    const input = createInput(length);
+    suite.add(`${name} ${length} nested template elements`, () => {
+      implementation.parse(input, options);
+    });
+  }
+}
+
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();

benchmark/babel-parser/many-template-elements/bench.mjs

@@ -0,0 +1,23 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+  return "`" + " ${0}".repeat(length) + "`";
+}
+function benchCases(name, implementation, options) {
+  for (const length of [128, 256, 512, 1024]) {
+    const input = createInput(length);
+    suite.add(`${name} ${length} template elements`, () => {
+      implementation.parse(input, options);
+    });
+  }
+}
+
+current.parse(createInput(1));
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();

eslint/babel-eslint-parser/src/convert/convertTokens.cjs

@@ -71,13 +71,6 @@ function convertTemplateType(tokens, tl) {
         templateTokens.push(token);
         break;
 
-      case tl.eof:
-        if (curlyBrace) {
-          result.push(curlyBrace);
-        }
-
-        break;
-
       default:
         if (curlyBrace) {
           result.push(curlyBrace);
@@ -186,6 +179,8 @@ function convertToken(token, source, tl) {
     token.value = `${token.value}n`;
   } else if (label === tl.privateName) {
     token.type = "PrivateIdentifier";
+  } else if (label === tl.templateNonTail || label === tl.templateTail) {
+    token.type = "Template";
   }
 
   if (typeof token.type !== "string") {
@@ -196,12 +191,16 @@ function convertToken(token, source, tl) {
 
 module.exports = function convertTokens(tokens, code, tl) {
   const result = [];
-
+  const templateTypeMergedTokens = process.env.BABEL_8_BREAKING
-  const withoutComments = convertTemplateType(tokens, tl).filter(
+    ? tokens
-    t => t.type !== "CommentLine" && t.type !== "CommentBlock",
+    : convertTemplateType(tokens, tl);
-  );
+  // The last token is always tt.eof and should be skipped
-  for (let i = 0, { length } = withoutComments; i < length; i++) {
+  for (let i = 0, { length } = templateTypeMergedTokens; i < length - 1; i++) {
-    const token = withoutComments[i];
+    const token = templateTypeMergedTokens[i];
+    const tokenType = token.type;
+    if (tokenType === "CommentLine" || tokenType === "CommentBlock") {
+      continue;
+    }
 
     if (!process.env.BABEL_8_BREAKING) {
       // Babel 8 already produces a single token
@@ -209,9 +208,9 @@ module.exports = function convertTokens(tokens, code, tl) {
       if (
         ESLINT_VERSION >= 8 &&
         i + 1 < length &&
-        token.type.label === tl.hash
+        tokenType.label === tl.hash
       ) {
-        const nextToken = withoutComments[i + 1];
+        const nextToken = templateTypeMergedTokens[i + 1];
 
         // We must disambiguate private identifier from the hack pipes topic token
         if (nextToken.type.label === tl.name && token.end === nextToken.start) {

packages/babel-parser/src/parser/expression.js

@@ -27,6 +27,7 @@ import {
   tokenIsPostfix,
   tokenIsPrefix,
   tokenIsRightAssociative,
+  tokenIsTemplate,
   tokenKeywordOrIdentifierIsKeyword,
   tokenLabelName,
   tokenOperatorPrecedence,
@@ -43,7 +44,7 @@ import {
   isIdentifierStart,
   canBeReservedWord,
 } from "../util/identifier";
-import { Position } from "../util/location";
+import { Position, createPositionWithColumnOffset } from "../util/location";
 import * as charCodes from "charcodes";
 import {
   BIND_OUTSIDE,
@@ -706,9 +707,10 @@ export default class ExpressionParser extends LValParser {
     noCalls: ?boolean,
     state: N.ParseSubscriptState,
   ): N.Expression {
-    if (!noCalls && this.eat(tt.doubleColon)) {
+    const { type } = this.state;
+    if (!noCalls && type === tt.doubleColon) {
       return this.parseBind(base, startPos, startLoc, noCalls, state);
-    } else if (this.match(tt.backQuote)) {
+    } else if (tokenIsTemplate(type)) {
       return this.parseTaggedTemplateExpression(
         base,
         startPos,
@@ -719,7 +721,7 @@ export default class ExpressionParser extends LValParser {
 
     let optional = false;
 
-    if (this.match(tt.questionDot)) {
+    if (type === tt.questionDot) {
       if (noCalls && this.lookaheadCharCode() === charCodes.leftParenthesis) {
         // stop at `?.` when parsing `new a?.()`
         state.stop = true;
@@ -801,6 +803,7 @@ export default class ExpressionParser extends LValParser {
   ): N.Expression {
     const node = this.startNodeAt(startPos, startLoc);
     node.object = base;
+    this.next(); // eat '::'
     node.callee = this.parseNoCallExpr();
     state.stop = true;
     return this.parseSubscripts(
@@ -1153,7 +1156,8 @@ export default class ExpressionParser extends LValParser {
       case tt._new:
         return this.parseNewOrNewTarget();
 
-      case tt.backQuote:
+      case tt.templateNonTail:
+      case tt.templateTail:
         return this.parseTemplate(false);
 
       // BindExpression[Yield]
@@ -1832,37 +1836,47 @@ export default class ExpressionParser extends LValParser {
   // Parse template expression.
 
   parseTemplateElement(isTagged: boolean): N.TemplateElement {
-    const elem = this.startNode();
+    const { start, end, value } = this.state;
-    if (this.state.value === null) {
+    const elemStart = start + 1;
+    const elem = this.startNodeAt(
+      elemStart,
+      createPositionWithColumnOffset(this.state.startLoc, 1),
+    );
+    if (value === null) {
       if (!isTagged) {
-        this.raise(this.state.start + 1, Errors.InvalidEscapeSequenceTemplate);
+        this.raise(start + 2, Errors.InvalidEscapeSequenceTemplate);
       }
     }
+
+    const isTail = this.match(tt.templateTail);
+    const endOffset = isTail ? -1 : -2;
+    const elemEnd = end + endOffset;
     elem.value = {
-      raw: this.input
+      raw: this.input.slice(elemStart, elemEnd).replace(/\r\n?/g, "\n"),
-        .slice(this.state.start, this.state.end)
+      cooked: value === null ? null : value.slice(1, endOffset),
-        .replace(/\r\n?/g, "\n"),
-      cooked: this.state.value,
     };
+    elem.tail = isTail;
     this.next();
-    elem.tail = this.match(tt.backQuote);
+    this.finishNode(elem, "TemplateElement");
-    return this.finishNode(elem, "TemplateElement");
+    this.resetEndLocation(
+      elem,
+      elemEnd,
+      createPositionWithColumnOffset(this.state.lastTokEndLoc, endOffset),
+    );
+    return elem;
   }
 
   // https://tc39.es/ecma262/#prod-TemplateLiteral
   parseTemplate(isTagged: boolean): N.TemplateLiteral {
     const node = this.startNode();
-    this.next();
     node.expressions = [];
     let curElt = this.parseTemplateElement(isTagged);
     node.quasis = [curElt];
     while (!curElt.tail) {
-      this.expect(tt.dollarBraceL);
       node.expressions.push(this.parseTemplateSubstitution());
-      this.expect(tt.braceR);
+      this.readTemplateContinuation();
       node.quasis.push((curElt = this.parseTemplateElement(isTagged)));
     }
-    this.next();
     return this.finishNode(node, "TemplateLiteral");
   }
 
@@ -2681,21 +2695,22 @@ export default class ExpressionParser extends LValParser {
   }
 
   isAmbiguousAwait(): boolean {
+    if (this.hasPrecedingLineBreak()) return true;
+    const { type } = this.state;
     return (
-      this.hasPrecedingLineBreak() ||
       // All the following expressions are ambiguous:
       //   await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await ``
-      this.match(tt.plusMin) ||
+      type === tt.plusMin ||
-      this.match(tt.parenL) ||
+      type === tt.parenL ||
-      this.match(tt.bracketL) ||
+      type === tt.bracketL ||
-      this.match(tt.backQuote) ||
+      tokenIsTemplate(type) ||
       // Sometimes the tokenizer generates tt.slash for regexps, and this is
       // handler by parseExprAtom
-      this.match(tt.regexp) ||
+      type === tt.regexp ||
-      this.match(tt.slash) ||
+      type === tt.slash ||
       // This code could be parsed both as a modulo operator or as an intrinsic:
       //   await %x(0)
-      (this.hasPlugin("v8intrinsic") && this.match(tt.modulo))
+      (this.hasPlugin("v8intrinsic") && type === tt.modulo)
     );
   }
 

packages/babel-parser/src/parser/statement.js

@@ -4,6 +4,7 @@ import * as N from "../types";
 import {
   tokenIsIdentifier,
   tokenIsLoop,
+  tokenIsTemplate,
   tt,
   type TokenType,
   getExportedToken,
@@ -39,7 +40,7 @@ import {
 } from "../util/expression-scope";
 import type { SourceType } from "../options";
 import { Token } from "../tokenizer";
-import { Position } from "../util/location";
+import { createPositionWithColumnOffset } from "../util/location";
 import { cloneStringLiteral, cloneIdentifier } from "./node";
 
 const loopLabel = { kind: "loop" },
@@ -55,7 +56,10 @@ const loneSurrogate = /[\uD800-\uDFFF]/u;
 const keywordRelationalOperator = /in(?:stanceof)?/y;
 
 /**
- * Convert tt.privateName to tt.hash + tt.name for backward Babel 7 compat.
+ * Convert tokens for backward Babel 7 compat.
+ * tt.privateName => tt.hash + tt.name
+ * tt.templateTail => tt.backquote/tt.braceR + tt.template + tt.backquote
+ * tt.templateNonTail => tt.backquote/tt.braceR + tt.template + tt.dollarBraceL
  * For performance reasons this routine mutates `tokens`, it is okay
  * here since we execute `parseTopLevel` once for every file.
  * @param {*} tokens
@@ -65,38 +69,116 @@ function babel7CompatTokens(tokens) {
   for (let i = 0; i < tokens.length; i++) {
     const token = tokens[i];
     const { type } = token;
-    if (type === tt.privateName) {
-      if (!process.env.BABEL_8_BREAKING) {
-        const { loc, start, value, end } = token;
-        const hashEndPos = start + 1;
-        const hashEndLoc = new Position(loc.start.line, loc.start.column + 1);
-        tokens.splice(
-          i,
-          1,
-          // $FlowIgnore: hacky way to create token
-          new Token({
-            type: getExportedToken(tt.hash),
-            value: "#",
-            start: start,
-            end: hashEndPos,
-            startLoc: loc.start,
-            endLoc: hashEndLoc,
-          }),
-          // $FlowIgnore: hacky way to create token
-          new Token({
-            type: getExportedToken(tt.name),
-            value: value,
-            start: hashEndPos,
-            end: end,
-            startLoc: hashEndLoc,
-            endLoc: loc.end,
-          }),
-        );
-        i++;
-        continue;
-      }
-    }
     if (typeof type === "number") {
+      if (!process.env.BABEL_8_BREAKING) {
+        if (type === tt.privateName) {
+          const { loc, start, value, end } = token;
+          const hashEndPos = start + 1;
+          const hashEndLoc = createPositionWithColumnOffset(loc.start, 1);
+          tokens.splice(
+            i,
+            1,
+            // $FlowIgnore: hacky way to create token
+            new Token({
+              type: getExportedToken(tt.hash),
+              value: "#",
+              start: start,
+              end: hashEndPos,
+              startLoc: loc.start,
+              endLoc: hashEndLoc,
+            }),
+            // $FlowIgnore: hacky way to create token
+            new Token({
+              type: getExportedToken(tt.name),
+              value: value,
+              start: hashEndPos,
+              end: end,
+              startLoc: hashEndLoc,
+              endLoc: loc.end,
+            }),
+          );
+          i++;
+          continue;
+        }
+
+        if (tokenIsTemplate(type)) {
+          const { loc, start, value, end } = token;
+          const backquoteEnd = start + 1;
+          const backquoteEndLoc = createPositionWithColumnOffset(loc.start, 1);
+          let startToken;
+          if (value.charCodeAt(0) === charCodes.graveAccent) {
+            // $FlowIgnore: hacky way to create token
+            startToken = new Token({
+              type: getExportedToken(tt.backQuote),
+              value: "`",
+              start: start,
+              end: backquoteEnd,
+              startLoc: loc.start,
+              endLoc: backquoteEndLoc,
+            });
+          } else {
+            // $FlowIgnore: hacky way to create token
+            startToken = new Token({
+              type: getExportedToken(tt.braceR),
+              value: "}",
+              start: start,
+              end: backquoteEnd,
+              startLoc: loc.start,
+              endLoc: backquoteEndLoc,
+            });
+          }
+          let templateValue,
+            templateElementEnd,
+            templateElementEndLoc,
+            endToken;
+          if (type === tt.templateTail) {
+            // ends with '`'
+            templateElementEnd = end - 1;
+            templateElementEndLoc = createPositionWithColumnOffset(loc.end, -1);
+            templateValue = value.slice(1, -1);
+            // $FlowIgnore: hacky way to create token
+            endToken = new Token({
+              type: getExportedToken(tt.backQuote),
+              value: "`",
+              start: templateElementEnd,
+              end: end,
+              startLoc: templateElementEndLoc,
+              endLoc: loc.end,
+            });
+          } else {
+            // ends with `${`
+            templateElementEnd = end - 2;
+            templateElementEndLoc = createPositionWithColumnOffset(loc.end, -2);
+            templateValue = value.slice(1, -2);
+            // $FlowIgnore: hacky way to create token
+            endToken = new Token({
+              type: getExportedToken(tt.dollarBraceL),
+              value: "${",
+              start: templateElementEnd,
+              end: end,
+              startLoc: templateElementEndLoc,
+              endLoc: loc.end,
+            });
+          }
+          tokens.splice(
+            i,
+            1,
+            startToken,
+            // $FlowIgnore: hacky way to create token
+            new Token({
+              type: getExportedToken(tt.template),
+              value: templateValue,
+              start: backquoteEnd,
+              end: templateElementEnd,
+              startLoc: backquoteEndLoc,
+              endLoc: templateElementEndLoc,
+            }),
+            endToken,
+          );
+          i += 2;
+          continue;
+        }
+      }
       // $FlowIgnore: we manipulate `token` for performance reasons
       token.type = getExportedToken(type);
     }

packages/babel-parser/src/plugins/flow/index.js

@@ -2807,11 +2807,9 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         // by parsing `jsxTagStart` to stop the JSX plugin from
         // messing with the tokens
         const { context } = this.state;
-        const curContext = context[context.length - 1];
+        const currentContext = context[context.length - 1];
-        if (curContext === tc.j_oTag) {
+        if (currentContext === tc.j_oTag || currentContext === tc.j_expr) {
-          context.length -= 2;
+          context.pop();
-        } else if (curContext === tc.j_expr) {
-          context.length -= 1;
         }
       }
 

packages/babel-parser/src/plugins/jsx/index.js

@@ -46,12 +46,6 @@ const JsxErrors = makeErrorTemplates(
 );
 /* eslint-disable sort-keys */
 
-// Be aware that this file is always executed and not only when the plugin is enabled.
-// Therefore the contexts do always exist.
-tc.j_oTag = new TokContext("<tag");
-tc.j_cTag = new TokContext("</tag");
-tc.j_expr = new TokContext("<tag>...</tag>", true);
-
 function isFragment(object: ?N.JSXElement): boolean {
   return object
     ? object.type === "JSXOpeningFragment" ||
@@ -301,8 +295,9 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       switch (this.state.type) {
         case tt.braceL:
           node = this.startNode();
+          this.setContext(tc.brace);
           this.next();
-          node = this.jsxParseExpressionContainer(node);
+          node = this.jsxParseExpressionContainer(node, tc.j_oTag);
           if (node.expression.type === "JSXEmptyExpression") {
             this.raise(node.start, JsxErrors.AttributeIsEmpty);
           }
@@ -339,6 +334,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
     jsxParseSpreadChild(node: N.JSXSpreadChild): N.JSXSpreadChild {
       this.next(); // ellipsis
       node.expression = this.parseExpression();
+      this.setContext(tc.j_oTag);
       this.expect(tt.braceR);
 
       return this.finishNode(node, "JSXSpreadChild");
@@ -348,6 +344,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
 
     jsxParseExpressionContainer(
       node: N.JSXExpressionContainer,
+      previousContext: TokContext,
     ): N.JSXExpressionContainer {
       if (this.match(tt.braceR)) {
         node.expression = this.jsxParseEmptyExpression();
@@ -368,6 +365,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
 
         node.expression = expression;
       }
+      this.setContext(previousContext);
       this.expect(tt.braceR);
 
       return this.finishNode(node, "JSXExpressionContainer");
@@ -377,9 +375,12 @@ export default (superClass: Class<Parser>): Class<Parser> =>
 
     jsxParseAttribute(): N.JSXAttribute {
       const node = this.startNode();
-      if (this.eat(tt.braceL)) {
+      if (this.match(tt.braceL)) {
+        this.setContext(tc.brace);
+        this.next();
         this.expect(tt.ellipsis);
         node.argument = this.parseMaybeAssignAllowIn();
+        this.setContext(tc.j_oTag);
         this.expect(tt.braceR);
         return this.finishNode(node, "JSXSpreadAttribute");
       }
@@ -464,11 +465,14 @@ export default (superClass: Class<Parser>): Class<Parser> =>
 
             case tt.braceL: {
               const node = this.startNode();
+              this.setContext(tc.brace);
               this.next();
               if (this.match(tt.ellipsis)) {
                 children.push(this.jsxParseSpreadChild(node));
               } else {
-                children.push(this.jsxParseExpressionContainer(node));
+                children.push(
+                  this.jsxParseExpressionContainer(node, tc.j_expr),
+                );
               }
 
               break;
@@ -537,6 +541,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       return this.jsxParseElementAt(startPos, startLoc);
     }
 
+    setContext(newContext: TokContext) {
+      const { context } = this.state;
+      context[context.length - 1] = newContext;
+    }
+
     // ==================================
     // Overrides
     // ==================================
@@ -559,6 +568,11 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       }
     }
 
+    skipSpace() {
+      const curContext = this.curContext();
+      if (!curContext.preserveSpace) super.skipSpace();
+    }
+
     getTokenFromCode(code: number): void {
       const context = this.curContext();
 
@@ -597,7 +611,6 @@ export default (superClass: Class<Parser>): Class<Parser> =>
     }
 
     updateContext(prevType: TokenType): void {
-      super.updateContext(prevType);
       const { context, type } = this.state;
       if (type === tt.slash && prevType === tt.jsxTagStart) {
         // do not consider JSX expr -> JSX open tag -> ... anymore
@@ -605,17 +618,16 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         context.splice(-2, 2, tc.j_cTag);
         this.state.canStartJSXElement = false;
       } else if (type === tt.jsxTagStart) {
-        context.push(
+        // start opening tag context
-          tc.j_expr, // treat as beginning of JSX expression
+        context.push(tc.j_oTag);
-          tc.j_oTag, // start opening tag context
-        );
       } else if (type === tt.jsxTagEnd) {
-        const out = context.pop();
+        const out = context[context.length - 1];
         if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) {
           context.pop();
           this.state.canStartJSXElement =
             context[context.length - 1] === tc.j_expr;
         } else {
+          this.setContext(tc.j_expr);
           this.state.canStartJSXElement = true;
         }
       } else {

packages/babel-parser/src/plugins/typescript/index.js

@@ -14,8 +14,9 @@ import {
   tokenIsKeywordOrIdentifier,
   tt,
   type TokenType,
+  tokenIsTemplate,
 } from "../../tokenizer/types";
-import { types as ct } from "../../tokenizer/context";
+import { types as tc } from "../../tokenizer/context";
 import * as N from "../../types";
 import type { Position } from "../../util/location";
 import type Parser from "../../parser";
@@ -1071,7 +1072,8 @@ export default (superClass: Class<Parser>): Class<Parser> =>
           }
 
           return this.tsParseParenthesizedType();
-        case tt.backQuote:
+        case tt.templateNonTail:
+        case tt.templateTail:
           return this.tsParseTemplateLiteralType();
         default: {
           const { type } = this.state;
@@ -2196,7 +2198,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
               }
 
               return this.finishCallExpression(node, state.optionalChainMember);
-            } else if (this.match(tt.backQuote)) {
+            } else if (tokenIsTemplate(this.state.type)) {
               const result = this.parseTaggedTemplateExpression(
                 base,
                 startPos,
@@ -2872,14 +2874,13 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         /*:: invariant(jsx.node != null) */
         if (!jsx.error) return jsx.node;
 
-        // Remove `tc.j_expr` and `tc.j_oTag` from context added
+        // Remove `tc.j_expr` or `tc.j_oTag` from context added
         // by parsing `jsxTagStart` to stop the JSX plugin from
         // messing with the tokens
         const { context } = this.state;
-        if (context[context.length - 1] === ct.j_oTag) {
+        const currentContext = context[context.length - 1];
-          context.length -= 2;
+        if (currentContext === tc.j_oTag || currentContext === tc.j_expr) {
-        } else if (context[context.length - 1] === ct.j_expr) {
+          context.pop();
-          context.length -= 1;
         }
       }
 

packages/babel-parser/src/tokenizer/context.js

@@ -1,7 +1,7 @@
 // @flow
 
-// The token context is used to track whether the apostrophe "`"
+// The token context is used in JSX plugin to track
-// starts or ends a string template
+// jsx tag / jsx text / normal JavaScript expression
 
 export class TokContext {
   constructor(token: string, preserveSpace?: boolean) {
@@ -13,9 +13,17 @@ export class TokContext {
   preserveSpace: boolean;
 }
 
-export const types: {
+const types: {
   [key: string]: TokContext,
 } = {
-  brace: new TokContext("{"),
+  brace: new TokContext("{"), // normal JavaScript expression
-  template: new TokContext("`", true),
+  j_oTag: new TokContext("<tag"), // JSX openning tag
+  j_cTag: new TokContext("</tag"), // JSX closing tag
+  j_expr: new TokContext("<tag>...</tag>", true), // JSX expressions
 };
+
+if (!process.env.BABEL_8_BREAKING) {
+  types.template = new TokContext("`", true);
+}
+
+export { types };

packages/babel-parser/src/tokenizer/index.js

@@ -13,7 +13,7 @@ import {
   keywords as keywordTypes,
   type TokenType,
 } from "./types";
-import { type TokContext, types as ct } from "./context";
+import { type TokContext } from "./context";
 import ParserErrors, { Errors, type ErrorTemplate } from "../parser/error";
 import { SourceLocation } from "../util/location";
 import {
@@ -296,8 +296,7 @@ export default class Tokenizer extends ParserErrors {
   // properties.
 
   nextToken(): void {
-    const curContext = this.curContext();
+    this.skipSpace();
-    if (!curContext.preserveSpace) this.skipSpace();
     this.state.start = this.state.pos;
     if (!this.isLookahead) this.state.startLoc = this.state.curPosition();
     if (this.state.pos >= this.length) {
@@ -305,11 +304,7 @@ export default class Tokenizer extends ParserErrors {
       return;
     }
 
-    if (curContext === ct.template) {
+    this.getTokenFromCode(this.codePointAtPos(this.state.pos));
-      this.readTmplToken();
-    } else {
-      this.getTokenFromCode(this.codePointAtPos(this.state.pos));
-    }
   }
 
   skipBlockComment(): N.CommentBlock | void {
@@ -921,8 +916,7 @@ export default class Tokenizer extends ParserErrors {
         return;
 
       case charCodes.graveAccent:
-        ++this.state.pos;
+        this.readTemplateToken();
-        this.finishToken(tt.backQuote);
         return;
 
       case charCodes.digit0: {
@@ -1375,36 +1369,40 @@ export default class Tokenizer extends ParserErrors {
     this.finishToken(tt.string, out);
   }
 
-  // Reads template string tokens.
+  // Reads tempalte continuation `}...`
+  readTemplateContinuation(): void {
+    if (!this.match(tt.braceR)) {
+      this.unexpected(this.state.start, tt.braceR);
+    }
+    // rewind pos to `}`
+    this.state.pos--;
+    this.readTemplateToken();
+  }
 
-  readTmplToken(): void {
+  // Reads template string tokens.
+  readTemplateToken(): void {
     let out = "",
-      chunkStart = this.state.pos,
+      chunkStart = this.state.pos, // eat '`' or `}`
       containsInvalid = false;
+    ++this.state.pos; // eat '`' or `}`
     for (;;) {
       if (this.state.pos >= this.length) {
-        throw this.raise(this.state.start, Errors.UnterminatedTemplate);
+        throw this.raise(this.state.start + 1, Errors.UnterminatedTemplate);
       }
       const ch = this.input.charCodeAt(this.state.pos);
-      if (
+      if (ch === charCodes.graveAccent) {
-        ch === charCodes.graveAccent ||
+        ++this.state.pos; // eat '`'
-        (ch === charCodes.dollarSign &&
-          this.input.charCodeAt(this.state.pos + 1) ===
-            charCodes.leftCurlyBrace)
-      ) {
-        if (this.state.pos === this.state.start && this.match(tt.template)) {
-          if (ch === charCodes.dollarSign) {
-            this.state.pos += 2;
-            this.finishToken(tt.dollarBraceL);
-            return;
-          } else {
-            ++this.state.pos;
-            this.finishToken(tt.backQuote);
-            return;
-          }
-        }
         out += this.input.slice(chunkStart, this.state.pos);
-        this.finishToken(tt.template, containsInvalid ? null : out);
+        this.finishToken(tt.templateTail, containsInvalid ? null : out);
+        return;
+      }
+      if (
+        ch === charCodes.dollarSign &&
+        this.input.charCodeAt(this.state.pos + 1) === charCodes.leftCurlyBrace
+      ) {
+        this.state.pos += 2; // eat '${'
+        out += this.input.slice(chunkStart, this.state.pos);
+        this.finishToken(tt.templateNonTail, containsInvalid ? null : out);
         return;
       }
       if (ch === charCodes.backslash) {
@@ -1633,44 +1631,7 @@ export default class Tokenizer extends ParserErrors {
     }
   }
 
-  // the prevType is required by the jsx plugin
+  // updateContext is used by the jsx plugin
   // eslint-disable-next-line no-unused-vars
-  updateContext(prevType: TokenType): void {
+  updateContext(prevType: TokenType): void {}
-    // Token-specific context update code
-    // Note that we should avoid accessing `this.prodParam` in context update,
-    // because it is executed immediately when last token is consumed, which may be
-    // before `this.prodParam` is updated. e.g.
-    // ```
-    // function *g() { () => yield / 2 }
-    // ```
-    // When `=>` is eaten, the context update of `yield` is executed, however,
-    // `this.prodParam` still has `[Yield]` production because it is not yet updated
-    const { context, type } = this.state;
-    switch (type) {
-      case tt.braceR:
-        context.pop();
-        break;
-      // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
-      // ideally only dollarBraceL "${" needs a non-template context
-      // in order to indicate that the last "`" in `${`" starts a new string template
-      // inside a template element within outer string template.
-      // but when we popped such context in `}`, we lost track of whether this
-      // `}` matches a `${` or other tokens matching `}`, so we have to push
-      // such context in every token that `}` will match.
-      case tt.braceL:
-      case tt.braceHashL:
-      case tt.dollarBraceL:
-        context.push(ct.brace);
-        break;
-      case tt.backQuote:
-        if (context[context.length - 1] === ct.template) {
-          context.pop();
-        } else {
-          context.push(ct.template);
-        }
-        break;
-      default:
-        break;
-    }
-  }
 }

packages/babel-parser/src/tokenizer/types.js

@@ -158,6 +158,10 @@ export const tt: { [name: string]: TokenType } = {
   ellipsis: createToken("...", { beforeExpr }),
   backQuote: createToken("`", { startsExpr }),
   dollarBraceL: createToken("${", { beforeExpr, startsExpr }),
+  // start: isTemplate
+  templateTail: createToken("...`", { startsExpr }),
+  templateNonTail: createToken("...${", { beforeExpr, startsExpr }),
+  // end: isTemplate
   at: createToken("@"),
   hash: createToken("#", { startsExpr }),
 
@@ -402,6 +406,10 @@ export function tokenIsRightAssociative(token: TokenType): boolean {
   return token === tt.exponent;
 }
 
+export function tokenIsTemplate(token: TokenType): boolean {
+  return token >= tt.templateTail && token <= tt.templateNonTail;
+}
+
 export function getExportedToken(token: TokenType): ExportedTokenType {
   return tokenTypes[token];
 }

packages/babel-parser/src/util/location.js

@@ -50,3 +50,22 @@ export function getLineInfo(input: string, offset: number): Position {
 
   return new Position(line, offset - lineStart);
 }
+
+/**
+ * creates a new position with a non-zero column offset from the given position.
+ * This function should be only be used when we create AST node out of the token
+ * boundaries, such as TemplateElement ends before tt.templateNonTail. This
+ * function does not skip whitespaces.
+ *
+ * @export
+ * @param {Position} position
+ * @param {number} columnOffset
+ * @returns {Position}
+ */
+export function createPositionWithColumnOffset(
+  position: Position,
+  columnOffset: number,
+) {
+  const { line, column } = position;
+  return new Position(line, column + columnOffset);
+}

packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js

@@ -0,0 +1 @@
+`${a}` // comment

packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json

@@ -0,0 +1,62 @@
+{
+  "type": "File",
+  "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+          "expressions": [
+            {
+              "type": "Identifier",
+              "start":3,"end":4,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4},"identifierName":"a"},
+              "name": "a"
+            }
+          ],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":1,"end":1,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":1}},
+              "value": {
+                "raw": "",
+                "cooked": ""
+              },
+              "tail": false
+            },
+            {
+              "type": "TemplateElement",
+              "start":5,"end":5,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":5}},
+              "value": {
+                "raw": "",
+                "cooked": ""
+              },
+              "tail": true
+            }
+          ]
+        },
+        "trailingComments": [
+          {
+            "type": "CommentLine",
+            "value": " comment",
+            "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}}
+          }
+        ]
+      }
+    ],
+    "directives": []
+  },
+  "comments": [
+    {
+      "type": "CommentLine",
+      "value": " comment",
+      "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js

@@ -0,0 +1,2 @@
+`before${x}middle${y}after`;
+`x`;

packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json

@@ -0,0 +1,345 @@
+{
+  "type": "File",
+  "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}},
+          "expressions": [
+            {
+              "type": "Identifier",
+              "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"},
+              "name": "x"
+            },
+            {
+              "type": "Identifier",
+              "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"},
+              "name": "y"
+            }
+          ],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}},
+              "value": {
+                "raw": "before",
+                "cooked": "before"
+              },
+              "tail": false
+            },
+            {
+              "type": "TemplateElement",
+              "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}},
+              "value": {
+                "raw": "middle",
+                "cooked": "middle"
+              },
+              "tail": false
+            },
+            {
+              "type": "TemplateElement",
+              "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}},
+              "value": {
+                "raw": "after",
+                "cooked": "after"
+              },
+              "tail": true
+            }
+          ]
+        }
+      },
+      {
+        "type": "ExpressionStatement",
+        "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}},
+          "expressions": [],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}},
+              "value": {
+                "raw": "x",
+                "cooked": "x"
+              },
+              "tail": true
+            }
+          ]
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`",
+      "start":0,"end":1,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":1}}
+    },
+    {
+      "type": {
+        "label": "template",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "before",
+      "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}}
+    },
+    {
+      "type": {
+        "label": "${",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "${",
+      "start":7,"end":9,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":9}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "x",
+      "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
+    },
+    {
+      "type": {
+        "label": "}",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "}",
+      "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}}
+    },
+    {
+      "type": {
+        "label": "template",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "middle",
+      "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}}
+    },
+    {
+      "type": {
+        "label": "${",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "${",
+      "start":17,"end":19,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":19}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "y",
+      "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}}
+    },
+    {
+      "type": {
+        "label": "}",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "}",
+      "start":20,"end":21,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":21}}
+    },
+    {
+      "type": {
+        "label": "template",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "after",
+      "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
+    },
+    {
+      "type": {
+        "label": "`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`",
+      "start":26,"end":27,"loc":{"start":{"line":1,"column":26},"end":{"line":1,"column":27}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
+    },
+    {
+      "type": {
+        "label": "`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`",
+      "start":29,"end":30,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":1}}
+    },
+    {
+      "type": {
+        "label": "template",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "value": "x",
+      "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}}
+    },
+    {
+      "type": {
+        "label": "`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`",
+      "start":31,"end":32,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":3}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json

@@ -0,0 +1,4 @@
+{
+  "tokens": true,
+  "BABEL_8_BREAKING": false
+}

packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js

@@ -0,0 +1,2 @@
+`before${x}middle${y}after`;
+`x`;

packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json

@@ -0,0 +1,216 @@
+{
+  "type": "File",
+  "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}},
+          "expressions": [
+            {
+              "type": "Identifier",
+              "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"},
+              "name": "x"
+            },
+            {
+              "type": "Identifier",
+              "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"},
+              "name": "y"
+            }
+          ],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}},
+              "value": {
+                "raw": "before",
+                "cooked": "before"
+              },
+              "tail": false
+            },
+            {
+              "type": "TemplateElement",
+              "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}},
+              "value": {
+                "raw": "middle",
+                "cooked": "middle"
+              },
+              "tail": false
+            },
+            {
+              "type": "TemplateElement",
+              "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}},
+              "value": {
+                "raw": "after",
+                "cooked": "after"
+              },
+              "tail": true
+            }
+          ]
+        }
+      },
+      {
+        "type": "ExpressionStatement",
+        "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}},
+          "expressions": [],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}},
+              "value": {
+                "raw": "x",
+                "cooked": "x"
+              },
+              "tail": true
+            }
+          ]
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "...${",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`before${",
+      "start":0,"end":9,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":9}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "x",
+      "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
+    },
+    {
+      "type": {
+        "label": "...${",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "}middle${",
+      "start":10,"end":19,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":19}}
+    },
+    {
+      "type": {
+        "label": "name",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "y",
+      "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}}
+    },
+    {
+      "type": {
+        "label": "...`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "}after`",
+      "start":20,"end":27,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":27}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
+    },
+    {
+      "type": {
+        "label": "...`",
+        "beforeExpr": false,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "value": "`x`",
+      "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}}
+    },
+    {
+      "type": {
+        "label": ";",
+        "beforeExpr": true,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}}
+    }
+  ]
+}

packages/babel-parser/test/fixtures/tokens/template-string/options.json

@@ -0,0 +1,4 @@
+{
+  "tokens": true,
+  "BABEL_8_BREAKING": true
+}