Parser refactoring (#11871)

* refactor: parseMaybeUnary => parseUnary

* refactor: extract shouldExitDescending method

* refactor: add parseUpdate

* refactor: avoid comparing with hardcoded token value

* refactor: add ParseNewOrNewTarget

* refactor: add parseCoverCallAndAsyncArrowHead

* add parseBind

* refactor: polish parseTaggedTemplateExpression interface

* refactor: add parseMember method

* refactor: add parseSuper method

* refactor: add parseAsyncArrowUnaryFunction method

* fix: disallow line break before async binding arrow

* refactor: simplify tt.name logic

* refactor: add parseDo method

* refactor: misc

* refactor: rename parseObjectMember by parsePropertyDefinition

* refactor: unify set/get/async keyword parsing in ObjectMethod

* refactor: misc

* refactor: add parseArrayLike method

* refactor: move fsharp epilogure and prologue inside parseObjectLike

* fixup

* refactor: rename parseFunctionExpression to parseFunctionOrFunctionSent

* refactor: remove redundant logic

* refactor: rename parseClassPropertyName by parseClassElementName

* refactor: avoid unecessary lookahead when parsing tt._export

* fix: export-default-from should support escaped async as export binding

* address review comments

* parseUnary -> parseMaybeUnary

packages/babel-parser/src/parser/error-message.js

@@ -80,6 +80,7 @@ export const ErrorMessages = Object.freeze({
   LabelRedeclaration: "Label '%0' is already declared",
   LetInLexicalBinding:
     "'let' is not allowed to be used as a name in 'let' or 'const' declarations.",
+  LineTerminatorBeforeArrow: "No line break is allowed before '=>'",
   MalformedRegExpFlags: "Invalid regular expression flag",
   MissingClassName: "A class name is required",
   MissingEqInAssignment:

packages/babel-parser/src/parser/lval.js

@@ -235,15 +235,18 @@ export default class LValParser extends NodeUtils {
     return this.finishNode(node, "SpreadElement");
   }
 
+  // https://tc39.es/ecma262/#prod-BindingRestProperty
+  // https://tc39.es/ecma262/#prod-BindingRestElement
   parseRestBinding(): RestElement {
     const node = this.startNode();
-    this.next();
+    this.next(); // eat `...`
     node.argument = this.parseBindingAtom();
     return this.finishNode(node, "RestElement");
   }
 
   // Parses lvalue (assignable) atom.
   parseBindingAtom(): Pattern {
+    // https://tc39.es/ecma262/#prod-BindingPattern
     switch (this.state.type) {
       case tt.bracketL: {
         const node = this.startNode();
@@ -257,12 +260,14 @@ export default class LValParser extends NodeUtils {
       }
 
       case tt.braceL:
-        return this.parseObj(tt.braceR, true);
+        return this.parseObjectLike(tt.braceR, true);
     }
 
+    // https://tc39.es/ecma262/#prod-BindingIdentifier
     return this.parseIdentifier();
   }
 
+  // https://tc39.es/ecma262/#prod-BindingElementList
   parseBindingList(
     close: TokenType,
     closeCharCode: $Values<typeof charCodes>,
@@ -292,6 +297,7 @@ export default class LValParser extends NodeUtils {
         if (this.match(tt.at) && this.hasPlugin("decorators")) {
           this.raise(this.state.start, Errors.UnsupportedParameterDecorator);
         }
+        // invariant: hasPlugin("decorators-legacy")
         while (this.match(tt.at)) {
           decorators.push(this.parseDecorator());
         }
@@ -314,20 +320,22 @@ export default class LValParser extends NodeUtils {
     return elt;
   }
 
+  // Used by flow/typescript plugin to add type annotations to binding elements
   parseAssignableListItemTypes(param: Pattern): Pattern {
     return param;
   }
 
   // Parses assignment pattern around given atom if possible.
-
+  // https://tc39.es/ecma262/#prod-BindingElement
   parseMaybeDefault(
     startPos?: ?number,
     startLoc?: ?Position,
     left?: ?Pattern,
   ): Pattern {
-    startLoc = startLoc || this.state.startLoc;
+    startLoc = startLoc ?? this.state.startLoc;
-    startPos = startPos || this.state.start;
+    startPos = startPos ?? this.state.start;
-    left = left || this.parseBindingAtom();
+    // $FlowIgnore
+    left = left ?? this.parseBindingAtom();
     if (!this.eat(tt.eq)) return left;
 
     const node = this.startNodeAt(startPos, startLoc);

packages/babel-parser/src/parser/statement.js

@@ -141,7 +141,9 @@ export default class StatementParser extends ExpressionParser {
   // regular expression literal. This is to handle cases like
   // `if (foo) /blah/.exec(foo)`, where looking at the previous token
   // does not help.
-
+  // https://tc39.es/ecma262/#prod-Statement
+  // ImportDeclaration and ExportDeclaration are also handled here so we can throw recoverable errors
+  // when they are not at the top level
   parseStatement(context: ?string, topLevel?: boolean): N.Statement {
     if (this.match(tt.at)) {
       this.parseDecorators(true);
@@ -216,21 +218,22 @@ export default class StatementParser extends ExpressionParser {
         return this.parseBlock();
       case tt.semi:
         return this.parseEmptyStatement(node);
-      case tt._export:
       case tt._import: {
         const nextTokenCharCode = this.lookaheadCharCode();
         if (
-          nextTokenCharCode === charCodes.leftParenthesis ||
+          nextTokenCharCode === charCodes.leftParenthesis || // import()
-          nextTokenCharCode === charCodes.dot
+          nextTokenCharCode === charCodes.dot // import.meta
         ) {
           break;
         }
-
+      }
+      // fall through
+      case tt._export: {
         if (!this.options.allowImportExportEverywhere && !topLevel) {
           this.raise(this.state.start, Errors.UnexpectedImportExport);
         }
 
-        this.next();
+        this.next(); // eat `import`/`export`
 
         let result;
         if (starttype === tt._import) {
@@ -847,6 +850,8 @@ export default class StatementParser extends ExpressionParser {
   }
 
   // Undefined directives means that directives are not allowed.
+  // https://tc39.es/ecma262/#prod-Block
+  // https://tc39.es/ecma262/#prod-ModuleBody
   parseBlockOrModuleBlockBody(
     body: N.Statement[],
     directives: ?(N.Directive[]),
@@ -1161,10 +1166,9 @@ export default class StatementParser extends ExpressionParser {
 
     this.parseClassId(node, isStatement, optionalId);
     this.parseClassSuper(node);
+    // this.state.strict is restored in parseClassBody
     node.body = this.parseClassBody(!!node.superClass, oldStrict);
 
-    this.state.strict = oldStrict;
-
     return this.finishNode(
       node,
       isStatement ? "ClassDeclaration" : "ClassExpression",
@@ -1188,6 +1192,7 @@ export default class StatementParser extends ExpressionParser {
     );
   }
 
+  // https://tc39.es/ecma262/#prod-ClassBody
   parseClassBody(
     constructorAllowsSuper: boolean,
     oldStrict?: boolean,
@@ -1238,11 +1243,9 @@ export default class StatementParser extends ExpressionParser {
       }
     });
 
-    if (!oldStrict) {
+    this.state.strict = oldStrict;
-      this.state.strict = false;
-    }
 
-    this.next();
+    this.next(); // eat `}`
 
     if (decorators.length) {
       throw this.raise(this.state.start, Errors.TrailingDecorator);
@@ -1253,13 +1256,26 @@ export default class StatementParser extends ExpressionParser {
     return this.finishNode(classBody, "ClassBody");
   }
 
+  // Check grammar production:
+  //   IdentifierName *_opt ClassElementName
+  // It is used in `parsePropertyDefinition` to detect AsyncMethod and Accessors
+  maybeClassModifier(prop: N.ObjectProperty): boolean {
+    return (
+      !prop.computed &&
+      prop.key.type === "Identifier" &&
+      (this.isLiteralPropertyName() ||
+        this.match(tt.bracketL) ||
+        this.match(tt.star) ||
+        this.match(tt.hash))
+    );
+  }
+
   // returns true if the current identifier is a method/field name,
   // false if it is a modifier
   parseClassMemberFromModifier(
     classBody: N.ClassBody,
     member: N.ClassMember,
   ): boolean {
-    const containsEsc = this.state.containsEsc;
     const key = this.parseIdentifier(true); // eats the modifier
 
     if (this.isClassMethod()) {
@@ -1288,10 +1304,7 @@ export default class StatementParser extends ExpressionParser {
       prop.static = false;
       classBody.body.push(this.parseClassProperty(prop));
       return true;
-    } else if (containsEsc) {
-      throw this.unexpected();
     }
-
     return false;
   }
 
@@ -1337,7 +1350,7 @@ export default class StatementParser extends ExpressionParser {
     if (this.eat(tt.star)) {
       // a generator
       method.kind = "method";
-      this.parseClassPropertyName(method);
+      this.parseClassElementName(method);
 
       if (method.key.type === "PrivateName") {
         // Private generator method
@@ -1362,7 +1375,7 @@ export default class StatementParser extends ExpressionParser {
     }
 
     const containsEsc = this.state.containsEsc;
-    const key = this.parseClassPropertyName(member);
+    const key = this.parseClassElementName(member);
     const isPrivate = key.type === "PrivateName";
     // Check the key is not a computed expression or string literal.
     const isSimple = key.type === "Identifier";
@@ -1421,7 +1434,7 @@ export default class StatementParser extends ExpressionParser {
 
       method.kind = "method";
       // The so-called parsed name would have been "async": get the real name.
-      this.parseClassPropertyName(method);
+      this.parseClassElementName(method);
       this.parsePostMemberNameModifiers(publicMember);
 
       if (method.key.type === "PrivateName") {
@@ -1456,7 +1469,7 @@ export default class StatementParser extends ExpressionParser {
       // a getter or setter
       method.kind = key.name;
       // The so-called parsed name would have been "get/set": get the real name.
-      this.parseClassPropertyName(publicMethod);
+      this.parseClassElementName(publicMethod);
 
       if (method.key.type === "PrivateName") {
         // private getter/setter
@@ -1488,7 +1501,8 @@ export default class StatementParser extends ExpressionParser {
     }
   }
 
-  parseClassPropertyName(member: N.ClassMember): N.Expression | N.Identifier {
+  // https://tc39.es/proposal-class-fields/#prod-ClassElementName
+  parseClassElementName(member: N.ClassMember): N.Expression | N.Identifier {
     const key = this.parsePropertyName(member, /* isPrivateNameAllowed */ true);
 
     if (
@@ -1660,11 +1674,13 @@ export default class StatementParser extends ExpressionParser {
     }
   }
 
+  // https://tc39.es/ecma262/#prod-ClassHeritage
   parseClassSuper(node: N.Class): void {
     node.superClass = this.eat(tt._extends) ? this.parseExprSubscripts() : null;
   }
 
   // Parses module export declaration.
+  // https://tc39.es/ecma262/#prod-ExportDeclaration
 
   parseExport(node: N.Node): N.AnyExport {
     const hasDefault = this.maybeParseExportDefaultSpecifier(node);
@@ -1839,7 +1855,7 @@ export default class StatementParser extends ExpressionParser {
   isExportDefaultSpecifier(): boolean {
     if (this.match(tt.name)) {
       const value = this.state.value;
-      if (value === "async" || value === "let") {
+      if ((value === "async" && !this.state.containsEsc) || value === "let") {
         return false;
       }
       if (
@@ -2068,6 +2084,7 @@ export default class StatementParser extends ExpressionParser {
   }
 
   // Parses import declaration.
+  // https://tc39.es/ecma262/#prod-ImportDeclaration
 
   parseImport(node: N.Node): N.AnyImport {
     // import '...'
@@ -2232,6 +2249,7 @@ export default class StatementParser extends ExpressionParser {
     }
   }
 
+  // https://tc39.es/ecma262/#prod-ImportSpecifier
   parseImportSpecifier(node: N.ImportDeclaration): void {
     const specifier = this.startNode();
     specifier.imported = this.parseIdentifier(true);

packages/babel-parser/src/plugins/estree.js

@@ -313,14 +313,14 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       isGenerator: boolean,
       isAsync: boolean,
       isPattern: boolean,
-      containsEsc: boolean,
+      isAccessor: boolean,
     ): ?N.ObjectMethod {
       const node: N.EstreeProperty = (super.parseObjectMethod(
         prop,
         isGenerator,
         isAsync,
         isPattern,
-        containsEsc,
+        isAccessor,
       ): any);
 
       if (node) {

packages/babel-parser/src/plugins/flow.js

@@ -2362,8 +2362,8 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       isGenerator: boolean,
       isAsync: boolean,
       isPattern: boolean,
+      isAccessor: boolean,
       refExpressionErrors: ?ExpressionErrors,
-      containsEsc: boolean,
     ): void {
       if ((prop: $FlowFixMe).variance) {
         this.unexpected((prop: $FlowFixMe).variance.start);
@@ -2373,7 +2373,7 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       let typeParameters;
 
       // method shorthand
-      if (this.isRelational("<")) {
+      if (this.isRelational("<") && !isAccessor) {
         typeParameters = this.flowParseTypeParameterDeclaration();
         if (!this.match(tt.parenL)) this.unexpected();
       }
@@ -2385,8 +2385,8 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         isGenerator,
         isAsync,
         isPattern,
+        isAccessor,
         refExpressionErrors,
-        containsEsc,
       );
 
       // add typeParameters if we found them

packages/babel-parser/src/plugins/typescript/index.js

@@ -1823,13 +1823,14 @@ export default (superClass: Class<Parser>): Class<Parser> =>
               node.typeParameters = typeArguments;
               return this.finishCallExpression(node, state.optionalChainMember);
             } else if (this.match(tt.backQuote)) {
-              return this.parseTaggedTemplateExpression(
+              const result = this.parseTaggedTemplateExpression(
+                base,
                 startPos,
                 startLoc,
-                base,
                 state,
-                typeArguments,
               );
+              result.typeParameters = typeArguments;
+              return result;
             }
           }
 

packages/babel-parser/src/types.js

@@ -389,6 +389,11 @@ export type ArrayExpression = NodeBase & {
   elements: $ReadOnlyArray<?(Expression | SpreadElement)>,
 };
 
+export type DoExpression = NodeBase & {
+  type: "DoExpression",
+  body: ?BlockStatement,
+};
+
 export type TupleExpression = NodeBase & {
   type: "TupleExpression",
   elements: $ReadOnlyArray<?(Expression | SpreadElement)>,

packages/babel-parser/test/fixtures/es2017/async-functions/invalid-escape-export-async-function/options.json

@@ -1,3 +1,3 @@
 {
   "throws": "Unexpected token, expected \"{\" (1:7)"
-}
+}

packages/babel-parser/test/fixtures/es2017/async-functions/newline-before-arrow/input.js

@@ -0,0 +1,2 @@
+async x
+=> x

packages/babel-parser/test/fixtures/es2017/async-functions/newline-before-arrow/output.json

@@ -0,0 +1,39 @@
+{
+  "type": "File",
+  "start":0,"end":12,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+  "errors": [
+    "SyntaxError: No line break is allowed before '=>' (2:2)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":12,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":12,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+        "expression": {
+          "type": "ArrowFunctionExpression",
+          "start":0,"end":12,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+          "id": null,
+          "generator": false,
+          "async": true,
+          "params": [
+            {
+              "type": "Identifier",
+              "start":6,"end":7,"loc":{"start":{"line":1,"column":6},"end":{"line":1,"column":7},"identifierName":"x"},
+              "name": "x"
+            }
+          ],
+          "body": {
+            "type": "Identifier",
+            "start":11,"end":12,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4},"identifierName":"x"},
+            "name": "x"
+          }
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-and-ns/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-default-asi/options.json

@@ -1,4 +0,0 @@
-{
-  "sourceType": "module",
-  "plugins": ["exportDefaultFrom"]
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-default/options.json

@@ -1,4 +0,0 @@
-{
-  "sourceType": "module",
-  "plugins": ["exportDefaultFrom"]
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-escaped/input.js

@@ -0,0 +1 @@
+export asyn\u{63} from "async";

packages/babel-parser/test/fixtures/experimental/export-extensions/default-escaped/output.json

@@ -0,0 +1,37 @@
+{
+  "type": "File",
+  "start":0,"end":31,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":31}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":31,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":31}},
+    "sourceType": "module",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExportNamedDeclaration",
+        "start":0,"end":31,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":31}},
+        "specifiers": [
+          {
+            "type": "ExportDefaultSpecifier",
+            "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}},
+            "exported": {
+              "type": "Identifier",
+              "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17},"identifierName":"async"},
+              "name": "async"
+            }
+          }
+        ],
+        "source": {
+          "type": "StringLiteral",
+          "start":23,"end":30,"loc":{"start":{"line":1,"column":23},"end":{"line":1,"column":30}},
+          "extra": {
+            "rawValue": "async",
+            "raw": "\"async\""
+          },
+          "value": "async"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-from-identifier-escaped/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-from-identifier-parenthesized/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default-type-without-flow/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/default/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/experimental/export-extensions/invalid-default-from-identifier/options.json

@@ -1,4 +0,0 @@
-{
-  "plugins": ["exportDefaultFrom"],
-  "sourceType": "module"
-}

packages/babel-parser/test/fixtures/flow/type-annotations/20/output.json

@@ -34,7 +34,6 @@
                 },
                 "computed": false,
                 "kind": "set",
-                "variance": null,
                 "id": null,
                 "generator": false,
                 "async": false,

packages/babel-parser/test/fixtures/flow/type-annotations/21/output.json

@@ -34,7 +34,6 @@
                 },
                 "computed": false,
                 "kind": "set",
-                "variance": null,
                 "id": null,
                 "generator": false,
                 "async": false,

packages/babel-parser/test/fixtures/flow/type-annotations/22/output.json

@@ -34,7 +34,6 @@
                 },
                 "computed": false,
                 "kind": "get",
-                "variance": null,
                 "id": null,
                 "generator": false,
                 "async": false,