Merge pull request #8289 from valtech-nyc/implement-smart-pipeline-in-parser

Implement Smart Pipeline proposal in @babel/parser

packages/babel-parser/src/parser/expression.js

@@ -297,18 +297,21 @@ export default class ExpressionParser extends LValParser {
         }
 
         const op = this.state.type;
-        if (op === tt.nullishCoalescing) {
-          this.expectPlugin("nullishCoalescingOperator");
-        } else if (op === tt.pipeline) {
+
+        if (op === tt.pipeline) {
           this.expectPlugin("pipelineOperator");
+          this.state.inPipeline = true;
+          this.checkPipelineAtInfixOperator(left, leftStartPos);
+        } else if (op === tt.nullishCoalescing) {
+          this.expectPlugin("nullishCoalescingOperator");
         }
 
         this.next();
 
-        const startPos = this.state.start;
-        const startLoc = this.state.startLoc;
-
-        if (op === tt.pipeline) {
+        if (
+          op === tt.pipeline &&
+          this.getPluginOption("pipelineOperator", "proposal") === "minimal"
+        ) {
           if (
             this.match(tt.name) &&
             this.state.value === "await" &&
@@ -321,13 +324,7 @@ export default class ExpressionParser extends LValParser {
           }
         }
 
-        node.right = this.parseExprOp(
-          this.parseMaybeUnary(),
-          startPos,
-          startLoc,
-          op.rightAssociative ? prec - 1 : prec,
-          noIn,
-        );
+        node.right = this.parseExprOpRightExpr(op, prec, noIn);
 
         this.finishNode(
           node,
@@ -337,6 +334,7 @@ export default class ExpressionParser extends LValParser {
             ? "LogicalExpression"
             : "BinaryExpression",
         );
+
         return this.parseExprOp(
           node,
           leftStartPos,
@@ -349,6 +347,54 @@ export default class ExpressionParser extends LValParser {
     return left;
   }
 
+  // Helper function for `parseExprOp`. Parse the right-hand side of binary-
+  // operator expressions, then apply any operator-specific functions.
+
+  parseExprOpRightExpr(
+    op: TokenType,
+    prec: number,
+    noIn: ?boolean,
+  ): N.Expression {
+    switch (op) {
+      case tt.pipeline:
+        if (this.getPluginOption("pipelineOperator", "proposal") === "smart") {
+          const startPos = this.state.start;
+          const startLoc = this.state.startLoc;
+          return this.withTopicPermittingContext(() => {
+            return this.parseSmartPipelineBody(
+              this.parseExprOpBaseRightExpr(op, prec, noIn),
+              startPos,
+              startLoc,
+            );
+          });
+        }
+      // falls through
+
+      default:
+        return this.parseExprOpBaseRightExpr(op, prec, noIn);
+    }
+  }
+
+  // Helper function for `parseExprOpRightExpr`. Parse the right-hand side of
+  // binary-operator expressions without applying any operator-specific functions.
+
+  parseExprOpBaseRightExpr(
+    op: TokenType,
+    prec: number,
+    noIn: ?boolean,
+  ): N.Expression {
+    const startPos = this.state.start;
+    const startLoc = this.state.startLoc;
+
+    return this.parseExprOp(
+      this.parseMaybeUnary(),
+      startPos,
+      startLoc,
+      op.rightAssociative ? prec - 1 : prec,
+      noIn,
+    );
+  }
+
   // Parse unary operators, both prefix and postfix.
 
   parseMaybeUnary(refShorthandDefaultPos: ?Pos): N.Expression {
@@ -936,6 +982,32 @@ export default class ExpressionParser extends LValParser {
         }
       }
 
+      case tt.hash: {
+        if (this.state.inPipeline) {
+          node = this.startNode();
+
+          if (
+            this.getPluginOption("pipelineOperator", "proposal") !== "smart"
+          ) {
+            this.raise(
+              node.start,
+              "Primary Topic Reference found but pipelineOperator not passed 'smart' for 'proposal' option.",
+            );
+          }
+
+          this.next();
+          if (this.primaryTopicReferenceIsAllowedInCurrentTopicContext()) {
+            this.registerTopicReference();
+            return this.finishNode(node, "PipelinePrimaryTopicReference");
+          } else {
+            throw this.raise(
+              node.start,
+              `Topic reference was used in a lexical context without topic binding`,
+            );
+          }
+        }
+      }
+
       default:
         throw this.unexpected();
     }
@@ -2037,4 +2109,180 @@ export default class ExpressionParser extends LValParser {
     }
     return this.finishNode(node, "YieldExpression");
   }
+
+  // Validates a pipeline (for any of the pipeline Babylon plugins) at the point
+  // of the infix operator `|>`.
+
+  checkPipelineAtInfixOperator(left: N.Expression, leftStartPos: number) {
+    if (this.getPluginOption("pipelineOperator", "proposal") === "smart") {
+      if (left.type === "SequenceExpression") {
+        // Ensure that the pipeline head is not a comma-delimited
+        // sequence expression.
+        throw this.raise(
+          leftStartPos,
+          `Pipeline head should not be a comma-separated sequence expression`,
+        );
+      }
+    }
+  }
+
+  parseSmartPipelineBody(
+    childExpression: N.Expression,
+    startPos: number,
+    startLoc: Position,
+  ): N.PipelineBody {
+    const pipelineStyle = this.checkSmartPipelineBodyStyle(childExpression);
+
+    this.checkSmartPipelineBodyEarlyErrors(
+      childExpression,
+      pipelineStyle,
+      startPos,
+    );
+
+    return this.parseSmartPipelineBodyInStyle(
+      childExpression,
+      pipelineStyle,
+      startPos,
+      startLoc,
+    );
+  }
+
+  checkSmartPipelineBodyEarlyErrors(
+    childExpression: N.Expression,
+    pipelineStyle: N.PipelineStyle,
+    startPos: number,
+  ): void {
+    if (this.match(tt.arrow)) {
+      // If the following token is invalidly `=>`, then throw a human-friendly error
+      // instead of something like 'Unexpected token, expected ";"'.
+      throw this.raise(
+        this.state.start,
+        `Unexpected arrow "=>" after pipeline body; arrow function in pipeline body must be parenthesized`,
+      );
+    } else if (
+      pipelineStyle === "PipelineTopicExpression" &&
+      childExpression.type === "SequenceExpression"
+    ) {
+      throw this.raise(
+        startPos,
+        `Pipeline body may not be a comma-separated sequence expression`,
+      );
+    }
+  }
+
+  parseSmartPipelineBodyInStyle(
+    childExpression: N.Expression,
+    pipelineStyle: N.PipelineStyle,
+    startPos: number,
+    startLoc: Position,
+  ): N.PipelineBody {
+    const bodyNode = this.startNodeAt(startPos, startLoc);
+    switch (pipelineStyle) {
+      case "PipelineBareFunction":
+        bodyNode.callee = childExpression;
+        break;
+      case "PipelineBareConstructor":
+        bodyNode.callee = childExpression.callee;
+        break;
+      case "PipelineBareAwaitedFunction":
+        bodyNode.callee = childExpression.argument;
+        break;
+      case "PipelineTopicExpression":
+        if (!this.topicReferenceWasUsedInCurrentTopicContext()) {
+          throw this.raise(
+            startPos,
+            `Pipeline is in topic style but does not use topic reference`,
+          );
+        }
+        bodyNode.expression = childExpression;
+        break;
+      default:
+        throw this.raise(startPos, `Unknown pipeline style ${pipelineStyle}`);
+    }
+    return this.finishNode(bodyNode, pipelineStyle);
+  }
+
+  checkSmartPipelineBodyStyle(expression: N.Expression): N.PipelineStyle {
+    switch (expression.type) {
+      default:
+        return this.isSimpleReference(expression)
+          ? "PipelineBareFunction"
+          : "PipelineTopicExpression";
+    }
+  }
+
+  isSimpleReference(expression: N.Expression): boolean {
+    switch (expression.type) {
+      case "MemberExpression":
+        return (
+          !expression.computed && this.isSimpleReference(expression.object)
+        );
+      case "Identifier":
+        return true;
+      default:
+        return false;
+    }
+  }
+
+  // Enable topic references from outer contexts within smart pipeline bodies.
+  // The function modifies the parser's topic-context state to enable or disable
+  // the use of topic references with the smartPipelines plugin. They then run a
+  // callback, then they reset the parser to the old topic-context state that it
+  // had before the function was called.
+
+  withTopicPermittingContext<T>(callback: () => T): T {
+    const outerContextTopicState = this.state.topicContext;
+    this.state.topicContext = {
+      // Enable the use of the primary topic reference.
+      maxNumOfResolvableTopics: 1,
+      // Hide the use of any topic references from outer contexts.
+      maxTopicIndex: null,
+    };
+
+    try {
+      return callback();
+    } finally {
+      this.state.topicContext = outerContextTopicState;
+    }
+  }
+
+  // Disable topic references from outer contexts within syntax constructs
+  // such as the bodies of iteration statements.
+  // The function modifies the parser's topic-context state to enable or disable
+  // the use of topic references with the smartPipelines plugin. They then run a
+  // callback, then they reset the parser to the old topic-context state that it
+  // had before the function was called.
+
+  withTopicForbiddingContext<T>(callback: () => T): T {
+    const outerContextTopicState = this.state.topicContext;
+    this.state.topicContext = {
+      // Disable the use of the primary topic reference.
+      maxNumOfResolvableTopics: 0,
+      // Hide the use of any topic references from outer contexts.
+      maxTopicIndex: null,
+    };
+
+    try {
+      return callback();
+    } finally {
+      this.state.topicContext = outerContextTopicState;
+    }
+  }
+
+  // Register the use of a primary topic reference (`#`) within the current
+  // topic context.
+  registerTopicReference(): void {
+    this.state.topicContext.maxTopicIndex = 0;
+  }
+
+  primaryTopicReferenceIsAllowedInCurrentTopicContext(): boolean {
+    return this.state.topicContext.maxNumOfResolvableTopics >= 1;
+  }
+
+  topicReferenceWasUsedInCurrentTopicContext(): boolean {
+    return (
+      this.state.topicContext.maxTopicIndex != null &&
+      this.state.topicContext.maxTopicIndex >= 0
+    );
+  }
 }

packages/babel-parser/src/parser/statement.js

@@ -369,8 +369,18 @@ export default class StatementParser extends ExpressionParser {
   parseDoStatement(node: N.DoWhileStatement): N.DoWhileStatement {
     this.next();
     this.state.labels.push(loopLabel);
-    node.body = this.parseStatement(false);
+
+    node.body =
+      // For the smartPipelines plugin: Disable topic references from outer
+      // contexts within the loop body. They are permitted in test expressions,
+      // outside of the loop body.
+      this.withTopicForbiddingContext(() =>
+        // Parse the loop body's body.
+        this.parseStatement(false),
+      );
+
     this.state.labels.pop();
+
     this.expect(tt._while);
     node.test = this.parseParenExpression();
     this.eat(tt.semi);
@@ -557,7 +567,17 @@ export default class StatementParser extends ExpressionParser {
       } else {
         clause.param = null;
       }
-      clause.body = this.parseBlock();
+
+      clause.body =
+        // For the smartPipelines plugin: Disable topic references from outer
+        // contexts within the function body. They are permitted in function
+        // default-parameter expressions, which are part of the outer context,
+        // outside of the function body.
+        this.withTopicForbiddingContext(() =>
+          // Parse the catch clause's body.
+          this.parseBlock(false),
+        );
+
       node.handler = this.finishNode(clause, "CatchClause");
     }
 
@@ -585,8 +605,18 @@ export default class StatementParser extends ExpressionParser {
     this.next();
     node.test = this.parseParenExpression();
     this.state.labels.push(loopLabel);
-    node.body = this.parseStatement(false);
+
+    node.body =
+      // For the smartPipelines plugin:
+      // Disable topic references from outer contexts within the loop body.
+      // They are permitted in test expressions, outside of the loop body.
+      this.withTopicForbiddingContext(() =>
+        // Parse loop body.
+        this.parseStatement(false),
+      );
+
     this.state.labels.pop();
+
     return this.finishNode(node, "WhileStatement");
   }
 
@@ -596,7 +626,17 @@ export default class StatementParser extends ExpressionParser {
     }
     this.next();
     node.object = this.parseParenExpression();
-    node.body = this.parseStatement(false);
+
+    node.body =
+      // For the smartPipelines plugin:
+      // Disable topic references from outer contexts within the function body.
+      // They are permitted in function default-parameter expressions, which are
+      // part of the outer context, outside of the function body.
+      this.withTopicForbiddingContext(() =>
+        // Parse the statement body.
+        this.parseStatement(false),
+      );
+
     return this.finishNode(node, "WithStatement");
   }
 
@@ -753,8 +793,18 @@ export default class StatementParser extends ExpressionParser {
     this.expect(tt.semi);
     node.update = this.match(tt.parenR) ? null : this.parseExpression();
     this.expect(tt.parenR);
-    node.body = this.parseStatement(false);
+
+    node.body =
+      // For the smartPipelines plugin: Disable topic references from outer
+      // contexts within the loop body. They are permitted in test expressions,
+      // outside of the loop body.
+      this.withTopicForbiddingContext(() =>
+        // Parse the loop body.
+        this.parseStatement(false),
+      );
+
     this.state.labels.pop();
+
     return this.finishNode(node, "ForStatement");
   }
 
@@ -778,8 +828,18 @@ export default class StatementParser extends ExpressionParser {
     node.left = init;
     node.right = this.parseExpression();
     this.expect(tt.parenR);
-    node.body = this.parseStatement(false);
+
+    node.body =
+      // For the smartPipelines plugin:
+      // Disable topic references from outer contexts within the loop body.
+      // They are permitted in test expressions, outside of the loop body.
+      this.withTopicForbiddingContext(() =>
+        // Parse loop body.
+        this.parseStatement(false),
+      );
+
     this.state.labels.pop();
+
     return this.finishNode(node, type);
   }
 
@@ -887,11 +947,18 @@ export default class StatementParser extends ExpressionParser {
     }
 
     this.parseFunctionParams(node);
-    this.parseFunctionBodyAndFinish(
-      node,
-      isStatement ? "FunctionDeclaration" : "FunctionExpression",
-      allowExpressionBody,
-    );
+
+    // For the smartPipelines plugin: Disable topic references from outer
+    // contexts within the function body. They are permitted in test
+    // expressions, outside of the function body.
+    this.withTopicForbiddingContext(() => {
+      // Parse the function body.
+      this.parseFunctionBodyAndFinish(
+        node,
+        isStatement ? "FunctionDeclaration" : "FunctionExpression",
+        allowExpressionBody,
+      );
+    });
 
     this.state.inFunction = oldInFunc;
     this.state.inMethod = oldInMethod;
@@ -966,44 +1033,49 @@ export default class StatementParser extends ExpressionParser {
 
     this.expect(tt.braceL);
 
-    while (!this.eat(tt.braceR)) {
-      if (this.eat(tt.semi)) {
-        if (decorators.length > 0) {
+    // For the smartPipelines plugin: Disable topic references from outer
+    // contexts within the class body. They are permitted in test expressions,
+    // outside of the class body.
+    this.withTopicForbiddingContext(() => {
+      while (!this.eat(tt.braceR)) {
+        if (this.eat(tt.semi)) {
+          if (decorators.length > 0) {
+            this.raise(
+              this.state.lastTokEnd,
+              "Decorators must not be followed by a semicolon",
+            );
+          }
+          continue;
+        }
+
+        if (this.match(tt.at)) {
+          decorators.push(this.parseDecorator());
+          continue;
+        }
+
+        const member = this.startNode();
+
+        // steal the decorators if there are any
+        if (decorators.length) {
+          member.decorators = decorators;
+          this.resetStartLocationFromNode(member, decorators[0]);
+          decorators = [];
+        }
+
+        this.parseClassMember(classBody, member, state);
+
+        if (
+          member.kind === "constructor" &&
+          member.decorators &&
+          member.decorators.length > 0
+        ) {
           this.raise(
-            this.state.lastTokEnd,
-            "Decorators must not be followed by a semicolon",
+            member.start,
+            "Decorators can't be used with a constructor. Did you mean '@dec class { ... }'?",
           );
         }
-        continue;
       }
-
-      if (this.match(tt.at)) {
-        decorators.push(this.parseDecorator());
-        continue;
-      }
-
-      const member = this.startNode();
-
-      // steal the decorators if there are any
-      if (decorators.length) {
-        member.decorators = decorators;
-        this.resetStartLocationFromNode(member, decorators[0]);
-        decorators = [];
-      }
-
-      this.parseClassMember(classBody, member, state);
-
-      if (
-        member.kind === "constructor" &&
-        member.decorators &&
-        member.decorators.length > 0
-      ) {
-        this.raise(
-          member.start,
-          "Decorators can't be used with a constructor. Did you mean '@dec class { ... }'?",
-        );
-      }
-    }
+    });
 
     if (decorators.length) {
       this.raise(

packages/babel-parser/src/plugin-utils.js

@@ -38,7 +38,7 @@ export function getPluginOption(
   return null;
 }
 
-const PIPELINE_PROPOSALS = ["minimal"];
+const PIPELINE_PROPOSALS = ["minimal", "smart"];
 
 export function validatePlugins(plugins: PluginList) {
   if (hasPlugin(plugins, "decorators")) {
@@ -77,7 +77,7 @@ export function validatePlugins(plugins: PluginList) {
   ) {
     throw new Error(
       "'pipelineOperator' requires 'proposal' option whose value should be one of: " +
-        PIPELINE_PROPOSALS.join(", "),
+        PIPELINE_PROPOSALS.map(p => `'${p}'`).join(", "),
     );
   }
 }

packages/babel-parser/src/tokenizer/index.js

@@ -374,7 +374,36 @@ export default class Tokenizer extends LocationParser {
   // into it.
   //
   // All in the name of speed.
-  //
+
+  // number sign is "#"
+  readToken_numberSign(): void {
+    if (this.state.pos === 0 && this.readToken_interpreter()) {
+      return;
+    }
+
+    const nextPos = this.state.pos + 1;
+    const next = this.input.charCodeAt(nextPos);
+    if (next >= charCodes.digit0 && next <= charCodes.digit9) {
+      this.raise(this.state.pos, "Unexpected digit after hash token");
+    }
+
+    if (
+      (this.hasPlugin("classPrivateProperties") ||
+        this.hasPlugin("classPrivateMethods")) &&
+      this.state.classLevel > 0
+    ) {
+      ++this.state.pos;
+      this.finishToken(tt.hash);
+      return;
+    } else if (
+      this.getPluginOption("pipelineOperator", "proposal") === "smart"
+    ) {
+      this.finishOp(tt.hash, 1);
+    } else {
+      this.raise(this.state.pos, "Unexpected character '#'");
+    }
+  }
+
   readToken_dot(): void {
     const next = this.input.charCodeAt(this.state.pos + 1);
     if (next >= charCodes.digit0 && next <= charCodes.digit9) {
@@ -623,24 +652,8 @@ export default class Tokenizer extends LocationParser {
   getTokenFromCode(code: number): void {
     switch (code) {
       case charCodes.numberSign:
-        if (this.state.pos === 0 && this.readToken_interpreter()) {
-          return;
-        }
-
-        if (
-          (this.hasPlugin("classPrivateProperties") ||
-            this.hasPlugin("classPrivateMethods")) &&
-          this.state.classLevel > 0
-        ) {
-          ++this.state.pos;
-          this.finishToken(tt.hash);
-          return;
-        } else {
-          this.raise(
-            this.state.pos,
-            `Unexpected character '${String.fromCodePoint(code)}'`,
-          );
-        }
+        this.readToken_numberSign();
+        return;
 
       // The interpretation of a dot depends on whether it is followed
       // by a digit or another two dots.

packages/babel-parser/src/tokenizer/state.js

@@ -6,7 +6,7 @@ import { Position } from "../util/location";
 
 import { types as ct, type TokContext } from "./context";
 import type { Token } from "./index";
-import { types as tt, type TokenType } from "./types";
+import { types as tt, type TokenType, type TopicContextState } from "./types";
 
 export default class State {
   init(options: Options, input: string): void {
@@ -26,6 +26,7 @@ export default class State {
     this.maybeInArrowParameters = false;
     this.inGenerator = false;
     this.inAsync = false;
+    this.inPipeline = false;
     this.inPropertyName = false;
     this.inType = false;
     this.inClassProperty = false;
@@ -33,6 +34,12 @@ export default class State {
     this.hasFlowComment = false;
     this.isIterator = false;
 
+    // Used by smartPipelines.
+    this.topicContext = {
+      maxNumOfResolvableTopics: 0,
+      maxTopicIndex: null,
+    };
+
     this.classLevel = 0;
 
     this.labels = [];
@@ -104,6 +111,7 @@ export default class State {
   inGenerator: boolean;
   inMethod: boolean | N.MethodKind;
   inAsync: boolean;
+  inPipeline: boolean;
   inType: boolean;
   noAnonFunctionType: boolean;
   inPropertyName: boolean;
@@ -111,6 +119,9 @@ export default class State {
   hasFlowComment: boolean;
   isIterator: boolean;
 
+  // For the smartPipelines plugin:
+  topicContext: TopicContextState;
+
   // Check whether we are in a (nested) class or not.
   classLevel: number;
 

packages/babel-parser/src/tokenizer/types.js

@@ -193,3 +193,17 @@ export const keywords = {
 Object.keys(keywords).forEach(name => {
   types["_" + name] = keywords[name];
 });
+
+// A type for the smartPipelines plugin.
+export type TopicContextState = {
+  // When a topic binding has been currently established,
+  // then this is 1. Otherwise, it is 0. This is forwards compatible
+  // with a future plugin for multiple lexical topics.
+  maxNumOfResolvableTopics: number,
+
+  // When a topic binding has been currently established, and if that binding
+  // has been used as a topic reference `#`, then this is 0. Otherwise, it is
+  // `null`. This is forwards compatible with a future plugin for multiple
+  // lexical topics.
+  maxTopicIndex: null | 0,
+};

packages/babel-parser/src/types.js

@@ -566,6 +566,38 @@ export type SequenceExpression = NodeBase & {
   expressions: $ReadOnlyArray<Expression>,
 };
 
+// Pipelines
+
+export type PipelineBody = NodeBase & {
+  type: "PipelineBody",
+};
+
+export type PipelineBareFunctionBody = NodeBase & {
+  type: "PipelineBareFunctionBody",
+  callee: Expression,
+};
+
+export type PipelineBareConstructorBody = NodeBase & {
+  type: "PipelineBareConstructorBody",
+  callee: Expression,
+};
+
+export type PipelineBareAwaitedFunctionBody = NodeBase & {
+  type: "PipelineBareAwaitedFunctionBody",
+  callee: Expression,
+};
+
+export type PipelineTopicBody = NodeBase & {
+  type: "PipelineTopicBody",
+  expression: Expression,
+};
+
+export type PipelineStyle =
+  | "PipelineBareFunction"
+  | "PipelineBareConstructor"
+  | "PipelineBareAwaitedFunction"
+  | "PipelineTopicExpression";
+
 // Template Literals
 
 export type TemplateLiteral = NodeBase & {