Add %%placeholders%% support to @babel/template (#9648)

This is the last step to make https://github.com/babel/babel/pull/9364 usable in Babel. I'm sorry for opening this PR so late, but I hope to get it in v7.4.

In this PR I added a new option to `@babel/template`, `syntacticPlaceholders: ?boolean`, which toggles between `%%foo%%` placeholders (when `true`) and `FOO` placeholders. If it isn't specified, Babel tries to be "smart" to avoid breaking backward compat: if `%%foo%%` is used `syntacticPlaceholders` defaults to `true`, otherwise to `false`.

0e58e252913efe84eba926cc9c9c19fb18d5c620 commit shows how some templates we used could be simplified by using this new placeholders syntax (we can't actually do it yet because we are importing `template` from `@babel/core` which could be an older version).

NOTE: Since I wanted to keep this PR as small as possible to make it easier to review, I didn't migrate `template.ast` to internally use the new syntax. It is an implementation detail, so it will be possible to change it in a patch release.

packages/babel-template/src/literal.js

@@ -74,6 +74,7 @@ function buildLiteralData<T>(
       ),
       placeholderPattern: opts.placeholderPattern,
       preserveComments: opts.preserveComments,
+      syntacticPlaceholders: opts.syntacticPlaceholders,
     });
   } while (
     metadata.placeholders.some(

packages/babel-template/src/options.js

@@ -1,5 +1,9 @@
 // @flow
 
+import type { Options as ParserOpts } from "@babel/parser/src/options";
+
+export type { ParserOpts };
+
 /**
  * These are the options that 'babel-template' actually accepts and typechecks
  * when called. All other options are passed through to the parser.
@@ -8,6 +12,8 @@ export type PublicOpts = {
   /**
    * A set of placeholder names to automatically accept, ignoring the given
    * pattern entirely.
+   *
+   * This option can be used when using %%foo%% style placeholders.
    */
   placeholderWhitelist?: ?Set<string>,
 
@@ -19,6 +25,8 @@ export type PublicOpts = {
    * 'placeholderWhitelist' value to find replacements.
    *
    * Defaults to /^[_$A-Z0-9]+$/.
+   *
+   * This option can be used when using %%foo%% style placeholders.
    */
   placeholderPattern?: ?(RegExp | false),
 
@@ -27,13 +35,22 @@ export type PublicOpts = {
    * or 'false' to automatically discard comments. Defaults to 'false'.
    */
   preserveComments?: ?boolean,
+
+  /**
+   * 'true' to use %%foo%% style placeholders, 'false' to use legacy placeholders
+   * described by placeholderPattern or placeholderWhitelist.
+   * When it is not set, it behaves as 'true' if there are syntactic placeholders,
+   * otherwise as 'false'.
+   */
+  syntacticPlaceholders?: ?boolean,
 };
 
 export type TemplateOpts = {|
-  parser: {},
+  parser: ParserOpts,
   placeholderWhitelist: Set<string> | void,
   placeholderPattern: RegExp | false | void,
   preserveComments: boolean | void,
+  syntacticPlaceholders: boolean | void,
 |};
 
 export function merge(a: TemplateOpts, b: TemplateOpts): TemplateOpts {
@@ -41,6 +58,7 @@ export function merge(a: TemplateOpts, b: TemplateOpts): TemplateOpts {
     placeholderWhitelist = a.placeholderWhitelist,
     placeholderPattern = a.placeholderPattern,
     preserveComments = a.preserveComments,
+    syntacticPlaceholders = a.syntacticPlaceholders,
   } = b;
 
   return {
@@ -51,6 +69,7 @@ export function merge(a: TemplateOpts, b: TemplateOpts): TemplateOpts {
     placeholderWhitelist,
     placeholderPattern,
     preserveComments,
+    syntacticPlaceholders,
   };
 }
 
@@ -63,6 +82,7 @@ export function validate(opts: mixed): TemplateOpts {
     placeholderWhitelist,
     placeholderPattern,
     preserveComments,
+    syntacticPlaceholders,
     ...parser
   } = opts || {};
 
@@ -88,12 +108,32 @@ export function validate(opts: mixed): TemplateOpts {
     );
   }
 
+  if (
+    syntacticPlaceholders != null &&
+    typeof syntacticPlaceholders !== "boolean"
+  ) {
+    throw new Error(
+      "'.syntacticPlaceholders' must be a boolean, null, or undefined",
+    );
+  }
+  if (
+    syntacticPlaceholders === true &&
+    (placeholderWhitelist != null || placeholderPattern != null)
+  ) {
+    throw new Error(
+      "'.placeholderWhitelist' and '.placeholderPattern' aren't compatible" +
+        " with '.syntacticPlaceholders: true'",
+    );
+  }
+
   return {
     parser,
     placeholderWhitelist: placeholderWhitelist || undefined,
     placeholderPattern:
       placeholderPattern == null ? undefined : placeholderPattern,
     preserveComments: preserveComments == null ? false : preserveComments,
+    syntacticPlaceholders:
+      syntacticPlaceholders == null ? undefined : syntacticPlaceholders,
   };
 }
 

packages/babel-template/src/parse.js

@@ -3,7 +3,7 @@ import * as t from "@babel/types";
 import type { TraversalAncestors, TraversalHandler } from "@babel/types";
 import { parse } from "@babel/parser";
 import { codeFrameColumns } from "@babel/code-frame";
-import type { TemplateOpts } from "./options";
+import type { TemplateOpts, ParserOpts } from "./options";
 import type { Formatter } from "./formatters";
 
 export type Metadata = {
@@ -31,8 +31,9 @@ export default function parseAndBuildMetadata<T>(
 
   const {
     placeholderWhitelist,
-    placeholderPattern = PATTERN,
+    placeholderPattern,
     preserveComments,
+    syntacticPlaceholders,
   } = opts;
 
   t.removePropertiesDeep(ast, {
@@ -41,20 +42,28 @@ export default function parseAndBuildMetadata<T>(
 
   formatter.validate(ast);
 
-  const placeholders = [];
+  const syntactic = {
-  const placeholderNames = new Set();
+    placeholders: [],
+    placeholderNames: new Set(),
+  };
+  const legacy = {
+    placeholders: [],
+    placeholderNames: new Set(),
+  };
+  const isLegacyRef = { value: undefined };
 
   t.traverse(ast, (placeholderVisitorHandler: TraversalHandler<*>), {
-    placeholders,
+    syntactic,
-    placeholderNames,
+    legacy,
+    isLegacyRef,
     placeholderWhitelist,
     placeholderPattern,
+    syntacticPlaceholders,
   });
 
   return {
     ast,
-    placeholders,
+    ...(isLegacyRef.value ? legacy : syntactic),
-    placeholderNames,
   };
 }
 
@@ -64,16 +73,45 @@ function placeholderVisitorHandler(
   state: MetadataState,
 ) {
   let name;
-  if (t.isIdentifier(node) || t.isJSXIdentifier(node)) {
+
+  if (t.isPlaceholder(node)) {
+    if (state.syntacticPlaceholders === false) {
+      throw new Error(
+        "%%foo%%-style placeholders can't be used when " +
+          "'.syntacticPlaceholders' is false.",
+      );
+    } else {
+      name = ((node: any).name: BabelNodeIdentifier).name;
+      state.isLegacyRef.value = false;
+    }
+  } else if (state.isLegacyRef.value === false || state.syntacticPlaceholders) {
+    return;
+  } else if (t.isIdentifier(node) || t.isJSXIdentifier(node)) {
     name = ((node: any): BabelNodeIdentifier).name;
+    state.isLegacyRef.value = true;
   } else if (t.isStringLiteral(node)) {
     name = ((node: any): BabelNodeStringLiteral).value;
+    state.isLegacyRef.value = true;
   } else {
     return;
   }
 
   if (
-    (!state.placeholderPattern || !state.placeholderPattern.test(name)) &&
+    !state.isLegacyRef.value &&
+    (state.placeholderPattern != null || state.placeholderWhitelist != null)
+  ) {
+    // This check is also in options.js. We need it there to handle the default
+    // .syntacticPlaceholders behavior.
+    throw new Error(
+      "'.placeholderWhitelist' and '.placeholderPattern' aren't compatible" +
+        " with '.syntacticPlaceholders: true'",
+    );
+  }
+
+  if (
+    state.isLegacyRef.value &&
+    (state.placeholderPattern === false ||
+      !(state.placeholderPattern || PATTERN).test(name)) &&
     (!state.placeholderWhitelist || !state.placeholderWhitelist.has(name))
   ) {
     return;
@@ -85,7 +123,10 @@ function placeholderVisitorHandler(
   const { node: parent, key } = ancestors[ancestors.length - 1];
 
   let type: PlaceholderType;
-  if (t.isStringLiteral(node)) {
+  if (
+    t.isStringLiteral(node) ||
+    t.isPlaceholder(node, { expectedNode: "StringLiteral" })
+  ) {
     type = "string";
   } else if (
     (t.isNewExpression(parent) && key === "arguments") ||
@@ -93,20 +134,26 @@ function placeholderVisitorHandler(
     (t.isFunction(parent) && key === "params")
   ) {
     type = "param";
-  } else if (t.isExpressionStatement(parent)) {
+  } else if (t.isExpressionStatement(parent) && !t.isPlaceholder(node)) {
     type = "statement";
     ancestors = ancestors.slice(0, -1);
+  } else if (t.isStatement(node) && t.isPlaceholder(node)) {
+    type = "statement";
   } else {
     type = "other";
   }
 
-  state.placeholders.push({
+  const { placeholders, placeholderNames } = state.isLegacyRef.value
+    ? state.legacy
+    : state.syntactic;
+
+  placeholders.push({
     name,
     type,
     resolve: ast => resolveAncestors(ast, ancestors),
-    isDuplicate: state.placeholderNames.has(name),
+    isDuplicate: placeholderNames.has(name),
   });
-  state.placeholderNames.add(name);
+  placeholderNames.add(name);
 }
 
 function resolveAncestors(ast: BabelNodeFile, ancestors: TraversalAncestors) {
@@ -127,18 +174,30 @@ function resolveAncestors(ast: BabelNodeFile, ancestors: TraversalAncestors) {
 }
 
 type MetadataState = {
+  syntactic: {
     placeholders: Array<Placeholder>,
     placeholderNames: Set<string>,
+  },
+  legacy: {
+    placeholders: Array<Placeholder>,
+    placeholderNames: Set<string>,
+  },
+  isLegacyRef: { value: boolean | void },
   placeholderWhitelist: Set<string> | void,
-  placeholderPattern: RegExp | false,
+  placeholderPattern: RegExp | false | void,
+  syntacticPlaceholders: boolean | void,
 };
 
-function parseWithCodeFrame(code: string, parserOpts: {}): BabelNodeFile {
+function parseWithCodeFrame(
+  code: string,
+  parserOpts: ParserOpts,
+): BabelNodeFile {
   parserOpts = {
     allowReturnOutsideFunction: true,
     allowSuperOutsideMethod: true,
     sourceType: "module",
     ...parserOpts,
+    plugins: (parserOpts.plugins || []).concat("placeholders"),
   };
 
   try {

packages/babel-template/test/index.js

@@ -218,4 +218,105 @@ describe("@babel/template", function() {
       expect(generator(result).code).toEqual("<div>{'content'}</div>");
     });
   });
+
+  describe.only(".syntacticPlaceholders", () => {
+    it("works in function body", () => {
+      const output = template(`function f() %%A%%`)({
+        A: t.blockStatement([]),
+      });
+      expect(generator(output).code).toMatchInlineSnapshot(`"function f() {}"`);
+    });
+
+    it("works in class body", () => {
+      const output = template(`class C %%A%%`)({
+        A: t.classBody([]),
+      });
+      expect(generator(output).code).toMatchInlineSnapshot(`"class C {}"`);
+    });
+
+    it("replaces lowercase names", () => {
+      const output = template(`%%foo%%`)({
+        foo: t.numericLiteral(1),
+      });
+      expect(generator(output).code).toMatchInlineSnapshot(`"1;"`);
+    });
+
+    it("pattern", () => {
+      expect(() => {
+        template(`%%A%% + %%B%%`, {
+          placeholderPattern: /B/,
+        })();
+      }).toThrow(/aren't compatible with '.syntacticPlaceholders: true'/);
+    });
+
+    it("whitelist", () => {
+      expect(() => {
+        template(`%%A%% + %%B%%`, {
+          placeholderPattern: false,
+          placeholderWhitelist: new Set(["B"]),
+        })();
+      }).toThrow(/aren't compatible with '.syntacticPlaceholders: true'/);
+    });
+
+    describe("option value", () => {
+      describe("true", () => {
+        it("allows placeholders", () => {
+          const output = template(`%%FOO%%`, { syntacticPlaceholders: true })({
+            FOO: t.numericLiteral(1),
+          });
+          expect(generator(output).code).toMatchInlineSnapshot(`"1;"`);
+        });
+
+        it("doesn't replace identifiers", () => {
+          expect(() => {
+            template(`FOO`, { syntacticPlaceholders: true })({
+              FOO: t.numericLiteral(1),
+            });
+          }).toThrow(/Unknown substitution/);
+        });
+      });
+
+      describe("false", () => {
+        it("disallow placeholders", () => {
+          expect(() => {
+            template(`%%FOO%%`, { syntacticPlaceholders: false })({
+              FOO: t.numericLiteral(1),
+            });
+          }).toThrow(/%%.*placeholders can't be used/);
+        });
+
+        it("replaces identifiers", () => {
+          const output = template(`FOO`, { syntacticPlaceholders: false })({
+            FOO: t.numericLiteral(1),
+          });
+          expect(generator(output).code).toMatchInlineSnapshot(`"1;"`);
+        });
+      });
+
+      describe("undefined", () => {
+        it("allows placeholders", () => {
+          const output = template(`%%FOO%%`)({
+            FOO: t.numericLiteral(1),
+          });
+          expect(generator(output).code).toMatchInlineSnapshot(`"1;"`);
+        });
+
+        it("replaces identifiers", () => {
+          expect(() => {
+            const output = template(`FOO`)({
+              FOO: t.numericLiteral(1),
+            });
+            expect(generator(output).code).toMatchInlineSnapshot(`"1;"`);
+          });
+        });
+
+        it("doesn't mix placeholder styles", () => {
+          const output = template(`FOO + %%FOO%%`)({
+            FOO: t.numericLiteral(1),
+          });
+          expect(generator(output).code).toMatchInlineSnapshot(`"FOO + 1;"`);
+        });
+      });
+    });
+  });
 });