Do not delete "fake" source map comments from strings (#9960)

Instead of using `convert-source-map`'s `removeComments` method before
parsing the file, we can first parse the file with `@babel/parser` and then
analyze the comments.
This is needed because it is not possible to reliabily detect comments in
JavaScript without fully parsing the file:
https://github.com/thlorenz/convert-source-map/issues/63

lib/third-party-libs.js.flow

@@ -181,7 +181,7 @@ declare module "convert-source-map" {
     fromJSON(str: string): Converter,
     fromBase64(str: string): Converter,
     fromComment(str: string): Converter,
-    fromMapFileComment(str: string): Converter,
+    fromMapFileComment(str: string, dir: string): Converter,
     fromSource(str: string): Converter,
     fromMapFileSource(str: string, dir: string): Converter,
     removeComments(str: string): string,

packages/babel-core/src/transformation/normalize-file.js

@@ -27,49 +27,6 @@ export default function normalizeFile(
 ): File {
   code = `${code || ""}`;
 
-  let inputMap = null;
-  if (options.inputSourceMap !== false) {
-    // If an explicit object is passed in, it overrides the processing of
-    // source maps that may be in the file itself.
-    if (typeof options.inputSourceMap === "object") {
-      inputMap = convertSourceMap.fromObject(options.inputSourceMap);
-    }
-
-    if (!inputMap) {
-      try {
-        inputMap = convertSourceMap.fromSource(code);
-
-        if (inputMap) {
-          code = convertSourceMap.removeComments(code);
-        }
-      } catch (err) {
-        debug("discarding unknown inline input sourcemap", err);
-        code = convertSourceMap.removeComments(code);
-      }
-    }
-
-    if (!inputMap) {
-      if (typeof options.filename === "string") {
-        try {
-          inputMap = convertSourceMap.fromMapFileSource(
-            code,
-            path.dirname(options.filename),
-          );
-
-          if (inputMap) {
-            code = convertSourceMap.removeMapFileComments(code);
-          }
-        } catch (err) {
-          debug("discarding unknown file input sourcemap", err);
-          code = convertSourceMap.removeMapFileComments(code);
-        }
-      } else {
-        debug("discarding un-loadable file input sourcemap");
-        code = convertSourceMap.removeMapFileComments(code);
-      }
-    }
-  }
-
   if (ast) {
     if (ast.type === "Program") {
       ast = t.file(ast, [], []);
@@ -84,6 +41,42 @@ export default function normalizeFile(
     ast = parser(pluginPasses, options, code);
   }
 
+  let inputMap = null;
+  if (options.inputSourceMap !== false) {
+    // If an explicit object is passed in, it overrides the processing of
+    // source maps that may be in the file itself.
+    if (typeof options.inputSourceMap === "object") {
+      inputMap = convertSourceMap.fromObject(options.inputSourceMap);
+    }
+
+    if (!inputMap) {
+      const lastComment = extractComments(INLINE_SOURCEMAP_REGEX, ast);
+      if (lastComment) {
+        try {
+          inputMap = convertSourceMap.fromComment(lastComment);
+        } catch (err) {
+          debug("discarding unknown inline input sourcemap", err);
+        }
+      }
+    }
+
+    if (!inputMap) {
+      const lastComment = extractComments(EXTERNAL_SOURCEMAP_REGEX, ast);
+      if (typeof options.filename === "string" && lastComment) {
+        try {
+          inputMap = convertSourceMap.fromMapFileComment(
+            lastComment,
+            path.dirname(options.filename),
+          );
+        } catch (err) {
+          debug("discarding unknown file input sourcemap", err);
+        }
+      } else if (lastComment) {
+        debug("discarding un-loadable file input sourcemap");
+      }
+    }
+  }
+
   return new File(options, {
     code,
     ast,
@@ -156,3 +149,48 @@ function parser(
     throw err;
   }
 }
+
+// These regexps are copied from the convert-source-map package,
+// but without // or /* at the beginning of the comment.
+
+// eslint-disable-next-line max-len
+const INLINE_SOURCEMAP_REGEX = /^[@#]\s+sourceMappingURL=data:(?:application|text)\/json;(?:charset[:=]\S+?;)?base64,(?:.*)$/;
+const EXTERNAL_SOURCEMAP_REGEX = /^[@#][ \t]+sourceMappingURL=([^\s'"`]+?)[ \t]*$/;
+
+function extractCommentsFromList(regex, comments, lastComment) {
+  if (comments) {
+    comments = comments.filter(({ value }) => {
+      if (regex.test(value)) {
+        lastComment = value;
+        return false;
+      }
+      return true;
+    });
+  }
+  return [comments, lastComment];
+}
+
+function extractComments(regex, ast) {
+  let lastComment = null;
+  t.traverseFast(ast, node => {
+    // $FlowIgnore destructuring with expressions is not supported
+    [node.leadingComments, lastComment] = extractCommentsFromList(
+      regex,
+      node.leadingComments,
+      lastComment,
+    );
+    // $FlowIgnore destructuring with expressions is not supported
+    [node.innerComments, lastComment] = extractCommentsFromList(
+      regex,
+      node.innerComments,
+      lastComment,
+    );
+    // $FlowIgnore destructuring with expressions is not supported
+    [node.trailingComments, lastComment] = extractCommentsFromList(
+      regex,
+      node.trailingComments,
+      lastComment,
+    );
+  });
+  return lastComment;
+}

packages/babel-core/test/fixtures/transformation/source-maps/comment-inside-string/input.js

@@ -0,0 +1,8 @@
+// https://github.com/babel/babel/issues/9790
+const comment = `//# sourceMappingURL=${path.basename(
+  sourceMapFilename
+)}`
+
+// https://github.com/babel/babel/issues/9956
+this.shadowRoot.innerHTML = `<style>div{display:block}
+/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmllbGQuaHRtbCIsInNvdXJjZXMiOlsiZmllbGQuaHRtbCJdLCJzb3VyY2VzQ29udGVudCI6WyI8c3ZlbHRlOm9wdGlvbnMgdGFnPVwiZGxzLWZpZWxkXCIgLz5cblxuPHN0eWxlPlxuICBkaXYgeyBkaXNwbGF5OiBibG9jazsgfVxuPC9zdHlsZT5cblxuPGRpdiBjbGFzcz1cImZpZWxkXCI+XG4gIDxkaXYgY2xhc3M9XCJfZmllbGRMYWJlbExheW91dFwiPlxuICAgIDxkaXYgY2xhc3M9XCJmaWVsZExhYmVsXCI+XG4gICAgICA8c2xvdCBuYW1lPVwiZmllbGQtbGFiZWxcIj48L3Nsb3Q+XG4gICAgPC9kaXY+XG4gICAgPGRpdiBjbGFzcz1cImZpZWxkT3B0aW9uYWxcIiBjbGFzczpvcHRpb25hbD5cbiAgICAgIE9wdGlvbmFsXG4gICAgPC9kaXY+XG4gIDwvZGl2PlxuICA8ZGl2IGNsYXNzPVwiZmllbGREZXRhaWxcIj5cbiAgICA8c2xvdCBuYW1lPVwiZmllbGQtZGV0YWlsXCI+PC9zbG90PlxuICA8L2Rpdj5cbiAgPGRpdiBjbGFzcz1cImZpZWxkQ29udHJvbFwiPlxuICAgIDxzbG90IG5hbWU9XCJmaWVsZC1jb250cm9sXCI+PC9zbG90PlxuICAgIDxzbG90Pjwvc2xvdD5cbiAgPC9kaXY+XG4gIDxkaXYgY2xhc3M9XCJmaWVsZE1lc3NhZ2VcIj5cbiAgICA8c2xvdCBuYW1lPVwiZmllbGQtbWVzc2FnZVwiPjwvc2xvdD5cbiAgPC9kaXY+XG48L2Rpdj5cblxuPHNjcmlwdD5cbiAgZXhwb3J0IGxldCB0aGVtZSA9ICcnO1xuICBleHBvcnQgbGV0IG9wdGlvbmFsID0gZmFsc2U7XG48L3NjcmlwdD5cbiJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFHRSxHQUFHLEFBQUMsQ0FBQyxBQUFDLE9BQU8sQ0FBRSxLQUFLLEFBQUUsQ0FBQyJ9 */</style>`;

packages/babel-core/test/fixtures/transformation/source-maps/comment-inside-string/options.json

@@ -0,0 +1,3 @@
+{
+  "inputSourceMap": true
+}

packages/babel-core/test/fixtures/transformation/source-maps/comment-inside-string/output.js

@@ -0,0 +1,5 @@
+// https://github.com/babel/babel/issues/9790
+const comment = `//# sourceMappingURL=${path.basename(sourceMapFilename)}`; // https://github.com/babel/babel/issues/9956
+
+this.shadowRoot.innerHTML = `<style>div{display:block}
+/*# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmllbGQuaHRtbCIsInNvdXJjZXMiOlsiZmllbGQuaHRtbCJdLCJzb3VyY2VzQ29udGVudCI6WyI8c3ZlbHRlOm9wdGlvbnMgdGFnPVwiZGxzLWZpZWxkXCIgLz5cblxuPHN0eWxlPlxuICBkaXYgeyBkaXNwbGF5OiBibG9jazsgfVxuPC9zdHlsZT5cblxuPGRpdiBjbGFzcz1cImZpZWxkXCI+XG4gIDxkaXYgY2xhc3M9XCJfZmllbGRMYWJlbExheW91dFwiPlxuICAgIDxkaXYgY2xhc3M9XCJmaWVsZExhYmVsXCI+XG4gICAgICA8c2xvdCBuYW1lPVwiZmllbGQtbGFiZWxcIj48L3Nsb3Q+XG4gICAgPC9kaXY+XG4gICAgPGRpdiBjbGFzcz1cImZpZWxkT3B0aW9uYWxcIiBjbGFzczpvcHRpb25hbD5cbiAgICAgIE9wdGlvbmFsXG4gICAgPC9kaXY+XG4gIDwvZGl2PlxuICA8ZGl2IGNsYXNzPVwiZmllbGREZXRhaWxcIj5cbiAgICA8c2xvdCBuYW1lPVwiZmllbGQtZGV0YWlsXCI+PC9zbG90PlxuICA8L2Rpdj5cbiAgPGRpdiBjbGFzcz1cImZpZWxkQ29udHJvbFwiPlxuICAgIDxzbG90IG5hbWU9XCJmaWVsZC1jb250cm9sXCI+PC9zbG90PlxuICAgIDxzbG90Pjwvc2xvdD5cbiAgPC9kaXY+XG4gIDxkaXYgY2xhc3M9XCJmaWVsZE1lc3NhZ2VcIj5cbiAgICA8c2xvdCBuYW1lPVwiZmllbGQtbWVzc2FnZVwiPjwvc2xvdD5cbiAgPC9kaXY+XG48L2Rpdj5cblxuPHNjcmlwdD5cbiAgZXhwb3J0IGxldCB0aGVtZSA9ICcnO1xuICBleHBvcnQgbGV0IG9wdGlvbmFsID0gZmFsc2U7XG48L3NjcmlwdD5cbiJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFHRSxHQUFHLEFBQUMsQ0FBQyxBQUFDLE9BQU8sQ0FBRSxLQUFLLEFBQUUsQ0FBQyJ9 */</style>`;

packages/babel-types/src/traverse/traverseFast.js

@@ -7,7 +7,7 @@ import { VISITOR_KEYS } from "../definitions";
  */
 export default function traverseFast(
   node: Object,
-  enter: (node: Node, opts?: Object) => void,
+  enter: (node: BabelNode, opts?: Object) => void,
   opts?: Object,
 ): void {
   if (!node) return;