Bundle standalone using rollup (#10779)

* chore: bundle babel-standalone via rollup

Co-authored-by: Daniel Tschinder <daniel.tschinder@researchgate.net>

* chore: build babel-preset-env-standalone via Rollup

# Conflicts:
#	yarn.lock

* chore: add babelEnvName

* chore: farewell webpack

* chore: add terser minification

* chore: dedupe some popular dependencies

* chore: remove unused devDependencies

* chore: move plugin-babel-source to scripts/

* make prettier happy

* fix: rewrite available-plugins into esm

* chore: add extra unambiguous sources when env is rollup/standalone

Gulpfile.js

@@ -10,12 +10,17 @@ const fancyLog = require("fancy-log");
 const filter = require("gulp-filter");
 const gulp = require("gulp");
 const path = require("path");
-const webpack = require("webpack");
 const rollup = require("rollup");
+const rollupAlias = require("@rollup/plugin-alias");
 const rollupBabel = require("rollup-plugin-babel");
+const rollupBabelSource = require("./scripts/rollup-plugin-babel-source");
+const rollupCommonJs = require("rollup-plugin-commonjs");
+const rollupJson = require("@rollup/plugin-json");
+const rollupNodeBuiltins = require("rollup-plugin-node-builtins");
+const rollupNodeGlobals = require("rollup-plugin-node-globals");
 const rollupNodeResolve = require("rollup-plugin-node-resolve");
 const rollupReplace = require("rollup-plugin-replace");
-const { registerStandalonePackageTask } = require("./scripts/gulp-tasks");
+const { terser: rollupTerser } = require("rollup-plugin-terser");
 
 const defaultSourcesGlob = "./@(codemods|packages|eslint)/*/src/**/*.js";
 
@@ -75,42 +80,203 @@ function buildBabel(exclude, sourcesGlob = defaultSourcesGlob) {
     .pipe(gulp.dest(base));
 }
 
+let babelVersion = require("./packages/babel-core/package.json").version;
 function buildRollup(packages) {
+  const sourcemap = process.env.NODE_ENV === "production";
+  const minify = !!process.env.IS_PUBLISH;
   return Promise.all(
-    packages.map(pkg => {
+    packages.map(
-      const input = getIndexFromPackage(pkg);
+      ({ src, format, dest, name, filename, version = babelVersion }) => {
-      fancyLog(`Compiling '${chalk.cyan(input)}' with rollup ...`);
+        const extraPlugins = [];
-      return rollup
+        let inputExternal = undefined,
-        .rollup({
+          outputGlobals = undefined,
-          input,
+          nodeResolveBrowser = false,
-          plugins: [
+          babelEnvName = "rollup";
-            rollupReplace({
+        switch (src) {
-              "process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV),
+          case "packages/babel-standalone":
-            }),
+            nodeResolveBrowser = true;
-            rollupBabel({
+            babelEnvName = "standalone";
-              envName: "babel-parser",
+            if (minify) {
-            }),
+              extraPlugins.push(
-            rollupNodeResolve(),
+                rollupTerser({
-          ],
+                  include: /^.+\.min\.js$/,
-        })
+                })
-        .then(bundle => {
+              );
-          return bundle.write({
+            }
-            file: path.join(pkg, "lib/index.js"),
+            break;
-            format: "cjs",
+          case "packages/babel-preset-env-standalone":
-            name: "babel-parser",
+            nodeResolveBrowser = true;
-            sourcemap: process.env.NODE_ENV !== "production",
+            babelEnvName = "standalone";
+            if (minify) {
+              extraPlugins.push(
+                rollupTerser({
+                  include: /^.+\.min\.js$/,
+                })
+              );
+            }
+            inputExternal = ["@babel/standalone"];
+            outputGlobals = {
+              "@babel/standalone": "Babel",
+            };
+            extraPlugins.push(
+              rollupAlias({
+                entries: [
+                  {
+                    find: "./available-plugins",
+                    replacement: require.resolve(
+                      path.join(__dirname, src, "./src/available-plugins")
+                    ),
+                  },
+                  {
+                    find: "caniuse-lite/data/regions",
+                    replacement: require.resolve(
+                      path.join(__dirname, src, "./src/caniuse-lite-regions")
+                    ),
+                  },
+                ],
+              })
+            );
+            break;
+        }
+        // If this build is part of a pull request, include the pull request number in
+        // the version number.
+        if (process.env.CIRCLE_PR_NUMBER) {
+          const prVersion = "+pr." + process.env.CIRCLE_PR_NUMBER;
+          babelVersion += prVersion;
+          version += prVersion;
+        }
+        const input = getIndexFromPackage(src);
+        fancyLog(`Compiling '${chalk.cyan(input)}' with rollup ...`);
+        return rollup
+          .rollup({
+            input,
+            external: inputExternal,
+            plugins: [
+              ...extraPlugins,
+              rollupBabelSource(),
+              rollupReplace({
+                "process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV),
+                BABEL_VERSION: JSON.stringify(babelVersion),
+                VERSION: JSON.stringify(version),
+              }),
+              rollupBabel({
+                envName: babelEnvName,
+                babelrc: false,
+                extends: "./babel.config.js",
+              }),
+              rollupNodeResolve({
+                browser: nodeResolveBrowser,
+                preferBuiltins: true,
+                //todo: When Yarn workspaces is enabled, remove `dedupe` option
+                dedupe(importee) {
+                  return (
+                    importee.startsWith("lodash/") ||
+                    [
+                      "babel-plugin-dynamic-import-node/utils",
+                      "esutils",
+                      "semver",
+                      "source-map",
+                    ].includes(importee)
+                  );
+                },
+              }),
+              rollupCommonJs({
+                include: [/node_modules/, "packages/babel-preset-env/data/**"],
+                namedExports: {
+                  "babel-plugin-dynamic-import-node/utils.js": [
+                    "createDynamicImportTransform",
+                    "getImportSource",
+                  ],
+                  "@babel/standalone": ["availablePlugins", "registerPlugin"],
+                },
+              }),
+              rollupJson(),
+              rollupNodeBuiltins(),
+              rollupNodeGlobals({ sourceMap: sourcemap }),
+            ],
+          })
+          .then(bundle => {
+            const outputFile = path.resolve(src, dest, filename || "index.js");
+            return bundle
+              .write({
+                file: outputFile,
+                format,
+                name,
+                globals: outputGlobals,
+                sourcemap: sourcemap,
+              })
+              .then(() => {
+                if (!process.env.IS_PUBLISH) {
+                  fancyLog(
+                    chalk.yellow(
+                      `Skipped minification of '${chalk.cyan(
+                        path.relative(path.join(__dirname, ".."), outputFile)
+                      )}' because not publishing`
+                    )
+                  );
+                  return undefined;
+                }
+                fancyLog(
+                  `Minifying '${chalk.cyan(
+                    path.relative(path.join(__dirname, ".."), outputFile)
+                  )}'...`
+                );
+
+                return bundle.write({
+                  file: outputFile.replace(/\.js$/, ".min.js"),
+                  format,
+                  name,
+                  globals: outputGlobals,
+                  sourcemap: sourcemap,
+                });
+              });
           });
-        });
+      }
-    })
+    )
   );
 }
 
-const bundles = ["packages/babel-parser"];
+const libBundles = [
+  {
+    src: "packages/babel-parser",
+    format: "cjs",
+    dest: "lib",
+    version: require("./packages/babel-parser/package").version,
+  },
+];
 
-gulp.task("build-rollup", () => buildRollup(bundles));
+const standaloneBundle = [
-gulp.task("build-babel", () => buildBabel(/* exclude */ bundles));
+  {
+    src: "packages/babel-standalone",
+    format: "umd",
+    name: "Babel",
+    filename: "babel.js",
+    dest: "",
+    version: require("./packages/babel-core/package").version,
+  },
+];
+
+const presetEnvStandaloneBundle = [
+  {
+    src: "packages/babel-preset-env-standalone",
+    format: "umd",
+    name: "BabelPresetEnv",
+    filename: "babel-preset-env.js",
+    dest: "",
+    version: require("./packages/babel-preset-env/package").version,
+  },
+];
+
+gulp.task("build-rollup", () => buildRollup(libBundles));
+gulp.task("build-babel-standalone", () => buildRollup(standaloneBundle));
+
+gulp.task("build-babel-preset-env-standalone", () =>
+  buildRollup(presetEnvStandaloneBundle)
+);
+
+gulp.task("build-babel", () => buildBabel(/* exclude */ libBundles));
 gulp.task("build-babel-types", () =>
-  buildBabel(/* exclude */ bundles, "packages/babel-types/src/**/*.js")
+  buildBabel(/* exclude */ libBundles, "packages/babel-types/src/**/*.js")
 );
 gulp.task("build", gulp.parallel("build-rollup", "build-babel"));
 
@@ -128,41 +294,3 @@ gulp.task(
     );
   })
 );
-
-registerStandalonePackageTask(
-  gulp,
-  "babel",
-  "Babel",
-  path.join(__dirname, "packages"),
-  require("./packages/babel-standalone/package.json").version
-);
-
-const presetEnvWebpackPlugins = [
-  new webpack.NormalModuleReplacementPlugin(
-    /\.\/available-plugins/,
-    require.resolve(
-      path.join(
-        __dirname,
-        "./packages/babel-preset-env-standalone/src/available-plugins"
-      )
-    )
-  ),
-  new webpack.NormalModuleReplacementPlugin(
-    /caniuse-lite\/data\/regions\/.+/,
-    require.resolve(
-      path.join(
-        __dirname,
-        "./packages/babel-preset-env-standalone/src/caniuse-lite-regions"
-      )
-    )
-  ),
-];
-
-registerStandalonePackageTask(
-  gulp,
-  "babel-preset-env",
-  "babelPresetEnv",
-  path.join(__dirname, "packages"),
-  require("./packages/babel-preset-env-standalone/package.json").version,
-  presetEnvWebpackPlugins
-);

babel.config.js

@@ -11,18 +11,33 @@ module.exports = function(api) {
     exclude: ["transform-typeof-symbol"],
   };
   const envOpts = Object.assign({}, envOptsNoTargets);
-  let transformRuntimeOpts = null;
 
   let convertESM = true;
   let ignoreLib = true;
-  let includeRuntime = false;
   const nodeVersion = "6.9";
+  // The vast majority of our src files are modules, but we use
+  // unambiguous to keep things simple until we get around to renaming
+  // the modules to be more easily distinguished from CommonJS
+  const unambiguousSources = [
+    "packages/*/src",
+    "packages/*/test",
+    "codemods/*/src",
+    "codemods/*/test",
+    "eslint/*/src",
+    "eslint/*/test",
+  ];
 
   switch (env) {
     // Configs used during bundling builds.
-    case "babel-parser":
+    case "rollup":
       convertESM = false;
       ignoreLib = false;
+      // rollup-commonjs will converts node_modules to ESM
+      unambiguousSources.push(
+        "**/node_modules",
+        // todo: remove this after it is rewritten into ESM
+        "packages/babel-preset-env/data"
+      );
       envOpts.targets = {
         node: nodeVersion,
       };
@@ -30,7 +45,11 @@ module.exports = function(api) {
     case "standalone":
       convertESM = false;
       ignoreLib = false;
-      includeRuntime = true;
+      unambiguousSources.push(
+        "**/node_modules",
+        "packages/babel-preset-env/data"
+      );
+      // targets to browserslists: defaults
       break;
     case "production":
       // Config during builds before publish.
@@ -51,17 +70,6 @@ module.exports = function(api) {
       break;
   }
 
-  if (includeRuntime) {
-    const babelRuntimePackageJSONPath = require.resolve(
-      "@babel/runtime/package.json"
-    );
-    const path = require("path");
-    transformRuntimeOpts = {
-      version: require(babelRuntimePackageJSONPath).version,
-      absoluteRuntime: path.dirname(babelRuntimePackageJSONPath),
-    };
-  }
-
   const config = {
     // Our dependencies are all standard CommonJS, along with all sorts of
     // other random files in Babel's codebase, so we use script as the default,
@@ -116,31 +124,9 @@ module.exports = function(api) {
         presets: [["@babel/env", envOptsNoTargets]],
       },
       {
-        // The vast majority of our src files are modules, but we use
+        test: unambiguousSources,
-        // unambiguous to keep things simple until we get around to renaming
-        // the modules to be more easily distinguished from CommonJS
-        test: [
-          "packages/*/src",
-          "packages/*/test",
-          "codemods/*/src",
-          "codemods/*/test",
-          "eslint/*/src",
-          "eslint/*/test",
-        ],
         sourceType: "unambiguous",
       },
-      {
-        // The runtime transform shouldn't process its own runtime or core-js.
-        exclude: [
-          "packages/babel-runtime",
-          /[\\/]node_modules[\\/](?:@babel\/runtime|babel-runtime|core-js)[\\/]/,
-        ],
-        plugins: [
-          includeRuntime
-            ? ["@babel/transform-runtime", transformRuntimeOpts]
-            : null,
-        ].filter(Boolean),
-      },
     ].filter(Boolean),
   };
 

package.json

@@ -27,16 +27,16 @@
     "@babel/preset-flow": "^7.0.0",
     "@babel/register": "^7.7.0",
     "@babel/runtime": "^7.7.2",
+    "@rollup/plugin-alias": "^2.2.0",
+    "@rollup/plugin-json": "^4.0.0",
     "babel-eslint": "^11.0.0-beta.1",
     "babel-jest": "^24.9.0",
-    "babel-loader": "^8.0.6",
     "babel-plugin-transform-charcodes": "^0.2.0",
     "browserify": "^16.2.3",
     "bundle-collapser": "^1.2.1",
     "chalk": "^2.4.2",
     "charcodes": "^0.2.0",
     "derequire": "^2.0.2",
-    "duplicate-package-checker-webpack-plugin": "^2.1.0",
     "enhanced-resolve": "^3.0.0",
     "eslint": "^6.0.1",
     "eslint-config-babel": "^9.0.0",
@@ -52,7 +52,6 @@
     "gulp-newer": "^1.0.0",
     "gulp-plumber": "^1.2.1",
     "gulp-rename": "^1.4.0",
-    "gulp-uglify": "^3.0.2",
     "gulp-watch": "^5.0.1",
     "husky": "^3.0.0",
     "jest": "^24.9.0",
@@ -65,17 +64,17 @@
     "prettier": "^1.19.1",
     "pump": "^3.0.0",
     "rimraf": "^2.6.3",
-    "rollup": "^1.12.0",
+    "rollup": "^1.27.5",
     "rollup-plugin-babel": "^4.0.0",
+    "rollup-plugin-commonjs": "^10.1.0",
+    "rollup-plugin-node-builtins": "^2.1.2",
+    "rollup-plugin-node-globals": "^1.4.0",
     "rollup-plugin-node-resolve": "^5.0.0",
     "rollup-plugin-replace": "^2.2.0",
+    "rollup-plugin-terser": "^5.1.2",
     "test262-stream": "^1.3.0",
     "through2": "^2.0.0",
-    "typescript": "^3.6.3",
+    "typescript": "^3.6.3"
-    "warnings-to-errors-webpack-plugin": "^2.0.0",
-    "webpack": "^3.4.1",
-    "webpack-dependency-suite": "^2.4.4",
-    "webpack-stream": "^4.0.0"
   },
   "resolutions": {
     "@lerna/**/@lerna/collect-updates": "https://github.com/babel/lerna.git#babel-collect-updates"

packages/babel-preset-env-standalone/src/available-plugins.js

@@ -1,12 +1,18 @@
 import { availablePlugins, registerPlugin } from "@babel/standalone";
+import proposalJsonStrings from "@babel/plugin-proposal-json-strings";
+import proposalDynamicImport from "@babel/plugin-proposal-dynamic-import";
+import syntaxJsonStrings from "@babel/plugin-syntax-json-strings";
+import syntaxTopLevelAwait from "@babel/plugin-syntax-top-level-await";
+import transformNamedCapturingGroupsRegex from "@babel/plugin-transform-named-capturing-groups-regex";
+import transformNewTarget from "@babel/plugin-transform-new-target";
 
 const notIncludedPlugins = {
-  "transform-named-capturing-groups-regex": require("@babel/plugin-transform-named-capturing-groups-regex"),
+  "proposal-dynamic-import": proposalDynamicImport,
-  "transform-new-target": require("@babel/plugin-transform-new-target"),
+  "proposal-json-strings": proposalJsonStrings,
-  "proposal-json-strings": require("@babel/plugin-proposal-json-strings"),
+  "syntax-json-strings": syntaxJsonStrings,
-  "proposal-dynamic-import": require("@babel/plugin-proposal-dynamic-import"),
+  "syntax-top-level-await": syntaxTopLevelAwait,
-  "syntax-json-strings": require("@babel/plugin-syntax-json-strings"),
+  "transform-named-capturing-groups-regex": transformNamedCapturingGroupsRegex,
-  "syntax-top-level-await": require("@babel/plugin-syntax-top-level-await"),
+  "transform-new-target": transformNewTarget,
 };
 
 Object.keys(notIncludedPlugins).forEach(pluginName => {

scripts/gulp-tasks.js

@@ -1,185 +0,0 @@
-"use strict";
-
-/**
- * This file contains the Gulp tasks for babel-standalone. Note that
- * babel-standalone is compiled using Webpack, and performs its own Babel
- * compilation of all the JavaScript files. This is because it targets web
- * browsers, so more transforms are needed than the regular Babel builds that
- * only target Node.js.
- *
- * The tasks in this file are designed to be reusable, so that they can be used
- * to make standalone builds of other Babel plugins/presets (such as babel-minify)
- */
-
-const fs = require("fs");
-const path = require("path");
-const pump = require("pump");
-const chalk = require("chalk");
-const through = require("through2");
-const fancyLog = require("fancy-log");
-const rename = require("gulp-rename");
-const webpack = require("webpack");
-const { RootMostResolvePlugin } = require("webpack-dependency-suite");
-const DuplicatePackageCheckerPlugin = require("duplicate-package-checker-webpack-plugin");
-const WarningsToErrorsPlugin = require("warnings-to-errors-webpack-plugin");
-const webpackStream = require("webpack-stream");
-const uglify = require("gulp-uglify");
-
-function generateResolveAlias() {
-  const alias = {};
-  const packagePath = path.resolve(process.cwd(), "packages");
-  fs.readdirSync(packagePath).forEach(folder => {
-    alias[folder.replace("babel-", "@babel/") + "$"] = path.resolve(
-      packagePath,
-      folder,
-      "src"
-    );
-  });
-  return alias;
-}
-
-function webpackBuild(opts) {
-  const plugins = opts.plugins || [];
-  let babelVersion = require("../packages/babel-core/package.json").version;
-  let version = opts.version || babelVersion;
-  // If this build is part of a pull request, include the pull request number in
-  // the version number.
-  if (process.env.CIRCLE_PR_NUMBER) {
-    const prVersion = "+pr." + process.env.CIRCLE_PR_NUMBER;
-    babelVersion += prVersion;
-    version += prVersion;
-  }
-
-  const config = {
-    module: {
-      rules: [
-        {
-          test: /\.js$/,
-          loader: "babel-loader",
-          options: {
-            // Use the bundled config so that module syntax is passed through
-            // for Webpack.
-            envName: "standalone",
-          },
-        },
-      ],
-    },
-    node: {
-      // Mock Node.js modules that Babel require()s but that we don't
-      // particularly care about.
-      fs: "empty",
-      module: "empty",
-      net: "empty",
-    },
-    output: {
-      filename: opts.filename,
-      library: opts.library,
-      libraryTarget: "umd",
-    },
-    plugins: [
-      new WarningsToErrorsPlugin(),
-      new DuplicatePackageCheckerPlugin({
-        exclude(instance) {
-          return instance.name === "semver";
-        },
-      }),
-      new webpack.DefinePlugin({
-        "process.env.NODE_ENV": '"production"',
-        BABEL_VERSION: JSON.stringify(babelVersion),
-        VERSION: JSON.stringify(version),
-      }),
-      /*new webpack.NormalModuleReplacementPlugin(
-        /..\/..\/package/,
-        "../../../../src/babel-package-shim"
-      ),*/
-      new webpack.optimize.ModuleConcatenationPlugin(),
-    ].concat(plugins),
-    resolve: {
-      //todo: remove resolve.alias when babel packages offer ESModule entry
-      alias: generateResolveAlias(),
-      plugins: [
-        // Dedupe packages that are used across multiple plugins.
-        // This replaces DedupePlugin from Webpack 1.x
-        new RootMostResolvePlugin(__dirname, true),
-      ],
-    },
-  };
-
-  if (opts.library !== "Babel") {
-    config.externals = {
-      "@babel/standalone": "Babel",
-    };
-  }
-
-  return webpackStream(config, webpack);
-  // To write JSON for debugging:
-  /*return webpackStream(config, webpack, (err, stats) => {
-    require("fancy-log")(stats.toString({ colors: true }));
-    require("fs").writeFileSync(
-      "webpack-debug.json",
-      JSON.stringify(stats.toJson())
-    );
-  });*/
-}
-
-function logUglify() {
-  return through.obj(function(file, enc, callback) {
-    fancyLog(
-      `Minifying '${chalk.cyan(
-        path.relative(path.join(__dirname, ".."), file.path)
-      )}'...`
-    );
-    callback(null, file);
-  });
-}
-
-function logNoUglify() {
-  return through.obj(function(file, enc, callback) {
-    fancyLog(
-      chalk.yellow(
-        `Skipped minification of '${chalk.cyan(
-          path.relative(path.join(__dirname, ".."), file.path)
-        )}' because not publishing`
-      )
-    );
-    callback(null, file);
-  });
-}
-
-function registerStandalonePackageTask(
-  gulp,
-  name,
-  exportName,
-  pathname,
-  version,
-  plugins
-) {
-  const standaloneName = name + "-standalone";
-  const standalonePath = path.join(pathname, standaloneName);
-  gulp.task("build-" + standaloneName, cb => {
-    pump(
-      [
-        gulp.src(path.join(standalonePath, "src/index.js")),
-        webpackBuild({
-          filename: name + ".js",
-          library: exportName,
-          version,
-          plugins,
-        }),
-        gulp.dest(standalonePath),
-      ].concat(
-        // Minification is super slow, so we skip it unless we are publishing
-        process.env.IS_PUBLISH ? logUglify() : logNoUglify(),
-        process.env.IS_PUBLISH ? uglify() : [],
-        rename({ extname: ".min.js" }),
-        gulp.dest(standalonePath)
-      ),
-      cb
-    );
-  });
-}
-
-module.exports = {
-  webpackBuild: webpackBuild,
-  registerStandalonePackageTask: registerStandalonePackageTask,
-};

scripts/rollup-plugin-babel-source.js

@@ -0,0 +1,66 @@
+const path = require("path");
+const fs = require("fs");
+const dirname = path.join(__dirname, "..");
+
+module.exports = function() {
+  return {
+    name: "babel-source",
+    load(id) {
+      const matches = id.match(/packages\/(babel-[^/]+)\/src\//);
+      if (matches) {
+        // check if browser field exists for this file and replace
+        const packageFolder = path.join(dirname, "packages", matches[1]);
+        const packageJson = require(path.join(packageFolder, "package.json"));
+
+        if (
+          packageJson["browser"] &&
+          typeof packageJson["browser"] === "object"
+        ) {
+          for (let nodeFile in packageJson["browser"]) {
+            const browserFile = packageJson["browser"][nodeFile].replace(
+              /^(\.\/)?lib\//,
+              "src/"
+            );
+            nodeFile = nodeFile.replace(/^(\.\/)?lib\//, "src/");
+            if (id.endsWith(nodeFile)) {
+              if (browserFile === false) {
+                return "";
+              }
+              return fs.readFileSync(
+                path.join(packageFolder, browserFile),
+                "UTF-8"
+              );
+            }
+          }
+        }
+      }
+      return null;
+    },
+    resolveId(importee) {
+      let packageFolderName;
+      const matches = importee.match(/^@babel\/([^/]+)$/);
+      if (matches) {
+        packageFolderName = `babel-${matches[1]}`;
+      }
+
+      if (packageFolderName) {
+        // resolve babel package names to their src index file
+        const packageFolder = path.join(dirname, "packages", packageFolderName);
+        const packageJson = require(path.join(packageFolder, "package.json"));
+
+        const filename =
+          typeof packageJson["browser"] === "string"
+            ? packageJson["browser"]
+            : packageJson["main"];
+
+        return path.join(
+          packageFolder,
+          // replace lib with src in the pkg.json entry
+          filename.replace(/^(\.\/)?lib\//, "src/")
+        );
+      }
+
+      return null;
+    },
+  };
+};