Merge pull request #5608 from loganfsmyth/config-cache

Cache configs based on mtime and allow .babelrc.js functions

packages/babel-core/src/config/caching.js

@@ -0,0 +1,187 @@
+// @flow
+
+type CacheConfigurator = CacheConfiguratorFn & CacheConfiguratorObj;
+
+type CacheConfiguratorFn = {
+  (boolean): void,
+  <T>(handler: () => T): T,
+};
+type CacheConfiguratorObj = {
+  forever: () => void,
+  never: () => void,
+  using: <T>(handler: () => T) => T,
+  invalidate: <T>(handler: () => T) => T,
+};
+
+type CacheEntry<ResultT> = Array<[ ResultT, () => boolean ]>;
+
+/**
+ * Given a function with a single argument, cache its results based on its argument and how it
+ * configures its caching behavior. Cached values are stored strongly.
+ */
+export function makeStrongCache<ArgT, ResultT>(
+  handler: (ArgT, CacheConfigurator) => ResultT,
+  autoPermacache?: boolean,
+): (ArgT) => ResultT {
+  return makeCachedFunction(new Map(), handler, autoPermacache);
+}
+
+/**
+ * Given a function with a single argument, cache its results based on its argument and how it
+ * configures its caching behavior. Cached values are stored weakly and the function argument must be
+ * an object type.
+ */
+export function makeWeakCache<ArgT: Object, ResultT>(
+  handler: (ArgT, CacheConfigurator) => ResultT,
+  autoPermacache?: boolean,
+): (ArgT) => ResultT {
+  return makeCachedFunction(new WeakMap(), handler, autoPermacache);
+}
+
+type CacheMap<ArgT, ResultT> = Map<ArgT, CacheEntry<ResultT>>|WeakMap<ArgT, CacheEntry<ResultT>>;
+
+function makeCachedFunction<ArgT, ResultT, Cache: CacheMap<ArgT, ResultT>>(
+  callCache: Cache,
+  handler: (ArgT, CacheConfigurator) => ResultT,
+  autoPermacache: boolean = true,
+): (ArgT) => ResultT {
+  return function cachedFunction(arg) {
+    let cachedValue: CacheEntry<ResultT>|void = callCache.get(arg);
+
+    if (cachedValue) {
+      for (const [ value, valid ] of cachedValue) {
+        if (valid()) return value;
+      }
+    }
+
+    const { cache, result, deactivate } = makeCacheConfig();
+
+    const value = handler(arg, cache);
+
+    if (autoPermacache && !result.configured) cache.forever();
+
+    deactivate();
+
+    if (!result.configured) {
+      // eslint-disable-next-line max-len
+      throw new Error([
+        "Caching was left unconfigured. Babel's plugins, presets, and .babelrc.js files can be configured",
+        "for various types of caching, using the first param of their handler functions:",
+        "",
+        "module.exports = function(api) {",
+        "  // The API exposes the following:",
+        "",
+        "  // Cache the returned value forever and don't call this function again.",
+        "  api.cache(true);",
+        "",
+        "  // Don't cache at all. Not recommended because it will be very slow.",
+        "  api.cache(false);",
+        "",
+        "  // Cached based on the value of some function. If this function returns a value different from",
+        "  // a previously-encountered value, the plugins will re-evaluate.",
+        "  var env = api.cache(() => process.env.NODE_ENV);",
+        "",
+        "  // If testing for a specific env, we recommend specifics to avoid instantiating a plugin for",
+        "  // any possible NODE_ENV value that might come up during plugin execution.",
+        "  var isProd = api.cache(() => process.env.NODE_ENV === \"production\");",
+        "",
+        "  // .cache(fn) will perform a linear search though instances to find the matching plugin based",
+        "  // based on previous instantiated plugins. If you want to recreate the plugin and discard the",
+        "  // previous instance whenever something changes, you may use:",
+        "  var isProd = api.cache.invalidate(() => process.env.NODE_ENV === \"production\");",
+        "",
+        "  // Note, we also expose the following more-verbose versions of the above examples:",
+        "  api.cache.forever(); // api.cache(true)",
+        "  api.cache.never();   // api.cache(false)",
+        "  api.cache.using(fn); // api.cache(fn)",
+        "",
+        "  // Return the value that will be cached.",
+        "  return { };",
+        "};",
+      ].join("\n"));
+    }
+
+    if (!result.never) {
+      if (result.forever) {
+        cachedValue = [
+          [value, () => true],
+        ];
+      } else if (result.invalidate) {
+        cachedValue = [
+          [value, result.valid],
+        ];
+      } else {
+        cachedValue = cachedValue || [];
+        cachedValue.push([ value, result.valid ]);
+      }
+      callCache.set(arg, cachedValue);
+    }
+
+    return value;
+  };
+}
+
+function makeCacheConfig(): { cache: CacheConfigurator, result: *, deactivate: () => void } {
+  const pairs = [];
+
+  const result = {
+    configured: false,
+    never: false,
+    forever: false,
+    invalidate: false,
+    valid: () => pairs.every(([key, fn]) => key === fn()),
+  };
+
+  let active = true;
+  const deactivate = () => {
+    active = false;
+  };
+
+  const cache: CacheConfigurator = Object.assign((function cacheFn(val) {
+    if (typeof val === "boolean") {
+      if (val) cache.forever();
+      else cache.never();
+      return;
+    }
+
+    return cache.using(val);
+  }: any), ({
+    forever() {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
+      if (result.never) throw new Error("Caching has already been configured with .never()");
+      result.forever = true;
+      result.configured = true;
+    },
+    never() {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
+      if (result.forever) throw new Error("Caching has already been configured with .forever()");
+      result.never = true;
+      result.configured = true;
+    },
+    using<T>(handler: () => T): T {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
+      if (result.never || result.forever) {
+        throw new Error("Caching has already been configured with .never or .forever()");
+      }
+      result.configured = true;
+
+      const key = handler();
+      pairs.push([ key, handler ]);
+      return key;
+    },
+    invalidate<T>(handler: () => T): T {
+      if (!active) throw new Error("Cannot change caching after evaluation has completed.");
+      if (result.never || result.forever) {
+        throw new Error("Caching has already been configured with .never or .forever()");
+      }
+      result.invalidate = true;
+      result.configured = true;
+
+      const key = handler();
+      pairs.push([ key, handler ]);
+      return key;
+    },
+  }: CacheConfiguratorObj));
+
+  return { cache, result, deactivate };
+}

packages/babel-core/src/config/loading/files/configuration.js

@@ -4,6 +4,8 @@ import path from "path";
 import fs from "fs";
 import json5 from "json5";
 import resolve from "resolve";
+import { getEnv } from "../../helpers/environment";
+import { makeStrongCache } from "../../caching";
 
 type ConfigFile = {
   filepath: string,
@@ -11,23 +13,11 @@ type ConfigFile = {
   options: Object,
 };
 
-const existsCache = {};
-const jsonCache = {};
-
 const BABELRC_FILENAME = ".babelrc";
 const BABELRC_JS_FILENAME = ".babelrc.js";
 const PACKAGE_FILENAME = "package.json";
 const BABELIGNORE_FILENAME = ".babelignore";
 
-function exists(filename) {
-  const cached = existsCache[filename];
-  if (cached == null) {
-    return existsCache[filename] = fs.existsSync(filename);
-  } else {
-    return cached;
-  }
-}
-
 export function findConfigs(dirname: string): Array<ConfigFile> {
   let foundConfig = false;
   let foundIgnore = false;
@@ -96,25 +86,11 @@ function readConfig(filepath) {
   return (path.extname(filepath) === ".js") ? readConfigJS(filepath) : readConfigFile(filepath);
 }
 
-function readIgnoreConfig(filepath) {
-  if (!exists(filepath)) return null;
-
-  const file = fs.readFileSync(filepath, "utf8");
-  let lines = file.split("\n");
-
-  lines = lines
-    .map((line) => line.replace(/#(.*?)$/, "").trim())
-    .filter((line) => !!line);
-
-  return {
-    filepath,
-    dirname: path.dirname(filepath),
-    options: { ignore: lines },
-  };
-}
-
-function readConfigJS(filepath) {
-  if (!exists(filepath)) return null;
+const readConfigJS = makeStrongCache((filepath, cache) => {
+  if (!fs.existsSync(filepath)) {
+    cache.forever();
+    return null;
+  }
 
   let options;
   try {
@@ -126,6 +102,16 @@ function readConfigJS(filepath) {
     throw err;
   }
 
+  if (typeof options === "function") {
+    options = options({
+      cache,
+      // Expose ".env()" so people can easily get the same env that we expose using the "env" key.
+      env: () => cache.using(() => getEnv()),
+    });
+  } else {
+    cache.forever();
+  }
+
   if (!options || typeof options !== "object" || Array.isArray(options)) {
     throw new Error(`${filepath}: Configuration should be an exported JavaScript object.`);
   }
@@ -135,19 +121,13 @@ function readConfigJS(filepath) {
     dirname: path.dirname(filepath),
     options,
   };
-}
-
-function readConfigFile(filepath) {
-  if (!exists(filepath)) return null;
-
-  const content = fs.readFileSync(filepath, "utf8");
+}, false /* autoPermacache */);
 
+const readConfigFile = makeStaticFileCache((filepath, content) => {
   let options;
   if (path.basename(filepath) === PACKAGE_FILENAME) {
     try {
-      const json = jsonCache[content] = jsonCache[content] || JSON.parse(content);
-
-      options = json.babel;
+      options = JSON.parse(content).babel;
     } catch (err) {
       err.message = `${filepath}: Error while parsing JSON - ${err.message}`;
       throw err;
@@ -155,7 +135,7 @@ function readConfigFile(filepath) {
     if (!options) return null;
   } else {
     try {
-      options = jsonCache[content] = jsonCache[content] || json5.parse(content);
+      options = json5.parse(content);
     } catch (err) {
       err.message = `${filepath}: Error while parsing config - ${err.message}`;
       throw err;
@@ -172,4 +152,38 @@ function readConfigFile(filepath) {
     dirname: path.dirname(filepath),
     options,
   };
+});
+
+const readIgnoreConfig = makeStaticFileCache((filepath, content) => {
+  const ignore = content
+    .split("\n")
+    .map((line) => line.replace(/#(.*?)$/, "").trim())
+    .filter((line) => !!line);
+
+  return {
+    filepath,
+    dirname: path.dirname(filepath),
+    options: { ignore },
+  };
+});
+
+function makeStaticFileCache<T>(fn: (string, string) => T): (string) => T|null {
+  return makeStrongCache((filepath, cache) => {
+    if (cache.invalidate(() => fileMtime(filepath)) === null) {
+      cache.forever();
+      return null;
+    }
+
+    return fn(filepath, fs.readFileSync(filepath, "utf8"));
+  });
+}
+
+function fileMtime(filepath: string): number|null {
+  try {
+    return +fs.statSync(filepath).mtime;
+  } catch (e) {
+    if (e.code !== "ENOENT") throw e;
+  }
+
+  return null;
 }

packages/babel-core/test/caching-api.js

@@ -0,0 +1,356 @@
+import assert from "assert";
+import { makeStrongCache } from "../lib/config/caching";
+
+describe("caching API", () => {
+  it("should allow permacaching with .forever()", () => {
+    let count = 0;
+
+    const fn = makeStrongCache((arg, cache) => {
+      cache.forever();
+      return { arg, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    assert.notEqual(fn("one"), fn("two"));
+  });
+
+  it("should allow permacaching with cache(true)", () => {
+    let count = 0;
+
+    const fn = makeStrongCache((arg, cache) => {
+      cache(true);
+      return { arg, count: count++ };
+    });
+
+
+    assert.deepEqual(fn("one"), { arg: "one", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    assert.notEqual(fn("one"), fn("two"));
+  });
+
+  it("should allow disabling caching with .never()", () => {
+    let count = 0;
+
+    const fn = makeStrongCache((arg, cache) => {
+      cache.never();
+      return { arg, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", count: 0 });
+    assert.deepEqual(fn("one"), { arg: "one", count: 1 });
+    assert.notEqual(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", count: 4 });
+    assert.deepEqual(fn("two"), { arg: "two", count: 5 });
+    assert.notEqual(fn("two"), fn("two"));
+
+    assert.notEqual(fn("one"), fn("two"));
+  });
+
+  it("should allow disabling caching with cache(false)", () => {
+    let count = 0;
+
+    const fn = makeStrongCache((arg, cache) => {
+      cache(false);
+      return { arg, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", count: 0 });
+    assert.deepEqual(fn("one"), { arg: "one", count: 1 });
+    assert.notEqual(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", count: 4 });
+    assert.deepEqual(fn("two"), { arg: "two", count: 5 });
+    assert.notEqual(fn("two"), fn("two"));
+
+    assert.notEqual(fn("one"), fn("two"));
+  });
+
+  it("should allow caching based on a value with .using(fn)", () => {
+    let count = 0;
+    let other = "default";
+
+    const fn = makeStrongCache((arg, cache) => {
+      const val = cache.using(() => other);
+
+      return { arg, val, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "default";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+  });
+
+  it("should allow caching based on a value with cache(fn)", () => {
+    let count = 0;
+    let other = "default";
+
+    const fn = makeStrongCache((arg, cache) => {
+      const val = cache(() => other);
+
+      return { arg, val, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "default";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+  });
+
+  it("should allow invalidation based on a value with .invalidate(fn)", () => {
+    let count = 0;
+    let other = "default";
+
+    const fn = makeStrongCache((arg, cache) => {
+      const val = cache.invalidate(() => other);
+
+      return { arg, val, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "default";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", count: 4 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", count: 5 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", count: 6 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", count: 7 });
+    assert.equal(fn("two"), fn("two"));
+  });
+
+  it("should allow invalidation with .using and .invalidate", () => {
+    let count = 0;
+    let other = "default";
+    let another = "another";
+
+    const fn = makeStrongCache((arg, cache) => {
+      const val = cache.using(() => other);
+      const val2 = cache.invalidate(() => another);
+
+      return { arg, val, val2, count: count++ };
+    });
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", val2: "another", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", val2: "another", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", val2: "another", count: 2 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", val2: "another", count: 3 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "default";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "default", val2: "another", count: 4 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "default", val2: "another", count: 5 });
+    assert.equal(fn("two"), fn("two"));
+
+    other = "new";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", val2: "another", count: 6 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", val2: "another", count: 7 });
+    assert.equal(fn("two"), fn("two"));
+
+    another = "second";
+
+    assert.deepEqual(fn("one"), { arg: "one", val: "new", val2: "second", count: 8 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", val: "new", val2: "second", count: 9 });
+    assert.equal(fn("two"), fn("two"));
+
+  });
+
+  it("should throw if caching is never configured and not defaulting", () => {
+    const fn = makeStrongCache(() => { }, false /* autoPermacache */);
+
+    assert.throws(() => fn(), /Error: Caching was left unconfigured./);
+  });
+
+  it("should auto-permacache by default", () => {
+    let count = 0;
+
+    const fn = makeStrongCache((arg) => ({ arg, count: count++ }));
+
+    assert.deepEqual(fn("one"), { arg: "one", count: 0 });
+    assert.equal(fn("one"), fn("one"));
+
+    assert.deepEqual(fn("two"), { arg: "two", count: 1 });
+    assert.equal(fn("two"), fn("two"));
+
+    assert.notEqual(fn("one"), fn("two"));
+  });
+
+  it("should throw if you set permacaching and use .using", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.forever();
+
+      cache.using(() => null);
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you set permacaching and use .invalidate", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.forever();
+
+      cache.invalidate(() => null);
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you set permacaching and use .never", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.forever();
+
+      cache.never();
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you set no caching and use .using", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.never();
+
+      cache.using(() => null);
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you set no caching and use .invalidate", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.never();
+
+      cache.invalidate(() => null);
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you set no caching and use .never", () => {
+    const fn = makeStrongCache((arg, cache) => {
+      cache.never();
+
+      cache.using(() => null);
+    });
+
+    assert.throws(() => fn(), /Caching has already been configured/);
+  });
+
+  it("should throw if you configure .forever after exiting", () => {
+    const fn = makeStrongCache((arg, cache) => cache);
+
+    assert.throws(() => fn().forever(), /Cannot change caching after evaluation/);
+  });
+
+  it("should throw if you configure .never after exiting", () => {
+    const fn = makeStrongCache((arg, cache) => cache);
+
+    assert.throws(() => fn().never(), /Cannot change caching after evaluation/);
+  });
+
+  it("should throw if you configure .using after exiting", () => {
+    const fn = makeStrongCache((arg, cache) => cache);
+
+    assert.throws(() => fn().using(() => null), /Cannot change caching after evaluation/);
+  });
+
+  it("should throw if you configure .invalidate after exiting", () => {
+    const fn = makeStrongCache((arg, cache) => cache);
+
+    assert.throws(() => fn().invalidate(() => null), /Cannot change caching after evaluation/);
+  });
+});

packages/babel-core/test/config-chain.js

@@ -418,6 +418,46 @@ describe("buildConfigChain", function () {
     assert.deepEqual(chain, expected);
   });
 
+  it("js-config-function", function () {
+    const chain = buildConfigChain({
+      filename: fixture("js-config-function", "src.js"),
+    });
+
+    const expected = [
+      {
+        type: "options",
+        options: {
+          ignore: [
+            "root-ignore",
+          ],
+        },
+        alias: fixture(".babelignore"),
+        loc: fixture(".babelignore"),
+        dirname: fixture(),
+      },
+      {
+        type: "options",
+        options: {
+          compact: true,
+        },
+        alias: fixture("js-config-function", ".babelrc.js"),
+        loc: fixture("js-config-function", ".babelrc.js"),
+        dirname: fixture("js-config-function"),
+      },
+      {
+        type: "arguments",
+        options: {
+          filename: fixture("js-config-function", "src.js"),
+        },
+        alias: "base",
+        loc: "base",
+        dirname: base(),
+      },
+    ];
+
+    assert.deepEqual(chain, expected);
+  });
+
   it("js-config-default - should read transpiled export default", function () {
     const chain = buildConfigChain({
       filename: fixture("js-config-default", "src.js"),

packages/babel-core/test/fixtures/config/js-config-function/.babelrc.js

@@ -0,0 +1,7 @@
+module.exports = function(api) {
+  api.cache(true);
+
+  return {
+    compact: true,
+  };
+};