feat(core): add ability to use globs as outputs (#10894)

* feat(core): add ability to use globs as outputs * fix(repo): adjust the way outputs are tracked * docs(core): improve outputs documentation
2022-07-14 18:31:12 -04:00 · 2022-07-14 18:31:12 -04:00 · 63b74d2a56
commit 63b74d2a56
parent 61e1931ded
5 changed files with 548 additions and 84 deletions
--- a/docs/shared/configuration/packagejson.md
+++ b/docs/shared/configuration/packagejson.md
@ -120,12 +120,49 @@ sources (non-test sources) of its dependencies. In other words, it treats test s

 ### outputs

-`"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts. The provided value
-is actually the default, so we can omit it in this case. `"outputs": []` tells Nx that the `test` target doesn't create
-any artifacts on disk.
+Targets may define outputs to tell Nx where the target is going to create file artifacts that Nx should cache. `"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts.

-This configuration is usually not needed. Nx comes with reasonable defaults (imported in `nx.json`) which implement the
-configuration above.
+This configuration is usually not needed. Nx comes with reasonable defaults (imported in `nx.json`) which implement the configuration above.
+
+#### Basic Example
+
+Usually, a target writes to a specific directory or a file. The following instructs Nx to cache `dist/libs/mylib` and `build/libs/mylib/main.js`:
+
+```json
+{
+  "build": {
+    "outputs": ["dist/libs/mylib", "build/libs/mylib/main.js"]
+  }
+}
+```
+
+#### Specifying Globs
+
+Sometimes, multiple targets might write to the same directory. When possible it is recommended to direct these targets into separate directories.
+
+```json
+{
+  "build-js": {
+    "outputs": ["dist/libs/mylib/js"]
+  },
+  "build-css": {
+    "outputs": ["dist/libs/mylib/css"]
+  }
+}
+```
+
+But if the above is not possible, globs can be specified as outputs to only cache a set of files rather than the whole directory.
+
+```json
+{
+  "build-js": {
+    "outputs": ["dist/libs/mylib/**/*.js"]
+  },
+  "build-css": {
+    "outputs": ["dist/libs/mylib/**/*.css"]
+  }
+}
+```

 ### dependsOn

--- a/docs/shared/configuration/projectjson.md
+++ b/docs/shared/configuration/projectjson.md
@ -271,9 +271,85 @@ sources (non-test sources) of its dependencies. In other words, it treats test s

 ### Outputs

-`"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts. The provided value
-is actually the default, so we can omit it in this case. `"outputs": []` tells Nx that the `test` target doesn't create
-any artifacts on disk.
+Targets may define outputs to tell Nx where the target is going to create file artifacts that Nx should cache. `"outputs": ["dist/libs/mylib"]` tells Nx where the `build` target is going to create file artifacts.
+
+#### Basic Example
+
+Usually, a target writes to a specific directory or a file. The following instructs Nx to cache `dist/libs/mylib` and `build/libs/mylib/main.js`:
+
+```json
+  {
+    "build": {
+      ...,
+      "outputs": ["dist/libs/mylib", "build/libs/mylib/main.js"],
+      "options": {
+        ...
+      },
+    }
+  }
+```
+
+#### Referencing Options
+
+Most commonly, targets have an option for an output file or directory. Rather than duplicating the information as seen above, options can be referenced using the below syntax:
+
+> When the `outputPath` option is changed, Nx will start caching the new path as well.
+
+```json
+{
+  "build": {
+    ...,
+    "outputs": ["{options.outputPath}"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  }
+}
+```
+
+#### Specifying Globs
+
+Sometimes, multiple targets might write to the same directory. When possible it is recommended to direct these targets into separate directories.
+
+```json
+{
+  "build-js": {
+    ...,
+    "outputs": ["dist/libs/mylib/js"],
+    "options": {
+      "outputPath": "dist/libs/mylib/js"
+    }
+  },
+  "build-css": {
+    ...,
+    "outputs": ["dist/libs/mylib/css"],
+    "options": {
+      "outputPath": "dist/libs/mylib/css"
+    }
+  }
+}
+```
+
+But if the above is not possible, globs can be specified as outputs to only cache a set of files rather than the whole directory.
+
+```json
+{
+  "build-js": {
+    ...,
+    "outputs": ["dist/libs/mylib/**/*.js"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  },
+  "build-css": {
+    ...,
+    "outputs": ["dist/libs/mylib/**/*.css"],
+    "options": {
+      "outputPath": "dist/libs/mylib"
+    }
+  }
+}
+```

 ### dependsOn

--- a/e2e/nx-run/src/cache.test.ts
+++ b/e2e/nx-run/src/cache.test.ts
@ -99,7 +99,7 @@ describe('cache', () => {
    expect(outputWithBothLintTasksCached).toContain(
      'read the output from the cache'
    );
-    expectCached(outputWithBothLintTasksCached, [
+    expectMatchedOutput(outputWithBothLintTasksCached, [
      myapp1,
      myapp2,
      `${myapp1}-e2e`,
@ -164,6 +164,80 @@ describe('cache', () => {
    updateFile('nx.json', (c) => originalNxJson);
  }, 120000);

+  it('should support using globs as outputs', async () => {
+    const mylib = uniq('mylib');
+    runCLI(`generate @nrwl/workspace:library ${mylib}`);
+    updateProjectConfig(mylib, (c) => {
+      c.targets.build = {
+        executor: 'nx:run-commands',
+        outputs: ['dist/*.txt'],
+        options: {
+          commands: [
+            'rm -rf dist',
+            'mkdir dist',
+            'echo a > dist/a.txt',
+            'echo b > dist/b.txt',
+            'echo c > dist/c.txt',
+            'echo d > dist/d.txt',
+            'echo e > dist/e.txt',
+            'echo f > dist/f.txt',
+          ],
+          parallel: false,
+        },
+      };
+      return c;
+    });
+
+    // Run without cache
+    const runWithoutCache = runCLI(`build ${mylib}`);
+    expect(runWithoutCache).not.toContain('read the output from the cache');
+
+    // Rerun without touching anything
+    const rerunWithUntouchedOutputs = runCLI(`build ${mylib}`);
+    expect(rerunWithUntouchedOutputs).toContain(
+      'existing outputs match the cache'
+    );
+    const outputsWithUntouchedOutputs = listFiles('dist');
+    expect(outputsWithUntouchedOutputs).toContain('a.txt');
+    expect(outputsWithUntouchedOutputs).toContain('b.txt');
+    expect(outputsWithUntouchedOutputs).toContain('c.txt');
+    expect(outputsWithUntouchedOutputs).toContain('d.txt');
+    expect(outputsWithUntouchedOutputs).toContain('e.txt');
+    expect(outputsWithUntouchedOutputs).toContain('f.txt');
+
+    // Create a file in the dist that does not match output glob
+    updateFile('dist/c.ts', '');
+
+    // Rerun
+    const rerunWithNewUnrelatedFile = runCLI(`build ${mylib}`);
+    expect(rerunWithNewUnrelatedFile).toContain(
+      'existing outputs match the cache'
+    );
+    const outputsAfterAddingUntouchedFileAndRerunning = listFiles('dist');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('a.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('b.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('d.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('e.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('f.txt');
+    expect(outputsAfterAddingUntouchedFileAndRerunning).toContain('c.ts');
+
+    // Clear Dist
+    rmDist();
+
+    // Rerun
+    const rerunWithoutOutputs = runCLI(`build ${mylib}`);
+    expect(rerunWithoutOutputs).toContain('read the output from the cache');
+    const outputsWithoutOutputs = listFiles('dist');
+    expect(outputsWithoutOutputs).toContain('a.txt');
+    expect(outputsWithoutOutputs).toContain('b.txt');
+    expect(outputsWithoutOutputs).toContain('c.txt');
+    expect(outputsWithoutOutputs).toContain('d.txt');
+    expect(outputsWithoutOutputs).toContain('e.txt');
+    expect(outputsWithoutOutputs).toContain('f.txt');
+    expect(outputsWithoutOutputs).not.toContain('c.ts');
+  });
+
  it('should use consider filesets when hashing', async () => {
    const parent = uniq('parent');
    const child1 = uniq('child1');
--- a/packages/nx/src/tasks-runner/cache.spec.ts
+++ b/packages/nx/src/tasks-runner/cache.spec.ts
@ -0,0 +1,112 @@
+import { Cache, collapseExpandedOutputs } from './cache';
+
+describe('Cache', () => {
+  describe('collapseExpandedOutputs', () => {
+    it('should handle no outputs', async () => {
+      const outputs = [];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual([]);
+    });
+
+    it('should keep files as is', async () => {
+      const outputs = ['dist/apps/app1/0.js'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1/0.js']);
+    });
+
+    it('should keep directories as is', async () => {
+      const outputs = ['dist/apps/app1'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should keep short lists of directories as is', async () => {
+      const outputs = ['test-results/apps/app1', 'coverage/apps/app1'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['test-results/apps/app1', 'coverage/apps/app1']);
+    });
+
+    it('should keep short lists of files as is', async () => {
+      const outputs = [
+        'test-results/apps/app1/results.xml',
+        'coverage/apps/app1/coverage.html',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual([
+        'test-results/apps/app1/results.xml',
+        'coverage/apps/app1/coverage.html',
+      ]);
+    });
+
+    it('should collapse long lists of directories', async () => {
+      const outputs = [
+        'dist/apps/app1/a',
+        'dist/apps/app1/b',
+        'dist/apps/app1/c',
+        'dist/apps/app1/d',
+        'dist/apps/app1/e',
+        'dist/apps/app1/f',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should collapse long lists of directories + files', async () => {
+      const outputs = [
+        'coverage/apps/app1',
+        'dist/apps/app1/a.txt',
+        'dist/apps/app1/b.txt',
+        'dist/apps/app1/c.txt',
+        'dist/apps/app1/d.txt',
+        'dist/apps/app1/e.txt',
+        'dist/apps/app1/f.txt',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['coverage/apps/app1', 'dist/apps/app1']);
+    });
+
+    it('should keep long lists of top-level directories', async () => {
+      const outputs = ['a', 'b', 'c', 'd', 'e', 'f'];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['a', 'b', 'c', 'd', 'e', 'f']);
+    });
+
+    it('should collapse long lists of files', async () => {
+      const outputs = [
+        'dist/apps/app1/a.js',
+        'dist/apps/app1/b.js',
+        'dist/apps/app1/c.js',
+        'dist/apps/app1/d.js',
+        'dist/apps/app1/e.js',
+        'dist/apps/app1/f.js',
+      ];
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+
+    it('should collapse long lists of files in nested directories', async () => {
+      const outputs = [];
+      // Create dist/apps/app1/n/m.js + dist/apps/app1/n/m.d.ts
+      for (let i = 0; i < 6; i++) {
+        outputs.push(`dist/apps/app1/${i}.js`);
+        outputs.push(`dist/apps/app1/${i}.d.ts`);
+        for (let j = 0; j < 6; j++) {
+          outputs.push(`dist/apps/app1/${i}/${j}.js`);
+          outputs.push(`dist/apps/app1/${i}/${j}.d.ts`);
+        }
+      }
+      const res = collapseExpandedOutputs(outputs);
+
+      expect(res).toEqual(['dist/apps/app1']);
+    });
+  });
+});
--- a/packages/nx/src/tasks-runner/cache.ts
+++ b/packages/nx/src/tasks-runner/cache.ts
@ -1,22 +1,23 @@
 import { workspaceRoot } from '../utils/workspace-root';
 import {
  copy,
+  lstat,
  mkdir,
  mkdirSync,
+  pathExists,
+  readdir,
  readFile,
  remove,
  unlink,
  writeFile,
-  pathExists,
-  lstat,
-  readdir,
 } from 'fs-extra';
-import { dirname, join, resolve, sep } from 'path';
+import { dirname, join, relative, resolve, sep } from 'path';
 import { DefaultTasksRunnerOptions } from './default-tasks-runner';
-import { spawn, execFile } from 'child_process';
+import { execFile, spawn } from 'child_process';
 import { cacheDir } from '../utils/cache-directory';
 import { platform } from 'os';
 import { Task } from '../config/task-graph';
+import * as fastGlob from 'fast-glob';

 export type CachedResult = {
  terminalOutput: string;
@ -94,12 +95,16 @@ export class Cache {
      );

      await mkdir(join(td, 'outputs'));
+      const expandedOutputs = await this.expandOutputsInWorkspace(outputs);
+      const collapsedOutputs = collapseExpandedOutputs(expandedOutputs);
+
      await Promise.all(
-        outputs.map(async (f) => {
+        expandedOutputs.map(async (f) => {
          const src = join(this.root, f);
          if (await pathExists(src)) {
-            const cached = join(td, 'outputs', f);
            const isFile = (await lstat(src)).isFile();
+
+            const cached = join(td, 'outputs', f);
            const directory = isFile ? dirname(cached) : cached;
            await mkdir(directory, { recursive: true });
            await this.copy(src, cached);
@ -117,7 +122,7 @@ export class Cache {
        await this.options.remoteCache.store(task.hash, this.cachePath);
      }

-      await this.recordOutputsHash(outputs, task.hash);
+      await this.recordOutputsHash(collapsedOutputs, task.hash);

      if (terminalOutput) {
        const outputPath = this.temporaryOutputPath(task);
@ -132,9 +137,16 @@ export class Cache {
    outputs: string[]
  ) {
    return this.tryAndRetry(async () => {
-      await this.removeRecordedOutputsHashes(outputs);
+      const expandedOutputs = await this.expandOutputsInCache(
+        outputs,
+        cachedResult
+      );
+      const collapsedOutputs = collapseExpandedOutputs(expandedOutputs);
+
+      await this.removeRecordedOutputsHashes(collapsedOutputs);
+
      await Promise.all(
-        outputs.map(async (f) => {
+        expandedOutputs.map(async (f) => {
          const cached = join(cachedResult.outputsPath, f);
          if (await pathExists(cached)) {
            const isFile = (await lstat(cached)).isFile();
@ -147,7 +159,8 @@ export class Cache {
          }
        })
      );
-      await this.recordOutputsHash(outputs, hash);
+
+      await this.recordOutputsHash(collapsedOutputs, hash);
    });
  }

@ -156,59 +169,94 @@ export class Cache {
  }

  async removeRecordedOutputsHashes(outputs: string[]): Promise<void> {
-    for (const output of outputs) {
-      const hashFile = this.getFileNameWithLatestRecordedHashForOutput(output);
-      try {
-        await unlink(hashFile);
-      } catch {}
-    }
+    await Promise.all(
+      outputs.map(async (output) => {
+        const hashFile =
+          this.getFileNameWithLatestRecordedHashForOutput(output);
+        try {
+          await unlink(hashFile);
+        } catch {}
+      })
+    );
  }

  async shouldCopyOutputsFromCache(
    taskWithCachedResult: TaskWithCachedResult,
    outputs: string[]
  ): Promise<boolean> {
-    return (
-      (await this.areLatestOutputsHashesDifferentThanTaskHash(
-        outputs,
-        taskWithCachedResult.task.hash
-      )) ||
-      (await this.isAnyOutputMissing(
-        taskWithCachedResult.cachedResult,
-        outputs
-      ))
-    );
+    const [outputsInCache, outputsInWorkspace] = await Promise.all([
+      this.expandOutputsInCache(outputs, taskWithCachedResult.cachedResult),
+      this.expandOutputsInWorkspace(outputs),
+    ]);
+
+    const collapsedOutputsInCache = collapseExpandedOutputs(outputsInCache);
+
+    const [latestHashesDifferent, outputMissing] = await Promise.all([
+      this.areLatestOutputsHashesDifferentThanTaskHash(
+        collapsedOutputsInCache,
+        taskWithCachedResult
+      ),
+      this.haveOutputsBeenAddedOrRemoved(
+        taskWithCachedResult,
+        outputsInCache,
+        outputsInWorkspace
+      ),
+    ]);
+    return latestHashesDifferent || outputMissing;
  }

-  private copy(src: string, directory: string): Promise<void> {
-    if (this.useFsExtraToCopyAndRemove) {
-      return copy(src, directory);
-    }
+  private async expandOutputsInWorkspace(outputs: string[]) {
+    return this._expandOutputs(outputs, workspaceRoot);
+  }

+  private async expandOutputsInCache(
+    outputs: string[],
+    cachedResult: CachedResult
+  ) {
+    return this._expandOutputs(outputs, cachedResult.outputsPath);
+  }
+
+  private async _expandOutputs(outputs: string[], cwd: string) {
+    return (
+      await Promise.all(
+        outputs.map(async (entry) => {
+          if (await pathExists(join(cwd, entry))) {
+            return entry;
+          }
+          return fastGlob(entry, { cwd });
+        })
+      )
+    ).flat();
+  }
+
+  private async copy(src: string, destination: string): Promise<void> {
+    if (this.useFsExtraToCopyAndRemove) {
+      return copy(src, destination);
+    }
    return new Promise((res, rej) => {
-      execFile('cp', ['-a', src, dirname(directory)], (error) => {
+      execFile('cp', ['-a', src, dirname(destination)], (error) => {
        if (!error) {
          res();
        } else {
          this.useFsExtraToCopyAndRemove = true;
-          copy(src, directory).then(res, rej);
+          copy(src, destination).then(res, rej);
        }
      });
    });
  }

-  private remove(folder: string): Promise<void> {
+  private async remove(path: string): Promise<void> {
    if (this.useFsExtraToCopyAndRemove) {
-      return remove(folder);
+      return remove(path);
    }

    return new Promise<void>((res, rej) => {
-      execFile('rm', ['-rf', folder], (error) => {
+      execFile('rm', ['-rf', path], (error) => {
        if (!error) {
          res();
        } else {
          this.useFsExtraToCopyAndRemove = true;
-          remove(folder).then(res, rej);
+          remove(path).then(res, rej);
        }
      });
    });
@ -218,22 +266,56 @@ export class Cache {
    outputs: string[],
    hash: string
  ): Promise<void> {
-    for (const output of outputs) {
-      const hashFile = this.getFileNameWithLatestRecordedHashForOutput(output);
-      try {
-        await mkdir(dirname(hashFile), { recursive: true });
-        await writeFile(hashFile, hash);
-      } catch {}
-    }
+    await mkdir(this.latestOutputsHashesDir, { recursive: true });
+
+    await Promise.all(
+      outputs.map(async (output) => {
+        const hashFile =
+          this.getFileNameWithLatestRecordedHashForOutput(output);
+        try {
+          await writeFile(hashFile, hash);
+        } catch {}
+      })
+    );
  }

  private async areLatestOutputsHashesDifferentThanTaskHash(
    outputs: string[],
-    hash: string
+    { task }: TaskWithCachedResult
  ) {
-    for (let output of outputs) {
-      if ((await this.getLatestRecordedHashForTask(output)) !== hash)
+    const latestExistingOutputHashes = (
+      await readdir(this.latestOutputsHashesDir)
+    ).map((m) => m.substring(0, m.length - 5));
+    // Purposely blocking
+    for (const output of outputs) {
+      const latestOutputFilename = this.getLatestOutputHashFilename(output);
+
+      const conflicts = latestExistingOutputHashes.filter((w) => {
+        // This is the exact same output
+        return (
+          w !== latestOutputFilename &&
+          // This is an child of the output
+          (latestOutputFilename.startsWith(w) ||
+            // This is a parent of the output
+            w.startsWith(latestOutputFilename))
+        );
+      });
+
+      if (conflicts.length > 0) {
+        // Clean up the conflicts
+        await Promise.all(
+          conflicts.map((conflict) =>
+            unlink(join(this.latestOutputsHashesDir, conflict + '.hash'))
+          )
+        );
        return true;
+      }
+
+      const hash = await this.getLatestRecordedHashForTask(output);
+
+      if (!!hash && hash !== task.hash) {
+        return true;
+      }
    }
    return false;
  }
@ -251,48 +333,83 @@ export class Cache {
    }
  }

-  private async isAnyOutputMissing(
-    cachedResult: CachedResult,
-    outputs: string[]
+  private async haveOutputsBeenAddedOrRemoved(
+    result: TaskWithCachedResult,
+    cachedOutputs: string[],
+    workspaceFiles: string[]
  ): Promise<boolean> {
-    for (let output of outputs) {
-      const cacheOutputPath = join(cachedResult.outputsPath, output);
-      const rootOutputPath = join(this.root, output);
+    const workspaceSet = new Set(workspaceFiles);

-      if (
-        (await pathExists(cacheOutputPath)) &&
-        (await lstat(cacheOutputPath)).isFile()
-      ) {
-        return (
-          (await pathExists(join(cachedResult.outputsPath, output))) &&
-          !(await pathExists(join(this.root, output)))
-        );
-      }
-
-      const haveDifferentAmountOfFiles =
-        (await pathExists(cacheOutputPath)) &&
-        (await pathExists(rootOutputPath)) &&
-        (await readdir(cacheOutputPath)).length !==
-          (await readdir(rootOutputPath)).length;
-
-      if (
-        ((await pathExists(cacheOutputPath)) &&
-          !(await pathExists(rootOutputPath))) ||
-        haveDifferentAmountOfFiles
-      ) {
+    for (const path of cachedOutputs) {
+      if (!(await pathExists(join(workspaceRoot, path)))) {
        return true;
      }
+      const isFile = (await lstat(path)).isFile();
+      if (!workspaceSet.has(path)) {
+        return true;
+      }
+      if (!isFile) {
+        const [cachedFiles, workspaceFiles] = await Promise.all([
+          this.getFilesInDirectory(join(result.cachedResult.outputsPath, path)),
+          this.getFilesInDirectory(join(workspaceRoot, path)),
+        ]);
+
+        if (workspaceFiles.size !== cachedFiles.size) {
+          return true;
+        }
+        for (const file of cachedFiles) {
+          if (!workspaceFiles.has(file)) {
+            return true;
+          }
+          workspaceFiles.delete(file);
+        }
+        if (workspaceFiles.size !== 0) {
+          return true;
+        }
+      }
+      workspaceSet.delete(path);
+    }
+    if (workspaceSet.size !== 0) {
+      return true;
    }
    return false;
  }

+  private async getFilesInDirectory(path: string): Promise<Set<string>> {
+    const paths = new Set<string>();
+    await this.visitDirectory(path, (entry) => {
+      paths.add(relative(path, entry));
+    });
+    return paths;
+  }
+
+  private async visitDirectory(path: string, visitor: (path: string) => void) {
+    const children = await readdir(join(path), {
+      withFileTypes: true,
+    });
+
+    await Promise.all(
+      children.map(async (child) => {
+        if (child.isFile()) {
+          visitor(join(path, child.name));
+        } else {
+          await this.visitDirectory(join(path, child.name), visitor);
+        }
+      })
+    );
+  }
+
  private getFileNameWithLatestRecordedHashForOutput(output: string): string {
    return join(
      this.latestOutputsHashesDir,
-      `${output.split(sep).join('-')}.hash`
+      `${this.getLatestOutputHashFilename(output)}.hash`
    );
  }

+  private getLatestOutputHashFilename(output: string) {
+    return output.split(sep).join('-');
+  }
+
  private async getFromLocalDir(task: Task) {
    const tdCommit = join(this.cachePath, `${task.hash}.commit`);
    const td = join(this.cachePath, task.hash);
@ -352,3 +469,51 @@ export class Cache {
    return _try();
  }
 }
+
+/**
+ * Heuristic to prevent writing too many hash files
+ */
+const MAX_OUTPUTS_TO_CHECK_HASHES = 5;
+
+/**
+ * Collapse Expanded Outputs back into a smaller set of directories/files to track
+ * Note: DO NOT USE, Only exported for unit testing
+ * */
+export function collapseExpandedOutputs(expandedOutputs: string[]) {
+  const tree: Set<string>[] = [];
+
+  // Create a Tree of directories/files
+  for (const output of expandedOutputs) {
+    const pathParts = [];
+    pathParts.unshift(output);
+    let dir = dirname(output);
+    while (dir !== dirname(dir)) {
+      pathParts.unshift(dir);
+
+      dir = dirname(dir);
+    }
+
+    for (let i = 0; i < pathParts.length; i++) {
+      tree[i] ??= new Set<string>();
+      tree[i].add(pathParts[i]);
+    }
+  }
+
+  // Find a level in the tree that has too many outputs
+  if (tree.length === 0) {
+    return [];
+  }
+
+  let j = 0;
+  let level = tree[j];
+  for (j = 0; j < tree.length; j++) {
+    level = tree[j];
+    if (level.size > MAX_OUTPUTS_TO_CHECK_HASHES) {
+      break;
+    }
+  }
+
+  // Return the level before the level with too many outputs
+  // If the first level has too many outputs, return that one.
+  return Array.from(tree[Math.max(0, j - 1)]);
+}