nx/packages/workspace/src/tasks-runner/cache.ts

import { appRootPath } from '../utils/app-root';
import { Task } from './tasks-runner';
import {
  existsSync,
  mkdirSync,
  readFileSync,
  writeFileSync,
  lstatSync,
} from 'fs';
import { join, resolve } from 'path';
import * as fsExtra from 'fs-extra';
import { DefaultTasksRunnerOptions } from './default-tasks-runner';
import { spawn } from 'child_process';

export type CachedResult = { terminalOutput: string; outputsPath: string };
export type TaskWithCachedResult = { task: Task; cachedResult: CachedResult };

class CacheConfig {
  constructor(private readonly options: DefaultTasksRunnerOptions) {}

  isCacheableTask(task: Task) {
    const cacheable =
      this.options.cacheableOperations || this.options.cacheableTargets;
    return (
      cacheable &&
      cacheable.indexOf(task.target.target) > -1 &&
      !this.longRunningTask(task)
    );
  }

  private longRunningTask(task: Task) {
    return !!task.overrides['watch'];
  }
}

export class Cache {
  root = appRootPath;
  cachePath = this.createCacheDir();
  terminalOutputsDir = this.createTerminalOutputsDir();
  cacheConfig = new CacheConfig(this.options);

  constructor(private readonly options: DefaultTasksRunnerOptions) {}

  removeOldCacheRecords() {
    /**
     * Even though spawning a process is fast, we don't want to do it every time
     * the user runs a command. Instead, we want to do it once in a while.
     */
    const shouldSpawnProcess = Math.floor(Math.random() * 50) === 1;
    if (shouldSpawnProcess) {
      const scriptPath = join(
        this.root,
        'node_modules',
        '@nrwl',
        'workspace',
        'src',
        'tasks-runner',
        'remove-old-cache-records.js'
      );
      try {
        const p = spawn('node', [`"${scriptPath}"`, `"${this.cachePath}"`], {
          stdio: 'ignore',
          detached: true,
        });
        p.unref();
      } catch (e) {
        console.log(`Unable to start remove-old-cache-records script:`);
        console.log(e.message);
      }
    }
  }

  async get(task: Task): Promise<CachedResult> {
    if (!this.cacheConfig.isCacheableTask(task)) return null;

    const res = this.getFromLocalDir(task);

    // didn't find it locally but we have a remote cache
    if (!res && this.options.remoteCache) {
      // attempt remote cache
      await this.options.remoteCache.retrieve(task.hash, this.cachePath);
      // try again from local cache
      return this.getFromLocalDir(task);
    } else {
      return res;
    }
  }

  async put(task: Task, terminalOutputPath: string, outputs: string[]) {
    const terminalOutput = readFileSync(terminalOutputPath).toString();
    const td = join(this.cachePath, task.hash);
    const tdCommit = join(this.cachePath, `${task.hash}.commit`);

    // might be left overs from partially-completed cache invocations
    if (existsSync(tdCommit)) {
      fsExtra.removeSync(tdCommit);
    }
    if (existsSync(td)) {
      fsExtra.removeSync(td);
    }

    mkdirSync(td);
    writeFileSync(join(td, 'terminalOutput'), terminalOutput);

    mkdirSync(join(td, 'outputs'));
    outputs.forEach((f) => {
      const src = join(this.root, f);
      if (existsSync(src)) {
        const cached = join(td, 'outputs', f);
        // Ensure parent directory is created if src is a file
        const isFile = lstatSync(src).isFile();
        const directory = isFile ? resolve(cached, '..') : cached;
        fsExtra.ensureDirSync(directory);

        fsExtra.copySync(src, cached);
      }
    });
    // we need this file to account for partial writes to the cache folder.
    // creating this file is atomic, whereas creating a folder is not.
    // so if the process gets terminated while we are copying stuff into cache,
    // the cache entry won't be used.
    writeFileSync(tdCommit, 'true');

    if (this.options.remoteCache) {
      await this.options.remoteCache.store(task.hash, this.cachePath);
    }
  }

  copyFilesFromCache(cachedResult: CachedResult, outputs: string[]) {
    outputs.forEach((f) => {
      const cached = join(cachedResult.outputsPath, f);
      if (existsSync(cached)) {
        const isFile = lstatSync(cached).isFile();
        const src = join(this.root, f);
        if (existsSync(src)) {
          fsExtra.removeSync(src);
        }
        // Ensure parent directory is created if src is a file
        const directory = isFile ? resolve(src, '..') : src;
        fsExtra.ensureDirSync(directory);
        fsExtra.copySync(cached, src);
      }
    });
  }

  temporaryOutputPath(task: Task) {
    if (this.cacheConfig.isCacheableTask(task)) {
      return join(this.terminalOutputsDir, task.hash);
    } else {
      return null;
    }
  }

  private getFromLocalDir(task: Task) {
    const tdCommit = join(this.cachePath, `${task.hash}.commit`);
    const td = join(this.cachePath, task.hash);

    if (existsSync(tdCommit)) {
      return {
        terminalOutput: readFileSync(join(td, 'terminalOutput')).toString(),
        outputsPath: join(td, 'outputs'),
      };
    } else {
      return null;
    }
  }

  private createCacheDir() {
    let dir;
    if (this.options.cacheDirectory) {
      if (this.options.cacheDirectory.startsWith('./')) {
        dir = join(this.root, this.options.cacheDirectory);
      } else {
        dir = this.options.cacheDirectory;
      }
    } else {
      dir = join(this.root, 'node_modules', '.cache', 'nx');
    }
    if (!existsSync(dir)) {
      fsExtra.ensureDirSync(dir);
    }
    return dir;
  }

  private createTerminalOutputsDir() {
    const path = join(this.cachePath, 'terminalOutputs');
    fsExtra.ensureDirSync(path);
    return path;
  }
}