feat(core): hash tasks early when possible to enable optimizations (#11533)

.gitignore

@@ -1,5 +1,6 @@
 node_modules
-.idea
+/.idea
+/.fleet
 /.vscode
 dist
 /build

docs/generated/devkit/index.md

@@ -942,6 +942,7 @@ stored in the daemon process. To reset both run: `nx reset`.
 | `tasks`                      | [`Task`](../../devkit/index#task)[]                                                   |
 | `options`                    | [`DefaultTasksRunnerOptions`](../../devkit/index#defaulttasksrunneroptions)           |
 | `context?`                   | `Object`                                                                              |
+| `context.hasher?`            | [`Hasher`](../../devkit/index#hasher)                                                 |
 | `context.initiatingProject?` | `string`                                                                              |
 | `context.nxArgs`             | `NxArgs`                                                                              |
 | `context.nxJson`             | [`NxJsonConfiguration`](../../devkit/index#nxjsonconfiguration)<`string`[] \| `"*"`\> |

packages/nx/src/hasher/hash-task.ts

@@ -0,0 +1,49 @@
+import { Task, TaskGraph } from '../config/task-graph';
+import { getCustomHasher } from '../tasks-runner/utils';
+import { readProjectsConfigurationFromProjectGraph } from '../project-graph/project-graph';
+import { Hasher } from './hasher';
+import { ProjectGraph } from '../config/project-graph';
+import { Workspaces } from '../config/workspaces';
+
+export function hashDependsOnOtherTasks(
+  workspaces: Workspaces,
+  hasher: Hasher,
+  projectGraph: ProjectGraph,
+  taskGraph: TaskGraph,
+  task: Task
+) {
+  const customHasher = getCustomHasher(
+    task,
+    workspaces,
+    workspaces.readNxJson(),
+    projectGraph
+  );
+  if (customHasher) return true;
+  return hasher.hashDependsOnOtherTasks(task);
+}
+
+export async function hashTask(
+  workspaces: Workspaces,
+  hasher: Hasher,
+  projectGraph: ProjectGraph,
+  taskGraph: TaskGraph,
+  task: Task
+) {
+  const customHasher = getCustomHasher(
+    task,
+    workspaces,
+    workspaces.readNxJson(),
+    projectGraph
+  );
+  const { value, details } = await (customHasher
+    ? customHasher(task, {
+        hasher,
+        projectGraph,
+        taskGraph,
+        workspaceConfig:
+          readProjectsConfigurationFromProjectGraph(projectGraph),
+      })
+    : hasher.hashTask(task));
+  task.hash = value;
+  task.hashDetails = details;
+}

packages/nx/src/hasher/hasher.ts

@@ -115,6 +115,12 @@ export class Hasher {
     };
   }
 
+  hashDependsOnOtherTasks(task: Task) {
+    const inputs = this.taskHasher.inputs(task);
+    // check here for outputs
+    return false;
+  }
+
   /**
    * @deprecated use hashTask instead
    */
@@ -212,10 +218,11 @@ class TaskHasher {
       if (!projectNode) {
         return this.hashExternalDependency(task);
       }
+
       const projectGraphDeps =
         this.projectGraph.dependencies[task.target.project] ?? [];
 
-      const { selfInputs, depsInputs } = this.inputs(task, projectNode);
+      const { selfInputs, depsInputs } = this.inputs(task);
       const self = await this.hashSelfInputs(task, selfInputs);
       const deps = await this.hashDepsTasks(
         depsInputs,
@@ -276,10 +283,11 @@ class TaskHasher {
       .filter((r) => !!r);
   }
 
-  private inputs(
-    task: Task,
-    projectNode: ProjectGraphProjectNode<any>
-  ): { depsInputs: { input: string }[]; selfInputs: ExpandedSelfInput[] } {
+  inputs(task: Task): {
+    depsInputs: { input: string }[];
+    selfInputs: ExpandedSelfInput[];
+  } {
+    const projectNode = this.projectGraph.nodes[task.target.project];
     const namedInputs = {
       default: [{ fileset: '{projectRoot}/**/*' }],
       ...this.nxJson.namedInputs,

packages/nx/src/tasks-runner/default-tasks-runner.ts

@@ -37,6 +37,7 @@ export const defaultTasksRunner: TasksRunner<
     nxJson: NxJsonConfiguration;
     nxArgs: NxArgs;
     taskGraph: TaskGraph;
+    hasher: Hasher;
   }
 ): Promise<{ [id: string]: TaskStatus }> => {
   if (
@@ -69,6 +70,7 @@ async function runAllTasks(
     nxJson: NxJsonConfiguration;
     nxArgs: NxArgs;
     taskGraph: TaskGraph;
+    hasher: Hasher;
   }
 ): Promise<{ [id: string]: TaskStatus }> {
   // TODO: vsavkin: remove this after Nx 16
@@ -81,10 +83,8 @@ async function runAllTasks(
     'task-graph-created'
   );
 
-  const hasher = new Hasher(context.projectGraph, context.nxJson, options);
-
   const orchestrator = new TaskOrchestrator(
-    hasher,
+    context.hasher,
     context.initiatingProject,
     context.projectGraph,
     context.taskGraph,

packages/nx/src/tasks-runner/run-command.ts

@@ -19,11 +19,14 @@ import {
   TargetDefaults,
   TargetDependencies,
 } from '../config/nx-json';
-import { Task } from '../config/task-graph';
+import { Task, TaskGraph } from '../config/task-graph';
 import { createTaskGraph } from './create-task-graph';
 import { findCycle, makeAcyclic } from './task-graph-utils';
 import { TargetDependencyConfig } from '../config/workspace-json-project-json';
 import { handleErrors } from '../utils/params';
+import { Workspaces } from 'nx/src/config/workspaces';
+import { Hasher } from 'nx/src/hasher/hasher';
+import { hashDependsOnOtherTasks, hashTask } from 'nx/src/hasher/hash-task';
 
 async function getTerminalOutputLifeCycle(
   initiatingProject: string,
@@ -82,6 +85,23 @@ async function getTerminalOutputLifeCycle(
   }
 }
 
+async function hashTasksThatDontDependOnOtherTasks(
+  workspaces: Workspaces,
+  hasher: Hasher,
+  projectGraph: ProjectGraph,
+  taskGraph: TaskGraph
+) {
+  const res = [] as Promise<void>[];
+  for (let t of Object.values(taskGraph.tasks)) {
+    if (
+      !hashDependsOnOtherTasks(workspaces, hasher, projectGraph, taskGraph, t)
+    ) {
+      res.push(hashTask(workspaces, hasher, projectGraph, taskGraph, t));
+    }
+  }
+  return Promise.all(res);
+}
+
 export async function runCommand(
   projectsToRun: ProjectGraphProjectNode[],
   projectGraph: ProjectGraph,
@@ -99,6 +119,7 @@ export async function runCommand(
       extraTargetDependencies
     );
     const projectNames = projectsToRun.map((t) => t.name);
+
     const taskGraph = createTaskGraph(
       projectGraph,
       defaultDependencyConfigs,
@@ -108,6 +129,14 @@ export async function runCommand(
       overrides
     );
 
+    const hasher = new Hasher(projectGraph, nxJson, runnerOptions);
+    await hashTasksThatDontDependOnOtherTasks(
+      new Workspaces(workspaceRoot),
+      hasher,
+      projectGraph,
+      taskGraph
+    );
+
     const cycle = findCycle(taskGraph);
     if (cycle) {
       if (nxArgs.nxIgnoreCycles) {
@@ -162,6 +191,7 @@ export async function runCommand(
         nxJson,
         nxArgs,
         taskGraph,
+        hasher,
       }
     );
     let anyFailures;

packages/nx/src/tasks-runner/tasks-runner.ts

@@ -2,6 +2,7 @@ import { NxJsonConfiguration } from '../config/nx-json';
 import { ProjectGraph } from '../config/project-graph';
 import { Task, TaskGraph } from '../config/task-graph';
 import { NxArgs } from '../utils/command-line-utils';
+import { Hasher } from '../hasher/hasher';
 
 export type TaskStatus =
   | 'success'
@@ -25,5 +26,6 @@ export type TasksRunner<T = unknown> = (
     nxJson: NxJsonConfiguration;
     nxArgs: NxArgs;
     taskGraph?: TaskGraph;
+    hasher?: Hasher;
   }
 ) => any | Promise<{ [id: string]: TaskStatus }>;

packages/nx/src/tasks-runner/tasks-schedule.spec.ts

@@ -49,6 +49,9 @@ describe('TasksSchedule', () => {
           batchImplementationFactory: jest.fn(),
         };
       },
+      readNxJson() {
+        return {};
+      },
     };
 
     const projectGraph: ProjectGraph = {

packages/nx/src/tasks-runner/tasks-schedule.ts

@@ -2,7 +2,6 @@ import { Workspaces } from '../config/workspaces';
 
 import {
   calculateReverseDeps,
-  getCustomHasher,
   getExecutorForTask,
   getExecutorNameForTask,
   removeTasksFromTaskGraph,
@@ -11,8 +10,8 @@ import { DefaultTasksRunnerOptions } from './default-tasks-runner';
 import { Hasher } from '../hasher/hasher';
 import { Task, TaskGraph } from '../config/task-graph';
 import { ProjectGraph } from '../config/project-graph';
-import { readProjectsConfigurationFromProjectGraph } from '../project-graph/project-graph';
 import { NxJsonConfiguration } from '../config/nx-json';
+import { hashTask } from '../hasher/hash-task';
 
 export interface Batch {
   executorName: string;
@@ -84,7 +83,16 @@ export class TasksSchedule {
 
   private async scheduleTask(taskId: string) {
     const task = this.taskGraph.tasks[taskId];
-    await this.hashTask(task);
+
+    if (!task.hash) {
+      await hashTask(
+        this.workspaces,
+        this.hasher,
+        this.projectGraph,
+        this.taskGraph,
+        task
+      );
+    }
 
     this.notScheduledTaskGraph = removeTasksFromTaskGraph(
       this.notScheduledTaskGraph,
@@ -170,25 +178,4 @@ export class TasksSchedule {
       this.completedTasks.has(id)
     );
   }
-
-  private async hashTask(task: Task) {
-    const customHasher = getCustomHasher(
-      task,
-      this.workspaces,
-      this.nxJson,
-      this.projectGraph
-    );
-    const { value, details } = await (customHasher
-      ? customHasher(task, {
-          hasher: this.hasher,
-          projectGraph: this.projectGraph,
-          taskGraph: this.taskGraph,
-          workspaceConfig: readProjectsConfigurationFromProjectGraph(
-            this.projectGraph
-          ),
-        })
-      : this.hasher.hashTask(task));
-    task.hash = value;
-    task.hashDetails = details;
-  }
 }