diff --git a/packages/nx/src/hasher/hash-plan-inspector.spec.ts b/packages/nx/src/hasher/hash-plan-inspector.spec.ts new file mode 100644 index 0000000000..251bcee42c --- /dev/null +++ b/packages/nx/src/hasher/hash-plan-inspector.spec.ts @@ -0,0 +1,479 @@ +import { HashPlanInspector } from './hash-plan-inspector'; +import { ProjectGraph } from '../config/project-graph'; +import { TempFs } from '../internal-testing-utils/temp-fs'; +import { ProjectGraphBuilder } from '../project-graph/project-graph-builder'; + +describe('HashPlanInspector', () => { + let tempFs: TempFs; + let inspector: HashPlanInspector; + let projectGraph: ProjectGraph; + + beforeEach(async () => { + tempFs = new TempFs('hash-plan-inspector'); + + // Create a minimal workspace structure + await tempFs.createFiles({ + 'package.json': JSON.stringify({ + name: 'test-workspace', + devDependencies: { + nx: '0.0.0', + }, + }), + 'nx.json': JSON.stringify({ + extends: 'nx/presets/npm.json', + targetDefaults: { + build: { + cache: true, + }, + test: { + cache: true, + }, + }, + namedInputs: { + default: ['{projectRoot}/**/*'], + production: ['default', '!{projectRoot}/**/*.spec.ts'], + }, + }), + 'apps/test-app/project.json': JSON.stringify({ + name: 'test-app', + sourceRoot: 'apps/test-app/src', + targets: { + build: { + executor: '@nx/webpack:webpack', + options: { + outputPath: 'dist/apps/test-app', + }, + }, + test: { + executor: '@nx/jest:jest', + options: { + jestConfig: 'apps/test-app/jest.config.ts', + }, + }, + }, + }), + 'apps/test-app/src/main.ts': 'console.log("Hello from test-app");', + 'libs/test-lib/project.json': JSON.stringify({ + name: 'test-lib', + sourceRoot: 'libs/test-lib/src', + targets: { + build: { + executor: '@nx/rollup:rollup', + options: { + outputPath: 'dist/libs/test-lib', + }, + }, + test: { + executor: '@nx/jest:jest', + options: { + jestConfig: 'libs/test-lib/jest.config.ts', + }, + }, + }, + }), + 'libs/test-lib/src/index.ts': 'export const lib = "test-lib";', + }); + + // Build project graph + const builder = new ProjectGraphBuilder(); + + builder.addNode({ + name: 'test-app', + type: 'app', + data: { + root: 'apps/test-app', + sourceRoot: 'apps/test-app/src', + targets: { + build: { + executor: '@nx/webpack:webpack', + options: { + outputPath: 'dist/apps/test-app', + }, + }, + test: { + executor: '@nx/jest:jest', + options: { + jestConfig: 'apps/test-app/jest.config.ts', + }, + }, + }, + }, + }); + + builder.addNode({ + name: 'test-lib', + type: 'lib', + data: { + root: 'libs/test-lib', + sourceRoot: 'libs/test-lib/src', + targets: { + build: { + executor: '@nx/rollup:rollup', + options: { + outputPath: 'dist/libs/test-lib', + }, + }, + test: { + executor: '@nx/jest:jest', + options: { + jestConfig: 'libs/test-lib/jest.config.ts', + }, + }, + }, + }, + }); + + builder.addImplicitDependency('test-app', 'test-lib'); + + projectGraph = builder.getUpdatedProjectGraph(); + + inspector = new HashPlanInspector(projectGraph, tempFs.tempDir); + }); + + afterEach(() => { + tempFs.reset(); + }); + + describe('inspectHashPlan', () => { + beforeEach(async () => { + await inspector.init(); + }); + + it('should inspect hash plan for single project and target', () => { + const result = inspector.inspectHashPlan(['test-app'], ['build']); + + // Should return a record mapping task IDs to arrays of hash instructions + expect(result).toBeDefined(); + expect(typeof result).toBe('object'); + + // Should only contain test-app:build (and not test-lib:build in this case) + expect(Object.keys(result)).toEqual(['test-app:build']); + + const testAppPlan = result['test-app:build']; + expect(Array.isArray(testAppPlan)).toBe(true); + expect(testAppPlan.length).toBeGreaterThan(0); + + // Should include nx.json + expect(testAppPlan).toContain('file:nx.json'); + + // Should include environment variable + expect(testAppPlan).toContain('env:NX_CLOUD_ENCRYPTION_KEY'); + + // Should include test-app files + expect(testAppPlan).toContain('file:apps/test-app/project.json'); + expect(testAppPlan).toContain('file:apps/test-app/src/main.ts'); + + // Should include test-lib files (due to dependency) + expect(testAppPlan).toContain('file:libs/test-lib/project.json'); + expect(testAppPlan).toContain('file:libs/test-lib/src/index.ts'); + + // Should include project configurations + expect(testAppPlan).toContain('test-app:ProjectConfiguration'); + expect(testAppPlan).toContain('test-lib:ProjectConfiguration'); + + // Should include TypeScript configurations + expect(testAppPlan).toContain('test-app:TsConfig'); + expect(testAppPlan).toContain('test-lib:TsConfig'); + }); + + it('should inspect hash plan for multiple projects', () => { + const result = inspector.inspectHashPlan( + ['test-app', 'test-lib'], + ['build'] + ); + + // Should have hash plans for both projects + expect(Object.keys(result).sort()).toEqual([ + 'test-app:build', + 'test-lib:build', + ]); + + // Check test-app:build hash plan + const testAppPlan = result['test-app:build']; + expect(Array.isArray(testAppPlan)).toBe(true); + + // test-app should include its own files + expect(testAppPlan).toContain('file:apps/test-app/project.json'); + expect(testAppPlan).toContain('file:apps/test-app/src/main.ts'); + + // test-app should also include test-lib files (due to dependency) + expect(testAppPlan).toContain('file:libs/test-lib/project.json'); + expect(testAppPlan).toContain('file:libs/test-lib/src/index.ts'); + + // Should include configurations for both projects + expect(testAppPlan).toContain('test-app:ProjectConfiguration'); + expect(testAppPlan).toContain('test-lib:ProjectConfiguration'); + expect(testAppPlan).toContain('test-app:TsConfig'); + expect(testAppPlan).toContain('test-lib:TsConfig'); + + // Should include common files + expect(testAppPlan).toContain('file:nx.json'); + expect(testAppPlan).toContain('env:NX_CLOUD_ENCRYPTION_KEY'); + + // Check test-lib:build hash plan + const testLibPlan = result['test-lib:build']; + expect(Array.isArray(testLibPlan)).toBe(true); + + // test-lib should only include its own files (no dependencies) + expect(testLibPlan).toContain('file:libs/test-lib/project.json'); + expect(testLibPlan).toContain('file:libs/test-lib/src/index.ts'); + + // Should not include test-app files + expect(testLibPlan).not.toContain('file:apps/test-app/project.json'); + expect(testLibPlan).not.toContain('file:apps/test-app/src/main.ts'); + + // Should include only test-lib configurations + expect(testLibPlan).toContain('test-lib:ProjectConfiguration'); + expect(testLibPlan).toContain('test-lib:TsConfig'); + expect(testLibPlan).not.toContain('test-app:ProjectConfiguration'); + expect(testLibPlan).not.toContain('test-app:TsConfig'); + + // Should include common files + expect(testLibPlan).toContain('file:nx.json'); + expect(testLibPlan).toContain('env:NX_CLOUD_ENCRYPTION_KEY'); + }); + + it('should inspect hash plan for multiple targets', () => { + const result = inspector.inspectHashPlan(['test-app'], ['build', 'test']); + + expect(Object.keys(result)).toContain('test-app:build'); + expect(Object.keys(result)).toContain('test-app:test'); + expect(Array.isArray(result['test-app:build'])).toBe(true); + expect(Array.isArray(result['test-app:test'])).toBe(true); + }); + + it('should handle configuration parameter', () => { + const result = inspector.inspectHashPlan( + ['test-app'], + ['build'], + 'production' + ); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should handle overrides parameter', () => { + const overrides = { watch: true }; + const result = inspector.inspectHashPlan( + ['test-app'], + ['build'], + undefined, + overrides + ); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should handle extraTargetDependencies parameter', () => { + const extraTargetDependencies = { + build: [{ target: 'test', projects: 'self' }], + }; + const result = inspector.inspectHashPlan( + ['test-app'], + ['build'], + undefined, + {}, + extraTargetDependencies + ); + + // Should include both build and test tasks due to extra dependency + expect(Object.keys(result)).toContain('test-app:build'); + expect(Object.keys(result)).toContain('test-app:test'); + }); + + it('should handle excludeTaskDependencies parameter', () => { + const result = inspector.inspectHashPlan( + ['test-app'], + ['build'], + undefined, + {}, + {}, + true + ); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should handle empty project names array', () => { + const result = inspector.inspectHashPlan([], ['build']); + + expect(Object.keys(result)).toHaveLength(0); + }); + + it('should handle empty targets array', () => { + const result = inspector.inspectHashPlan(['test-app'], []); + + expect(Object.keys(result)).toHaveLength(0); + }); + }); + + describe('inspectTask', () => { + beforeEach(async () => { + await inspector.init(); + }); + + it('should inspect a single task', () => { + const target = { project: 'test-app', target: 'build' }; + const result = inspector.inspectTask(target); + + // Should only contain test-app:build + expect(Object.keys(result)).toEqual(['test-app:build']); + + const testAppPlan = result['test-app:build']; + expect(Array.isArray(testAppPlan)).toBe(true); + + // Should include test-app files + expect(testAppPlan).toContain('file:apps/test-app/project.json'); + expect(testAppPlan).toContain('file:apps/test-app/src/main.ts'); + + // Should include test-lib files (due to dependency) + expect(testAppPlan).toContain('file:libs/test-lib/project.json'); + expect(testAppPlan).toContain('file:libs/test-lib/src/index.ts'); + + // Should include common files + expect(testAppPlan).toContain('file:nx.json'); + expect(testAppPlan).toContain('env:NX_CLOUD_ENCRYPTION_KEY'); + + // Should include configurations + expect(testAppPlan).toContain('test-app:TsConfig'); + expect(testAppPlan).toContain('test-lib:TsConfig'); + expect(testAppPlan).toContain('test-app:ProjectConfiguration'); + expect(testAppPlan).toContain('test-lib:ProjectConfiguration'); + }); + + it('should inspect task with configuration', () => { + const target = { + project: 'test-app', + target: 'build', + configuration: 'production', + }; + const result = inspector.inspectTask(target); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should handle parsed args parameter', () => { + const target = { project: 'test-app', target: 'build' }; + const parsedArgs = { watch: true, verbose: true }; + const result = inspector.inspectTask(target, parsedArgs); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should handle extraTargetDependencies parameter', () => { + const target = { project: 'test-app', target: 'build' }; + const extraTargetDependencies = { + build: [{ target: 'test', projects: 'self' }], + }; + const result = inspector.inspectTask(target, {}, extraTargetDependencies); + + // Should include both build and test tasks due to extra dependency + expect(Object.keys(result)).toContain('test-app:build'); + expect(Object.keys(result)).toContain('test-app:test'); + }); + + it('should handle excludeTaskDependencies parameter', () => { + const target = { project: 'test-app', target: 'build' }; + const result = inspector.inspectTask(target, {}, {}, true); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + + it('should inspect test target', () => { + const target = { project: 'test-app', target: 'test' }; + const result = inspector.inspectTask(target); + + expect(result['test-app:test']).toBeDefined(); + expect(Array.isArray(result['test-app:test'])).toBe(true); + }); + + it('should inspect library project task', () => { + const target = { project: 'test-lib', target: 'build' }; + const result = inspector.inspectTask(target); + + expect(result['test-lib:build']).toBeDefined(); + expect(Array.isArray(result['test-lib:build'])).toBe(true); + }); + + it('should handle complex parsed args with configuration override', () => { + const target = { project: 'test-app', target: 'build' }; + const parsedArgs = { + configuration: 'development', + targets: ['build'], + parallel: 3, + maxParallel: 3, + }; + const result = inspector.inspectTask(target, parsedArgs); + + expect(result['test-app:build']).toBeDefined(); + expect(Array.isArray(result['test-app:build'])).toBe(true); + }); + }); + + describe('integration scenarios', () => { + beforeEach(async () => { + await inspector.init(); + }); + + it('should handle both inspectHashPlan and inspectTask on same instance', () => { + const hashPlanResult = inspector.inspectHashPlan(['test-app'], ['build']); + const taskResult = inspector.inspectTask({ + project: 'test-app', + target: 'build', + }); + + expect(hashPlanResult['test-app:build']).toBeDefined(); + expect(taskResult['test-app:build']).toBeDefined(); + expect(Array.isArray(hashPlanResult['test-app:build'])).toBe(true); + expect(Array.isArray(taskResult['test-app:build'])).toBe(true); + }); + + it('should work with project dependencies', () => { + // test-app depends on test-lib, so building test-app should include test-lib + const result = inspector.inspectHashPlan(['test-app'], ['build']); + + // Should only contain test-app:build (test-lib:build is not included as a separate task) + expect(Object.keys(result)).toEqual(['test-app:build']); + + const testAppPlan = result['test-app:build']; + + // Should include test-lib files in the hash plan due to the dependency + expect(testAppPlan).toContain('file:libs/test-lib/project.json'); + expect(testAppPlan).toContain('file:libs/test-lib/src/index.ts'); + + // Should include both project configurations + expect(testAppPlan).toContain('test-app:ProjectConfiguration'); + expect(testAppPlan).toContain('test-lib:ProjectConfiguration'); + + // Should include both TypeScript configurations + expect(testAppPlan).toContain('test-app:TsConfig'); + expect(testAppPlan).toContain('test-lib:TsConfig'); + + // Should include common files + expect(testAppPlan).toContain('env:NX_CLOUD_ENCRYPTION_KEY'); + expect(testAppPlan).toContain('file:nx.json'); + + // Should include test-app files + expect(testAppPlan).toContain('file:apps/test-app/project.json'); + expect(testAppPlan).toContain('file:apps/test-app/src/main.ts'); + }); + + it('should throw error for non-existent project', () => { + expect(() => { + inspector.inspectHashPlan(['non-existent-project'], ['build']); + }).toThrow(); + }); + + it('should throw error for non-existent target', () => { + expect(() => { + inspector.inspectHashPlan(['test-app'], ['non-existent-target']); + }).toThrow(); + }); + }); +}); diff --git a/packages/nx/src/hasher/hash-plan-inspector.ts b/packages/nx/src/hasher/hash-plan-inspector.ts new file mode 100644 index 0000000000..386dfea998 --- /dev/null +++ b/packages/nx/src/hasher/hash-plan-inspector.ts @@ -0,0 +1,122 @@ +import { + HashPlanInspector as NativeHashPlanInspector, + HashPlanner, + transferProjectGraph, + ExternalObject, + ProjectGraph as NativeProjectGraph, +} from '../native'; +import { readNxJson, NxJsonConfiguration } from '../config/nx-json'; +import { transformProjectGraphForRust } from '../native/transform-objects'; +import { ProjectGraph } from '../config/project-graph'; +import { workspaceRoot } from '../utils/workspace-root'; +import { createProjectRootMappings } from '../project-graph/utils/find-project-for-path'; +import { createTaskGraph } from '../tasks-runner/create-task-graph'; +import type { Target } from '../command-line/run/run'; +import { TargetDependencies } from '../config/nx-json'; +import { TargetDependencyConfig } from '../config/workspace-json-project-json'; +import { splitArgsIntoNxArgsAndOverrides } from '../utils/command-line-utils'; +import { getNxWorkspaceFilesFromContext } from '../utils/workspace-context'; + +export class HashPlanInspector { + private readonly projectGraphRef: ExternalObject; + private planner: HashPlanner; + private inspector: NativeHashPlanInspector; + private readonly nxJson: NxJsonConfiguration; + + constructor( + private projectGraph: ProjectGraph, + private readonly workspaceRootPath: string = workspaceRoot, + nxJson?: NxJsonConfiguration + ) { + this.nxJson = nxJson ?? readNxJson(this.workspaceRootPath); + this.projectGraphRef = transferProjectGraph( + transformProjectGraphForRust(this.projectGraph) + ); + this.planner = new HashPlanner(this.nxJson, this.projectGraphRef); + } + + async init() { + const projectRootMap = createProjectRootMappings(this.projectGraph.nodes); + const map = Object.fromEntries(projectRootMap.entries()); + const { externalReferences } = await getNxWorkspaceFilesFromContext( + this.workspaceRootPath, + map, + false + ); + this.inspector = new NativeHashPlanInspector( + externalReferences.allWorkspaceFiles, + this.projectGraphRef, + externalReferences.projectFiles + ); + } + + /** + * This is a lower level method which will inspect the hash plan for a set of tasks. + */ + inspectHashPlan( + projectNames: string[], + targets: string[], + configuration?: string, + overrides: Object = {}, + extraTargetDependencies: TargetDependencies = {}, + excludeTaskDependencies: boolean = false + ) { + const taskGraph = createTaskGraph( + this.projectGraph, + extraTargetDependencies, + projectNames, + targets, + configuration, + overrides, + excludeTaskDependencies + ); + // Generate task IDs for ALL tasks in the task graph (including dependencies) + const taskIds = Object.keys(taskGraph.tasks); + + const plansReference = this.planner.getPlansReference(taskIds, taskGraph); + + return this.inspector.inspect(plansReference); + } + + /** + * This inspects tasks involved in the execution of a task, including its dependencies by default. + */ + inspectTask( + { project, target, configuration }: Target, + parsedArgs: { [k: string]: any } = {}, + extraTargetDependencies: Record< + string, + (TargetDependencyConfig | string)[] + > = {}, + excludeTaskDependencies: boolean = false + ) { + // Mirror the exact flow from run-one.ts + const { nxArgs, overrides } = splitArgsIntoNxArgsAndOverrides( + { + ...parsedArgs, + configuration: configuration, + targets: [target], + }, + 'run-one', + { printWarnings: false }, + this.nxJson + ); + + // Create task graph exactly like run-one.ts does via createTaskGraphAndRunValidations + const taskGraph = createTaskGraph( + this.projectGraph, + extraTargetDependencies, + [project], + nxArgs.targets, + nxArgs.configuration, + overrides, + excludeTaskDependencies + ); + + // Generate task IDs for ALL tasks in the task graph (including dependencies) + const taskIds = Object.keys(taskGraph.tasks); + + const plansReference = this.planner.getPlansReference(taskIds, taskGraph); + return this.inspector.inspect(plansReference); + } +} diff --git a/packages/nx/src/native/glob.rs b/packages/nx/src/native/glob.rs index 6f4e995815..ee2c751e72 100644 --- a/packages/nx/src/native/glob.rs +++ b/packages/nx/src/native/glob.rs @@ -1,3 +1,4 @@ +pub mod glob_files; mod glob_group; mod glob_parser; pub mod glob_transform; diff --git a/packages/nx/src/native/workspace/config_files.rs b/packages/nx/src/native/glob/glob_files.rs similarity index 97% rename from packages/nx/src/native/workspace/config_files.rs rename to packages/nx/src/native/glob/glob_files.rs index 146abf3831..c069c9ad28 100644 --- a/packages/nx/src/native/workspace/config_files.rs +++ b/packages/nx/src/native/glob/glob_files.rs @@ -4,7 +4,7 @@ use crate::native::glob::build_glob_set; use crate::native::types::FileData; /// Get workspace config files based on provided globs -pub(super) fn glob_files( +pub fn glob_files( files: &[FileData], globs: Vec, exclude: Option>, diff --git a/packages/nx/src/native/index.d.ts b/packages/nx/src/native/index.d.ts index c7a25b48fc..52447e28cd 100644 --- a/packages/nx/src/native/index.d.ts +++ b/packages/nx/src/native/index.d.ts @@ -41,6 +41,11 @@ export declare class FileLock { lock(): void } +export declare class HashPlanInspector { + constructor(allWorkspaceFiles: ExternalObject>, projectGraph: ExternalObject, projectFileMap: ExternalObject>>) + inspect(hashPlans: ExternalObject>>): Record +} + export declare class HashPlanner { constructor(nxJson: NxJson, projectGraph: ExternalObject) getPlans(taskIds: Array, taskGraph: TaskGraph): Record diff --git a/packages/nx/src/native/native-bindings.js b/packages/nx/src/native/native-bindings.js index 63c62c84b3..87c94ffef7 100644 --- a/packages/nx/src/native/native-bindings.js +++ b/packages/nx/src/native/native-bindings.js @@ -364,6 +364,7 @@ if (!nativeBinding) { module.exports.AppLifeCycle = nativeBinding.AppLifeCycle module.exports.ChildProcess = nativeBinding.ChildProcess module.exports.FileLock = nativeBinding.FileLock +module.exports.HashPlanInspector = nativeBinding.HashPlanInspector module.exports.HashPlanner = nativeBinding.HashPlanner module.exports.HttpRemoteCache = nativeBinding.HttpRemoteCache module.exports.ImportResult = nativeBinding.ImportResult diff --git a/packages/nx/src/native/tasks/hash_plan_inspector.rs b/packages/nx/src/native/tasks/hash_plan_inspector.rs new file mode 100644 index 0000000000..0b321db570 --- /dev/null +++ b/packages/nx/src/native/tasks/hash_plan_inspector.rs @@ -0,0 +1,81 @@ +use crate::native::project_graph::types::ProjectGraph; +use crate::native::tasks::hashers::{collect_project_files, get_workspace_files}; +use crate::native::tasks::types::HashInstruction; +use crate::native::types::FileData; +use anyhow::anyhow; +use napi::bindgen_prelude::*; +use rayon::prelude::*; +use std::collections::HashMap; + +#[napi] +pub struct HashPlanInspector { + all_workspace_files: External>, + project_graph: External, + project_file_map: External>>, +} + +#[napi] +impl HashPlanInspector { + #[napi(constructor)] + pub fn new( + all_workspace_files: External>, + project_graph: External, + project_file_map: External>>, + ) -> Self { + Self { + all_workspace_files, + project_graph, + project_file_map, + } + } + + #[napi(ts_return_type = "Record")] + pub fn inspect( + &self, + hash_plans: External>>, + ) -> anyhow::Result>> { + let a: Vec<(&String, Vec)> = hash_plans + .iter() + .flat_map(|(task_id, instructions)| { + instructions + .iter() + .map(move |instruction| (task_id, instruction)) + }) + .par_bridge() + .map(|(task_id, instruction)| match instruction { + HashInstruction::WorkspaceFileSet(workspace_file_set) => { + let files = get_workspace_files(workspace_file_set, &self.all_workspace_files)? + .map(|x| format!("file:{}", x.file)) + .collect(); + + Ok::<_, anyhow::Error>((task_id, files)) + } + HashInstruction::ProjectFileSet(project_name, file_sets) => { + let project = self + .project_graph + .nodes + .get(project_name) + .ok_or_else(|| anyhow!("project {} not found", project_name))?; + + let files = collect_project_files( + project_name, + &project.root, + file_sets, + &self.project_file_map, + )? + .iter() + .map(|x| format!("file:{}", x.file)) + .collect(); + Ok::<_, anyhow::Error>((task_id, files)) + } + _ => Ok::<_, anyhow::Error>((task_id, vec![instruction.to_string()])), + }) + .collect::>()?; + + Ok(a.into_iter() + .fold(HashMap::new(), |mut acc, (task_id, files)| { + acc.entry(task_id.clone()).or_default().extend(files); + acc + })) + } +} diff --git a/packages/nx/src/native/tasks/hashers/hash_project_files.rs b/packages/nx/src/native/tasks/hashers/hash_project_files.rs index d524b4d173..4fc62819c6 100644 --- a/packages/nx/src/native/tasks/hashers/hash_project_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_project_files.rs @@ -13,7 +13,8 @@ pub fn hash_project_files( project_file_map: &HashMap>, ) -> Result { let _span = trace_span!("hash_project_files", project_name).entered(); - let collected_files = collect_files(project_name, project_root, file_sets, project_file_map)?; + let collected_files = + collect_project_files(project_name, project_root, file_sets, project_file_map)?; trace!("collected_files: {:?}", collected_files.len()); let mut hasher = xxhash_rust::xxh3::Xxh3::new(); for file in collected_files { @@ -24,7 +25,7 @@ pub fn hash_project_files( } /// base function that should be testable (to make sure that we're getting the proper files back) -fn collect_files<'a>( +pub fn collect_project_files<'a>( project_name: &str, project_root: &str, file_sets: &[String], @@ -100,11 +101,11 @@ mod tests { ], ); - let result = collect_files(proj_name, proj_root, file_sets, &file_map).unwrap(); + let result = collect_project_files(proj_name, proj_root, file_sets, &file_map).unwrap(); assert_eq!(result, vec![&tsfile_1, &tsfile_2]); - let result = collect_files( + let result = collect_project_files( proj_name, proj_root, &["!{projectRoot}/**/*.spec.ts".into()], diff --git a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs index 53d07ce720..072aa0586f 100644 --- a/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs +++ b/packages/nx/src/native/tasks/hashers/hash_workspace_files.rs @@ -1,18 +1,16 @@ +use rayon::prelude::*; use std::sync::Arc; +use crate::native::glob::build_glob_set; +use crate::native::glob::glob_files::glob_files; +use crate::native::hasher::hash; +use crate::native::types::FileData; use anyhow::*; use dashmap::DashMap; use tracing::{debug, debug_span, trace, warn}; -use crate::native::types::FileData; -use crate::native::{glob::build_glob_set, hasher::hash}; - -pub fn hash_workspace_files( - workspace_file_sets: &[String], - all_workspace_files: &[FileData], - cache: Arc>, -) -> Result { - let globs: Vec = workspace_file_sets +fn globs_from_workspace_inputs(workspace_file_sets: &[String]) -> Vec { + workspace_file_sets .iter() .inspect(|&x| trace!("Workspace file set: {}", x)) .filter_map(|x| { @@ -33,7 +31,23 @@ pub fn hash_workspace_files( None } }) - .collect(); + .collect() +} + +pub fn get_workspace_files<'a, 'b>( + workspace_file_sets: &'a [String], + all_workspace_files: &'b [FileData], +) -> napi::Result> { + let globs = globs_from_workspace_inputs(workspace_file_sets); + glob_files(all_workspace_files, globs, None) +} + +pub fn hash_workspace_files( + workspace_file_sets: &[String], + all_workspace_files: &[FileData], + cache: Arc>, +) -> Result { + let globs = globs_from_workspace_inputs(workspace_file_sets); if globs.is_empty() { return Ok(hash(b"")); @@ -133,7 +147,6 @@ mod test { file: "packages/project/project.json".into(), hash: "abc".into(), }; - for i in 0..1000 { let result = hash_workspace_files( &["{workspaceRoot}/**/*".to_string()], diff --git a/packages/nx/src/native/tasks/mod.rs b/packages/nx/src/native/tasks/mod.rs index 20af6ddb82..964b67ab2f 100644 --- a/packages/nx/src/native/tasks/mod.rs +++ b/packages/nx/src/native/tasks/mod.rs @@ -1,4 +1,5 @@ mod dep_outputs; +mod hash_plan_inspector; mod hash_planner; pub mod hashers; mod inputs; diff --git a/packages/nx/src/native/tasks/task_hasher.rs b/packages/nx/src/native/tasks/task_hasher.rs index 4c52843eaa..0edd20f363 100644 --- a/packages/nx/src/native/tasks/task_hasher.rs +++ b/packages/nx/src/native/tasks/task_hasher.rs @@ -21,7 +21,7 @@ use crate::native::{ }; use anyhow::anyhow; use dashmap::DashMap; -use napi::bindgen_prelude::{Buffer, External}; +use napi::bindgen_prelude::*; use rayon::prelude::*; use tracing::{debug, trace, trace_span}; diff --git a/packages/nx/src/native/tasks/types.rs b/packages/nx/src/native/tasks/types.rs index 0e3fdfb776..ff75c0d11e 100644 --- a/packages/nx/src/native/tasks/types.rs +++ b/packages/nx/src/native/tasks/types.rs @@ -42,7 +42,7 @@ pub struct TaskGraph { pub dependencies: HashMap>, } -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] pub enum HashInstruction { WorkspaceFileSet(Vec), Runtime(String), diff --git a/packages/nx/src/native/workspace/context.rs b/packages/nx/src/native/workspace/context.rs index f066ecf52f..834cf38f85 100644 --- a/packages/nx/src/native/workspace/context.rs +++ b/packages/nx/src/native/workspace/context.rs @@ -4,6 +4,7 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; +use crate::native::glob::glob_files::glob_files; use crate::native::hasher::hash; use crate::native::logger::enable_logger; use crate::native::project_graph::utils::{ProjectRootMappings, find_project_for_path}; @@ -14,7 +15,7 @@ use crate::native::workspace::files_hashing::{full_files_hash, selective_files_h use crate::native::workspace::types::{ FileMap, NxWorkspaceFilesExternals, ProjectFiles, UpdatedWorkspaceFiles, }; -use crate::native::workspace::{config_files, types::NxWorkspaceFiles, workspace_files}; +use crate::native::workspace::{types::NxWorkspaceFiles, workspace_files}; use napi::bindgen_prelude::External; use rayon::prelude::*; use tracing::{trace, warn}; @@ -229,7 +230,7 @@ impl WorkspaceContext { exclude: Option>, ) -> napi::Result> { let file_data = self.all_file_data(); - let globbed_files = config_files::glob_files(&file_data, globs, exclude)?; + let globbed_files = glob_files(&file_data, globs, exclude)?; Ok(globbed_files.map(|file| file.file.to_owned()).collect()) } @@ -248,8 +249,7 @@ impl WorkspaceContext { globs .into_iter() .map(|glob| { - let globbed_files = - config_files::glob_files(&file_data, vec![glob], exclude.clone())?; + let globbed_files = glob_files(&file_data, vec![glob], exclude.clone())?; Ok(globbed_files.map(|file| file.file.to_owned()).collect()) }) .collect() @@ -264,8 +264,7 @@ impl WorkspaceContext { let hashes = glob_groups .into_iter() .map(|globs| { - let globbed_files = - config_files::glob_files(files, globs, None)?.collect::>(); + let globbed_files = glob_files(files, globs, None)?.collect::>(); let mut hasher = xxh3::Xxh3::new(); for file in globbed_files { hasher.update(file.file.as_bytes()); @@ -285,7 +284,7 @@ impl WorkspaceContext { exclude: Option>, ) -> napi::Result { let files = &self.all_file_data(); - let globbed_files = config_files::glob_files(files, globs, exclude)?.collect::>(); + let globbed_files = glob_files(files, globs, exclude)?.collect::>(); let mut hasher = xxh3::Xxh3::new(); for file in globbed_files { diff --git a/packages/nx/src/native/workspace/mod.rs b/packages/nx/src/native/workspace/mod.rs index 4a3e0b4c51..c130d2faaa 100644 --- a/packages/nx/src/native/workspace/mod.rs +++ b/packages/nx/src/native/workspace/mod.rs @@ -3,7 +3,6 @@ use crate::native::workspace::types::NxWorkspaceFilesExternals; use napi::bindgen_prelude::External; use std::collections::HashMap; -pub mod config_files; pub mod context; mod errors; mod files_archive; diff --git a/packages/nx/src/utils/workspace-context.ts b/packages/nx/src/utils/workspace-context.ts index 6cf6ef5fbc..293479a645 100644 --- a/packages/nx/src/utils/workspace-context.ts +++ b/packages/nx/src/utils/workspace-context.ts @@ -24,9 +24,10 @@ export function setupWorkspaceContext(workspaceRoot: string) { export async function getNxWorkspaceFilesFromContext( workspaceRoot: string, - projectRootMap: Record + projectRootMap: Record, + useDaemonProcess: boolean = true ) { - if (isOnDaemon() || !daemonClient.enabled()) { + if (!useDaemonProcess || isOnDaemon() || !daemonClient.enabled()) { ensureContextAvailable(workspaceRoot); return workspaceContext.getWorkspaceFiles(projectRootMap); }