From 99fcb7873eddfaac04e87367777e51aae55658ab Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Wed, 27 Apr 2022 12:44:48 -0400 Subject: [PATCH] feat(nx-plugin): add --includeHasher option to generate hasher boilerplate (#9891) Co-authored-by: James Henry --- docs/generated/api-nx-devkit/index.md | 29 ++++++++++ docs/generated/packages/nx-plugin.json | 5 ++ docs/map.json | 2 +- docs/shared/tools-workspace-builders.md | 54 ++++++++++++++++++ e2e/nx-plugin/src/nx-plugin.test.ts | 9 ++- packages/devkit/index.ts | 2 + packages/jest/src/executors/jest/hasher.ts | 17 +----- .../src/generators/executor/executor.spec.ts | 55 +++++++++++++++++++ .../src/generators/executor/executor.ts | 42 ++++++++++++-- .../__fileName__/hasher.spec.ts__tmpl__ | 22 ++++++++ .../hasher/__fileName__/hasher.ts__tmpl__ | 12 ++++ .../src/generators/executor/schema.d.ts | 1 + .../src/generators/executor/schema.json | 5 ++ .../nx-plugin/src/generators/plugin/plugin.ts | 1 + packages/nx/src/config/misc-interfaces.ts | 21 ++++++- packages/nx/src/config/workspaces.ts | 3 +- 16 files changed, 254 insertions(+), 26 deletions(-) create mode 100644 packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.spec.ts__tmpl__ create mode 100644 packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.ts__tmpl__ diff --git a/docs/generated/api-nx-devkit/index.md b/docs/generated/api-nx-devkit/index.md index 4d41e3ba2f..3321ef43be 100644 --- a/docs/generated/api-nx-devkit/index.md +++ b/docs/generated/api-nx-devkit/index.md @@ -70,6 +70,7 @@ It only uses language primitives and immutable objects - [ExecutorContext](../../nx-devkit/index#executorcontext) - [ExecutorsJson](../../nx-devkit/index#executorsjson) - [GeneratorsJson](../../nx-devkit/index#generatorsjson) +- [HasherContext](../../nx-devkit/index#hashercontext) - [ImplicitJsonSubsetDependency](../../nx-devkit/index#implicitjsonsubsetdependency) - [MigrationsJson](../../nx-devkit/index#migrationsjson) - [NxAffectedConfig](../../nx-devkit/index#nxaffectedconfig) @@ -104,6 +105,7 @@ It only uses language primitives and immutable objects ### Workspace Type aliases +- [CustomHasher](../../nx-devkit/index#customhasher) - [Executor](../../nx-devkit/index#executor) - [Generator](../../nx-devkit/index#generator) - [GeneratorCallback](../../nx-devkit/index#generatorcallback) @@ -374,6 +376,12 @@ A plugin for Nx --- +### HasherContext + +• **HasherContext**: `Object` + +--- + ### ImplicitJsonSubsetDependency • **ImplicitJsonSubsetDependency**<`T`\>: `Object` @@ -512,6 +520,27 @@ A plugin for Nx ## Workspace Type aliases +### CustomHasher + +Ƭ **CustomHasher**: (`task`: [`Task`](../../nx-devkit/index#task), `context`: [`HasherContext`](../../nx-devkit/index#hashercontext)) => `Promise`<[`Hash`](../../nx-devkit/index#hash)\> + +#### Type declaration + +▸ (`task`, `context`): `Promise`<[`Hash`](../../nx-devkit/index#hash)\> + +##### Parameters + +| Name | Type | +| :-------- | :----------------------------------------------------- | +| `task` | [`Task`](../../nx-devkit/index#task) | +| `context` | [`HasherContext`](../../nx-devkit/index#hashercontext) | + +##### Returns + +`Promise`<[`Hash`](../../nx-devkit/index#hash)\> + +--- + ### Executor Ƭ **Executor**<`T`\>: (`options`: `T`, `context`: [`ExecutorContext`](../../nx-devkit/index#executorcontext)) => `Promise`<`Object`\> \| `AsyncIterableIterator`<`Object`\> diff --git a/docs/generated/packages/nx-plugin.json b/docs/generated/packages/nx-plugin.json index c1e3a1b845..a966ea0412 100644 --- a/docs/generated/packages/nx-plugin.json +++ b/docs/generated/packages/nx-plugin.json @@ -290,6 +290,11 @@ "enum": ["jest", "none"], "description": "Test runner to use for unit tests.", "default": "jest" + }, + "includeHasher": { + "type": "boolean", + "default": false, + "description": "Should the boilerplate for a custom hasher be generated?" } }, "required": ["project", "name"], diff --git a/docs/map.json b/docs/map.json index 041fec9d94..80f4bdadfa 100644 --- a/docs/map.json +++ b/docs/map.json @@ -1281,7 +1281,7 @@ "file": "shared/running-custom-commands" }, { - "name": "Creating Custom Builders", + "name": "Creating Custom Executors", "id": "creating-custom-builders", "file": "shared/tools-workspace-builders" } diff --git a/docs/shared/tools-workspace-builders.md b/docs/shared/tools-workspace-builders.md index 14cf604866..1c207ee047 100644 --- a/docs/shared/tools-workspace-builders.md +++ b/docs/shared/tools-workspace-builders.md @@ -213,3 +213,57 @@ export default async function multipleExecutor( For other ideas on how to create your own executors, you can always check out Nx's own open-source executors as well! (For example, our [cypress executor](https://github.com/nrwl/nx/blob/master/packages/cypress/src/executors/cypress/cypress.impl.ts)) + +## Using Custom Hashers + +For most executors, the default hashing in Nx makes sense. The output of the executor is dependent on the files in the project that it is being run for, or that project's dependencies, and nothing else. Changing a miscellaneous file at the workspace root will not affect that executor, and changing _*any*_ file inside of the project may affect the executor. When dealing with targets which only depend on a small subset of the files in a project, or may depend on arbitrary data that is not stored within the project, the default hasher may not make sense anymore. In these cases, the target will either experience more frequent cache misses than necessary or not be able to be cached. + +Executors can provide a custom hasher that Nx uses when determining if a target run should be a cache hit, or if it must be run. When generating an executor for a plugin, you can use `nx g @nrwl/nx-plugin:executor my-executor --project my-plugin --includeHasher` to automatically add a custom hasher. + +If you want to add a custom hasher manually, create a new file beside your executor's implementation. We will use `hasher.ts` as an example here. You'll also need to update `executors.json`, so that it resembles something like this: + +```json +{ + "executors": { + "echo": { + "implementation": "./src/executors/my-executor/executor", + "hasher": "./src/executors/my-executor/hasher", + "schema": "./src/executors/my-executor/schema.json" + } + } +} +``` + +This would allow you to write a custom function in `hasher.ts`, which Nx would use to calculate the target's hash. As an example, consider the below hasher which mimics the behavior of Nx's default hashing algorithm. + +```typescript +import { CustomHasher, Task, HasherContext } from '@nrwl/devkit'; + +export const mimicNxHasher: CustomHasher = async ( + task: Task, + context: HasherContext +) => { + return context.hasher.hashTaskWithDepsAndContext(task); +}; + +export default mimicNxHasher; +``` + +The hash function can do anything it wants, but it is important to remember that the hasher replaces the hashing done normally by Nx. If you change the hasher, Nx may return cache hits when you do not anticipate it. Imagine the below custom hasher: + +```typescript +import { CustomHasher, Task, HasherContext } from '@nrwl/devkit'; + +export const badHasher: CustomHasher = async ( + task: Task, + context: HasherContext +) => { + return { + value: 'my-static-hash', + }; +}; + +export default badHasher; +``` + +This hasher would never return a different hash, so every run of a task that consumes the executor would be a cache hit. It is important that anything that would change the result of your executor's implementation is accounted for in the hasher. diff --git a/e2e/nx-plugin/src/nx-plugin.test.ts b/e2e/nx-plugin/src/nx-plugin.test.ts index aa113d2a9f..430324ada0 100644 --- a/e2e/nx-plugin/src/nx-plugin.test.ts +++ b/e2e/nx-plugin/src/nx-plugin.test.ts @@ -147,7 +147,9 @@ describe('Nx Plugin', () => { const executor = uniq('executor'); runCLI(`generate @nrwl/nx-plugin:plugin ${plugin} --linter=eslint`); - runCLI(`generate @nrwl/nx-plugin:executor ${executor} --project=${plugin}`); + runCLI( + `generate @nrwl/nx-plugin:executor ${executor} --project=${plugin} --includeHasher` + ); const lintResults = runCLI(`lint ${plugin}`); expect(lintResults).toContain('All files pass linting.'); @@ -160,16 +162,19 @@ describe('Nx Plugin', () => { `libs/${plugin}/src/executors/${executor}/schema.d.ts`, `libs/${plugin}/src/executors/${executor}/schema.json`, `libs/${plugin}/src/executors/${executor}/executor.ts`, + `libs/${plugin}/src/executors/${executor}/hasher.ts`, `libs/${plugin}/src/executors/${executor}/executor.spec.ts`, `dist/libs/${plugin}/src/executors/${executor}/schema.d.ts`, `dist/libs/${plugin}/src/executors/${executor}/schema.json`, - `dist/libs/${plugin}/src/executors/${executor}/executor.js` + `dist/libs/${plugin}/src/executors/${executor}/executor.js`, + `dist/libs/${plugin}/src/executors/${executor}/hasher.js` ); const executorsJson = readJson(`libs/${plugin}/executors.json`); expect(executorsJson).toMatchObject({ executors: expect.objectContaining({ [executor]: { implementation: `./src/executors/${executor}/executor`, + hasher: `./src/executors/${executor}/hasher`, schema: `./src/executors/${executor}/schema.json`, description: `${executor} executor`, }, diff --git a/packages/devkit/index.ts b/packages/devkit/index.ts index 4a78298318..3c0cf827bf 100644 --- a/packages/devkit/index.ts +++ b/packages/devkit/index.ts @@ -38,6 +38,8 @@ export type { GeneratorsJson, ExecutorsJson, MigrationsJson, + CustomHasher, + HasherContext, } from 'nx/src/config/misc-interfaces'; /** diff --git a/packages/jest/src/executors/jest/hasher.ts b/packages/jest/src/executors/jest/hasher.ts index 770a4eefee..61728d17a0 100644 --- a/packages/jest/src/executors/jest/hasher.ts +++ b/packages/jest/src/executors/jest/hasher.ts @@ -1,21 +1,8 @@ -import { - NxJsonConfiguration, - ProjectGraph, - Task, - TaskGraph, - WorkspaceJsonConfiguration, - Hasher, - Hash, -} from '@nrwl/devkit'; +import { Task, Hash, HasherContext } from '@nrwl/devkit'; export default async function run( task: Task, - context: { - hasher: Hasher; - projectGraph: ProjectGraph; - taskGraph: TaskGraph; - workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration; - } + context: HasherContext ): Promise { const jestPluginConfig = context.workspaceConfig.pluginsConfig ? (context.workspaceConfig.pluginsConfig['@nrwl/jest'] as any) diff --git a/packages/nx-plugin/src/generators/executor/executor.spec.ts b/packages/nx-plugin/src/generators/executor/executor.spec.ts index 3b4803d22d..92a3662f01 100644 --- a/packages/nx-plugin/src/generators/executor/executor.spec.ts +++ b/packages/nx-plugin/src/generators/executor/executor.spec.ts @@ -21,6 +21,7 @@ describe('NxPlugin Executor Generator', () => { project: projectName, name: 'my-executor', unitTestRunner: 'jest', + includeHasher: false, }); expect( @@ -43,6 +44,7 @@ describe('NxPlugin Executor Generator', () => { name: 'my-executor', description: 'my-executor description', unitTestRunner: 'jest', + includeHasher: false, }); const executorJson = readJson(tree, 'libs/my-plugin/executors.json'); @@ -63,6 +65,7 @@ describe('NxPlugin Executor Generator', () => { project: projectName, name: 'my-executor', unitTestRunner: 'jest', + includeHasher: false, }); const executorsJson = readJson(tree, 'libs/my-plugin/executors.json'); @@ -78,6 +81,7 @@ describe('NxPlugin Executor Generator', () => { name: 'my-executor', description: 'my-executor custom description', unitTestRunner: 'jest', + includeHasher: false, }); const executorsJson = readJson(tree, 'libs/my-plugin/executors.json'); @@ -95,6 +99,7 @@ describe('NxPlugin Executor Generator', () => { name: 'my-executor', description: 'my-executor description', unitTestRunner: 'none', + includeHasher: true, }); expect( @@ -102,7 +107,57 @@ describe('NxPlugin Executor Generator', () => { 'libs/my-plugin/src/executors/my-executor/executor.spec.ts' ) ).toBeFalsy(); + expect( + tree.exists('libs/my-plugin/src/executors/my-executor/hasher.spec.ts') + ).toBeFalsy(); }); }); }); + + describe('--includeHasher', () => { + it('should generate hasher files', async () => { + await executorGenerator(tree, { + project: projectName, + name: 'my-executor', + includeHasher: true, + unitTestRunner: 'jest', + }); + expect( + tree.exists('libs/my-plugin/src/executors/my-executor/hasher.spec.ts') + ).toBeTruthy(); + expect( + tree + .read('libs/my-plugin/src/executors/my-executor/hasher.ts') + .toString() + ).toMatchInlineSnapshot(` + "import { CustomHasher } from '@nrwl/devkit'; + + /** + * This is a boilerplate custom hasher that matches + * the default Nx hasher. If you need to extend the behavior, + * you can consume workspace details from the context. + */ + export const myExecutorHasher: CustomHasher = async (task, context) => { + return context.hasher.hashTaskWithDepsAndContext(task) + }; + + export default myExecutorHasher; + " + `); + }); + + it('should update executors.json', async () => { + await executorGenerator(tree, { + project: projectName, + name: 'my-executor', + includeHasher: true, + unitTestRunner: 'jest', + }); + + const executorsJson = readJson(tree, 'libs/my-plugin/executors.json'); + expect(executorsJson.executors['my-executor'].hasher).toEqual( + './src/executors/my-executor/hasher' + ); + }); + }); }); diff --git a/packages/nx-plugin/src/generators/executor/executor.ts b/packages/nx-plugin/src/generators/executor/executor.ts index 67c5ea0818..5a90663a6e 100644 --- a/packages/nx-plugin/src/generators/executor/executor.ts +++ b/packages/nx-plugin/src/generators/executor/executor.ts @@ -5,6 +5,7 @@ import { generateFiles, updateJson, getWorkspaceLayout, + joinPathFragments, } from '@nrwl/devkit'; import type { Tree } from '@nrwl/devkit'; import type { Schema } from './schema'; @@ -13,6 +14,7 @@ import * as path from 'path'; interface NormalizedSchema extends Schema { fileName: string; className: string; + propertyName: string; projectRoot: string; projectSourceRoot: string; npmScope: string; @@ -31,7 +33,7 @@ function addFiles(host: Tree, options: NormalizedSchema) { if (options.unitTestRunner === 'none') { host.delete( - path.join( + joinPathFragments( options.projectSourceRoot, 'executors', options.fileName, @@ -41,6 +43,29 @@ function addFiles(host: Tree, options: NormalizedSchema) { } } +function addHasherFiles(host: Tree, options: NormalizedSchema) { + generateFiles( + host, + path.join(__dirname, './files/hasher'), + `${options.projectSourceRoot}/executors`, + { + ...options, + tmpl: '', + } + ); + + if (options.unitTestRunner === 'none') { + host.delete( + joinPathFragments( + options.projectSourceRoot, + 'executors', + options.fileName, + 'hasher.spec.ts' + ) + ); + } +} + function updateExecutorJson(host: Tree, options: NormalizedSchema) { let executorPath: string; if (host.exists(path.join(options.projectRoot, 'executors.json'))) { @@ -53,10 +78,15 @@ function updateExecutorJson(host: Tree, options: NormalizedSchema) { let executors = json.executors ?? json.builders; executors ||= {}; executors[options.name] = { - implementation: `./src/executors/${options.name}/executor`, - schema: `./src/executors/${options.name}/schema.json`, + implementation: `./src/executors/${options.fileName}/executor`, + schema: `./src/executors/${options.fileName}/schema.json`, description: options.description, }; + if (options.includeHasher) { + executors[ + options.name + ].hasher = `./src/executors/${options.fileName}/hasher`; + } json.executors = executors; return json; @@ -65,7 +95,7 @@ function updateExecutorJson(host: Tree, options: NormalizedSchema) { function normalizeOptions(host: Tree, options: Schema): NormalizedSchema { const { npmScope } = getWorkspaceLayout(host); - const { fileName, className } = names(options.name); + const { fileName, className, propertyName } = names(options.name); const { root: projectRoot, sourceRoot: projectSourceRoot } = readProjectConfiguration(host, options.project); @@ -81,6 +111,7 @@ function normalizeOptions(host: Tree, options: Schema): NormalizedSchema { ...options, fileName, className, + propertyName, description, projectRoot, projectSourceRoot, @@ -92,6 +123,9 @@ export async function executorGenerator(host: Tree, schema: Schema) { const options = normalizeOptions(host, schema); addFiles(host, options); + if (options.includeHasher) { + addHasherFiles(host, options); + } updateExecutorJson(host, options); } diff --git a/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.spec.ts__tmpl__ b/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.spec.ts__tmpl__ new file mode 100644 index 0000000000..7b8d147f37 --- /dev/null +++ b/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.spec.ts__tmpl__ @@ -0,0 +1,22 @@ +import { Hasher, HasherContext } from '@nrwl/devkit'; + +import { <%=propertyName%>Hasher } from './hasher'; + +describe('<%=propertyName%>Hasher', () => { + it('should generate hash', async () => { + const mockHasher: Hasher = { + hashTaskWithDepsAndContext: jest.fn().mockReturnValue({value: 'hashed-task'}) + } as unknown as Hasher + const hash = await <%=propertyName%>Hasher({ + id: 'my-task-id', + target: { + project: 'proj', + target: 'target' + }, + overrides: {} + }, { + hasher: mockHasher + } as unknown as HasherContext) + expect(hash).toEqual({value: 'hashed-task'}) + }) +}) \ No newline at end of file diff --git a/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.ts__tmpl__ b/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.ts__tmpl__ new file mode 100644 index 0000000000..c57cc29b5d --- /dev/null +++ b/packages/nx-plugin/src/generators/executor/files/hasher/__fileName__/hasher.ts__tmpl__ @@ -0,0 +1,12 @@ +import { CustomHasher } from '@nrwl/devkit'; + +/** + * This is a boilerplate custom hasher that matches + * the default Nx hasher. If you need to extend the behavior, + * you can consume workspace details from the context. + */ +export const <%=propertyName%>Hasher: CustomHasher = async (task, context) => { + return context.hasher.hashTaskWithDepsAndContext(task) +}; + +export default <%=propertyName%>Hasher; diff --git a/packages/nx-plugin/src/generators/executor/schema.d.ts b/packages/nx-plugin/src/generators/executor/schema.d.ts index c0088067da..7b67bda919 100644 --- a/packages/nx-plugin/src/generators/executor/schema.d.ts +++ b/packages/nx-plugin/src/generators/executor/schema.d.ts @@ -3,4 +3,5 @@ export interface Schema { name: string; description?: string; unitTestRunner: 'jest' | 'none'; + includeHasher: boolean; } diff --git a/packages/nx-plugin/src/generators/executor/schema.json b/packages/nx-plugin/src/generators/executor/schema.json index 18188901a6..5e6187353c 100644 --- a/packages/nx-plugin/src/generators/executor/schema.json +++ b/packages/nx-plugin/src/generators/executor/schema.json @@ -40,6 +40,11 @@ "enum": ["jest", "none"], "description": "Test runner to use for unit tests.", "default": "jest" + }, + "includeHasher": { + "type": "boolean", + "default": false, + "description": "Should the boilerplate for a custom hasher be generated?" } }, "required": ["project", "name"], diff --git a/packages/nx-plugin/src/generators/plugin/plugin.ts b/packages/nx-plugin/src/generators/plugin/plugin.ts index 69ad664e7b..7e416439bf 100644 --- a/packages/nx-plugin/src/generators/plugin/plugin.ts +++ b/packages/nx-plugin/src/generators/plugin/plugin.ts @@ -84,6 +84,7 @@ async function addFiles(host: Tree, options: NormalizedSchema) { project: options.name, name: 'build', unitTestRunner: options.unitTestRunner, + includeHasher: false, }); } diff --git a/packages/nx/src/config/misc-interfaces.ts b/packages/nx/src/config/misc-interfaces.ts index 6a738f5163..8cdd213ddf 100644 --- a/packages/nx/src/config/misc-interfaces.ts +++ b/packages/nx/src/config/misc-interfaces.ts @@ -1,9 +1,12 @@ -import type { NxJsonConfiguration } from './nx-json'; -import { TaskGraph } from './task-graph'; +import { Hash, Hasher } from '../hasher/hasher'; +import { ProjectGraph } from './project-graph'; +import { Task, TaskGraph } from './task-graph'; import { TargetConfiguration, WorkspaceJsonConfiguration, } from './workspace-json-project-json'; + +import type { NxJsonConfiguration } from './nx-json'; /** * A callback function that is executed after changes are made to the file system */ @@ -82,7 +85,7 @@ export interface ExecutorsJson { export interface ExecutorConfig { schema: any; - hasherFactory?: () => any; + hasherFactory?: () => CustomHasher; implementationFactory: () => Executor; batchImplementationFactory?: () => TaskGraphExecutor; } @@ -100,6 +103,18 @@ export type Executor = ( | Promise<{ success: boolean }> | AsyncIterableIterator<{ success: boolean }>; +export interface HasherContext { + hasher: Hasher; + projectGraph: ProjectGraph; + taskGraph: TaskGraph; + workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration; +} + +export type CustomHasher = ( + task: Task, + context: HasherContext +) => Promise; + /** * Implementation of a target of a project that handles multiple projects to be batched */ diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts index 7db0ec1c70..c09ef1df89 100644 --- a/packages/nx/src/config/workspaces.ts +++ b/packages/nx/src/config/workspaces.ts @@ -22,6 +22,7 @@ import { Generator, GeneratorsJson, ExecutorsJson, + CustomHasher, } from './misc-interfaces'; import { PackageJson } from '../utils/package-json'; @@ -126,7 +127,7 @@ export class Workspaces { : null; const hasherFactory = executorConfig.hasher - ? this.getImplementationFactory( + ? this.getImplementationFactory( executorConfig.hasher, executorsDir )