diff --git a/e2e/affected.test.ts b/e2e/affected.test.ts index 47fc018bd7..49c01adc08 100644 --- a/e2e/affected.test.ts +++ b/e2e/affected.test.ts @@ -240,4 +240,54 @@ forEachCli(() => { expect(interpolatedTests).toContain(`Running target \"test\" succeeded`); }, 1000000); }); + + describe('build in the right order', () => { + let myapp, mypublishablelib; + beforeEach(() => { + ensureProject(); + + // create my app depending on mypublishablelib + myapp = uniq('myapp'); + mypublishablelib = uniq('mypublishablelib'); + runCLI(`generate @nrwl/angular:app ${myapp}`); + runCLI(`generate @nrwl/angular:lib ${mypublishablelib} --publishable`); + updateFile( + `apps/${myapp}/src/app/app.component.spec.ts`, + ` + import '@proj/${mypublishablelib}'; + describe('sample test', () => { + it('should test', () => { + expect(1).toEqual(1); + }); + }); + ` + ); + }); + + it('should wait for deps to be built before continuing', () => { + const build = runCommand( + `npm run affected:build -- --files="apps/${myapp}/src/main.ts,libs/${mypublishablelib}/src/index.ts" --parallel` + ); + console.log(build); + // make sure that the package is done building before we start building the app + expect( + build.indexOf('Built Angular Package!') < + build.indexOf(`"build" "${myapp}"`) + ).toBeTruthy(); + }); + + it('should not invoke build for projects who deps fail', () => { + updateFile( + `libs/${mypublishablelib}/src/index.ts`, + ` + const x: number = 'string'; + ` + ); + + const build = runCommand( + `npm run affected:build -- --files="apps/${myapp}/src/main.ts,libs/${mypublishablelib}/src/index.ts" --parallel` + ); + expect(build.indexOf(`"build" "${myapp}"`)).toEqual(-1); + }); + }); }); diff --git a/packages/insights/src/insights-task-runner.ts b/packages/insights/src/insights-task-runner.ts index a2127a7c76..f55b9007ef 100644 --- a/packages/insights/src/insights-task-runner.ts +++ b/packages/insights/src/insights-task-runner.ts @@ -10,7 +10,6 @@ import { DefaultTasksRunnerOptions } from '@nrwl/workspace/src/tasks-runner/default-tasks-runner'; import * as fs from 'fs'; -import { TasksMap } from '@nrwl/workspace/src/tasks-runner/run-command'; import { ProjectGraph } from '@nrwl/workspace/src/core/project-graph'; const axios = require('axios'); @@ -20,7 +19,6 @@ interface InsightsTaskRunnerOptions extends DefaultTasksRunnerOptions { type Context = { projectGraph: ProjectGraph; - tasksMap: TasksMap; target: string; }; diff --git a/packages/workspace/src/tasks-runner/default-tasks-runner.spec.ts b/packages/workspace/src/tasks-runner/default-tasks-runner.spec.ts index 807181ccf7..44b9b30049 100644 --- a/packages/workspace/src/tasks-runner/default-tasks-runner.spec.ts +++ b/packages/workspace/src/tasks-runner/default-tasks-runner.spec.ts @@ -1,7 +1,11 @@ -import defaultTaskRunner from './default-tasks-runner'; -import { AffectedEventType, Task } from './tasks-runner'; -jest.mock('npm-run-all', () => jest.fn()); +import defaultTaskRunner, { + splitTasksIntoStages +} from './default-tasks-runner'; +import { AffectedEventType } from './tasks-runner'; import * as runAll from 'npm-run-all'; +import { DependencyType } from '@nrwl/workspace/src/core/project-graph'; + +jest.mock('npm-run-all', () => jest.fn()); jest.mock('../core/file-utils', () => ({ cliCommand: () => 'nx' })); @@ -134,4 +138,77 @@ describe('defaultTasksRunner', () => { complete: done }); }); + + describe('splitTasksIntoStages', () => { + it('should return empty for an empty array', () => { + const stages = splitTasksIntoStages([], { nodes: {}, dependencies: {} }); + expect(stages).toEqual([]); + }); + + it('should split tasks into stages based on their dependencies', () => { + const stages = splitTasksIntoStages( + [ + { + target: { project: 'parent' } + }, + { + target: { project: 'child1' } + }, + { + target: { project: 'child2' } + }, + { + target: { project: 'grandparent' } + } + ] as any, + { + nodes: {}, + dependencies: { + child1: [], + child2: [], + parent: [ + { + source: 'parent', + target: 'child1', + type: DependencyType.static + }, + { + source: 'parent', + target: 'child2', + type: DependencyType.static + } + ], + grandparent: [ + { + source: 'grandparent', + target: 'parent', + type: DependencyType.static + } + ] + } + } + ); + + expect(stages).toEqual([ + [ + { + target: { project: 'child1' } + }, + { + target: { project: 'child2' } + } + ], + [ + { + target: { project: 'parent' } + } + ], + [ + { + target: { project: 'grandparent' } + } + ] + ]); + }); + }); }); diff --git a/packages/workspace/src/tasks-runner/default-tasks-runner.ts b/packages/workspace/src/tasks-runner/default-tasks-runner.ts index 03a24425df..cb2b2298fb 100644 --- a/packages/workspace/src/tasks-runner/default-tasks-runner.ts +++ b/packages/workspace/src/tasks-runner/default-tasks-runner.ts @@ -11,46 +11,79 @@ import { output } from '../utils/output'; import { readJsonFile } from '../utils/fileutils'; import { getCommand } from './utils'; import { cliCommand } from '../core/file-utils'; +import { ProjectGraph } from '../core/project-graph'; export interface DefaultTasksRunnerOptions { parallel?: boolean; maxParallel?: number; } +function taskDependsOnDeps( + task: Task, + deps: Task[], + projectGraph: ProjectGraph +) { + function hasDep(source: string, target: string, visitedProjects: string[]) { + if (!projectGraph.dependencies[source]) { + return false; + } + + if (projectGraph.dependencies[source].find(d => d.target === target)) { + return true; + } + + return !!projectGraph.dependencies[source].find(r => { + if (visitedProjects.indexOf(r.target) > -1) return null; + return hasDep(r.target, target, [...visitedProjects, r.target]); + }); + } + + return !!deps.find(dep => + hasDep(task.target.project, dep.target.project, []) + ); +} + +function topologicallySortTasks(tasks: Task[], projectGraph: ProjectGraph) { + const sortedTasks = [...tasks]; + sortedTasks.sort((a, b) => { + if (taskDependsOnDeps(a, [b], projectGraph)) return 1; + if (taskDependsOnDeps(b, [a], projectGraph)) return -1; + return 0; + }); + return sortedTasks; +} + +export function splitTasksIntoStages( + tasks: Task[], + projectGraph: ProjectGraph +) { + if (tasks.length === 0) return []; + const res = []; + topologicallySortTasks(tasks, projectGraph).forEach(t => { + const stageWithNoDeps = res.find( + tasksInStage => !taskDependsOnDeps(t, tasksInStage, projectGraph) + ); + if (stageWithNoDeps) { + stageWithNoDeps.push(t); + } else { + res.push([t]); + } + }); + return res; +} + export const defaultTasksRunner: TasksRunner = ( tasks: Task[], - options: DefaultTasksRunnerOptions + options: DefaultTasksRunnerOptions, + context: { target: string; projectGraph: ProjectGraph } ): Observable => { - const cli = cliCommand(); - const isYarn = basename(process.env.npm_execpath || 'npm').startsWith('yarn'); - assertPackageJsonScriptExists(cli); - const commands = tasks.map(t => getCommand(cli, isYarn, t)); return new Observable(subscriber => { - runAll(commands, { - parallel: options.parallel || false, - maxParallel: options.maxParallel || 3, - continueOnError: true, - stdin: process.stdin, - stdout: process.stdout, - stderr: process.stderr - }) - .then(() => { - tasks.forEach(task => { - subscriber.next({ - task: task, - type: AffectedEventType.TaskComplete, - success: true - }); - }); - }) + runTasks(tasks, options, context) + .then(data => data.forEach(d => subscriber.next(d))) .catch(e => { - e.results.forEach((result, i) => { - subscriber.next({ - task: tasks[i], - type: AffectedEventType.TaskComplete, - success: result.code === 0 - }); - }); + console.error('Unexpected error:'); + console.error(e); + process.exit(1); }) .finally(() => { subscriber.complete(); @@ -60,6 +93,60 @@ export const defaultTasksRunner: TasksRunner = ( }); }; +async function runTasks( + tasks: Task[], + options: DefaultTasksRunnerOptions, + context: { target: string; projectGraph: ProjectGraph } +): Promise> { + const cli = cliCommand(); + assertPackageJsonScriptExists(cli); + const isYarn = basename(process.env.npm_execpath || 'npm').startsWith('yarn'); + const stages = + context.target === 'build' + ? splitTasksIntoStages(tasks, context.projectGraph) + : [tasks]; + + const res = []; + for (let i = 0; i < stages.length; ++i) { + const tasksInStage = stages[i]; + try { + const commands = tasksInStage.map(t => getCommand(cli, isYarn, t)); + await runAll(commands, { + parallel: options.parallel || false, + maxParallel: options.maxParallel || 3, + continueOnError: true, + stdin: process.stdin, + stdout: process.stdout, + stderr: process.stderr + }); + res.push(...tasksToStatuses(tasksInStage, true)); + } catch (e) { + e.results.forEach((result, i) => { + res.push({ + task: tasksInStage[i], + type: AffectedEventType.TaskComplete, + success: result.code === 0 + }); + }); + res.push(...markStagesAsNotSuccessful(stages.splice(i + 1))); + return res; + } + } + return res; +} + +function markStagesAsNotSuccessful(stages: Task[][]) { + return stages.reduce((m, c) => [...m, ...tasksToStatuses(c, false)], []); +} + +function tasksToStatuses(tasks: Task[], success: boolean) { + return tasks.map(task => ({ + task, + type: AffectedEventType.TaskComplete, + success + })); +} + function assertPackageJsonScriptExists(cli: string) { // Make sure the `package.json` has the `nx: "nx"` command needed by `npm-run-all` const packageJson = readJsonFile('./package.json'); diff --git a/packages/workspace/src/tasks-runner/run-command.ts b/packages/workspace/src/tasks-runner/run-command.ts index 0db548bbb1..4e9c43602a 100644 --- a/packages/workspace/src/tasks-runner/run-command.ts +++ b/packages/workspace/src/tasks-runner/run-command.ts @@ -12,13 +12,8 @@ import { DefaultReporter, ReporterArgs } from './default-reporter'; import * as yargs from 'yargs'; import { ProjectGraph, ProjectGraphNode } from '../core/project-graph'; import { Environment, NxJson } from '../core/shared-interfaces'; -import { projectHasTargetAndConfiguration } from '../utils/project-has-target-and-configuration'; import { NxArgs } from '@nrwl/workspace/src/command-line/utils'; -export interface TasksMap { - [projectName: string]: { [targetName: string]: Task }; -} - type RunArgs = yargs.Arguments & ReporterArgs; export function runCommand( @@ -39,25 +34,6 @@ export function runCommand( }) ); - const tasksMap: TasksMap = {}; - Object.entries(projectGraph.nodes).forEach(([projectName, project]) => { - const runnable = projectHasTargetAndConfiguration( - project, - nxArgs.target, - nxArgs.configuration - ); - if (runnable) { - tasksMap[projectName] = { - [nxArgs.target]: createTask({ - project: project, - target: nxArgs.target, - configuration: nxArgs.configuration, - overrides: overrides - }) - }; - } - }); - const { tasksRunner, tasksOptions } = getRunner( nxArgs.runner, nxJson, @@ -65,8 +41,7 @@ export function runCommand( ); tasksRunner(tasks, tasksOptions, { target: nxArgs.target, - projectGraph, - tasksMap + projectGraph }).subscribe({ next: (event: TaskCompleteEvent) => { switch (event.type) { @@ -107,17 +82,14 @@ export function createTask({ configuration, overrides }: TaskParams): Task { + const qualifiedTarget = { + project: project.name, + target, + configuration + }; return { - id: getId({ - project: project.name, - target: target, - configuration: configuration - }), - target: { - project: project.name, - target, - configuration - }, + id: getId(qualifiedTarget), + target: qualifiedTarget, overrides: interpolateOverrides(overrides, project.name, project.data) }; } diff --git a/packages/workspace/src/tasks-runner/tasks-runner.ts b/packages/workspace/src/tasks-runner/tasks-runner.ts index 5ce45c1308..f608d80366 100644 --- a/packages/workspace/src/tasks-runner/tasks-runner.ts +++ b/packages/workspace/src/tasks-runner/tasks-runner.ts @@ -25,14 +25,9 @@ export interface TaskCompleteEvent extends AffectedEvent { export type TasksRunner = ( tasks: Task[], - options?: T, + options: T, context?: { target?: string; projectGraph: ProjectGraph; - tasksMap: { - [projectName: string]: { - [targetName: string]: Task; - }; - }; } ) => Observable;