feat(core): refactor most of js graph work into its own folder (#15365)

This commit is contained in:
Jason Jean 2023-03-28 14:57:06 -04:00 committed by GitHub
parent 16b3fa0931
commit 8cf8f18c1c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
101 changed files with 551 additions and 462 deletions

View File

@ -12,7 +12,7 @@ packages/express/src/schematics/**/files/**/*.json
packages/nest/src/schematics/**/files/**/*.json
packages/react/src/schematics/**/files/**/*.json
packages/jest/src/schematics/**/files/**/*.json
packages/nx/src/lock-file/__fixtures__/**/*.*
packages/nx/src/plugins/js/lock-file/__fixtures__/**/*.*
packages/**/schematics/**/files/**/*.html
packages/**/generators/**/files/**/*.html
packages/nx/src/native/

View File

@ -26,7 +26,7 @@ import {
} from '@nrwl/e2e/utils';
import { exec, execSync } from 'child_process';
import * as http from 'http';
import { getLockFileName } from 'nx/src/lock-file/lock-file';
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
import { satisfies } from 'semver';
function getData(port, path = '/api'): Promise<any> {

View File

@ -9,7 +9,7 @@ import { DependencyType } from '@nrwl/devkit';
import * as parser from '@typescript-eslint/parser';
import { TSESLint } from '@typescript-eslint/utils';
import { vol } from 'memfs';
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
import enforceModuleBoundaries, {
RULE_NAME as enforceModuleBoundariesRuleName,
} from '../../src/rules/enforce-module-boundaries';

View File

@ -33,7 +33,7 @@ import {
isComboDepConstraint,
} from '../utils/runtime-lint-utils';
import { AST_NODE_TYPES, TSESTree } from '@typescript-eslint/utils';
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
import { basename, dirname, relative } from 'path';
import {
getBarrelEntryPointByImportScope,

View File

@ -13,7 +13,7 @@ import {
} from '@nrwl/devkit';
import { getPath, pathExists } from './graph-utils';
import { readFileIfExisting } from 'nx/src/utils/fileutils';
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
import {
findProjectForPath,
ProjectRootMappings,

View File

@ -11,5 +11,5 @@ export * from './utils/assets';
export * from './utils/package-json/update-package-json';
export { libraryGenerator } from './generators/library/library';
export { initGenerator } from './generators/init/init';
export { createLockFile } from 'nx/src/lock-file/lock-file';
export { createPackageJson } from 'nx/src/utils/create-package-json';
export { createLockFile } from 'nx/src/plugins/js/lock-file/lock-file';
export { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';

View File

@ -1,5 +1,8 @@
import { createLockFile, getLockFileName } from 'nx/src/lock-file/lock-file';
import { createPackageJson } from 'nx/src/utils/create-package-json';
import {
createLockFile,
getLockFileName,
} from 'nx/src/plugins/js/lock-file/lock-file';
import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
import {
ExecutorContext,
getOutputsForTargetAndConfiguration,

View File

@ -18,7 +18,7 @@ import { createNextConfigFile } from './lib/create-next-config-file';
import { checkPublicDirectory } from './lib/check-project';
import { NextBuildBuilderOptions } from '../../utils/types';
import { getLockFileName } from 'nx/src/lock-file/lock-file';
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
export default async function buildExecutor(
options: NextBuildBuilderOptions,

View File

@ -226,8 +226,8 @@ export { Hash, Hasher } from './hasher/hasher';
*/
export { cacheDir } from './utils/cache-directory';
import { createLockFile as _createLockFile } from './lock-file/lock-file';
import { createPackageJson as _createPackageJson } from './utils/create-package-json';
import { createLockFile as _createLockFile } from './plugins/js/lock-file/lock-file';
import { createPackageJson as _createPackageJson } from './plugins/js/package-json/create-package-json';
/**
* @category Package Manager

View File

@ -1,11 +1,88 @@
import { ProjectGraphProcessor } from '../../config/project-graph';
import { ProjectGraphBuilder } from '../../project-graph/project-graph-builder';
import { buildNpmPackageNodes } from './project-graph/build-nodes/build-npm-package-nodes';
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
import { readNxJson } from '../../config/configuration';
import { fileExists, readJsonFile } from '../../utils/fileutils';
import { PackageJson } from '../../utils/package-json';
import {
lockFileExists,
lockFileHash,
parseLockFile,
} from './lock-file/lock-file';
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json';
import { dirname, join } from 'path';
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
import { readFileSync, writeFileSync } from 'fs';
import { workspaceRoot } from '../../utils/workspace-root';
import { ensureDirSync } from 'fs-extra';
import { removeNpmNodes } from 'nx/src/plugins/js/lock-file/remove-npm-nodes';
export const processProjectGraph: ProjectGraphProcessor = (graph) => {
export const processProjectGraph: ProjectGraphProcessor = (graph, context) => {
const builder = new ProjectGraphBuilder(graph);
const lockHash = lockFileHash() ?? 'n/a';
// during the create-nx-workspace lock file might not exists yet
if (lockFileExists()) {
if (lockFileNeedsReprocessing(lockHash)) {
removeNpmNodes(graph, builder);
parseLockFile(builder);
}
writeLastProcessedLockfileHash(lockHash);
}
buildNpmPackageNodes(builder);
buildExplicitDependencies(jsPluginConfig(readNxJson()), context, builder);
return builder.getUpdatedProjectGraph();
};
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
function lockFileNeedsReprocessing(lockHash: string) {
try {
return readFileSync(lockFileHashFile).toString() !== lockHash;
} catch {
return true;
}
}
function writeLastProcessedLockfileHash(hash: string) {
ensureDirSync(dirname(lockFileHashFile));
writeFileSync(lockFileHashFile, hash);
}
function jsPluginConfig(nxJson: NxJsonConfiguration): NrwlJsPluginConfig {
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
return nxJson?.pluginsConfig?.['@nrwl/js'];
}
if (!fileExists(join(workspaceRoot, 'package.json'))) {
return {
analyzePackageJson: false,
analyzeSourceFiles: false,
};
}
const packageJson = readJsonFile<PackageJson>(
join(workspaceRoot, 'package.json')
);
const packageJsonDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
if (
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return { analyzePackageJson: true, analyzeSourceFiles: true };
} else {
return { analyzePackageJson: true, analyzeSourceFiles: false };
}
}

View File

@ -13,9 +13,9 @@ const packageNames = [];
function processNodeModules(path = '.') {
if (existsSync(`${path}/node_modules`)) {
readdirSync(`${path}/node_modules`).forEach(folder => {
readdirSync(`${path}/node_modules`).forEach((folder) => {
if (folder.startsWith('@')) {
readdirSync(`${path}/node_modules/${folder}`).forEach(subfolder => {
readdirSync(`${path}/node_modules/${folder}`).forEach((subfolder) => {
packageNames.push(`${path}/node_modules/${folder}/${subfolder}`);
processNodeModules(`${path}/node_modules/${folder}/${subfolder}`);
});
@ -29,15 +29,15 @@ function processNodeModules(path = '.') {
processNodeModules();
packageNames.forEach(path => {
packageNames.forEach((path) => {
const filePath = `${path}/package.json`;
if (existsSync(filePath)) {
const content = readFileSync(filePath, 'utf-8');
const peerDependencies = JSON.parse(content).peerDependencies;
const peerDependenciesMeta = JSON.parse(content).peerDependenciesMeta;
const output = JSON.stringify({
...peerDependencies && { peerDependencies },
...peerDependenciesMeta && { peerDependenciesMeta },
...(peerDependencies && { peerDependencies }),
...(peerDependenciesMeta && { peerDependenciesMeta }),
});
if (output === '{}') return;
report += `'${filePath.slice(2)}': '${output}',\n`;
@ -59,10 +59,10 @@ const existsSync = require('fs').existsSync;
let report = '';
const packageNames = [];
readdirSync('node_modules').forEach(folder => {
readdirSync('node_modules').forEach((folder) => {
if (folder === '.pnpm') return;
if (folder.startsWith('@')) {
readdirSync(`node_modules/${folder}`).forEach(subfolder => {
readdirSync(`node_modules/${folder}`).forEach((subfolder) => {
packageNames.push(`${folder}/${subfolder}`);
});
} else {
@ -70,7 +70,7 @@ readdirSync('node_modules').forEach(folder => {
}
});
packageNames.forEach(packageName => {
packageNames.forEach((packageName) => {
const path = `node_modules/${packageName}/package.json`;
if (existsSync(path)) {
const content = readFileSync(path, 'utf-8');

View File

@ -1103,9 +1103,7 @@
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"os": ["darwin"],
"peer": true,
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"

View File

@ -1103,9 +1103,7 @@
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"os": ["darwin"],
"peer": true,
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"

View File

@ -6,18 +6,22 @@
import { readFileSync, existsSync } from 'fs';
import { join } from 'path';
import { detectPackageManager, PackageManager } from '../utils/package-manager';
import { workspaceRoot } from '../utils/workspace-root';
import { ProjectGraph } from '../config/project-graph';
import { PackageJson } from '../utils/package-json';
import { defaultHashing } from '../hasher/hashing-impl';
import {
detectPackageManager,
PackageManager,
} from '../../../utils/package-manager';
import { workspaceRoot } from '../../../utils/workspace-root';
import { ProjectGraph } from '../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { PackageJson } from '../../../utils/package-json';
import { defaultHashing } from '../../../hasher/hashing-impl';
import { output } from '../../../utils/output';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { normalizePackageJson } from './utils/package-json';
import { output } from '../utils/output';
const YARN_LOCK_FILE = 'yarn.lock';
const NPM_LOCK_FILE = 'package-lock.json';
@ -76,20 +80,24 @@ export function lockFileHash(
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function parseLockFile(
builder: ProjectGraphBuilder,
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): ProjectGraph {
try {
if (packageManager === 'yarn') {
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
return parseYarnLockfile(content);
parseYarnLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'pnpm') {
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
return parsePnpmLockfile(content);
parsePnpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'npm') {
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
return parseNpmLockfile(content);
parseNpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
} catch (e) {
if (!isPostInstallProcess()) {
@ -138,19 +146,26 @@ export function createLockFile(
const normalizedPackageJson = normalizePackageJson(packageJson);
const content = readFileSync(getLockFileName(packageManager), 'utf8');
const builder = new ProjectGraphBuilder();
try {
if (packageManager === 'yarn') {
const graph = parseYarnLockfile(content);
parseYarnLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
}
if (packageManager === 'pnpm') {
const graph = parsePnpmLockfile(content);
parsePnpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
}
if (packageManager === 'npm') {
const graph = parseNpmLockfile(content);
parseNpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
}

View File

@ -1,20 +1,12 @@
import { joinPathFragments } from '../utils/path';
import { joinPathFragments } from '../../../utils/path';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs';
import { ProjectGraph } from '../config/project-graph';
import { ProjectGraph } from '../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
jest.mock('fs', () => require('memfs').fs);
jest.mock('@nrwl/devkit', () => ({
...jest.requireActual<any>('@nrwl/devkit'),
workspaceRoot: '/root',
}));
jest.mock('nx/src/utils/workspace-root', () => ({
workspaceRoot: '/root',
}));
describe('NPM lock file utility', () => {
afterEach(() => {
vol.reset();
@ -29,7 +21,9 @@ describe('NPM lock file utility', () => {
let graph: ProjectGraph;
beforeEach(() => {
graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
graph = builder.getUpdatedProjectGraph();
});
it('should parse root lock file', async () => {
@ -47,7 +41,9 @@ describe('NPM lock file utility', () => {
));
// this is original generated lock file
const appGraph = parseNpmLockfile(JSON.stringify(appLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(appLockFile), builder);
const appGraph = builder.getUpdatedProjectGraph();
expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
// this is our pruned lock file structure
@ -93,7 +89,9 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
@ -150,7 +148,9 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock-v2.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -246,7 +246,9 @@ describe('NPM lock file utility', () => {
cleanupTypes(prunedV2LockFile.packages);
cleanupTypes(prunedV2LockFile.dependencies, true);
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -331,7 +333,9 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock-v1.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369); // 338
});
it('should parse v3', async () => {
@ -340,7 +344,9 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369); //338
});
});
@ -355,7 +361,9 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/optional/package.json'
));
const graph = parseNpmLockfile(JSON.stringify(lockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8);
const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -378,7 +386,9 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/pruning/typescript/package.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -403,7 +413,9 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/pruning/devkit-yargs/package.json'
));
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -432,7 +444,9 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/workspaces/package-lock.json'
));
const result = parseNpmLockfile(JSON.stringify(lockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const result = builder.getUpdatedProjectGraph();
expect(Object.keys(result.externalNodes).length).toEqual(5);
});
@ -441,7 +455,9 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/workspaces/package-lock.v1.json'
));
const result = parseNpmLockfile(JSON.stringify(lockFile));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const result = builder.getUpdatedProjectGraph();
expect(Object.keys(result.externalNodes).length).toEqual(5);
});
});

View File

@ -1,14 +1,14 @@
import { existsSync, readFileSync } from 'fs';
import { satisfies } from 'semver';
import { workspaceRoot } from '../utils/workspace-root';
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
import { reverse } from '../project-graph/operators';
import { workspaceRoot } from '../../../utils/workspace-root';
import { reverse } from '../../../project-graph/operators';
import { NormalizedPackageJson } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
ProjectGraph,
ProjectGraphExternalNode,
} from '../config/project-graph';
import { NormalizedPackageJson } from './utils/package-json';
import { defaultHashing } from '../hasher/hashing-impl';
} from '../../../config/project-graph';
import { defaultHashing } from '../../../hasher/hashing-impl';
/**
* NPM
@ -50,16 +50,16 @@ type NpmLockFile = {
dependencies?: Record<string, NpmDependencyV1>;
};
export function parseNpmLockfile(lockFileContent: string): ProjectGraph {
export function parseNpmLockfile(
lockFileContent: string,
builder: ProjectGraphBuilder
) {
const data = JSON.parse(lockFileContent) as NpmLockFile;
const builder = new ProjectGraphBuilder();
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
addNodes(data, builder, keyMap);
addDependencies(data, builder, keyMap);
return builder.getUpdatedProjectGraph();
}
function addNodes(

View File

@ -1,17 +1,13 @@
import { joinPathFragments } from '../utils/path';
import { joinPathFragments } from '../../../utils/path';
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
import { ProjectGraph } from '../config/project-graph';
import { ProjectGraph } from '../../../config/project-graph';
import { vol } from 'memfs';
import { pruneProjectGraph } from './project-graph-pruning';
import { ProjectGraphBuilder } from 'nx/src/project-graph/project-graph-builder';
jest.mock('fs', () => require('memfs').fs);
jest.mock('@nrwl/devkit', () => ({
...jest.requireActual<any>('@nrwl/devkit'),
workspaceRoot: '/root',
}));
jest.mock('nx/src/utils/workspace-root', () => ({
jest.mock('../../../utils/workspace-root', () => ({
workspaceRoot: '/root',
}));
@ -121,7 +117,9 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/nextjs/pnpm-lock.yaml'
)).default;
graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
graph = builder.getUpdatedProjectGraph();
});
it('should parse root lock file', async () => {
@ -188,7 +186,9 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
)).default;
const graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(213); //202
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -268,7 +268,9 @@ describe('pnpm LockFile utility', () => {
},
};
const graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
const result = stringifyPnpmLockfile(
prunedGraph,
@ -305,7 +307,9 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/duplicate-package/pnpm-lock.yaml'
)).default;
const graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(370); //337
expect(Object.keys(graph.dependencies).length).toEqual(213);
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
@ -329,7 +333,9 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/optional/pnpm-lock.yaml'
)).default;
const graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8);
const packageJson = require(joinPathFragments(
@ -364,7 +370,9 @@ describe('pnpm LockFile utility', () => {
'__fixtures__/pruning/pnpm-lock.yaml'
)).default;
graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
graph = builder.getUpdatedProjectGraph();
});
it('should prune single package', () => {
@ -426,7 +434,9 @@ describe('pnpm LockFile utility', () => {
});
it('should parse lock file', async () => {
const graph = parsePnpmLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
});
});

View File

@ -11,24 +11,24 @@ import {
} from './utils/pnpm-normalizer';
import { getHoistedPackageVersion } from './utils/package-json';
import { NormalizedPackageJson } from './utils/package-json';
import { sortObjectByKeys } from '../utils/object-sort';
import { sortObjectByKeys } from '../../../utils/object-sort';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
ProjectGraph,
ProjectGraphExternalNode,
} from '../config/project-graph';
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
import { defaultHashing } from '../hasher/hashing-impl';
} from '../../../config/project-graph';
import { defaultHashing } from '../../../hasher/hashing-impl';
export function parsePnpmLockfile(lockFileContent: string): ProjectGraph {
export function parsePnpmLockfile(
lockFileContent: string,
builder: ProjectGraphBuilder
): void {
const data = parseAndNormalizePnpmLockfile(lockFileContent);
const builder = new ProjectGraphBuilder();
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
addNodes(data, builder, keyMap);
addDependencies(data, builder, keyMap);
return builder.getUpdatedProjectGraph();
}
function addNodes(

View File

@ -1,11 +1,11 @@
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
import {
ProjectGraph,
ProjectGraphExternalNode,
} from '../config/project-graph';
import { PackageJson } from '../utils/package-json';
import { reverse } from '../project-graph/operators';
} from '../../../config/project-graph';
import { satisfies, gte } from 'semver';
import { PackageJson } from '../../../utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { reverse } from '../../../project-graph/operators';
/**
* Prune project graph's external nodes and their dependencies

View File

@ -0,0 +1,11 @@
import { ProjectGraph } from '../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
export function removeNpmNodes(
graph: ProjectGraph,
builder: ProjectGraphBuilder
) {
for (const externalNode in graph.externalNodes) {
builder.removeNode(externalNode);
}
}

View File

@ -1,6 +1,6 @@
import { existsSync, readFileSync } from 'fs';
import { PackageJson } from '../../utils/package-json';
import { workspaceRoot } from '../../utils/workspace-root';
import { PackageJson } from '../../../../utils/package-json';
import { workspaceRoot } from '../../../../utils/workspace-root';
/**
* Get version of hoisted package if available

View File

@ -10,7 +10,7 @@ import type {
} from '@pnpm/lockfile-types';
import { dump, load } from '@zkochan/js-yaml';
import { existsSync, readFileSync } from 'fs';
import { workspaceRoot } from '../../utils/workspace-root';
import { workspaceRoot } from '../../../../utils/workspace-root';
const LOCKFILE_YAML_FORMAT = {
blankLines: true,

View File

@ -1,9 +1,10 @@
import { joinPathFragments } from '../utils/path';
import { joinPathFragments } from '../../../utils/path';
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs';
import { ProjectGraph } from '../config/project-graph';
import { PackageJson } from '../utils/package-json';
import { ProjectGraph } from '../../../config/project-graph';
import { PackageJson } from '../../../utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
jest.mock('fs', () => require('memfs').fs);
@ -12,7 +13,7 @@ jest.mock('@nrwl/devkit', () => ({
workspaceRoot: '/root',
}));
jest.mock('nx/src/utils/workspace-root', () => ({
jest.mock('../../../utils/workspace-root', () => ({
workspaceRoot: '/root',
}));
@ -156,11 +157,13 @@ describe('yarn LockFile utility', () => {
let graph: ProjectGraph;
beforeEach(() => {
const builder = new ProjectGraphBuilder();
lockFile = require(joinPathFragments(
__dirname,
'__fixtures__/nextjs/yarn.lock'
)).default;
graph = parseYarnLockfile(lockFile);
parseYarnLockfile(lockFile, builder);
graph = builder.getUpdatedProjectGraph();
});
it('should parse root lock file', async () => {
@ -225,7 +228,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/yarn.lock'
)).default;
const graph = parseYarnLockfile(classicLockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(classicLockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(127); // 124 hoisted
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -301,7 +306,9 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -337,7 +344,9 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -357,7 +366,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/yarn-berry.lock'
)).default;
const graph = parseYarnLockfile(berryLockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(berryLockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(128); //124 hoisted
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -433,7 +444,9 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn-berry.lock.pruned'
)).default;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -489,7 +502,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/duplicate-package/yarn.lock'
)).default;
const graph = parseYarnLockfile(classicLockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(classicLockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(371); //337 hoisted
});
});
@ -517,7 +532,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/optional/package.json'
));
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(103);
const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -694,7 +711,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/pruning/typescript/package.json'
));
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -719,7 +738,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/pruning/devkit-yargs/package.json'
));
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -750,7 +771,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/workspaces/yarn.lock'
)).default;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
});
@ -759,7 +782,9 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/workspaces/yarn.lock.berry'
)).default;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
});
});
@ -800,7 +825,9 @@ type-fest@^0.20.2:
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
`;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
Object {
"data": Object {
@ -873,7 +900,9 @@ __metadata:
languageName: node
linkType: hard
`;
const graph = parseYarnLockfile(lockFile);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
Object {
"data": Object {

View File

@ -1,15 +1,15 @@
import { parseSyml, stringifySyml } from '@yarnpkg/parsers';
import { stringify } from '@yarnpkg/lockfile';
import { sortObjectByKeys } from '../utils/object-sort';
import { getHoistedPackageVersion } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { satisfies } from 'semver';
import { NormalizedPackageJson } from './utils/package-json';
import {
ProjectGraph,
ProjectGraphExternalNode,
} from '../config/project-graph';
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
import { satisfies } from 'semver';
import { NormalizedPackageJson } from './utils/package-json';
import { defaultHashing } from '../hasher/hashing-impl';
} from '../../../config/project-graph';
import { defaultHashing } from '../../../hasher/hashing-impl';
import { sortObjectByKeys } from '../../../utils/object-sort';
/**
* Yarn
@ -37,16 +37,16 @@ type YarnDependency = {
linkType?: 'soft' | 'hard';
};
export function parseYarnLockfile(lockFileContent: string): ProjectGraph {
export function parseYarnLockfile(
lockFileContent: string,
builder: ProjectGraphBuilder
) {
const data = parseSyml(lockFileContent);
const builder = new ProjectGraphBuilder();
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
addNodes(data, builder, keyMap);
addDependencies(data, builder, keyMap);
return builder.getUpdatedProjectGraph();
}
function addNodes(

View File

@ -1,10 +1,10 @@
import * as fs from 'fs';
import * as configModule from '../config/configuration';
import { DependencyType } from '../config/project-graph';
import * as hashModule from '../hasher/hasher';
import * as configModule from '../../../config/configuration';
import { DependencyType } from '../../../config/project-graph';
import * as hashModule from '../../../hasher/hasher';
import { createPackageJson } from './create-package-json';
import * as fileutilsModule from './fileutils';
import * as fileutilsModule from '../../../utils/fileutils';
describe('createPackageJson', () => {
it('should add additional dependencies', () => {

View File

@ -1,11 +1,17 @@
import { readJsonFile } from './fileutils';
import { sortObjectByKeys } from './object-sort';
import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
import { PackageJson } from './package-json';
import { readJsonFile } from '../../../utils/fileutils';
import { sortObjectByKeys } from '../../../utils/object-sort';
import {
ProjectGraph,
ProjectGraphProjectNode,
} from '../../../config/project-graph';
import { PackageJson } from '../../../utils/package-json';
import { existsSync } from 'fs';
import { workspaceRoot } from './workspace-root';
import { filterUsingGlobPatterns, getTargetInputs } from '../hasher/hasher';
import { readNxJson } from '../config/configuration';
import { workspaceRoot } from '../../../utils/workspace-root';
import {
filterUsingGlobPatterns,
getTargetInputs,
} from '../../../hasher/hasher';
import { readNxJson } from '../../../config/configuration';
interface NpmDeps {
readonly dependencies: Record<string, string>;

View File

@ -0,0 +1,189 @@
import {
DependencyType,
ProjectFileMap,
ProjectGraphProcessorContext,
} from '../../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
import { join } from 'path';
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
import * as os from 'os';
export function buildExplicitDependencies(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
// using workers has an overhead, so we only do it when the number of
// files we need to process is >= 100 and there are more than 2 CPUs
// to be able to use at least 2 workers (1 worker per CPU and
// 1 CPU for the main thread)
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
return buildExplicitDependenciesWithoutWorkers(
jsPluginConfig,
ctx,
builder
);
} else {
return buildExplicitDependenciesUsingWorkers(
jsPluginConfig,
ctx,
totalNumOfFilesToProcess,
builder
);
}
}
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
let totalNumOfFilesToProcess = 0;
Object.values(ctx.filesToProcess).forEach(
(t) => (totalNumOfFilesToProcess += t.length)
);
return totalNumOfFilesToProcess;
}
function splitFilesIntoBins(
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
numberOfWorkers: number
) {
// we want to have numberOfWorkers * 5 bins
const filesPerBin =
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
const bins: ProjectFileMap[] = [];
let currentProjectFileMap = {};
let currentNumberOfFiles = 0;
for (const source of Object.keys(ctx.filesToProcess)) {
for (const f of Object.values(ctx.filesToProcess[source])) {
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
currentProjectFileMap[source].push(f);
currentNumberOfFiles++;
if (currentNumberOfFiles >= filesPerBin) {
bins.push(currentProjectFileMap);
currentProjectFileMap = {};
currentNumberOfFiles = 0;
}
}
}
bins.push(currentProjectFileMap);
return bins;
}
function createWorkerPool(numberOfWorkers: number) {
const res = [];
for (let i = 0; i < numberOfWorkers; ++i) {
res.push(
new (require('worker_threads').Worker)(
join(__dirname, './project-graph-worker.js'),
{
env: process.env,
}
)
);
}
return res;
}
function buildExplicitDependenciesWithoutWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
buildExplicitTypescriptAndPackageJsonDependencies(
jsPluginConfig,
ctx.nxJsonConfiguration,
ctx.projectsConfigurations,
builder.graph,
ctx.filesToProcess
).forEach((r) => {
if (r.type === DependencyType.static) {
builder.addStaticDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
} else {
builder.addDynamicDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
}
});
}
function buildExplicitDependenciesUsingWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
builder: ProjectGraphBuilder
) {
const numberOfWorkers = Math.min(
totalNumOfFilesToProcess,
getNumberOfWorkers()
);
const bins = splitFilesIntoBins(
ctx,
totalNumOfFilesToProcess,
numberOfWorkers
);
const workers = createWorkerPool(numberOfWorkers);
let numberOfExpectedResponses = bins.length;
return new Promise((res, reject) => {
for (let w of workers) {
w.on('message', (explicitDependencies) => {
explicitDependencies.forEach((r) => {
builder.addExplicitDependency(
r.sourceProjectName,
r.sourceProjectFile,
r.targetProjectName
);
});
if (bins.length > 0) {
w.postMessage({ filesToProcess: bins.shift() });
}
// we processed all the bins
if (--numberOfExpectedResponses === 0) {
for (let w of workers) {
w.terminate();
}
res(null);
}
});
w.on('error', reject);
w.on('exit', (code) => {
if (code !== 0) {
reject(
new Error(
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
)
);
}
});
w.postMessage({
nxJsonConfiguration: ctx.nxJsonConfiguration,
projectsConfigurations: ctx.projectsConfigurations,
projectGraph: builder.graph,
jsPluginConfig,
});
w.postMessage({ filesToProcess: bins.shift() });
}
});
}
function getNumberOfWorkers(): number {
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
}

View File

@ -3,9 +3,9 @@ import {
ExplicitDependency,
} from './explicit-project-dependencies';
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
import { ProjectFileMap, ProjectGraph } from '../../config/project-graph';
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
import { NxJsonConfiguration } from '../../config/nx-json';
import { ProjectFileMap, ProjectGraph } from '../../../../config/project-graph';
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
import { NxJsonConfiguration } from '../../../../config/nx-json';
export function buildExplicitTypescriptAndPackageJsonDependencies(
jsPluginConfig: {

View File

@ -1,15 +1,15 @@
import { TempFs } from '../../utils/testing/temp-fs';
import { TempFs } from '../../../../utils/testing/temp-fs';
const tempFs = new TempFs('explicit-package-json');
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
import { createProjectFileMap } from '../file-map-utils';
import { defaultFileHasher } from '../../hasher/file-hasher';
import { defaultFileHasher } from '../../../../hasher/file-hasher';
import {
ProjectGraphProcessorContext,
ProjectGraphProjectNode,
} from '../../config/project-graph';
import { ProjectGraphBuilder } from '../project-graph-builder';
} from '../../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
describe('explicit package json dependencies', () => {
let ctx: ProjectGraphProcessorContext;

View File

@ -1,14 +1,14 @@
import { defaultFileRead } from '../file-utils';
import { defaultFileRead } from '../../../../project-graph/file-utils';
import { join } from 'path';
import {
DependencyType,
ProjectFileMap,
ProjectGraph,
} from '../../config/project-graph';
import { parseJson } from '../../utils/json';
import { getImportPath, joinPathFragments } from '../../utils/path';
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
import { NxJsonConfiguration } from '../../config/nx-json';
} from '../../../../config/project-graph';
import { parseJson } from '../../../../utils/json';
import { getImportPath, joinPathFragments } from '../../../../utils/path';
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
import { NxJsonConfiguration } from '../../../../config/nx-json';
import { ExplicitDependency } from './explicit-project-dependencies';
class ProjectGraphNodeRecords {}

View File

@ -1,9 +1,9 @@
import { TempFs } from '../../utils/testing/temp-fs';
import { TempFs } from '../../../../utils/testing/temp-fs';
const tempFs = new TempFs('explicit-project-deps');
import { defaultFileHasher } from '../../hasher/file-hasher';
import { createProjectFileMap } from '../file-map-utils';
import { ProjectGraphBuilder } from '../project-graph-builder';
import { defaultFileHasher } from '../../../../hasher/file-hasher';
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies';
// projectName => tsconfig import path

View File

@ -1,10 +1,10 @@
import { TypeScriptImportLocator } from './typescript-import-locator';
import { TargetProjectLocator } from '../../utils/target-project-locator';
import { TargetProjectLocator } from './target-project-locator';
import {
DependencyType,
ProjectFileMap,
ProjectGraph,
} from '../../config/project-graph';
} from '../../../../config/project-graph';
export type ExplicitDependency = {
sourceProjectName: string;

View File

@ -1,8 +1,8 @@
import { parentPort } from 'worker_threads';
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
import { ProjectGraph } from '../config/project-graph';
import { ProjectsConfigurations } from '../config/workspace-json-project-json';
import { NxJsonConfiguration } from '../config/nx-json';
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
import { NxJsonConfiguration } from '../../../../config/nx-json';
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
import { ProjectGraph } from '../../../../config/project-graph';
let nxJsonConfiguration: NxJsonConfiguration | null;
let projectsConfigurations: ProjectsConfigurations | null;

View File

@ -4,7 +4,7 @@ import {
ProjectGraphExternalNode,
ProjectGraphProcessorContext,
ProjectGraphProjectNode,
} from '../config/project-graph';
} from '../../../../config/project-graph';
jest.mock('nx/src/utils/workspace-root', () => ({
workspaceRoot: '/root',

View File

@ -1,16 +1,19 @@
import { getRootTsConfigFileName, resolveModuleByImport } from './typescript';
import { isRelativePath, readJsonFile } from './fileutils';
import {
getRootTsConfigFileName,
resolveModuleByImport,
} from '../../../../utils/typescript';
import { isRelativePath, readJsonFile } from '../../../../utils/fileutils';
import { dirname, join, posix } from 'path';
import { workspaceRoot } from './workspace-root';
import { workspaceRoot } from '../../../../utils/workspace-root';
import {
ProjectGraphExternalNode,
ProjectGraphProjectNode,
} from '../config/project-graph';
} from '../../../../config/project-graph';
import { builtinModules } from 'module';
import {
createProjectRootMappings,
findProjectForPath,
} from '../project-graph/utils/find-project-for-path';
import { builtinModules } from 'module';
} from '../../../../project-graph/utils/find-project-for-path';
const builtInModuleSet = new Set<string>([
...builtinModules,

View File

@ -1,8 +1,8 @@
import type * as ts from 'typescript';
import * as path from 'path';
import { stripSourceCode } from '../../utils/strip-source-code';
import { defaultFileRead } from '../file-utils';
import { DependencyType } from '../../config/project-graph';
import { stripSourceCode } from './strip-source-code';
import { DependencyType } from '../../../../config/project-graph';
import { defaultFileRead } from '../../../../project-graph/file-utils';
let tsModule: typeof import('typescript');

View File

@ -1,3 +1 @@
export * from './implicit-project-dependencies';
export * from './explicit-project-dependencies';
export * from './explicit-package-json-dependencies';

View File

@ -201,17 +201,6 @@ describe('project graph', () => {
jest.spyOn(fastGlob, 'sync').mockImplementation(() => globResults);
});
it('should throw an appropriate error for an invalid json config', async () => {
tempFs.appendFile('tsconfig.base.json', 'invalid');
try {
await buildProjectGraph();
fail('Invalid tsconfigs should cause project graph to throw error');
} catch (e) {
expect(e.message).toContain(`${tempFs.tempDir}/tsconfig.base.json`);
expect(e.message).toContain(`invalid`);
}
});
it('should create nodes and dependencies with workspace projects', async () => {
const graph = await buildProjectGraph();

View File

@ -11,13 +11,8 @@ import {
shouldRecomputeWholeGraph,
writeCache,
} from './nx-deps-cache';
import {
buildImplicitProjectDependencies,
ExplicitDependency,
} from './build-dependencies';
import { buildImplicitProjectDependencies } from './build-dependencies';
import { buildWorkspaceProjectNodes } from './build-nodes';
import * as os from 'os';
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
import { loadNxPlugins } from '../utils/nx-plugin';
import { defaultFileHasher } from '../hasher/file-hasher';
import { createProjectFileMap } from './file-map-utils';
@ -28,7 +23,7 @@ import {
ProjectGraphProcessorContext,
} from '../config/project-graph';
import { readJsonFile } from '../utils/fileutils';
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../config/nx-json';
import { NxJsonConfiguration } from '../config/nx-json';
import { logger } from '../utils/logger';
import { ProjectGraphBuilder } from './project-graph-builder';
import {
@ -36,11 +31,6 @@ import {
ProjectsConfigurations,
} from '../config/workspace-json-project-json';
import { readNxJson } from '../config/configuration';
import {
lockFileExists,
lockFileHash,
parseLockFile,
} from '../lock-file/lock-file';
import { Workspaces } from '../config/workspaces';
import { existsSync } from 'fs';
import { PackageJson } from '../utils/package-json';
@ -102,17 +92,7 @@ export async function buildProjectGraphUsingProjectFileMap(
filesToProcess = projectFileMap;
cachedFileData = {};
}
let partialGraph: ProjectGraph;
let lockHash = 'n/a';
// during the create-nx-workspace lock file might not exists yet
if (lockFileExists()) {
lockHash = lockFileHash();
if (cache && cache.lockFileHash === lockHash) {
partialGraph = isolatePartialGraphFromCache(cache);
} else {
partialGraph = parseLockFile();
}
}
const context = createContext(
projectsConfigurations,
nxJson,
@ -124,15 +104,13 @@ export async function buildProjectGraphUsingProjectFileMap(
context,
cachedFileData,
projectGraphVersion,
partialGraph,
packageJsonDeps
cache
);
const projectGraphCache = createCache(
nxJson,
packageJsonDeps,
projectGraph,
rootTsConfig,
lockHash
rootTsConfig
);
if (shouldWriteCache) {
writeCache(projectGraphCache);
@ -168,33 +146,24 @@ function readCombinedDeps() {
};
}
// extract only external nodes and their dependencies
function isolatePartialGraphFromCache(cache: ProjectGraphCache): ProjectGraph {
const dependencies = {};
Object.keys(cache.dependencies).forEach((k) => {
if (cache.externalNodes[k]) {
dependencies[k] = cache.dependencies[k];
}
});
return {
nodes: {},
dependencies,
externalNodes: cache.externalNodes,
};
}
async function buildProjectGraphUsingContext(
nxJson: NxJsonConfiguration,
ctx: ProjectGraphProcessorContext,
cachedFileData: { [project: string]: { [file: string]: FileData } },
projectGraphVersion: string,
partialGraph: ProjectGraph,
packageJsonDeps: Record<string, string>
cache: ProjectGraphCache | null
) {
performance.mark('build project graph:start');
const builder = new ProjectGraphBuilder(partialGraph);
const builder = new ProjectGraphBuilder(
cache
? {
nodes: cache.nodes,
externalNodes: cache.externalNodes,
dependencies: cache.dependencies,
}
: null
);
builder.setVersion(projectGraphVersion);
await buildWorkspaceProjectNodes(ctx, builder, nxJson);
@ -212,12 +181,6 @@ async function buildProjectGraphUsingContext(
}
}
await buildExplicitDependencies(
jsPluginConfig(nxJson, packageJsonDeps),
ctx,
updatedBuilder
);
buildImplicitProjectDependencies(ctx, updatedBuilder);
const finalGraph = updatedBuilder.getUpdatedProjectGraph();
@ -232,216 +195,6 @@ async function buildProjectGraphUsingContext(
return finalGraph;
}
function jsPluginConfig(
nxJson: NxJsonConfiguration,
packageJsonDeps: { [packageName: string]: string }
): NrwlJsPluginConfig {
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
return nxJson?.pluginsConfig?.['@nrwl/js'];
}
if (
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return { analyzePackageJson: true, analyzeSourceFiles: true };
} else {
return { analyzePackageJson: true, analyzeSourceFiles: false };
}
}
function buildExplicitDependencies(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
// using workers has an overhead, so we only do it when the number of
// files we need to process is >= 100 and there are more than 2 CPUs
// to be able to use at least 2 workers (1 worker per CPU and
// 1 CPU for the main thread)
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
return buildExplicitDependenciesWithoutWorkers(
jsPluginConfig,
ctx,
builder
);
} else {
return buildExplicitDependenciesUsingWorkers(
jsPluginConfig,
ctx,
totalNumOfFilesToProcess,
builder
);
}
}
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
let totalNumOfFilesToProcess = 0;
Object.values(ctx.filesToProcess).forEach(
(t) => (totalNumOfFilesToProcess += t.length)
);
return totalNumOfFilesToProcess;
}
function splitFilesIntoBins(
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
numberOfWorkers: number
) {
// we want to have numberOfWorkers * 5 bins
const filesPerBin =
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
const bins: ProjectFileMap[] = [];
let currentProjectFileMap = {};
let currentNumberOfFiles = 0;
for (const source of Object.keys(ctx.filesToProcess)) {
for (const f of Object.values(ctx.filesToProcess[source])) {
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
currentProjectFileMap[source].push(f);
currentNumberOfFiles++;
if (currentNumberOfFiles >= filesPerBin) {
bins.push(currentProjectFileMap);
currentProjectFileMap = {};
currentNumberOfFiles = 0;
}
}
}
bins.push(currentProjectFileMap);
return bins;
}
function createWorkerPool(numberOfWorkers: number) {
const res = [];
for (let i = 0; i < numberOfWorkers; ++i) {
res.push(
new (require('worker_threads').Worker)(
join(__dirname, './project-graph-worker.js'),
{
env: process.env,
}
)
);
}
return res;
}
function buildExplicitDependenciesWithoutWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
builder: ProjectGraphBuilder
) {
buildExplicitTypescriptAndPackageJsonDependencies(
jsPluginConfig,
ctx.nxJsonConfiguration,
ctx.projectsConfigurations,
builder.graph,
ctx.filesToProcess
).forEach((r) => {
if (r.type === 'static') {
builder.addStaticDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
} else {
builder.addDynamicDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
}
});
}
function buildExplicitDependenciesUsingWorkers(
jsPluginConfig: {
analyzeSourceFiles?: boolean;
analyzePackageJson?: boolean;
},
ctx: ProjectGraphProcessorContext,
totalNumOfFilesToProcess: number,
builder: ProjectGraphBuilder
) {
const numberOfWorkers = Math.min(
totalNumOfFilesToProcess,
getNumberOfWorkers()
);
const bins = splitFilesIntoBins(
ctx,
totalNumOfFilesToProcess,
numberOfWorkers
);
const workers = createWorkerPool(numberOfWorkers);
let numberOfExpectedResponses = bins.length;
return new Promise((res, reject) => {
for (let w of workers) {
w.on('message', (explicitDependencies) => {
explicitDependencies.forEach((r: ExplicitDependency) => {
if (r.type === 'static') {
builder.addStaticDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
} else {
builder.addDynamicDependency(
r.sourceProjectName,
r.targetProjectName,
r.sourceProjectFile
);
}
});
if (bins.length > 0) {
w.postMessage({ filesToProcess: bins.shift() });
}
// we processed all the bins
if (--numberOfExpectedResponses === 0) {
for (let w of workers) {
w.terminate();
}
res(null);
}
});
w.on('error', reject);
w.on('exit', (code) => {
if (code !== 0) {
reject(
new Error(
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
)
);
}
});
w.postMessage({
nxJsonConfiguration: ctx.nxJsonConfiguration,
projectsConfigurations: ctx.projectsConfigurations,
projectGraph: builder.graph,
jsPluginConfig,
});
w.postMessage({ filesToProcess: bins.shift() });
}
});
}
function getNumberOfWorkers(): number {
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
}
function createContext(
projectsConfigurations: ProjectsConfigurations,
nxJson: NxJsonConfiguration,

View File

@ -19,8 +19,8 @@ export function createProjectFileMap(
projectFileMap[projectName] ??= [];
}
for (const f of allWorkspaceFiles) {
const matchingProjectFiles =
projectFileMap[findProjectForPath(f.file, projectRootMappings)];
const projectFileMapKey = findProjectForPath(f.file, projectRootMappings);
const matchingProjectFiles = projectFileMap[projectFileMapKey];
if (matchingProjectFiles) {
matchingProjectFiles.push(f);
}

View File

@ -298,8 +298,7 @@ describe('nx deps utils', () => {
createNxJson({}),
createPackageJsonDeps({}),
createCache({}) as ProjectGraph,
{},
'abcd1234'
{}
);
});
@ -308,8 +307,7 @@ describe('nx deps utils', () => {
createNxJson({}),
createPackageJsonDeps({}),
createCache({}) as ProjectGraph,
undefined,
'abcd1234'
undefined
);
expect(result).toBeDefined();
@ -323,7 +321,6 @@ describe('nx deps utils', () => {
'@nrwl/workspace': '12.0.0',
plugin: '1.0.0',
},
lockFileHash: 'abcd1234',
pathMappings: {
mylib: ['libs/mylib/index.ts'],
},

View File

@ -23,7 +23,6 @@ import {
export interface ProjectGraphCache {
version: string;
deps: Record<string, string>;
lockFileHash: string;
pathMappings: Record<string, any>;
nxJsonPlugins: { name: string; version: string }[];
pluginsConfig?: any;
@ -86,8 +85,7 @@ export function createCache(
nxJson: NxJsonConfiguration<'*' | string[]>,
packageJsonDeps: Record<string, string>,
projectGraph: ProjectGraph,
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } },
lockFileHash: string
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } }
) {
const nxJsonPlugins = (nxJson.plugins || []).map((p) => ({
name: p,
@ -96,7 +94,6 @@ export function createCache(
const newValue: ProjectGraphCache = {
version: projectGraph.version || '5.1',
deps: packageJsonDeps,
lockFileHash,
// compilerOptions may not exist, especially for repos converted through add-nx-to-monorepo
pathMappings: tsConfig?.compilerOptions?.paths || {},
nxJsonPlugins,

Some files were not shown because too many files have changed in this diff Show More