feat(core): refactor most of js graph work into its own folder (#15365)
This commit is contained in:
parent
16b3fa0931
commit
8cf8f18c1c
@ -12,7 +12,7 @@ packages/express/src/schematics/**/files/**/*.json
|
||||
packages/nest/src/schematics/**/files/**/*.json
|
||||
packages/react/src/schematics/**/files/**/*.json
|
||||
packages/jest/src/schematics/**/files/**/*.json
|
||||
packages/nx/src/lock-file/__fixtures__/**/*.*
|
||||
packages/nx/src/plugins/js/lock-file/__fixtures__/**/*.*
|
||||
packages/**/schematics/**/files/**/*.html
|
||||
packages/**/generators/**/files/**/*.html
|
||||
packages/nx/src/native/
|
||||
|
||||
@ -26,7 +26,7 @@ import {
|
||||
} from '@nrwl/e2e/utils';
|
||||
import { exec, execSync } from 'child_process';
|
||||
import * as http from 'http';
|
||||
import { getLockFileName } from 'nx/src/lock-file/lock-file';
|
||||
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||
import { satisfies } from 'semver';
|
||||
|
||||
function getData(port, path = '/api'): Promise<any> {
|
||||
|
||||
@ -9,7 +9,7 @@ import { DependencyType } from '@nrwl/devkit';
|
||||
import * as parser from '@typescript-eslint/parser';
|
||||
import { TSESLint } from '@typescript-eslint/utils';
|
||||
import { vol } from 'memfs';
|
||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
||||
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||
import enforceModuleBoundaries, {
|
||||
RULE_NAME as enforceModuleBoundariesRuleName,
|
||||
} from '../../src/rules/enforce-module-boundaries';
|
||||
|
||||
@ -33,7 +33,7 @@ import {
|
||||
isComboDepConstraint,
|
||||
} from '../utils/runtime-lint-utils';
|
||||
import { AST_NODE_TYPES, TSESTree } from '@typescript-eslint/utils';
|
||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
||||
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||
import { basename, dirname, relative } from 'path';
|
||||
import {
|
||||
getBarrelEntryPointByImportScope,
|
||||
|
||||
@ -13,7 +13,7 @@ import {
|
||||
} from '@nrwl/devkit';
|
||||
import { getPath, pathExists } from './graph-utils';
|
||||
import { readFileIfExisting } from 'nx/src/utils/fileutils';
|
||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
||||
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||
import {
|
||||
findProjectForPath,
|
||||
ProjectRootMappings,
|
||||
|
||||
@ -11,5 +11,5 @@ export * from './utils/assets';
|
||||
export * from './utils/package-json/update-package-json';
|
||||
export { libraryGenerator } from './generators/library/library';
|
||||
export { initGenerator } from './generators/init/init';
|
||||
export { createLockFile } from 'nx/src/lock-file/lock-file';
|
||||
export { createPackageJson } from 'nx/src/utils/create-package-json';
|
||||
export { createLockFile } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||
export { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
|
||||
|
||||
@ -1,5 +1,8 @@
|
||||
import { createLockFile, getLockFileName } from 'nx/src/lock-file/lock-file';
|
||||
import { createPackageJson } from 'nx/src/utils/create-package-json';
|
||||
import {
|
||||
createLockFile,
|
||||
getLockFileName,
|
||||
} from 'nx/src/plugins/js/lock-file/lock-file';
|
||||
import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
|
||||
import {
|
||||
ExecutorContext,
|
||||
getOutputsForTargetAndConfiguration,
|
||||
|
||||
@ -18,7 +18,7 @@ import { createNextConfigFile } from './lib/create-next-config-file';
|
||||
import { checkPublicDirectory } from './lib/check-project';
|
||||
import { NextBuildBuilderOptions } from '../../utils/types';
|
||||
|
||||
import { getLockFileName } from 'nx/src/lock-file/lock-file';
|
||||
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||
|
||||
export default async function buildExecutor(
|
||||
options: NextBuildBuilderOptions,
|
||||
|
||||
@ -226,8 +226,8 @@ export { Hash, Hasher } from './hasher/hasher';
|
||||
*/
|
||||
export { cacheDir } from './utils/cache-directory';
|
||||
|
||||
import { createLockFile as _createLockFile } from './lock-file/lock-file';
|
||||
import { createPackageJson as _createPackageJson } from './utils/create-package-json';
|
||||
import { createLockFile as _createLockFile } from './plugins/js/lock-file/lock-file';
|
||||
import { createPackageJson as _createPackageJson } from './plugins/js/package-json/create-package-json';
|
||||
|
||||
/**
|
||||
* @category Package Manager
|
||||
|
||||
@ -1,11 +1,88 @@
|
||||
import { ProjectGraphProcessor } from '../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../project-graph/project-graph-builder';
|
||||
import { buildNpmPackageNodes } from './project-graph/build-nodes/build-npm-package-nodes';
|
||||
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
|
||||
import { readNxJson } from '../../config/configuration';
|
||||
import { fileExists, readJsonFile } from '../../utils/fileutils';
|
||||
import { PackageJson } from '../../utils/package-json';
|
||||
import {
|
||||
lockFileExists,
|
||||
lockFileHash,
|
||||
parseLockFile,
|
||||
} from './lock-file/lock-file';
|
||||
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json';
|
||||
import { dirname, join } from 'path';
|
||||
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
|
||||
import { readFileSync, writeFileSync } from 'fs';
|
||||
import { workspaceRoot } from '../../utils/workspace-root';
|
||||
import { ensureDirSync } from 'fs-extra';
|
||||
import { removeNpmNodes } from 'nx/src/plugins/js/lock-file/remove-npm-nodes';
|
||||
|
||||
export const processProjectGraph: ProjectGraphProcessor = (graph) => {
|
||||
export const processProjectGraph: ProjectGraphProcessor = (graph, context) => {
|
||||
const builder = new ProjectGraphBuilder(graph);
|
||||
|
||||
const lockHash = lockFileHash() ?? 'n/a';
|
||||
// during the create-nx-workspace lock file might not exists yet
|
||||
if (lockFileExists()) {
|
||||
if (lockFileNeedsReprocessing(lockHash)) {
|
||||
removeNpmNodes(graph, builder);
|
||||
parseLockFile(builder);
|
||||
}
|
||||
writeLastProcessedLockfileHash(lockHash);
|
||||
}
|
||||
|
||||
buildNpmPackageNodes(builder);
|
||||
|
||||
buildExplicitDependencies(jsPluginConfig(readNxJson()), context, builder);
|
||||
|
||||
return builder.getUpdatedProjectGraph();
|
||||
};
|
||||
|
||||
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
|
||||
function lockFileNeedsReprocessing(lockHash: string) {
|
||||
try {
|
||||
return readFileSync(lockFileHashFile).toString() !== lockHash;
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
function writeLastProcessedLockfileHash(hash: string) {
|
||||
ensureDirSync(dirname(lockFileHashFile));
|
||||
writeFileSync(lockFileHashFile, hash);
|
||||
}
|
||||
|
||||
function jsPluginConfig(nxJson: NxJsonConfiguration): NrwlJsPluginConfig {
|
||||
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
|
||||
return nxJson?.pluginsConfig?.['@nrwl/js'];
|
||||
}
|
||||
|
||||
if (!fileExists(join(workspaceRoot, 'package.json'))) {
|
||||
return {
|
||||
analyzePackageJson: false,
|
||||
analyzeSourceFiles: false,
|
||||
};
|
||||
}
|
||||
|
||||
const packageJson = readJsonFile<PackageJson>(
|
||||
join(workspaceRoot, 'package.json')
|
||||
);
|
||||
|
||||
const packageJsonDeps = {
|
||||
...packageJson.dependencies,
|
||||
...packageJson.devDependencies,
|
||||
};
|
||||
if (
|
||||
packageJsonDeps['@nrwl/workspace'] ||
|
||||
packageJsonDeps['@nrwl/js'] ||
|
||||
packageJsonDeps['@nrwl/node'] ||
|
||||
packageJsonDeps['@nrwl/next'] ||
|
||||
packageJsonDeps['@nrwl/react'] ||
|
||||
packageJsonDeps['@nrwl/angular'] ||
|
||||
packageJsonDeps['@nrwl/web']
|
||||
) {
|
||||
return { analyzePackageJson: true, analyzeSourceFiles: true };
|
||||
} else {
|
||||
return { analyzePackageJson: true, analyzeSourceFiles: false };
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,9 +13,9 @@ const packageNames = [];
|
||||
|
||||
function processNodeModules(path = '.') {
|
||||
if (existsSync(`${path}/node_modules`)) {
|
||||
readdirSync(`${path}/node_modules`).forEach(folder => {
|
||||
readdirSync(`${path}/node_modules`).forEach((folder) => {
|
||||
if (folder.startsWith('@')) {
|
||||
readdirSync(`${path}/node_modules/${folder}`).forEach(subfolder => {
|
||||
readdirSync(`${path}/node_modules/${folder}`).forEach((subfolder) => {
|
||||
packageNames.push(`${path}/node_modules/${folder}/${subfolder}`);
|
||||
processNodeModules(`${path}/node_modules/${folder}/${subfolder}`);
|
||||
});
|
||||
@ -29,15 +29,15 @@ function processNodeModules(path = '.') {
|
||||
|
||||
processNodeModules();
|
||||
|
||||
packageNames.forEach(path => {
|
||||
packageNames.forEach((path) => {
|
||||
const filePath = `${path}/package.json`;
|
||||
if (existsSync(filePath)) {
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
const peerDependencies = JSON.parse(content).peerDependencies;
|
||||
const peerDependenciesMeta = JSON.parse(content).peerDependenciesMeta;
|
||||
const output = JSON.stringify({
|
||||
...peerDependencies && { peerDependencies },
|
||||
...peerDependenciesMeta && { peerDependenciesMeta },
|
||||
...(peerDependencies && { peerDependencies }),
|
||||
...(peerDependenciesMeta && { peerDependenciesMeta }),
|
||||
});
|
||||
if (output === '{}') return;
|
||||
report += `'${filePath.slice(2)}': '${output}',\n`;
|
||||
@ -59,10 +59,10 @@ const existsSync = require('fs').existsSync;
|
||||
let report = '';
|
||||
|
||||
const packageNames = [];
|
||||
readdirSync('node_modules').forEach(folder => {
|
||||
readdirSync('node_modules').forEach((folder) => {
|
||||
if (folder === '.pnpm') return;
|
||||
if (folder.startsWith('@')) {
|
||||
readdirSync(`node_modules/${folder}`).forEach(subfolder => {
|
||||
readdirSync(`node_modules/${folder}`).forEach((subfolder) => {
|
||||
packageNames.push(`${folder}/${subfolder}`);
|
||||
});
|
||||
} else {
|
||||
@ -70,7 +70,7 @@ readdirSync('node_modules').forEach(folder => {
|
||||
}
|
||||
});
|
||||
|
||||
packageNames.forEach(packageName => {
|
||||
packageNames.forEach((packageName) => {
|
||||
const path = `node_modules/${packageName}/package.json`;
|
||||
if (existsSync(path)) {
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
@ -1103,9 +1103,7 @@
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"os": ["darwin"],
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
@ -1103,9 +1103,7 @@
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"hasInstallScript": true,
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"os": ["darwin"],
|
||||
"peer": true,
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
@ -6,18 +6,22 @@
|
||||
import { readFileSync, existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
import { detectPackageManager, PackageManager } from '../utils/package-manager';
|
||||
import { workspaceRoot } from '../utils/workspace-root';
|
||||
import { ProjectGraph } from '../config/project-graph';
|
||||
import { PackageJson } from '../utils/package-json';
|
||||
import { defaultHashing } from '../hasher/hashing-impl';
|
||||
import {
|
||||
detectPackageManager,
|
||||
PackageManager,
|
||||
} from '../../../utils/package-manager';
|
||||
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||
import { ProjectGraph } from '../../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
import { PackageJson } from '../../../utils/package-json';
|
||||
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||
import { output } from '../../../utils/output';
|
||||
|
||||
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
||||
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
||||
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
||||
import { pruneProjectGraph } from './project-graph-pruning';
|
||||
import { normalizePackageJson } from './utils/package-json';
|
||||
import { output } from '../utils/output';
|
||||
|
||||
const YARN_LOCK_FILE = 'yarn.lock';
|
||||
const NPM_LOCK_FILE = 'package-lock.json';
|
||||
@ -76,20 +80,24 @@ export function lockFileHash(
|
||||
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
|
||||
*/
|
||||
export function parseLockFile(
|
||||
builder: ProjectGraphBuilder,
|
||||
packageManager: PackageManager = detectPackageManager(workspaceRoot)
|
||||
): ProjectGraph {
|
||||
try {
|
||||
if (packageManager === 'yarn') {
|
||||
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
|
||||
return parseYarnLockfile(content);
|
||||
parseYarnLockfile(content, builder);
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
if (packageManager === 'pnpm') {
|
||||
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
|
||||
return parsePnpmLockfile(content);
|
||||
parsePnpmLockfile(content, builder);
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
if (packageManager === 'npm') {
|
||||
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
|
||||
return parseNpmLockfile(content);
|
||||
parseNpmLockfile(content, builder);
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
} catch (e) {
|
||||
if (!isPostInstallProcess()) {
|
||||
@ -138,19 +146,26 @@ export function createLockFile(
|
||||
const normalizedPackageJson = normalizePackageJson(packageJson);
|
||||
const content = readFileSync(getLockFileName(packageManager), 'utf8');
|
||||
|
||||
const builder = new ProjectGraphBuilder();
|
||||
|
||||
try {
|
||||
if (packageManager === 'yarn') {
|
||||
const graph = parseYarnLockfile(content);
|
||||
parseYarnLockfile(content, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
|
||||
}
|
||||
if (packageManager === 'pnpm') {
|
||||
const graph = parsePnpmLockfile(content);
|
||||
parsePnpmLockfile(content, builder);
|
||||
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
|
||||
}
|
||||
if (packageManager === 'npm') {
|
||||
const graph = parseNpmLockfile(content);
|
||||
parseNpmLockfile(content, builder);
|
||||
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
|
||||
}
|
||||
@ -1,20 +1,12 @@
|
||||
import { joinPathFragments } from '../utils/path';
|
||||
import { joinPathFragments } from '../../../utils/path';
|
||||
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
||||
import { pruneProjectGraph } from './project-graph-pruning';
|
||||
import { vol } from 'memfs';
|
||||
import { ProjectGraph } from '../config/project-graph';
|
||||
import { ProjectGraph } from '../../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
|
||||
jest.mock('fs', () => require('memfs').fs);
|
||||
|
||||
jest.mock('@nrwl/devkit', () => ({
|
||||
...jest.requireActual<any>('@nrwl/devkit'),
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
describe('NPM lock file utility', () => {
|
||||
afterEach(() => {
|
||||
vol.reset();
|
||||
@ -29,7 +21,9 @@ describe('NPM lock file utility', () => {
|
||||
let graph: ProjectGraph;
|
||||
|
||||
beforeEach(() => {
|
||||
graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
graph = builder.getUpdatedProjectGraph();
|
||||
});
|
||||
|
||||
it('should parse root lock file', async () => {
|
||||
@ -47,7 +41,9 @@ describe('NPM lock file utility', () => {
|
||||
));
|
||||
|
||||
// this is original generated lock file
|
||||
const appGraph = parseNpmLockfile(JSON.stringify(appLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(appLockFile), builder);
|
||||
const appGraph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
|
||||
|
||||
// this is our pruned lock file structure
|
||||
@ -93,7 +89,9 @@ describe('NPM lock file utility', () => {
|
||||
'__fixtures__/auxiliary-packages/package-lock.json'
|
||||
));
|
||||
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
||||
|
||||
@ -150,7 +148,9 @@ describe('NPM lock file utility', () => {
|
||||
'__fixtures__/auxiliary-packages/package-lock-v2.json'
|
||||
));
|
||||
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
||||
|
||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||
@ -246,7 +246,9 @@ describe('NPM lock file utility', () => {
|
||||
cleanupTypes(prunedV2LockFile.packages);
|
||||
cleanupTypes(prunedV2LockFile.dependencies, true);
|
||||
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||
const result = stringifyNpmLockfile(
|
||||
prunedGraph,
|
||||
@ -331,7 +333,9 @@ describe('NPM lock file utility', () => {
|
||||
'__fixtures__/duplicate-package/package-lock-v1.json'
|
||||
));
|
||||
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(369); // 338
|
||||
});
|
||||
it('should parse v3', async () => {
|
||||
@ -340,7 +344,9 @@ describe('NPM lock file utility', () => {
|
||||
'__fixtures__/duplicate-package/package-lock.json'
|
||||
));
|
||||
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(369); //338
|
||||
});
|
||||
});
|
||||
@ -355,7 +361,9 @@ describe('NPM lock file utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/optional/package.json'
|
||||
));
|
||||
const graph = parseNpmLockfile(JSON.stringify(lockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
||||
|
||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||
@ -378,7 +386,9 @@ describe('NPM lock file utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/pruning/typescript/package.json'
|
||||
));
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
||||
const result = stringifyNpmLockfile(
|
||||
prunedGraph,
|
||||
@ -403,7 +413,9 @@ describe('NPM lock file utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/pruning/devkit-yargs/package.json'
|
||||
));
|
||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
||||
const result = stringifyNpmLockfile(
|
||||
prunedGraph,
|
||||
@ -432,7 +444,9 @@ describe('NPM lock file utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/workspaces/package-lock.json'
|
||||
));
|
||||
const result = parseNpmLockfile(JSON.stringify(lockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||
const result = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
||||
});
|
||||
|
||||
@ -441,7 +455,9 @@ describe('NPM lock file utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/workspaces/package-lock.v1.json'
|
||||
));
|
||||
const result = parseNpmLockfile(JSON.stringify(lockFile));
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||
const result = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
||||
});
|
||||
});
|
||||
@ -1,14 +1,14 @@
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { satisfies } from 'semver';
|
||||
import { workspaceRoot } from '../utils/workspace-root';
|
||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
||||
import { reverse } from '../project-graph/operators';
|
||||
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||
import { reverse } from '../../../project-graph/operators';
|
||||
import { NormalizedPackageJson } from './utils/package-json';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
import {
|
||||
ProjectGraph,
|
||||
ProjectGraphExternalNode,
|
||||
} from '../config/project-graph';
|
||||
import { NormalizedPackageJson } from './utils/package-json';
|
||||
import { defaultHashing } from '../hasher/hashing-impl';
|
||||
} from '../../../config/project-graph';
|
||||
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||
|
||||
/**
|
||||
* NPM
|
||||
@ -50,16 +50,16 @@ type NpmLockFile = {
|
||||
dependencies?: Record<string, NpmDependencyV1>;
|
||||
};
|
||||
|
||||
export function parseNpmLockfile(lockFileContent: string): ProjectGraph {
|
||||
export function parseNpmLockfile(
|
||||
lockFileContent: string,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
const data = JSON.parse(lockFileContent) as NpmLockFile;
|
||||
const builder = new ProjectGraphBuilder();
|
||||
|
||||
// we use key => node map to avoid duplicate work when parsing keys
|
||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||
addNodes(data, builder, keyMap);
|
||||
addDependencies(data, builder, keyMap);
|
||||
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
|
||||
function addNodes(
|
||||
@ -1,17 +1,13 @@
|
||||
import { joinPathFragments } from '../utils/path';
|
||||
import { joinPathFragments } from '../../../utils/path';
|
||||
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
||||
import { ProjectGraph } from '../config/project-graph';
|
||||
import { ProjectGraph } from '../../../config/project-graph';
|
||||
import { vol } from 'memfs';
|
||||
import { pruneProjectGraph } from './project-graph-pruning';
|
||||
import { ProjectGraphBuilder } from 'nx/src/project-graph/project-graph-builder';
|
||||
|
||||
jest.mock('fs', () => require('memfs').fs);
|
||||
|
||||
jest.mock('@nrwl/devkit', () => ({
|
||||
...jest.requireActual<any>('@nrwl/devkit'),
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
||||
jest.mock('../../../utils/workspace-root', () => ({
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
@ -121,7 +117,9 @@ describe('pnpm LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/nextjs/pnpm-lock.yaml'
|
||||
)).default;
|
||||
graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
graph = builder.getUpdatedProjectGraph();
|
||||
});
|
||||
|
||||
it('should parse root lock file', async () => {
|
||||
@ -188,7 +186,9 @@ describe('pnpm LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
|
||||
)).default;
|
||||
const graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(213); //202
|
||||
|
||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||
@ -268,7 +268,9 @@ describe('pnpm LockFile utility', () => {
|
||||
},
|
||||
};
|
||||
|
||||
const graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
|
||||
const result = stringifyPnpmLockfile(
|
||||
prunedGraph,
|
||||
@ -305,7 +307,9 @@ describe('pnpm LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/duplicate-package/pnpm-lock.yaml'
|
||||
)).default;
|
||||
const graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(370); //337
|
||||
expect(Object.keys(graph.dependencies).length).toEqual(213);
|
||||
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
|
||||
@ -329,7 +333,9 @@ describe('pnpm LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/optional/pnpm-lock.yaml'
|
||||
)).default;
|
||||
const graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
||||
|
||||
const packageJson = require(joinPathFragments(
|
||||
@ -364,7 +370,9 @@ describe('pnpm LockFile utility', () => {
|
||||
'__fixtures__/pruning/pnpm-lock.yaml'
|
||||
)).default;
|
||||
|
||||
graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
graph = builder.getUpdatedProjectGraph();
|
||||
});
|
||||
|
||||
it('should prune single package', () => {
|
||||
@ -426,7 +434,9 @@ describe('pnpm LockFile utility', () => {
|
||||
});
|
||||
|
||||
it('should parse lock file', async () => {
|
||||
const graph = parsePnpmLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parsePnpmLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||
});
|
||||
});
|
||||
@ -11,24 +11,24 @@ import {
|
||||
} from './utils/pnpm-normalizer';
|
||||
import { getHoistedPackageVersion } from './utils/package-json';
|
||||
import { NormalizedPackageJson } from './utils/package-json';
|
||||
import { sortObjectByKeys } from '../utils/object-sort';
|
||||
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
import {
|
||||
ProjectGraph,
|
||||
ProjectGraphExternalNode,
|
||||
} from '../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
||||
import { defaultHashing } from '../hasher/hashing-impl';
|
||||
} from '../../../config/project-graph';
|
||||
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||
|
||||
export function parsePnpmLockfile(lockFileContent: string): ProjectGraph {
|
||||
export function parsePnpmLockfile(
|
||||
lockFileContent: string,
|
||||
builder: ProjectGraphBuilder
|
||||
): void {
|
||||
const data = parseAndNormalizePnpmLockfile(lockFileContent);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
|
||||
// we use key => node map to avoid duplicate work when parsing keys
|
||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||
addNodes(data, builder, keyMap);
|
||||
addDependencies(data, builder, keyMap);
|
||||
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
|
||||
function addNodes(
|
||||
@ -1,11 +1,11 @@
|
||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
||||
import {
|
||||
ProjectGraph,
|
||||
ProjectGraphExternalNode,
|
||||
} from '../config/project-graph';
|
||||
import { PackageJson } from '../utils/package-json';
|
||||
import { reverse } from '../project-graph/operators';
|
||||
} from '../../../config/project-graph';
|
||||
import { satisfies, gte } from 'semver';
|
||||
import { PackageJson } from '../../../utils/package-json';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
import { reverse } from '../../../project-graph/operators';
|
||||
|
||||
/**
|
||||
* Prune project graph's external nodes and their dependencies
|
||||
11
packages/nx/src/plugins/js/lock-file/remove-npm-nodes.ts
Normal file
11
packages/nx/src/plugins/js/lock-file/remove-npm-nodes.ts
Normal file
@ -0,0 +1,11 @@
|
||||
import { ProjectGraph } from '../../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
|
||||
export function removeNpmNodes(
|
||||
graph: ProjectGraph,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
for (const externalNode in graph.externalNodes) {
|
||||
builder.removeNode(externalNode);
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { PackageJson } from '../../utils/package-json';
|
||||
import { workspaceRoot } from '../../utils/workspace-root';
|
||||
import { PackageJson } from '../../../../utils/package-json';
|
||||
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||
|
||||
/**
|
||||
* Get version of hoisted package if available
|
||||
@ -10,7 +10,7 @@ import type {
|
||||
} from '@pnpm/lockfile-types';
|
||||
import { dump, load } from '@zkochan/js-yaml';
|
||||
import { existsSync, readFileSync } from 'fs';
|
||||
import { workspaceRoot } from '../../utils/workspace-root';
|
||||
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||
|
||||
const LOCKFILE_YAML_FORMAT = {
|
||||
blankLines: true,
|
||||
@ -1,9 +1,10 @@
|
||||
import { joinPathFragments } from '../utils/path';
|
||||
import { joinPathFragments } from '../../../utils/path';
|
||||
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
||||
import { pruneProjectGraph } from './project-graph-pruning';
|
||||
import { vol } from 'memfs';
|
||||
import { ProjectGraph } from '../config/project-graph';
|
||||
import { PackageJson } from '../utils/package-json';
|
||||
import { ProjectGraph } from '../../../config/project-graph';
|
||||
import { PackageJson } from '../../../utils/package-json';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
|
||||
jest.mock('fs', () => require('memfs').fs);
|
||||
|
||||
@ -12,7 +13,7 @@ jest.mock('@nrwl/devkit', () => ({
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
||||
jest.mock('../../../utils/workspace-root', () => ({
|
||||
workspaceRoot: '/root',
|
||||
}));
|
||||
|
||||
@ -156,11 +157,13 @@ describe('yarn LockFile utility', () => {
|
||||
let graph: ProjectGraph;
|
||||
|
||||
beforeEach(() => {
|
||||
const builder = new ProjectGraphBuilder();
|
||||
lockFile = require(joinPathFragments(
|
||||
__dirname,
|
||||
'__fixtures__/nextjs/yarn.lock'
|
||||
)).default;
|
||||
graph = parseYarnLockfile(lockFile);
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
graph = builder.getUpdatedProjectGraph();
|
||||
});
|
||||
|
||||
it('should parse root lock file', async () => {
|
||||
@ -225,7 +228,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/auxiliary-packages/yarn.lock'
|
||||
)).default;
|
||||
const graph = parseYarnLockfile(classicLockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(classicLockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(127); // 124 hoisted
|
||||
|
||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||
@ -301,7 +306,9 @@ describe('yarn LockFile utility', () => {
|
||||
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
||||
)).default;
|
||||
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||
const result = stringifyYarnLockfile(
|
||||
prunedGraph,
|
||||
@ -337,7 +344,9 @@ describe('yarn LockFile utility', () => {
|
||||
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
||||
)).default;
|
||||
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||
const result = stringifyYarnLockfile(
|
||||
prunedGraph,
|
||||
@ -357,7 +366,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/auxiliary-packages/yarn-berry.lock'
|
||||
)).default;
|
||||
const graph = parseYarnLockfile(berryLockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(berryLockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(128); //124 hoisted
|
||||
|
||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||
@ -433,7 +444,9 @@ describe('yarn LockFile utility', () => {
|
||||
'__fixtures__/auxiliary-packages/yarn-berry.lock.pruned'
|
||||
)).default;
|
||||
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||
const result = stringifyYarnLockfile(
|
||||
prunedGraph,
|
||||
@ -489,7 +502,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/duplicate-package/yarn.lock'
|
||||
)).default;
|
||||
const graph = parseYarnLockfile(classicLockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(classicLockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(371); //337 hoisted
|
||||
});
|
||||
});
|
||||
@ -517,7 +532,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/optional/package.json'
|
||||
));
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(103);
|
||||
|
||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||
@ -694,7 +711,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/pruning/typescript/package.json'
|
||||
));
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
||||
const result = stringifyYarnLockfile(
|
||||
prunedGraph,
|
||||
@ -719,7 +738,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/pruning/devkit-yargs/package.json'
|
||||
));
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
||||
const result = stringifyYarnLockfile(
|
||||
prunedGraph,
|
||||
@ -750,7 +771,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/workspaces/yarn.lock'
|
||||
)).default;
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||
});
|
||||
|
||||
@ -759,7 +782,9 @@ describe('yarn LockFile utility', () => {
|
||||
__dirname,
|
||||
'__fixtures__/workspaces/yarn.lock.berry'
|
||||
)).default;
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||
});
|
||||
});
|
||||
@ -800,7 +825,9 @@ type-fest@^0.20.2:
|
||||
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
|
||||
`;
|
||||
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"data": Object {
|
||||
@ -873,7 +900,9 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
`;
|
||||
const graph = parseYarnLockfile(lockFile);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
parseYarnLockfile(lockFile, builder);
|
||||
const graph = builder.getUpdatedProjectGraph();
|
||||
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"data": Object {
|
||||
@ -1,15 +1,15 @@
|
||||
import { parseSyml, stringifySyml } from '@yarnpkg/parsers';
|
||||
import { stringify } from '@yarnpkg/lockfile';
|
||||
import { sortObjectByKeys } from '../utils/object-sort';
|
||||
import { getHoistedPackageVersion } from './utils/package-json';
|
||||
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||
import { satisfies } from 'semver';
|
||||
import { NormalizedPackageJson } from './utils/package-json';
|
||||
import {
|
||||
ProjectGraph,
|
||||
ProjectGraphExternalNode,
|
||||
} from '../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
||||
import { satisfies } from 'semver';
|
||||
import { NormalizedPackageJson } from './utils/package-json';
|
||||
import { defaultHashing } from '../hasher/hashing-impl';
|
||||
} from '../../../config/project-graph';
|
||||
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||
|
||||
/**
|
||||
* Yarn
|
||||
@ -37,16 +37,16 @@ type YarnDependency = {
|
||||
linkType?: 'soft' | 'hard';
|
||||
};
|
||||
|
||||
export function parseYarnLockfile(lockFileContent: string): ProjectGraph {
|
||||
export function parseYarnLockfile(
|
||||
lockFileContent: string,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
const data = parseSyml(lockFileContent);
|
||||
const builder = new ProjectGraphBuilder();
|
||||
|
||||
// we use key => node map to avoid duplicate work when parsing keys
|
||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||
addNodes(data, builder, keyMap);
|
||||
addDependencies(data, builder, keyMap);
|
||||
|
||||
return builder.getUpdatedProjectGraph();
|
||||
}
|
||||
|
||||
function addNodes(
|
||||
@ -1,10 +1,10 @@
|
||||
import * as fs from 'fs';
|
||||
|
||||
import * as configModule from '../config/configuration';
|
||||
import { DependencyType } from '../config/project-graph';
|
||||
import * as hashModule from '../hasher/hasher';
|
||||
import * as configModule from '../../../config/configuration';
|
||||
import { DependencyType } from '../../../config/project-graph';
|
||||
import * as hashModule from '../../../hasher/hasher';
|
||||
import { createPackageJson } from './create-package-json';
|
||||
import * as fileutilsModule from './fileutils';
|
||||
import * as fileutilsModule from '../../../utils/fileutils';
|
||||
|
||||
describe('createPackageJson', () => {
|
||||
it('should add additional dependencies', () => {
|
||||
@ -1,11 +1,17 @@
|
||||
import { readJsonFile } from './fileutils';
|
||||
import { sortObjectByKeys } from './object-sort';
|
||||
import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
|
||||
import { PackageJson } from './package-json';
|
||||
import { readJsonFile } from '../../../utils/fileutils';
|
||||
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||
import {
|
||||
ProjectGraph,
|
||||
ProjectGraphProjectNode,
|
||||
} from '../../../config/project-graph';
|
||||
import { PackageJson } from '../../../utils/package-json';
|
||||
import { existsSync } from 'fs';
|
||||
import { workspaceRoot } from './workspace-root';
|
||||
import { filterUsingGlobPatterns, getTargetInputs } from '../hasher/hasher';
|
||||
import { readNxJson } from '../config/configuration';
|
||||
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||
import {
|
||||
filterUsingGlobPatterns,
|
||||
getTargetInputs,
|
||||
} from '../../../hasher/hasher';
|
||||
import { readNxJson } from '../../../config/configuration';
|
||||
|
||||
interface NpmDeps {
|
||||
readonly dependencies: Record<string, string>;
|
||||
@ -0,0 +1,189 @@
|
||||
import {
|
||||
DependencyType,
|
||||
ProjectFileMap,
|
||||
ProjectGraphProcessorContext,
|
||||
} from '../../../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||
import { join } from 'path';
|
||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
|
||||
import * as os from 'os';
|
||||
|
||||
export function buildExplicitDependencies(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
|
||||
// using workers has an overhead, so we only do it when the number of
|
||||
// files we need to process is >= 100 and there are more than 2 CPUs
|
||||
// to be able to use at least 2 workers (1 worker per CPU and
|
||||
// 1 CPU for the main thread)
|
||||
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
|
||||
return buildExplicitDependenciesWithoutWorkers(
|
||||
jsPluginConfig,
|
||||
ctx,
|
||||
builder
|
||||
);
|
||||
} else {
|
||||
return buildExplicitDependenciesUsingWorkers(
|
||||
jsPluginConfig,
|
||||
ctx,
|
||||
totalNumOfFilesToProcess,
|
||||
builder
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
|
||||
let totalNumOfFilesToProcess = 0;
|
||||
Object.values(ctx.filesToProcess).forEach(
|
||||
(t) => (totalNumOfFilesToProcess += t.length)
|
||||
);
|
||||
return totalNumOfFilesToProcess;
|
||||
}
|
||||
|
||||
function splitFilesIntoBins(
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
totalNumOfFilesToProcess: number,
|
||||
numberOfWorkers: number
|
||||
) {
|
||||
// we want to have numberOfWorkers * 5 bins
|
||||
const filesPerBin =
|
||||
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
|
||||
const bins: ProjectFileMap[] = [];
|
||||
let currentProjectFileMap = {};
|
||||
let currentNumberOfFiles = 0;
|
||||
for (const source of Object.keys(ctx.filesToProcess)) {
|
||||
for (const f of Object.values(ctx.filesToProcess[source])) {
|
||||
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
|
||||
currentProjectFileMap[source].push(f);
|
||||
currentNumberOfFiles++;
|
||||
|
||||
if (currentNumberOfFiles >= filesPerBin) {
|
||||
bins.push(currentProjectFileMap);
|
||||
currentProjectFileMap = {};
|
||||
currentNumberOfFiles = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
bins.push(currentProjectFileMap);
|
||||
return bins;
|
||||
}
|
||||
|
||||
function createWorkerPool(numberOfWorkers: number) {
|
||||
const res = [];
|
||||
for (let i = 0; i < numberOfWorkers; ++i) {
|
||||
res.push(
|
||||
new (require('worker_threads').Worker)(
|
||||
join(__dirname, './project-graph-worker.js'),
|
||||
{
|
||||
env: process.env,
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
function buildExplicitDependenciesWithoutWorkers(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
buildExplicitTypescriptAndPackageJsonDependencies(
|
||||
jsPluginConfig,
|
||||
ctx.nxJsonConfiguration,
|
||||
ctx.projectsConfigurations,
|
||||
builder.graph,
|
||||
ctx.filesToProcess
|
||||
).forEach((r) => {
|
||||
if (r.type === DependencyType.static) {
|
||||
builder.addStaticDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
} else {
|
||||
builder.addDynamicDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function buildExplicitDependenciesUsingWorkers(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
totalNumOfFilesToProcess: number,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
const numberOfWorkers = Math.min(
|
||||
totalNumOfFilesToProcess,
|
||||
getNumberOfWorkers()
|
||||
);
|
||||
const bins = splitFilesIntoBins(
|
||||
ctx,
|
||||
totalNumOfFilesToProcess,
|
||||
numberOfWorkers
|
||||
);
|
||||
const workers = createWorkerPool(numberOfWorkers);
|
||||
let numberOfExpectedResponses = bins.length;
|
||||
|
||||
return new Promise((res, reject) => {
|
||||
for (let w of workers) {
|
||||
w.on('message', (explicitDependencies) => {
|
||||
explicitDependencies.forEach((r) => {
|
||||
builder.addExplicitDependency(
|
||||
r.sourceProjectName,
|
||||
r.sourceProjectFile,
|
||||
r.targetProjectName
|
||||
);
|
||||
});
|
||||
if (bins.length > 0) {
|
||||
w.postMessage({ filesToProcess: bins.shift() });
|
||||
}
|
||||
// we processed all the bins
|
||||
if (--numberOfExpectedResponses === 0) {
|
||||
for (let w of workers) {
|
||||
w.terminate();
|
||||
}
|
||||
res(null);
|
||||
}
|
||||
});
|
||||
w.on('error', reject);
|
||||
w.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
reject(
|
||||
new Error(
|
||||
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
w.postMessage({
|
||||
nxJsonConfiguration: ctx.nxJsonConfiguration,
|
||||
projectsConfigurations: ctx.projectsConfigurations,
|
||||
projectGraph: builder.graph,
|
||||
jsPluginConfig,
|
||||
});
|
||||
w.postMessage({ filesToProcess: bins.shift() });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getNumberOfWorkers(): number {
|
||||
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
|
||||
}
|
||||
@ -3,9 +3,9 @@ import {
|
||||
ExplicitDependency,
|
||||
} from './explicit-project-dependencies';
|
||||
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
||||
import { ProjectFileMap, ProjectGraph } from '../../config/project-graph';
|
||||
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
|
||||
import { NxJsonConfiguration } from '../../config/nx-json';
|
||||
import { ProjectFileMap, ProjectGraph } from '../../../../config/project-graph';
|
||||
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||
|
||||
export function buildExplicitTypescriptAndPackageJsonDependencies(
|
||||
jsPluginConfig: {
|
||||
@ -1,15 +1,15 @@
|
||||
import { TempFs } from '../../utils/testing/temp-fs';
|
||||
import { TempFs } from '../../../../utils/testing/temp-fs';
|
||||
const tempFs = new TempFs('explicit-package-json');
|
||||
|
||||
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
||||
|
||||
import { createProjectFileMap } from '../file-map-utils';
|
||||
import { defaultFileHasher } from '../../hasher/file-hasher';
|
||||
import { defaultFileHasher } from '../../../../hasher/file-hasher';
|
||||
import {
|
||||
ProjectGraphProcessorContext,
|
||||
ProjectGraphProjectNode,
|
||||
} from '../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../project-graph-builder';
|
||||
} from '../../../../config/project-graph';
|
||||
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
|
||||
|
||||
describe('explicit package json dependencies', () => {
|
||||
let ctx: ProjectGraphProcessorContext;
|
||||
@ -1,14 +1,14 @@
|
||||
import { defaultFileRead } from '../file-utils';
|
||||
import { defaultFileRead } from '../../../../project-graph/file-utils';
|
||||
import { join } from 'path';
|
||||
import {
|
||||
DependencyType,
|
||||
ProjectFileMap,
|
||||
ProjectGraph,
|
||||
} from '../../config/project-graph';
|
||||
import { parseJson } from '../../utils/json';
|
||||
import { getImportPath, joinPathFragments } from '../../utils/path';
|
||||
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
|
||||
import { NxJsonConfiguration } from '../../config/nx-json';
|
||||
} from '../../../../config/project-graph';
|
||||
import { parseJson } from '../../../../utils/json';
|
||||
import { getImportPath, joinPathFragments } from '../../../../utils/path';
|
||||
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||
import { ExplicitDependency } from './explicit-project-dependencies';
|
||||
|
||||
class ProjectGraphNodeRecords {}
|
||||
@ -1,9 +1,9 @@
|
||||
import { TempFs } from '../../utils/testing/temp-fs';
|
||||
import { TempFs } from '../../../../utils/testing/temp-fs';
|
||||
const tempFs = new TempFs('explicit-project-deps');
|
||||
|
||||
import { defaultFileHasher } from '../../hasher/file-hasher';
|
||||
import { createProjectFileMap } from '../file-map-utils';
|
||||
import { ProjectGraphBuilder } from '../project-graph-builder';
|
||||
import { defaultFileHasher } from '../../../../hasher/file-hasher';
|
||||
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
|
||||
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||
import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies';
|
||||
|
||||
// projectName => tsconfig import path
|
||||
@ -1,10 +1,10 @@
|
||||
import { TypeScriptImportLocator } from './typescript-import-locator';
|
||||
import { TargetProjectLocator } from '../../utils/target-project-locator';
|
||||
import { TargetProjectLocator } from './target-project-locator';
|
||||
import {
|
||||
DependencyType,
|
||||
ProjectFileMap,
|
||||
ProjectGraph,
|
||||
} from '../../config/project-graph';
|
||||
} from '../../../../config/project-graph';
|
||||
|
||||
export type ExplicitDependency = {
|
||||
sourceProjectName: string;
|
||||
@ -1,8 +1,8 @@
|
||||
import { parentPort } from 'worker_threads';
|
||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
|
||||
import { ProjectGraph } from '../config/project-graph';
|
||||
import { ProjectsConfigurations } from '../config/workspace-json-project-json';
|
||||
import { NxJsonConfiguration } from '../config/nx-json';
|
||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
|
||||
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||
import { ProjectGraph } from '../../../../config/project-graph';
|
||||
|
||||
let nxJsonConfiguration: NxJsonConfiguration | null;
|
||||
let projectsConfigurations: ProjectsConfigurations | null;
|
||||
@ -4,7 +4,7 @@ import {
|
||||
ProjectGraphExternalNode,
|
||||
ProjectGraphProcessorContext,
|
||||
ProjectGraphProjectNode,
|
||||
} from '../config/project-graph';
|
||||
} from '../../../../config/project-graph';
|
||||
|
||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
||||
workspaceRoot: '/root',
|
||||
@ -1,16 +1,19 @@
|
||||
import { getRootTsConfigFileName, resolveModuleByImport } from './typescript';
|
||||
import { isRelativePath, readJsonFile } from './fileutils';
|
||||
import {
|
||||
getRootTsConfigFileName,
|
||||
resolveModuleByImport,
|
||||
} from '../../../../utils/typescript';
|
||||
import { isRelativePath, readJsonFile } from '../../../../utils/fileutils';
|
||||
import { dirname, join, posix } from 'path';
|
||||
import { workspaceRoot } from './workspace-root';
|
||||
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||
import {
|
||||
ProjectGraphExternalNode,
|
||||
ProjectGraphProjectNode,
|
||||
} from '../config/project-graph';
|
||||
} from '../../../../config/project-graph';
|
||||
import { builtinModules } from 'module';
|
||||
import {
|
||||
createProjectRootMappings,
|
||||
findProjectForPath,
|
||||
} from '../project-graph/utils/find-project-for-path';
|
||||
import { builtinModules } from 'module';
|
||||
} from '../../../../project-graph/utils/find-project-for-path';
|
||||
|
||||
const builtInModuleSet = new Set<string>([
|
||||
...builtinModules,
|
||||
@ -1,8 +1,8 @@
|
||||
import type * as ts from 'typescript';
|
||||
import * as path from 'path';
|
||||
import { stripSourceCode } from '../../utils/strip-source-code';
|
||||
import { defaultFileRead } from '../file-utils';
|
||||
import { DependencyType } from '../../config/project-graph';
|
||||
import { stripSourceCode } from './strip-source-code';
|
||||
import { DependencyType } from '../../../../config/project-graph';
|
||||
import { defaultFileRead } from '../../../../project-graph/file-utils';
|
||||
|
||||
let tsModule: typeof import('typescript');
|
||||
|
||||
@ -1,3 +1 @@
|
||||
export * from './implicit-project-dependencies';
|
||||
export * from './explicit-project-dependencies';
|
||||
export * from './explicit-package-json-dependencies';
|
||||
|
||||
@ -201,17 +201,6 @@ describe('project graph', () => {
|
||||
jest.spyOn(fastGlob, 'sync').mockImplementation(() => globResults);
|
||||
});
|
||||
|
||||
it('should throw an appropriate error for an invalid json config', async () => {
|
||||
tempFs.appendFile('tsconfig.base.json', 'invalid');
|
||||
try {
|
||||
await buildProjectGraph();
|
||||
fail('Invalid tsconfigs should cause project graph to throw error');
|
||||
} catch (e) {
|
||||
expect(e.message).toContain(`${tempFs.tempDir}/tsconfig.base.json`);
|
||||
expect(e.message).toContain(`invalid`);
|
||||
}
|
||||
});
|
||||
|
||||
it('should create nodes and dependencies with workspace projects', async () => {
|
||||
const graph = await buildProjectGraph();
|
||||
|
||||
|
||||
@ -11,13 +11,8 @@ import {
|
||||
shouldRecomputeWholeGraph,
|
||||
writeCache,
|
||||
} from './nx-deps-cache';
|
||||
import {
|
||||
buildImplicitProjectDependencies,
|
||||
ExplicitDependency,
|
||||
} from './build-dependencies';
|
||||
import { buildImplicitProjectDependencies } from './build-dependencies';
|
||||
import { buildWorkspaceProjectNodes } from './build-nodes';
|
||||
import * as os from 'os';
|
||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
|
||||
import { loadNxPlugins } from '../utils/nx-plugin';
|
||||
import { defaultFileHasher } from '../hasher/file-hasher';
|
||||
import { createProjectFileMap } from './file-map-utils';
|
||||
@ -28,7 +23,7 @@ import {
|
||||
ProjectGraphProcessorContext,
|
||||
} from '../config/project-graph';
|
||||
import { readJsonFile } from '../utils/fileutils';
|
||||
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../config/nx-json';
|
||||
import { NxJsonConfiguration } from '../config/nx-json';
|
||||
import { logger } from '../utils/logger';
|
||||
import { ProjectGraphBuilder } from './project-graph-builder';
|
||||
import {
|
||||
@ -36,11 +31,6 @@ import {
|
||||
ProjectsConfigurations,
|
||||
} from '../config/workspace-json-project-json';
|
||||
import { readNxJson } from '../config/configuration';
|
||||
import {
|
||||
lockFileExists,
|
||||
lockFileHash,
|
||||
parseLockFile,
|
||||
} from '../lock-file/lock-file';
|
||||
import { Workspaces } from '../config/workspaces';
|
||||
import { existsSync } from 'fs';
|
||||
import { PackageJson } from '../utils/package-json';
|
||||
@ -102,17 +92,7 @@ export async function buildProjectGraphUsingProjectFileMap(
|
||||
filesToProcess = projectFileMap;
|
||||
cachedFileData = {};
|
||||
}
|
||||
let partialGraph: ProjectGraph;
|
||||
let lockHash = 'n/a';
|
||||
// during the create-nx-workspace lock file might not exists yet
|
||||
if (lockFileExists()) {
|
||||
lockHash = lockFileHash();
|
||||
if (cache && cache.lockFileHash === lockHash) {
|
||||
partialGraph = isolatePartialGraphFromCache(cache);
|
||||
} else {
|
||||
partialGraph = parseLockFile();
|
||||
}
|
||||
}
|
||||
|
||||
const context = createContext(
|
||||
projectsConfigurations,
|
||||
nxJson,
|
||||
@ -124,15 +104,13 @@ export async function buildProjectGraphUsingProjectFileMap(
|
||||
context,
|
||||
cachedFileData,
|
||||
projectGraphVersion,
|
||||
partialGraph,
|
||||
packageJsonDeps
|
||||
cache
|
||||
);
|
||||
const projectGraphCache = createCache(
|
||||
nxJson,
|
||||
packageJsonDeps,
|
||||
projectGraph,
|
||||
rootTsConfig,
|
||||
lockHash
|
||||
rootTsConfig
|
||||
);
|
||||
if (shouldWriteCache) {
|
||||
writeCache(projectGraphCache);
|
||||
@ -168,33 +146,24 @@ function readCombinedDeps() {
|
||||
};
|
||||
}
|
||||
|
||||
// extract only external nodes and their dependencies
|
||||
function isolatePartialGraphFromCache(cache: ProjectGraphCache): ProjectGraph {
|
||||
const dependencies = {};
|
||||
Object.keys(cache.dependencies).forEach((k) => {
|
||||
if (cache.externalNodes[k]) {
|
||||
dependencies[k] = cache.dependencies[k];
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
nodes: {},
|
||||
dependencies,
|
||||
externalNodes: cache.externalNodes,
|
||||
};
|
||||
}
|
||||
|
||||
async function buildProjectGraphUsingContext(
|
||||
nxJson: NxJsonConfiguration,
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
cachedFileData: { [project: string]: { [file: string]: FileData } },
|
||||
projectGraphVersion: string,
|
||||
partialGraph: ProjectGraph,
|
||||
packageJsonDeps: Record<string, string>
|
||||
cache: ProjectGraphCache | null
|
||||
) {
|
||||
performance.mark('build project graph:start');
|
||||
|
||||
const builder = new ProjectGraphBuilder(partialGraph);
|
||||
const builder = new ProjectGraphBuilder(
|
||||
cache
|
||||
? {
|
||||
nodes: cache.nodes,
|
||||
externalNodes: cache.externalNodes,
|
||||
dependencies: cache.dependencies,
|
||||
}
|
||||
: null
|
||||
);
|
||||
builder.setVersion(projectGraphVersion);
|
||||
|
||||
await buildWorkspaceProjectNodes(ctx, builder, nxJson);
|
||||
@ -212,12 +181,6 @@ async function buildProjectGraphUsingContext(
|
||||
}
|
||||
}
|
||||
|
||||
await buildExplicitDependencies(
|
||||
jsPluginConfig(nxJson, packageJsonDeps),
|
||||
ctx,
|
||||
updatedBuilder
|
||||
);
|
||||
|
||||
buildImplicitProjectDependencies(ctx, updatedBuilder);
|
||||
|
||||
const finalGraph = updatedBuilder.getUpdatedProjectGraph();
|
||||
@ -232,216 +195,6 @@ async function buildProjectGraphUsingContext(
|
||||
return finalGraph;
|
||||
}
|
||||
|
||||
function jsPluginConfig(
|
||||
nxJson: NxJsonConfiguration,
|
||||
packageJsonDeps: { [packageName: string]: string }
|
||||
): NrwlJsPluginConfig {
|
||||
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
|
||||
return nxJson?.pluginsConfig?.['@nrwl/js'];
|
||||
}
|
||||
if (
|
||||
packageJsonDeps['@nrwl/workspace'] ||
|
||||
packageJsonDeps['@nrwl/js'] ||
|
||||
packageJsonDeps['@nrwl/node'] ||
|
||||
packageJsonDeps['@nrwl/next'] ||
|
||||
packageJsonDeps['@nrwl/react'] ||
|
||||
packageJsonDeps['@nrwl/angular'] ||
|
||||
packageJsonDeps['@nrwl/web']
|
||||
) {
|
||||
return { analyzePackageJson: true, analyzeSourceFiles: true };
|
||||
} else {
|
||||
return { analyzePackageJson: true, analyzeSourceFiles: false };
|
||||
}
|
||||
}
|
||||
|
||||
function buildExplicitDependencies(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
|
||||
// using workers has an overhead, so we only do it when the number of
|
||||
// files we need to process is >= 100 and there are more than 2 CPUs
|
||||
// to be able to use at least 2 workers (1 worker per CPU and
|
||||
// 1 CPU for the main thread)
|
||||
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
|
||||
return buildExplicitDependenciesWithoutWorkers(
|
||||
jsPluginConfig,
|
||||
ctx,
|
||||
builder
|
||||
);
|
||||
} else {
|
||||
return buildExplicitDependenciesUsingWorkers(
|
||||
jsPluginConfig,
|
||||
ctx,
|
||||
totalNumOfFilesToProcess,
|
||||
builder
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
|
||||
let totalNumOfFilesToProcess = 0;
|
||||
Object.values(ctx.filesToProcess).forEach(
|
||||
(t) => (totalNumOfFilesToProcess += t.length)
|
||||
);
|
||||
return totalNumOfFilesToProcess;
|
||||
}
|
||||
|
||||
function splitFilesIntoBins(
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
totalNumOfFilesToProcess: number,
|
||||
numberOfWorkers: number
|
||||
) {
|
||||
// we want to have numberOfWorkers * 5 bins
|
||||
const filesPerBin =
|
||||
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
|
||||
const bins: ProjectFileMap[] = [];
|
||||
let currentProjectFileMap = {};
|
||||
let currentNumberOfFiles = 0;
|
||||
for (const source of Object.keys(ctx.filesToProcess)) {
|
||||
for (const f of Object.values(ctx.filesToProcess[source])) {
|
||||
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
|
||||
currentProjectFileMap[source].push(f);
|
||||
currentNumberOfFiles++;
|
||||
|
||||
if (currentNumberOfFiles >= filesPerBin) {
|
||||
bins.push(currentProjectFileMap);
|
||||
currentProjectFileMap = {};
|
||||
currentNumberOfFiles = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
bins.push(currentProjectFileMap);
|
||||
return bins;
|
||||
}
|
||||
|
||||
function createWorkerPool(numberOfWorkers: number) {
|
||||
const res = [];
|
||||
for (let i = 0; i < numberOfWorkers; ++i) {
|
||||
res.push(
|
||||
new (require('worker_threads').Worker)(
|
||||
join(__dirname, './project-graph-worker.js'),
|
||||
{
|
||||
env: process.env,
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
function buildExplicitDependenciesWithoutWorkers(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
buildExplicitTypescriptAndPackageJsonDependencies(
|
||||
jsPluginConfig,
|
||||
ctx.nxJsonConfiguration,
|
||||
ctx.projectsConfigurations,
|
||||
builder.graph,
|
||||
ctx.filesToProcess
|
||||
).forEach((r) => {
|
||||
if (r.type === 'static') {
|
||||
builder.addStaticDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
} else {
|
||||
builder.addDynamicDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function buildExplicitDependenciesUsingWorkers(
|
||||
jsPluginConfig: {
|
||||
analyzeSourceFiles?: boolean;
|
||||
analyzePackageJson?: boolean;
|
||||
},
|
||||
ctx: ProjectGraphProcessorContext,
|
||||
totalNumOfFilesToProcess: number,
|
||||
builder: ProjectGraphBuilder
|
||||
) {
|
||||
const numberOfWorkers = Math.min(
|
||||
totalNumOfFilesToProcess,
|
||||
getNumberOfWorkers()
|
||||
);
|
||||
const bins = splitFilesIntoBins(
|
||||
ctx,
|
||||
totalNumOfFilesToProcess,
|
||||
numberOfWorkers
|
||||
);
|
||||
const workers = createWorkerPool(numberOfWorkers);
|
||||
let numberOfExpectedResponses = bins.length;
|
||||
|
||||
return new Promise((res, reject) => {
|
||||
for (let w of workers) {
|
||||
w.on('message', (explicitDependencies) => {
|
||||
explicitDependencies.forEach((r: ExplicitDependency) => {
|
||||
if (r.type === 'static') {
|
||||
builder.addStaticDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
} else {
|
||||
builder.addDynamicDependency(
|
||||
r.sourceProjectName,
|
||||
r.targetProjectName,
|
||||
r.sourceProjectFile
|
||||
);
|
||||
}
|
||||
});
|
||||
if (bins.length > 0) {
|
||||
w.postMessage({ filesToProcess: bins.shift() });
|
||||
}
|
||||
// we processed all the bins
|
||||
if (--numberOfExpectedResponses === 0) {
|
||||
for (let w of workers) {
|
||||
w.terminate();
|
||||
}
|
||||
res(null);
|
||||
}
|
||||
});
|
||||
w.on('error', reject);
|
||||
w.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
reject(
|
||||
new Error(
|
||||
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
w.postMessage({
|
||||
nxJsonConfiguration: ctx.nxJsonConfiguration,
|
||||
projectsConfigurations: ctx.projectsConfigurations,
|
||||
projectGraph: builder.graph,
|
||||
jsPluginConfig,
|
||||
});
|
||||
w.postMessage({ filesToProcess: bins.shift() });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getNumberOfWorkers(): number {
|
||||
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
|
||||
}
|
||||
|
||||
function createContext(
|
||||
projectsConfigurations: ProjectsConfigurations,
|
||||
nxJson: NxJsonConfiguration,
|
||||
|
||||
@ -19,8 +19,8 @@ export function createProjectFileMap(
|
||||
projectFileMap[projectName] ??= [];
|
||||
}
|
||||
for (const f of allWorkspaceFiles) {
|
||||
const matchingProjectFiles =
|
||||
projectFileMap[findProjectForPath(f.file, projectRootMappings)];
|
||||
const projectFileMapKey = findProjectForPath(f.file, projectRootMappings);
|
||||
const matchingProjectFiles = projectFileMap[projectFileMapKey];
|
||||
if (matchingProjectFiles) {
|
||||
matchingProjectFiles.push(f);
|
||||
}
|
||||
|
||||
@ -298,8 +298,7 @@ describe('nx deps utils', () => {
|
||||
createNxJson({}),
|
||||
createPackageJsonDeps({}),
|
||||
createCache({}) as ProjectGraph,
|
||||
{},
|
||||
'abcd1234'
|
||||
{}
|
||||
);
|
||||
});
|
||||
|
||||
@ -308,8 +307,7 @@ describe('nx deps utils', () => {
|
||||
createNxJson({}),
|
||||
createPackageJsonDeps({}),
|
||||
createCache({}) as ProjectGraph,
|
||||
undefined,
|
||||
'abcd1234'
|
||||
undefined
|
||||
);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
@ -323,7 +321,6 @@ describe('nx deps utils', () => {
|
||||
'@nrwl/workspace': '12.0.0',
|
||||
plugin: '1.0.0',
|
||||
},
|
||||
lockFileHash: 'abcd1234',
|
||||
pathMappings: {
|
||||
mylib: ['libs/mylib/index.ts'],
|
||||
},
|
||||
|
||||
@ -23,7 +23,6 @@ import {
|
||||
export interface ProjectGraphCache {
|
||||
version: string;
|
||||
deps: Record<string, string>;
|
||||
lockFileHash: string;
|
||||
pathMappings: Record<string, any>;
|
||||
nxJsonPlugins: { name: string; version: string }[];
|
||||
pluginsConfig?: any;
|
||||
@ -86,8 +85,7 @@ export function createCache(
|
||||
nxJson: NxJsonConfiguration<'*' | string[]>,
|
||||
packageJsonDeps: Record<string, string>,
|
||||
projectGraph: ProjectGraph,
|
||||
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } },
|
||||
lockFileHash: string
|
||||
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } }
|
||||
) {
|
||||
const nxJsonPlugins = (nxJson.plugins || []).map((p) => ({
|
||||
name: p,
|
||||
@ -96,7 +94,6 @@ export function createCache(
|
||||
const newValue: ProjectGraphCache = {
|
||||
version: projectGraph.version || '5.1',
|
||||
deps: packageJsonDeps,
|
||||
lockFileHash,
|
||||
// compilerOptions may not exist, especially for repos converted through add-nx-to-monorepo
|
||||
pathMappings: tsConfig?.compilerOptions?.paths || {},
|
||||
nxJsonPlugins,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user