feat(core): refactor most of js graph work into its own folder (#15365)
This commit is contained in:
parent
16b3fa0931
commit
8cf8f18c1c
@ -12,7 +12,7 @@ packages/express/src/schematics/**/files/**/*.json
|
|||||||
packages/nest/src/schematics/**/files/**/*.json
|
packages/nest/src/schematics/**/files/**/*.json
|
||||||
packages/react/src/schematics/**/files/**/*.json
|
packages/react/src/schematics/**/files/**/*.json
|
||||||
packages/jest/src/schematics/**/files/**/*.json
|
packages/jest/src/schematics/**/files/**/*.json
|
||||||
packages/nx/src/lock-file/__fixtures__/**/*.*
|
packages/nx/src/plugins/js/lock-file/__fixtures__/**/*.*
|
||||||
packages/**/schematics/**/files/**/*.html
|
packages/**/schematics/**/files/**/*.html
|
||||||
packages/**/generators/**/files/**/*.html
|
packages/**/generators/**/files/**/*.html
|
||||||
packages/nx/src/native/
|
packages/nx/src/native/
|
||||||
|
|||||||
@ -26,7 +26,7 @@ import {
|
|||||||
} from '@nrwl/e2e/utils';
|
} from '@nrwl/e2e/utils';
|
||||||
import { exec, execSync } from 'child_process';
|
import { exec, execSync } from 'child_process';
|
||||||
import * as http from 'http';
|
import * as http from 'http';
|
||||||
import { getLockFileName } from 'nx/src/lock-file/lock-file';
|
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||||
import { satisfies } from 'semver';
|
import { satisfies } from 'semver';
|
||||||
|
|
||||||
function getData(port, path = '/api'): Promise<any> {
|
function getData(port, path = '/api'): Promise<any> {
|
||||||
|
|||||||
@ -9,7 +9,7 @@ import { DependencyType } from '@nrwl/devkit';
|
|||||||
import * as parser from '@typescript-eslint/parser';
|
import * as parser from '@typescript-eslint/parser';
|
||||||
import { TSESLint } from '@typescript-eslint/utils';
|
import { TSESLint } from '@typescript-eslint/utils';
|
||||||
import { vol } from 'memfs';
|
import { vol } from 'memfs';
|
||||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||||
import enforceModuleBoundaries, {
|
import enforceModuleBoundaries, {
|
||||||
RULE_NAME as enforceModuleBoundariesRuleName,
|
RULE_NAME as enforceModuleBoundariesRuleName,
|
||||||
} from '../../src/rules/enforce-module-boundaries';
|
} from '../../src/rules/enforce-module-boundaries';
|
||||||
|
|||||||
@ -33,7 +33,7 @@ import {
|
|||||||
isComboDepConstraint,
|
isComboDepConstraint,
|
||||||
} from '../utils/runtime-lint-utils';
|
} from '../utils/runtime-lint-utils';
|
||||||
import { AST_NODE_TYPES, TSESTree } from '@typescript-eslint/utils';
|
import { AST_NODE_TYPES, TSESTree } from '@typescript-eslint/utils';
|
||||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||||
import { basename, dirname, relative } from 'path';
|
import { basename, dirname, relative } from 'path';
|
||||||
import {
|
import {
|
||||||
getBarrelEntryPointByImportScope,
|
getBarrelEntryPointByImportScope,
|
||||||
|
|||||||
@ -13,7 +13,7 @@ import {
|
|||||||
} from '@nrwl/devkit';
|
} from '@nrwl/devkit';
|
||||||
import { getPath, pathExists } from './graph-utils';
|
import { getPath, pathExists } from './graph-utils';
|
||||||
import { readFileIfExisting } from 'nx/src/utils/fileutils';
|
import { readFileIfExisting } from 'nx/src/utils/fileutils';
|
||||||
import { TargetProjectLocator } from 'nx/src/utils/target-project-locator';
|
import { TargetProjectLocator } from 'nx/src/plugins/js/project-graph/build-dependencies/target-project-locator';
|
||||||
import {
|
import {
|
||||||
findProjectForPath,
|
findProjectForPath,
|
||||||
ProjectRootMappings,
|
ProjectRootMappings,
|
||||||
|
|||||||
@ -11,5 +11,5 @@ export * from './utils/assets';
|
|||||||
export * from './utils/package-json/update-package-json';
|
export * from './utils/package-json/update-package-json';
|
||||||
export { libraryGenerator } from './generators/library/library';
|
export { libraryGenerator } from './generators/library/library';
|
||||||
export { initGenerator } from './generators/init/init';
|
export { initGenerator } from './generators/init/init';
|
||||||
export { createLockFile } from 'nx/src/lock-file/lock-file';
|
export { createLockFile } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||||
export { createPackageJson } from 'nx/src/utils/create-package-json';
|
export { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
|
||||||
|
|||||||
@ -1,5 +1,8 @@
|
|||||||
import { createLockFile, getLockFileName } from 'nx/src/lock-file/lock-file';
|
import {
|
||||||
import { createPackageJson } from 'nx/src/utils/create-package-json';
|
createLockFile,
|
||||||
|
getLockFileName,
|
||||||
|
} from 'nx/src/plugins/js/lock-file/lock-file';
|
||||||
|
import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
|
||||||
import {
|
import {
|
||||||
ExecutorContext,
|
ExecutorContext,
|
||||||
getOutputsForTargetAndConfiguration,
|
getOutputsForTargetAndConfiguration,
|
||||||
|
|||||||
@ -18,7 +18,7 @@ import { createNextConfigFile } from './lib/create-next-config-file';
|
|||||||
import { checkPublicDirectory } from './lib/check-project';
|
import { checkPublicDirectory } from './lib/check-project';
|
||||||
import { NextBuildBuilderOptions } from '../../utils/types';
|
import { NextBuildBuilderOptions } from '../../utils/types';
|
||||||
|
|
||||||
import { getLockFileName } from 'nx/src/lock-file/lock-file';
|
import { getLockFileName } from 'nx/src/plugins/js/lock-file/lock-file';
|
||||||
|
|
||||||
export default async function buildExecutor(
|
export default async function buildExecutor(
|
||||||
options: NextBuildBuilderOptions,
|
options: NextBuildBuilderOptions,
|
||||||
|
|||||||
@ -226,8 +226,8 @@ export { Hash, Hasher } from './hasher/hasher';
|
|||||||
*/
|
*/
|
||||||
export { cacheDir } from './utils/cache-directory';
|
export { cacheDir } from './utils/cache-directory';
|
||||||
|
|
||||||
import { createLockFile as _createLockFile } from './lock-file/lock-file';
|
import { createLockFile as _createLockFile } from './plugins/js/lock-file/lock-file';
|
||||||
import { createPackageJson as _createPackageJson } from './utils/create-package-json';
|
import { createPackageJson as _createPackageJson } from './plugins/js/package-json/create-package-json';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @category Package Manager
|
* @category Package Manager
|
||||||
|
|||||||
@ -1,11 +1,88 @@
|
|||||||
import { ProjectGraphProcessor } from '../../config/project-graph';
|
import { ProjectGraphProcessor } from '../../config/project-graph';
|
||||||
import { ProjectGraphBuilder } from '../../project-graph/project-graph-builder';
|
import { ProjectGraphBuilder } from '../../project-graph/project-graph-builder';
|
||||||
import { buildNpmPackageNodes } from './project-graph/build-nodes/build-npm-package-nodes';
|
import { buildNpmPackageNodes } from './project-graph/build-nodes/build-npm-package-nodes';
|
||||||
|
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
|
||||||
|
import { readNxJson } from '../../config/configuration';
|
||||||
|
import { fileExists, readJsonFile } from '../../utils/fileutils';
|
||||||
|
import { PackageJson } from '../../utils/package-json';
|
||||||
|
import {
|
||||||
|
lockFileExists,
|
||||||
|
lockFileHash,
|
||||||
|
parseLockFile,
|
||||||
|
} from './lock-file/lock-file';
|
||||||
|
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json';
|
||||||
|
import { dirname, join } from 'path';
|
||||||
|
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
|
||||||
|
import { readFileSync, writeFileSync } from 'fs';
|
||||||
|
import { workspaceRoot } from '../../utils/workspace-root';
|
||||||
|
import { ensureDirSync } from 'fs-extra';
|
||||||
|
import { removeNpmNodes } from 'nx/src/plugins/js/lock-file/remove-npm-nodes';
|
||||||
|
|
||||||
export const processProjectGraph: ProjectGraphProcessor = (graph) => {
|
export const processProjectGraph: ProjectGraphProcessor = (graph, context) => {
|
||||||
const builder = new ProjectGraphBuilder(graph);
|
const builder = new ProjectGraphBuilder(graph);
|
||||||
|
|
||||||
|
const lockHash = lockFileHash() ?? 'n/a';
|
||||||
|
// during the create-nx-workspace lock file might not exists yet
|
||||||
|
if (lockFileExists()) {
|
||||||
|
if (lockFileNeedsReprocessing(lockHash)) {
|
||||||
|
removeNpmNodes(graph, builder);
|
||||||
|
parseLockFile(builder);
|
||||||
|
}
|
||||||
|
writeLastProcessedLockfileHash(lockHash);
|
||||||
|
}
|
||||||
|
|
||||||
buildNpmPackageNodes(builder);
|
buildNpmPackageNodes(builder);
|
||||||
|
|
||||||
|
buildExplicitDependencies(jsPluginConfig(readNxJson()), context, builder);
|
||||||
|
|
||||||
return builder.getUpdatedProjectGraph();
|
return builder.getUpdatedProjectGraph();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
|
||||||
|
function lockFileNeedsReprocessing(lockHash: string) {
|
||||||
|
try {
|
||||||
|
return readFileSync(lockFileHashFile).toString() !== lockHash;
|
||||||
|
} catch {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function writeLastProcessedLockfileHash(hash: string) {
|
||||||
|
ensureDirSync(dirname(lockFileHashFile));
|
||||||
|
writeFileSync(lockFileHashFile, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
function jsPluginConfig(nxJson: NxJsonConfiguration): NrwlJsPluginConfig {
|
||||||
|
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
|
||||||
|
return nxJson?.pluginsConfig?.['@nrwl/js'];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!fileExists(join(workspaceRoot, 'package.json'))) {
|
||||||
|
return {
|
||||||
|
analyzePackageJson: false,
|
||||||
|
analyzeSourceFiles: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const packageJson = readJsonFile<PackageJson>(
|
||||||
|
join(workspaceRoot, 'package.json')
|
||||||
|
);
|
||||||
|
|
||||||
|
const packageJsonDeps = {
|
||||||
|
...packageJson.dependencies,
|
||||||
|
...packageJson.devDependencies,
|
||||||
|
};
|
||||||
|
if (
|
||||||
|
packageJsonDeps['@nrwl/workspace'] ||
|
||||||
|
packageJsonDeps['@nrwl/js'] ||
|
||||||
|
packageJsonDeps['@nrwl/node'] ||
|
||||||
|
packageJsonDeps['@nrwl/next'] ||
|
||||||
|
packageJsonDeps['@nrwl/react'] ||
|
||||||
|
packageJsonDeps['@nrwl/angular'] ||
|
||||||
|
packageJsonDeps['@nrwl/web']
|
||||||
|
) {
|
||||||
|
return { analyzePackageJson: true, analyzeSourceFiles: true };
|
||||||
|
} else {
|
||||||
|
return { analyzePackageJson: true, analyzeSourceFiles: false };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -13,9 +13,9 @@ const packageNames = [];
|
|||||||
|
|
||||||
function processNodeModules(path = '.') {
|
function processNodeModules(path = '.') {
|
||||||
if (existsSync(`${path}/node_modules`)) {
|
if (existsSync(`${path}/node_modules`)) {
|
||||||
readdirSync(`${path}/node_modules`).forEach(folder => {
|
readdirSync(`${path}/node_modules`).forEach((folder) => {
|
||||||
if (folder.startsWith('@')) {
|
if (folder.startsWith('@')) {
|
||||||
readdirSync(`${path}/node_modules/${folder}`).forEach(subfolder => {
|
readdirSync(`${path}/node_modules/${folder}`).forEach((subfolder) => {
|
||||||
packageNames.push(`${path}/node_modules/${folder}/${subfolder}`);
|
packageNames.push(`${path}/node_modules/${folder}/${subfolder}`);
|
||||||
processNodeModules(`${path}/node_modules/${folder}/${subfolder}`);
|
processNodeModules(`${path}/node_modules/${folder}/${subfolder}`);
|
||||||
});
|
});
|
||||||
@ -29,15 +29,15 @@ function processNodeModules(path = '.') {
|
|||||||
|
|
||||||
processNodeModules();
|
processNodeModules();
|
||||||
|
|
||||||
packageNames.forEach(path => {
|
packageNames.forEach((path) => {
|
||||||
const filePath = `${path}/package.json`;
|
const filePath = `${path}/package.json`;
|
||||||
if (existsSync(filePath)) {
|
if (existsSync(filePath)) {
|
||||||
const content = readFileSync(filePath, 'utf-8');
|
const content = readFileSync(filePath, 'utf-8');
|
||||||
const peerDependencies = JSON.parse(content).peerDependencies;
|
const peerDependencies = JSON.parse(content).peerDependencies;
|
||||||
const peerDependenciesMeta = JSON.parse(content).peerDependenciesMeta;
|
const peerDependenciesMeta = JSON.parse(content).peerDependenciesMeta;
|
||||||
const output = JSON.stringify({
|
const output = JSON.stringify({
|
||||||
...peerDependencies && { peerDependencies },
|
...(peerDependencies && { peerDependencies }),
|
||||||
...peerDependenciesMeta && { peerDependenciesMeta },
|
...(peerDependenciesMeta && { peerDependenciesMeta }),
|
||||||
});
|
});
|
||||||
if (output === '{}') return;
|
if (output === '{}') return;
|
||||||
report += `'${filePath.slice(2)}': '${output}',\n`;
|
report += `'${filePath.slice(2)}': '${output}',\n`;
|
||||||
@ -59,10 +59,10 @@ const existsSync = require('fs').existsSync;
|
|||||||
let report = '';
|
let report = '';
|
||||||
|
|
||||||
const packageNames = [];
|
const packageNames = [];
|
||||||
readdirSync('node_modules').forEach(folder => {
|
readdirSync('node_modules').forEach((folder) => {
|
||||||
if (folder === '.pnpm') return;
|
if (folder === '.pnpm') return;
|
||||||
if (folder.startsWith('@')) {
|
if (folder.startsWith('@')) {
|
||||||
readdirSync(`node_modules/${folder}`).forEach(subfolder => {
|
readdirSync(`node_modules/${folder}`).forEach((subfolder) => {
|
||||||
packageNames.push(`${folder}/${subfolder}`);
|
packageNames.push(`${folder}/${subfolder}`);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
@ -70,7 +70,7 @@ readdirSync('node_modules').forEach(folder => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
packageNames.forEach(packageName => {
|
packageNames.forEach((packageName) => {
|
||||||
const path = `node_modules/${packageName}/package.json`;
|
const path = `node_modules/${packageName}/package.json`;
|
||||||
if (existsSync(path)) {
|
if (existsSync(path)) {
|
||||||
const content = readFileSync(path, 'utf-8');
|
const content = readFileSync(path, 'utf-8');
|
||||||
@ -1103,9 +1103,7 @@
|
|||||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||||
"hasInstallScript": true,
|
"hasInstallScript": true,
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": ["darwin"],
|
||||||
"darwin"
|
|
||||||
],
|
|
||||||
"peer": true,
|
"peer": true,
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||||
@ -1103,9 +1103,7 @@
|
|||||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||||
"hasInstallScript": true,
|
"hasInstallScript": true,
|
||||||
"optional": true,
|
"optional": true,
|
||||||
"os": [
|
"os": ["darwin"],
|
||||||
"darwin"
|
|
||||||
],
|
|
||||||
"peer": true,
|
"peer": true,
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||||
@ -6,18 +6,22 @@
|
|||||||
import { readFileSync, existsSync } from 'fs';
|
import { readFileSync, existsSync } from 'fs';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
|
|
||||||
import { detectPackageManager, PackageManager } from '../utils/package-manager';
|
import {
|
||||||
import { workspaceRoot } from '../utils/workspace-root';
|
detectPackageManager,
|
||||||
import { ProjectGraph } from '../config/project-graph';
|
PackageManager,
|
||||||
import { PackageJson } from '../utils/package-json';
|
} from '../../../utils/package-manager';
|
||||||
import { defaultHashing } from '../hasher/hashing-impl';
|
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||||
|
import { ProjectGraph } from '../../../config/project-graph';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
import { PackageJson } from '../../../utils/package-json';
|
||||||
|
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||||
|
import { output } from '../../../utils/output';
|
||||||
|
|
||||||
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
||||||
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
||||||
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
||||||
import { pruneProjectGraph } from './project-graph-pruning';
|
import { pruneProjectGraph } from './project-graph-pruning';
|
||||||
import { normalizePackageJson } from './utils/package-json';
|
import { normalizePackageJson } from './utils/package-json';
|
||||||
import { output } from '../utils/output';
|
|
||||||
|
|
||||||
const YARN_LOCK_FILE = 'yarn.lock';
|
const YARN_LOCK_FILE = 'yarn.lock';
|
||||||
const NPM_LOCK_FILE = 'package-lock.json';
|
const NPM_LOCK_FILE = 'package-lock.json';
|
||||||
@ -76,20 +80,24 @@ export function lockFileHash(
|
|||||||
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
|
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
|
||||||
*/
|
*/
|
||||||
export function parseLockFile(
|
export function parseLockFile(
|
||||||
|
builder: ProjectGraphBuilder,
|
||||||
packageManager: PackageManager = detectPackageManager(workspaceRoot)
|
packageManager: PackageManager = detectPackageManager(workspaceRoot)
|
||||||
): ProjectGraph {
|
): ProjectGraph {
|
||||||
try {
|
try {
|
||||||
if (packageManager === 'yarn') {
|
if (packageManager === 'yarn') {
|
||||||
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
|
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
|
||||||
return parseYarnLockfile(content);
|
parseYarnLockfile(content, builder);
|
||||||
|
return builder.getUpdatedProjectGraph();
|
||||||
}
|
}
|
||||||
if (packageManager === 'pnpm') {
|
if (packageManager === 'pnpm') {
|
||||||
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
|
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
|
||||||
return parsePnpmLockfile(content);
|
parsePnpmLockfile(content, builder);
|
||||||
|
return builder.getUpdatedProjectGraph();
|
||||||
}
|
}
|
||||||
if (packageManager === 'npm') {
|
if (packageManager === 'npm') {
|
||||||
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
|
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
|
||||||
return parseNpmLockfile(content);
|
parseNpmLockfile(content, builder);
|
||||||
|
return builder.getUpdatedProjectGraph();
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (!isPostInstallProcess()) {
|
if (!isPostInstallProcess()) {
|
||||||
@ -138,19 +146,26 @@ export function createLockFile(
|
|||||||
const normalizedPackageJson = normalizePackageJson(packageJson);
|
const normalizedPackageJson = normalizePackageJson(packageJson);
|
||||||
const content = readFileSync(getLockFileName(packageManager), 'utf8');
|
const content = readFileSync(getLockFileName(packageManager), 'utf8');
|
||||||
|
|
||||||
|
const builder = new ProjectGraphBuilder();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
if (packageManager === 'yarn') {
|
if (packageManager === 'yarn') {
|
||||||
const graph = parseYarnLockfile(content);
|
parseYarnLockfile(content, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||||
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
|
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
|
||||||
}
|
}
|
||||||
if (packageManager === 'pnpm') {
|
if (packageManager === 'pnpm') {
|
||||||
const graph = parsePnpmLockfile(content);
|
parsePnpmLockfile(content, builder);
|
||||||
|
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||||
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
|
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
|
||||||
}
|
}
|
||||||
if (packageManager === 'npm') {
|
if (packageManager === 'npm') {
|
||||||
const graph = parseNpmLockfile(content);
|
parseNpmLockfile(content, builder);
|
||||||
|
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||||
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
|
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
|
||||||
}
|
}
|
||||||
@ -1,20 +1,12 @@
|
|||||||
import { joinPathFragments } from '../utils/path';
|
import { joinPathFragments } from '../../../utils/path';
|
||||||
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
|
||||||
import { pruneProjectGraph } from './project-graph-pruning';
|
import { pruneProjectGraph } from './project-graph-pruning';
|
||||||
import { vol } from 'memfs';
|
import { vol } from 'memfs';
|
||||||
import { ProjectGraph } from '../config/project-graph';
|
import { ProjectGraph } from '../../../config/project-graph';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
|
||||||
jest.mock('fs', () => require('memfs').fs);
|
jest.mock('fs', () => require('memfs').fs);
|
||||||
|
|
||||||
jest.mock('@nrwl/devkit', () => ({
|
|
||||||
...jest.requireActual<any>('@nrwl/devkit'),
|
|
||||||
workspaceRoot: '/root',
|
|
||||||
}));
|
|
||||||
|
|
||||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
|
||||||
workspaceRoot: '/root',
|
|
||||||
}));
|
|
||||||
|
|
||||||
describe('NPM lock file utility', () => {
|
describe('NPM lock file utility', () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
vol.reset();
|
vol.reset();
|
||||||
@ -29,7 +21,9 @@ describe('NPM lock file utility', () => {
|
|||||||
let graph: ProjectGraph;
|
let graph: ProjectGraph;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
graph = builder.getUpdatedProjectGraph();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should parse root lock file', async () => {
|
it('should parse root lock file', async () => {
|
||||||
@ -47,7 +41,9 @@ describe('NPM lock file utility', () => {
|
|||||||
));
|
));
|
||||||
|
|
||||||
// this is original generated lock file
|
// this is original generated lock file
|
||||||
const appGraph = parseNpmLockfile(JSON.stringify(appLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(appLockFile), builder);
|
||||||
|
const appGraph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
|
expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
|
||||||
|
|
||||||
// this is our pruned lock file structure
|
// this is our pruned lock file structure
|
||||||
@ -93,7 +89,9 @@ describe('NPM lock file utility', () => {
|
|||||||
'__fixtures__/auxiliary-packages/package-lock.json'
|
'__fixtures__/auxiliary-packages/package-lock.json'
|
||||||
));
|
));
|
||||||
|
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
|
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
||||||
|
|
||||||
@ -150,7 +148,9 @@ describe('NPM lock file utility', () => {
|
|||||||
'__fixtures__/auxiliary-packages/package-lock-v2.json'
|
'__fixtures__/auxiliary-packages/package-lock-v2.json'
|
||||||
));
|
));
|
||||||
|
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
|
||||||
|
|
||||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||||
@ -246,7 +246,9 @@ describe('NPM lock file utility', () => {
|
|||||||
cleanupTypes(prunedV2LockFile.packages);
|
cleanupTypes(prunedV2LockFile.packages);
|
||||||
cleanupTypes(prunedV2LockFile.dependencies, true);
|
cleanupTypes(prunedV2LockFile.dependencies, true);
|
||||||
|
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootV2LockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||||
const result = stringifyNpmLockfile(
|
const result = stringifyNpmLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -331,7 +333,9 @@ describe('NPM lock file utility', () => {
|
|||||||
'__fixtures__/duplicate-package/package-lock-v1.json'
|
'__fixtures__/duplicate-package/package-lock-v1.json'
|
||||||
));
|
));
|
||||||
|
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(369); // 338
|
expect(Object.keys(graph.externalNodes).length).toEqual(369); // 338
|
||||||
});
|
});
|
||||||
it('should parse v3', async () => {
|
it('should parse v3', async () => {
|
||||||
@ -340,7 +344,9 @@ describe('NPM lock file utility', () => {
|
|||||||
'__fixtures__/duplicate-package/package-lock.json'
|
'__fixtures__/duplicate-package/package-lock.json'
|
||||||
));
|
));
|
||||||
|
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(369); //338
|
expect(Object.keys(graph.externalNodes).length).toEqual(369); //338
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -355,7 +361,9 @@ describe('NPM lock file utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/optional/package.json'
|
'__fixtures__/optional/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseNpmLockfile(JSON.stringify(lockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
||||||
|
|
||||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||||
@ -378,7 +386,9 @@ describe('NPM lock file utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/pruning/typescript/package.json'
|
'__fixtures__/pruning/typescript/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
||||||
const result = stringifyNpmLockfile(
|
const result = stringifyNpmLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -403,7 +413,9 @@ describe('NPM lock file utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/pruning/devkit-yargs/package.json'
|
'__fixtures__/pruning/devkit-yargs/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseNpmLockfile(JSON.stringify(rootLockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
||||||
const result = stringifyNpmLockfile(
|
const result = stringifyNpmLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -432,7 +444,9 @@ describe('NPM lock file utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/workspaces/package-lock.json'
|
'__fixtures__/workspaces/package-lock.json'
|
||||||
));
|
));
|
||||||
const result = parseNpmLockfile(JSON.stringify(lockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||||
|
const result = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -441,7 +455,9 @@ describe('NPM lock file utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/workspaces/package-lock.v1.json'
|
'__fixtures__/workspaces/package-lock.v1.json'
|
||||||
));
|
));
|
||||||
const result = parseNpmLockfile(JSON.stringify(lockFile));
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseNpmLockfile(JSON.stringify(lockFile), builder);
|
||||||
|
const result = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
expect(Object.keys(result.externalNodes).length).toEqual(5);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -1,14 +1,14 @@
|
|||||||
import { existsSync, readFileSync } from 'fs';
|
import { existsSync, readFileSync } from 'fs';
|
||||||
import { satisfies } from 'semver';
|
import { satisfies } from 'semver';
|
||||||
import { workspaceRoot } from '../utils/workspace-root';
|
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
import { reverse } from '../../../project-graph/operators';
|
||||||
import { reverse } from '../project-graph/operators';
|
import { NormalizedPackageJson } from './utils/package-json';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
import {
|
import {
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../config/project-graph';
|
||||||
import { NormalizedPackageJson } from './utils/package-json';
|
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||||
import { defaultHashing } from '../hasher/hashing-impl';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* NPM
|
* NPM
|
||||||
@ -50,16 +50,16 @@ type NpmLockFile = {
|
|||||||
dependencies?: Record<string, NpmDependencyV1>;
|
dependencies?: Record<string, NpmDependencyV1>;
|
||||||
};
|
};
|
||||||
|
|
||||||
export function parseNpmLockfile(lockFileContent: string): ProjectGraph {
|
export function parseNpmLockfile(
|
||||||
|
lockFileContent: string,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
const data = JSON.parse(lockFileContent) as NpmLockFile;
|
const data = JSON.parse(lockFileContent) as NpmLockFile;
|
||||||
const builder = new ProjectGraphBuilder();
|
|
||||||
|
|
||||||
// we use key => node map to avoid duplicate work when parsing keys
|
// we use key => node map to avoid duplicate work when parsing keys
|
||||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||||
addNodes(data, builder, keyMap);
|
addNodes(data, builder, keyMap);
|
||||||
addDependencies(data, builder, keyMap);
|
addDependencies(data, builder, keyMap);
|
||||||
|
|
||||||
return builder.getUpdatedProjectGraph();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function addNodes(
|
function addNodes(
|
||||||
@ -1,17 +1,13 @@
|
|||||||
import { joinPathFragments } from '../utils/path';
|
import { joinPathFragments } from '../../../utils/path';
|
||||||
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
|
||||||
import { ProjectGraph } from '../config/project-graph';
|
import { ProjectGraph } from '../../../config/project-graph';
|
||||||
import { vol } from 'memfs';
|
import { vol } from 'memfs';
|
||||||
import { pruneProjectGraph } from './project-graph-pruning';
|
import { pruneProjectGraph } from './project-graph-pruning';
|
||||||
|
import { ProjectGraphBuilder } from 'nx/src/project-graph/project-graph-builder';
|
||||||
|
|
||||||
jest.mock('fs', () => require('memfs').fs);
|
jest.mock('fs', () => require('memfs').fs);
|
||||||
|
|
||||||
jest.mock('@nrwl/devkit', () => ({
|
jest.mock('../../../utils/workspace-root', () => ({
|
||||||
...jest.requireActual<any>('@nrwl/devkit'),
|
|
||||||
workspaceRoot: '/root',
|
|
||||||
}));
|
|
||||||
|
|
||||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
|
||||||
workspaceRoot: '/root',
|
workspaceRoot: '/root',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@ -121,7 +117,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/nextjs/pnpm-lock.yaml'
|
'__fixtures__/nextjs/pnpm-lock.yaml'
|
||||||
)).default;
|
)).default;
|
||||||
graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
graph = builder.getUpdatedProjectGraph();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should parse root lock file', async () => {
|
it('should parse root lock file', async () => {
|
||||||
@ -188,7 +186,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
|
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(213); //202
|
expect(Object.keys(graph.externalNodes).length).toEqual(213); //202
|
||||||
|
|
||||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||||
@ -268,7 +268,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
|
||||||
const result = stringifyPnpmLockfile(
|
const result = stringifyPnpmLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -305,7 +307,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/duplicate-package/pnpm-lock.yaml'
|
'__fixtures__/duplicate-package/pnpm-lock.yaml'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(370); //337
|
expect(Object.keys(graph.externalNodes).length).toEqual(370); //337
|
||||||
expect(Object.keys(graph.dependencies).length).toEqual(213);
|
expect(Object.keys(graph.dependencies).length).toEqual(213);
|
||||||
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
|
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
|
||||||
@ -329,7 +333,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/optional/pnpm-lock.yaml'
|
'__fixtures__/optional/pnpm-lock.yaml'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
expect(Object.keys(graph.externalNodes).length).toEqual(8);
|
||||||
|
|
||||||
const packageJson = require(joinPathFragments(
|
const packageJson = require(joinPathFragments(
|
||||||
@ -364,7 +370,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
'__fixtures__/pruning/pnpm-lock.yaml'
|
'__fixtures__/pruning/pnpm-lock.yaml'
|
||||||
)).default;
|
)).default;
|
||||||
|
|
||||||
graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
graph = builder.getUpdatedProjectGraph();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should prune single package', () => {
|
it('should prune single package', () => {
|
||||||
@ -426,7 +434,9 @@ describe('pnpm LockFile utility', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should parse lock file', async () => {
|
it('should parse lock file', async () => {
|
||||||
const graph = parsePnpmLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parsePnpmLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -11,24 +11,24 @@ import {
|
|||||||
} from './utils/pnpm-normalizer';
|
} from './utils/pnpm-normalizer';
|
||||||
import { getHoistedPackageVersion } from './utils/package-json';
|
import { getHoistedPackageVersion } from './utils/package-json';
|
||||||
import { NormalizedPackageJson } from './utils/package-json';
|
import { NormalizedPackageJson } from './utils/package-json';
|
||||||
import { sortObjectByKeys } from '../utils/object-sort';
|
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
import {
|
import {
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../config/project-graph';
|
||||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||||
import { defaultHashing } from '../hasher/hashing-impl';
|
|
||||||
|
|
||||||
export function parsePnpmLockfile(lockFileContent: string): ProjectGraph {
|
export function parsePnpmLockfile(
|
||||||
|
lockFileContent: string,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
): void {
|
||||||
const data = parseAndNormalizePnpmLockfile(lockFileContent);
|
const data = parseAndNormalizePnpmLockfile(lockFileContent);
|
||||||
const builder = new ProjectGraphBuilder();
|
|
||||||
|
|
||||||
// we use key => node map to avoid duplicate work when parsing keys
|
// we use key => node map to avoid duplicate work when parsing keys
|
||||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||||
addNodes(data, builder, keyMap);
|
addNodes(data, builder, keyMap);
|
||||||
addDependencies(data, builder, keyMap);
|
addDependencies(data, builder, keyMap);
|
||||||
|
|
||||||
return builder.getUpdatedProjectGraph();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function addNodes(
|
function addNodes(
|
||||||
@ -1,11 +1,11 @@
|
|||||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
|
||||||
import {
|
import {
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../config/project-graph';
|
||||||
import { PackageJson } from '../utils/package-json';
|
|
||||||
import { reverse } from '../project-graph/operators';
|
|
||||||
import { satisfies, gte } from 'semver';
|
import { satisfies, gte } from 'semver';
|
||||||
|
import { PackageJson } from '../../../utils/package-json';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
import { reverse } from '../../../project-graph/operators';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prune project graph's external nodes and their dependencies
|
* Prune project graph's external nodes and their dependencies
|
||||||
11
packages/nx/src/plugins/js/lock-file/remove-npm-nodes.ts
Normal file
11
packages/nx/src/plugins/js/lock-file/remove-npm-nodes.ts
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
import { ProjectGraph } from '../../../config/project-graph';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
|
||||||
|
export function removeNpmNodes(
|
||||||
|
graph: ProjectGraph,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
|
for (const externalNode in graph.externalNodes) {
|
||||||
|
builder.removeNode(externalNode);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -1,6 +1,6 @@
|
|||||||
import { existsSync, readFileSync } from 'fs';
|
import { existsSync, readFileSync } from 'fs';
|
||||||
import { PackageJson } from '../../utils/package-json';
|
import { PackageJson } from '../../../../utils/package-json';
|
||||||
import { workspaceRoot } from '../../utils/workspace-root';
|
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get version of hoisted package if available
|
* Get version of hoisted package if available
|
||||||
@ -10,7 +10,7 @@ import type {
|
|||||||
} from '@pnpm/lockfile-types';
|
} from '@pnpm/lockfile-types';
|
||||||
import { dump, load } from '@zkochan/js-yaml';
|
import { dump, load } from '@zkochan/js-yaml';
|
||||||
import { existsSync, readFileSync } from 'fs';
|
import { existsSync, readFileSync } from 'fs';
|
||||||
import { workspaceRoot } from '../../utils/workspace-root';
|
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||||
|
|
||||||
const LOCKFILE_YAML_FORMAT = {
|
const LOCKFILE_YAML_FORMAT = {
|
||||||
blankLines: true,
|
blankLines: true,
|
||||||
@ -1,9 +1,10 @@
|
|||||||
import { joinPathFragments } from '../utils/path';
|
import { joinPathFragments } from '../../../utils/path';
|
||||||
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
|
||||||
import { pruneProjectGraph } from './project-graph-pruning';
|
import { pruneProjectGraph } from './project-graph-pruning';
|
||||||
import { vol } from 'memfs';
|
import { vol } from 'memfs';
|
||||||
import { ProjectGraph } from '../config/project-graph';
|
import { ProjectGraph } from '../../../config/project-graph';
|
||||||
import { PackageJson } from '../utils/package-json';
|
import { PackageJson } from '../../../utils/package-json';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
|
||||||
jest.mock('fs', () => require('memfs').fs);
|
jest.mock('fs', () => require('memfs').fs);
|
||||||
|
|
||||||
@ -12,7 +13,7 @@ jest.mock('@nrwl/devkit', () => ({
|
|||||||
workspaceRoot: '/root',
|
workspaceRoot: '/root',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
jest.mock('../../../utils/workspace-root', () => ({
|
||||||
workspaceRoot: '/root',
|
workspaceRoot: '/root',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
@ -156,11 +157,13 @@ describe('yarn LockFile utility', () => {
|
|||||||
let graph: ProjectGraph;
|
let graph: ProjectGraph;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
|
const builder = new ProjectGraphBuilder();
|
||||||
lockFile = require(joinPathFragments(
|
lockFile = require(joinPathFragments(
|
||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/nextjs/yarn.lock'
|
'__fixtures__/nextjs/yarn.lock'
|
||||||
)).default;
|
)).default;
|
||||||
graph = parseYarnLockfile(lockFile);
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
graph = builder.getUpdatedProjectGraph();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should parse root lock file', async () => {
|
it('should parse root lock file', async () => {
|
||||||
@ -225,7 +228,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/auxiliary-packages/yarn.lock'
|
'__fixtures__/auxiliary-packages/yarn.lock'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parseYarnLockfile(classicLockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(classicLockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(127); // 124 hoisted
|
expect(Object.keys(graph.externalNodes).length).toEqual(127); // 124 hoisted
|
||||||
|
|
||||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||||
@ -301,7 +306,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
||||||
)).default;
|
)).default;
|
||||||
|
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||||
const result = stringifyYarnLockfile(
|
const result = stringifyYarnLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -337,7 +344,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
|
||||||
)).default;
|
)).default;
|
||||||
|
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||||
const result = stringifyYarnLockfile(
|
const result = stringifyYarnLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -357,7 +366,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/auxiliary-packages/yarn-berry.lock'
|
'__fixtures__/auxiliary-packages/yarn-berry.lock'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parseYarnLockfile(berryLockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(berryLockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(128); //124 hoisted
|
expect(Object.keys(graph.externalNodes).length).toEqual(128); //124 hoisted
|
||||||
|
|
||||||
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
|
||||||
@ -433,7 +444,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
'__fixtures__/auxiliary-packages/yarn-berry.lock.pruned'
|
'__fixtures__/auxiliary-packages/yarn-berry.lock.pruned'
|
||||||
)).default;
|
)).default;
|
||||||
|
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
|
||||||
const result = stringifyYarnLockfile(
|
const result = stringifyYarnLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -489,7 +502,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/duplicate-package/yarn.lock'
|
'__fixtures__/duplicate-package/yarn.lock'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parseYarnLockfile(classicLockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(classicLockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(371); //337 hoisted
|
expect(Object.keys(graph.externalNodes).length).toEqual(371); //337 hoisted
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -517,7 +532,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/optional/package.json'
|
'__fixtures__/optional/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(103);
|
expect(Object.keys(graph.externalNodes).length).toEqual(103);
|
||||||
|
|
||||||
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
const prunedGraph = pruneProjectGraph(graph, packageJson);
|
||||||
@ -694,7 +711,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/pruning/typescript/package.json'
|
'__fixtures__/pruning/typescript/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
|
||||||
const result = stringifyYarnLockfile(
|
const result = stringifyYarnLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -719,7 +738,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/pruning/devkit-yargs/package.json'
|
'__fixtures__/pruning/devkit-yargs/package.json'
|
||||||
));
|
));
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
|
||||||
const result = stringifyYarnLockfile(
|
const result = stringifyYarnLockfile(
|
||||||
prunedGraph,
|
prunedGraph,
|
||||||
@ -750,7 +771,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/workspaces/yarn.lock'
|
'__fixtures__/workspaces/yarn.lock'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -759,7 +782,9 @@ describe('yarn LockFile utility', () => {
|
|||||||
__dirname,
|
__dirname,
|
||||||
'__fixtures__/workspaces/yarn.lock.berry'
|
'__fixtures__/workspaces/yarn.lock.berry'
|
||||||
)).default;
|
)).default;
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
expect(Object.keys(graph.externalNodes).length).toEqual(5);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -800,7 +825,9 @@ type-fest@^0.20.2:
|
|||||||
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
|
integrity sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
||||||
Object {
|
Object {
|
||||||
"data": Object {
|
"data": Object {
|
||||||
@ -873,7 +900,9 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
`;
|
`;
|
||||||
const graph = parseYarnLockfile(lockFile);
|
const builder = new ProjectGraphBuilder();
|
||||||
|
parseYarnLockfile(lockFile, builder);
|
||||||
|
const graph = builder.getUpdatedProjectGraph();
|
||||||
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
|
||||||
Object {
|
Object {
|
||||||
"data": Object {
|
"data": Object {
|
||||||
@ -1,15 +1,15 @@
|
|||||||
import { parseSyml, stringifySyml } from '@yarnpkg/parsers';
|
import { parseSyml, stringifySyml } from '@yarnpkg/parsers';
|
||||||
import { stringify } from '@yarnpkg/lockfile';
|
import { stringify } from '@yarnpkg/lockfile';
|
||||||
import { sortObjectByKeys } from '../utils/object-sort';
|
|
||||||
import { getHoistedPackageVersion } from './utils/package-json';
|
import { getHoistedPackageVersion } from './utils/package-json';
|
||||||
|
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
|
||||||
|
import { satisfies } from 'semver';
|
||||||
|
import { NormalizedPackageJson } from './utils/package-json';
|
||||||
import {
|
import {
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../config/project-graph';
|
||||||
import { ProjectGraphBuilder } from '../project-graph/project-graph-builder';
|
import { defaultHashing } from '../../../hasher/hashing-impl';
|
||||||
import { satisfies } from 'semver';
|
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||||
import { NormalizedPackageJson } from './utils/package-json';
|
|
||||||
import { defaultHashing } from '../hasher/hashing-impl';
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Yarn
|
* Yarn
|
||||||
@ -37,16 +37,16 @@ type YarnDependency = {
|
|||||||
linkType?: 'soft' | 'hard';
|
linkType?: 'soft' | 'hard';
|
||||||
};
|
};
|
||||||
|
|
||||||
export function parseYarnLockfile(lockFileContent: string): ProjectGraph {
|
export function parseYarnLockfile(
|
||||||
|
lockFileContent: string,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
const data = parseSyml(lockFileContent);
|
const data = parseSyml(lockFileContent);
|
||||||
const builder = new ProjectGraphBuilder();
|
|
||||||
|
|
||||||
// we use key => node map to avoid duplicate work when parsing keys
|
// we use key => node map to avoid duplicate work when parsing keys
|
||||||
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
const keyMap = new Map<string, ProjectGraphExternalNode>();
|
||||||
addNodes(data, builder, keyMap);
|
addNodes(data, builder, keyMap);
|
||||||
addDependencies(data, builder, keyMap);
|
addDependencies(data, builder, keyMap);
|
||||||
|
|
||||||
return builder.getUpdatedProjectGraph();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function addNodes(
|
function addNodes(
|
||||||
@ -1,10 +1,10 @@
|
|||||||
import * as fs from 'fs';
|
import * as fs from 'fs';
|
||||||
|
|
||||||
import * as configModule from '../config/configuration';
|
import * as configModule from '../../../config/configuration';
|
||||||
import { DependencyType } from '../config/project-graph';
|
import { DependencyType } from '../../../config/project-graph';
|
||||||
import * as hashModule from '../hasher/hasher';
|
import * as hashModule from '../../../hasher/hasher';
|
||||||
import { createPackageJson } from './create-package-json';
|
import { createPackageJson } from './create-package-json';
|
||||||
import * as fileutilsModule from './fileutils';
|
import * as fileutilsModule from '../../../utils/fileutils';
|
||||||
|
|
||||||
describe('createPackageJson', () => {
|
describe('createPackageJson', () => {
|
||||||
it('should add additional dependencies', () => {
|
it('should add additional dependencies', () => {
|
||||||
@ -1,11 +1,17 @@
|
|||||||
import { readJsonFile } from './fileutils';
|
import { readJsonFile } from '../../../utils/fileutils';
|
||||||
import { sortObjectByKeys } from './object-sort';
|
import { sortObjectByKeys } from '../../../utils/object-sort';
|
||||||
import { ProjectGraph, ProjectGraphProjectNode } from '../config/project-graph';
|
import {
|
||||||
import { PackageJson } from './package-json';
|
ProjectGraph,
|
||||||
|
ProjectGraphProjectNode,
|
||||||
|
} from '../../../config/project-graph';
|
||||||
|
import { PackageJson } from '../../../utils/package-json';
|
||||||
import { existsSync } from 'fs';
|
import { existsSync } from 'fs';
|
||||||
import { workspaceRoot } from './workspace-root';
|
import { workspaceRoot } from '../../../utils/workspace-root';
|
||||||
import { filterUsingGlobPatterns, getTargetInputs } from '../hasher/hasher';
|
import {
|
||||||
import { readNxJson } from '../config/configuration';
|
filterUsingGlobPatterns,
|
||||||
|
getTargetInputs,
|
||||||
|
} from '../../../hasher/hasher';
|
||||||
|
import { readNxJson } from '../../../config/configuration';
|
||||||
|
|
||||||
interface NpmDeps {
|
interface NpmDeps {
|
||||||
readonly dependencies: Record<string, string>;
|
readonly dependencies: Record<string, string>;
|
||||||
@ -0,0 +1,189 @@
|
|||||||
|
import {
|
||||||
|
DependencyType,
|
||||||
|
ProjectFileMap,
|
||||||
|
ProjectGraphProcessorContext,
|
||||||
|
} from '../../../../config/project-graph';
|
||||||
|
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
|
||||||
|
import * as os from 'os';
|
||||||
|
|
||||||
|
export function buildExplicitDependencies(
|
||||||
|
jsPluginConfig: {
|
||||||
|
analyzeSourceFiles?: boolean;
|
||||||
|
analyzePackageJson?: boolean;
|
||||||
|
},
|
||||||
|
ctx: ProjectGraphProcessorContext,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
|
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
|
||||||
|
// using workers has an overhead, so we only do it when the number of
|
||||||
|
// files we need to process is >= 100 and there are more than 2 CPUs
|
||||||
|
// to be able to use at least 2 workers (1 worker per CPU and
|
||||||
|
// 1 CPU for the main thread)
|
||||||
|
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
|
||||||
|
return buildExplicitDependenciesWithoutWorkers(
|
||||||
|
jsPluginConfig,
|
||||||
|
ctx,
|
||||||
|
builder
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
return buildExplicitDependenciesUsingWorkers(
|
||||||
|
jsPluginConfig,
|
||||||
|
ctx,
|
||||||
|
totalNumOfFilesToProcess,
|
||||||
|
builder
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
|
||||||
|
let totalNumOfFilesToProcess = 0;
|
||||||
|
Object.values(ctx.filesToProcess).forEach(
|
||||||
|
(t) => (totalNumOfFilesToProcess += t.length)
|
||||||
|
);
|
||||||
|
return totalNumOfFilesToProcess;
|
||||||
|
}
|
||||||
|
|
||||||
|
function splitFilesIntoBins(
|
||||||
|
ctx: ProjectGraphProcessorContext,
|
||||||
|
totalNumOfFilesToProcess: number,
|
||||||
|
numberOfWorkers: number
|
||||||
|
) {
|
||||||
|
// we want to have numberOfWorkers * 5 bins
|
||||||
|
const filesPerBin =
|
||||||
|
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
|
||||||
|
const bins: ProjectFileMap[] = [];
|
||||||
|
let currentProjectFileMap = {};
|
||||||
|
let currentNumberOfFiles = 0;
|
||||||
|
for (const source of Object.keys(ctx.filesToProcess)) {
|
||||||
|
for (const f of Object.values(ctx.filesToProcess[source])) {
|
||||||
|
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
|
||||||
|
currentProjectFileMap[source].push(f);
|
||||||
|
currentNumberOfFiles++;
|
||||||
|
|
||||||
|
if (currentNumberOfFiles >= filesPerBin) {
|
||||||
|
bins.push(currentProjectFileMap);
|
||||||
|
currentProjectFileMap = {};
|
||||||
|
currentNumberOfFiles = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bins.push(currentProjectFileMap);
|
||||||
|
return bins;
|
||||||
|
}
|
||||||
|
|
||||||
|
function createWorkerPool(numberOfWorkers: number) {
|
||||||
|
const res = [];
|
||||||
|
for (let i = 0; i < numberOfWorkers; ++i) {
|
||||||
|
res.push(
|
||||||
|
new (require('worker_threads').Worker)(
|
||||||
|
join(__dirname, './project-graph-worker.js'),
|
||||||
|
{
|
||||||
|
env: process.env,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildExplicitDependenciesWithoutWorkers(
|
||||||
|
jsPluginConfig: {
|
||||||
|
analyzeSourceFiles?: boolean;
|
||||||
|
analyzePackageJson?: boolean;
|
||||||
|
},
|
||||||
|
ctx: ProjectGraphProcessorContext,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
|
buildExplicitTypescriptAndPackageJsonDependencies(
|
||||||
|
jsPluginConfig,
|
||||||
|
ctx.nxJsonConfiguration,
|
||||||
|
ctx.projectsConfigurations,
|
||||||
|
builder.graph,
|
||||||
|
ctx.filesToProcess
|
||||||
|
).forEach((r) => {
|
||||||
|
if (r.type === DependencyType.static) {
|
||||||
|
builder.addStaticDependency(
|
||||||
|
r.sourceProjectName,
|
||||||
|
r.targetProjectName,
|
||||||
|
r.sourceProjectFile
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
builder.addDynamicDependency(
|
||||||
|
r.sourceProjectName,
|
||||||
|
r.targetProjectName,
|
||||||
|
r.sourceProjectFile
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildExplicitDependenciesUsingWorkers(
|
||||||
|
jsPluginConfig: {
|
||||||
|
analyzeSourceFiles?: boolean;
|
||||||
|
analyzePackageJson?: boolean;
|
||||||
|
},
|
||||||
|
ctx: ProjectGraphProcessorContext,
|
||||||
|
totalNumOfFilesToProcess: number,
|
||||||
|
builder: ProjectGraphBuilder
|
||||||
|
) {
|
||||||
|
const numberOfWorkers = Math.min(
|
||||||
|
totalNumOfFilesToProcess,
|
||||||
|
getNumberOfWorkers()
|
||||||
|
);
|
||||||
|
const bins = splitFilesIntoBins(
|
||||||
|
ctx,
|
||||||
|
totalNumOfFilesToProcess,
|
||||||
|
numberOfWorkers
|
||||||
|
);
|
||||||
|
const workers = createWorkerPool(numberOfWorkers);
|
||||||
|
let numberOfExpectedResponses = bins.length;
|
||||||
|
|
||||||
|
return new Promise((res, reject) => {
|
||||||
|
for (let w of workers) {
|
||||||
|
w.on('message', (explicitDependencies) => {
|
||||||
|
explicitDependencies.forEach((r) => {
|
||||||
|
builder.addExplicitDependency(
|
||||||
|
r.sourceProjectName,
|
||||||
|
r.sourceProjectFile,
|
||||||
|
r.targetProjectName
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (bins.length > 0) {
|
||||||
|
w.postMessage({ filesToProcess: bins.shift() });
|
||||||
|
}
|
||||||
|
// we processed all the bins
|
||||||
|
if (--numberOfExpectedResponses === 0) {
|
||||||
|
for (let w of workers) {
|
||||||
|
w.terminate();
|
||||||
|
}
|
||||||
|
res(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
w.on('error', reject);
|
||||||
|
w.on('exit', (code) => {
|
||||||
|
if (code !== 0) {
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
w.postMessage({
|
||||||
|
nxJsonConfiguration: ctx.nxJsonConfiguration,
|
||||||
|
projectsConfigurations: ctx.projectsConfigurations,
|
||||||
|
projectGraph: builder.graph,
|
||||||
|
jsPluginConfig,
|
||||||
|
});
|
||||||
|
w.postMessage({ filesToProcess: bins.shift() });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNumberOfWorkers(): number {
|
||||||
|
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||||
|
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
||||||
|
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
|
||||||
|
}
|
||||||
@ -3,9 +3,9 @@ import {
|
|||||||
ExplicitDependency,
|
ExplicitDependency,
|
||||||
} from './explicit-project-dependencies';
|
} from './explicit-project-dependencies';
|
||||||
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
||||||
import { ProjectFileMap, ProjectGraph } from '../../config/project-graph';
|
import { ProjectFileMap, ProjectGraph } from '../../../../config/project-graph';
|
||||||
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
|
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||||
import { NxJsonConfiguration } from '../../config/nx-json';
|
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||||
|
|
||||||
export function buildExplicitTypescriptAndPackageJsonDependencies(
|
export function buildExplicitTypescriptAndPackageJsonDependencies(
|
||||||
jsPluginConfig: {
|
jsPluginConfig: {
|
||||||
@ -1,15 +1,15 @@
|
|||||||
import { TempFs } from '../../utils/testing/temp-fs';
|
import { TempFs } from '../../../../utils/testing/temp-fs';
|
||||||
const tempFs = new TempFs('explicit-package-json');
|
const tempFs = new TempFs('explicit-package-json');
|
||||||
|
|
||||||
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
import { buildExplicitPackageJsonDependencies } from './explicit-package-json-dependencies';
|
||||||
|
|
||||||
import { createProjectFileMap } from '../file-map-utils';
|
import { defaultFileHasher } from '../../../../hasher/file-hasher';
|
||||||
import { defaultFileHasher } from '../../hasher/file-hasher';
|
|
||||||
import {
|
import {
|
||||||
ProjectGraphProcessorContext,
|
ProjectGraphProcessorContext,
|
||||||
ProjectGraphProjectNode,
|
ProjectGraphProjectNode,
|
||||||
} from '../../config/project-graph';
|
} from '../../../../config/project-graph';
|
||||||
import { ProjectGraphBuilder } from '../project-graph-builder';
|
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||||
|
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
|
||||||
|
|
||||||
describe('explicit package json dependencies', () => {
|
describe('explicit package json dependencies', () => {
|
||||||
let ctx: ProjectGraphProcessorContext;
|
let ctx: ProjectGraphProcessorContext;
|
||||||
@ -1,14 +1,14 @@
|
|||||||
import { defaultFileRead } from '../file-utils';
|
import { defaultFileRead } from '../../../../project-graph/file-utils';
|
||||||
import { join } from 'path';
|
import { join } from 'path';
|
||||||
import {
|
import {
|
||||||
DependencyType,
|
DependencyType,
|
||||||
ProjectFileMap,
|
ProjectFileMap,
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
} from '../../config/project-graph';
|
} from '../../../../config/project-graph';
|
||||||
import { parseJson } from '../../utils/json';
|
import { parseJson } from '../../../../utils/json';
|
||||||
import { getImportPath, joinPathFragments } from '../../utils/path';
|
import { getImportPath, joinPathFragments } from '../../../../utils/path';
|
||||||
import { ProjectsConfigurations } from '../../config/workspace-json-project-json';
|
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||||
import { NxJsonConfiguration } from '../../config/nx-json';
|
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||||
import { ExplicitDependency } from './explicit-project-dependencies';
|
import { ExplicitDependency } from './explicit-project-dependencies';
|
||||||
|
|
||||||
class ProjectGraphNodeRecords {}
|
class ProjectGraphNodeRecords {}
|
||||||
@ -1,9 +1,9 @@
|
|||||||
import { TempFs } from '../../utils/testing/temp-fs';
|
import { TempFs } from '../../../../utils/testing/temp-fs';
|
||||||
const tempFs = new TempFs('explicit-project-deps');
|
const tempFs = new TempFs('explicit-project-deps');
|
||||||
|
|
||||||
import { defaultFileHasher } from '../../hasher/file-hasher';
|
import { defaultFileHasher } from '../../../../hasher/file-hasher';
|
||||||
import { createProjectFileMap } from '../file-map-utils';
|
import { createProjectFileMap } from '../../../../project-graph/file-map-utils';
|
||||||
import { ProjectGraphBuilder } from '../project-graph-builder';
|
import { ProjectGraphBuilder } from '../../../../project-graph/project-graph-builder';
|
||||||
import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies';
|
import { buildExplicitTypeScriptDependencies } from './explicit-project-dependencies';
|
||||||
|
|
||||||
// projectName => tsconfig import path
|
// projectName => tsconfig import path
|
||||||
@ -1,10 +1,10 @@
|
|||||||
import { TypeScriptImportLocator } from './typescript-import-locator';
|
import { TypeScriptImportLocator } from './typescript-import-locator';
|
||||||
import { TargetProjectLocator } from '../../utils/target-project-locator';
|
import { TargetProjectLocator } from './target-project-locator';
|
||||||
import {
|
import {
|
||||||
DependencyType,
|
DependencyType,
|
||||||
ProjectFileMap,
|
ProjectFileMap,
|
||||||
ProjectGraph,
|
ProjectGraph,
|
||||||
} from '../../config/project-graph';
|
} from '../../../../config/project-graph';
|
||||||
|
|
||||||
export type ExplicitDependency = {
|
export type ExplicitDependency = {
|
||||||
sourceProjectName: string;
|
sourceProjectName: string;
|
||||||
@ -1,8 +1,8 @@
|
|||||||
import { parentPort } from 'worker_threads';
|
import { parentPort } from 'worker_threads';
|
||||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
|
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-explicit-typescript-and-package-json-dependencies';
|
||||||
import { ProjectGraph } from '../config/project-graph';
|
import { NxJsonConfiguration } from '../../../../config/nx-json';
|
||||||
import { ProjectsConfigurations } from '../config/workspace-json-project-json';
|
import { ProjectsConfigurations } from '../../../../config/workspace-json-project-json';
|
||||||
import { NxJsonConfiguration } from '../config/nx-json';
|
import { ProjectGraph } from '../../../../config/project-graph';
|
||||||
|
|
||||||
let nxJsonConfiguration: NxJsonConfiguration | null;
|
let nxJsonConfiguration: NxJsonConfiguration | null;
|
||||||
let projectsConfigurations: ProjectsConfigurations | null;
|
let projectsConfigurations: ProjectsConfigurations | null;
|
||||||
@ -4,7 +4,7 @@ import {
|
|||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
ProjectGraphProcessorContext,
|
ProjectGraphProcessorContext,
|
||||||
ProjectGraphProjectNode,
|
ProjectGraphProjectNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../../config/project-graph';
|
||||||
|
|
||||||
jest.mock('nx/src/utils/workspace-root', () => ({
|
jest.mock('nx/src/utils/workspace-root', () => ({
|
||||||
workspaceRoot: '/root',
|
workspaceRoot: '/root',
|
||||||
@ -1,16 +1,19 @@
|
|||||||
import { getRootTsConfigFileName, resolveModuleByImport } from './typescript';
|
import {
|
||||||
import { isRelativePath, readJsonFile } from './fileutils';
|
getRootTsConfigFileName,
|
||||||
|
resolveModuleByImport,
|
||||||
|
} from '../../../../utils/typescript';
|
||||||
|
import { isRelativePath, readJsonFile } from '../../../../utils/fileutils';
|
||||||
import { dirname, join, posix } from 'path';
|
import { dirname, join, posix } from 'path';
|
||||||
import { workspaceRoot } from './workspace-root';
|
import { workspaceRoot } from '../../../../utils/workspace-root';
|
||||||
import {
|
import {
|
||||||
ProjectGraphExternalNode,
|
ProjectGraphExternalNode,
|
||||||
ProjectGraphProjectNode,
|
ProjectGraphProjectNode,
|
||||||
} from '../config/project-graph';
|
} from '../../../../config/project-graph';
|
||||||
|
import { builtinModules } from 'module';
|
||||||
import {
|
import {
|
||||||
createProjectRootMappings,
|
createProjectRootMappings,
|
||||||
findProjectForPath,
|
findProjectForPath,
|
||||||
} from '../project-graph/utils/find-project-for-path';
|
} from '../../../../project-graph/utils/find-project-for-path';
|
||||||
import { builtinModules } from 'module';
|
|
||||||
|
|
||||||
const builtInModuleSet = new Set<string>([
|
const builtInModuleSet = new Set<string>([
|
||||||
...builtinModules,
|
...builtinModules,
|
||||||
@ -1,8 +1,8 @@
|
|||||||
import type * as ts from 'typescript';
|
import type * as ts from 'typescript';
|
||||||
import * as path from 'path';
|
import * as path from 'path';
|
||||||
import { stripSourceCode } from '../../utils/strip-source-code';
|
import { stripSourceCode } from './strip-source-code';
|
||||||
import { defaultFileRead } from '../file-utils';
|
import { DependencyType } from '../../../../config/project-graph';
|
||||||
import { DependencyType } from '../../config/project-graph';
|
import { defaultFileRead } from '../../../../project-graph/file-utils';
|
||||||
|
|
||||||
let tsModule: typeof import('typescript');
|
let tsModule: typeof import('typescript');
|
||||||
|
|
||||||
@ -1,3 +1 @@
|
|||||||
export * from './implicit-project-dependencies';
|
export * from './implicit-project-dependencies';
|
||||||
export * from './explicit-project-dependencies';
|
|
||||||
export * from './explicit-package-json-dependencies';
|
|
||||||
|
|||||||
@ -201,17 +201,6 @@ describe('project graph', () => {
|
|||||||
jest.spyOn(fastGlob, 'sync').mockImplementation(() => globResults);
|
jest.spyOn(fastGlob, 'sync').mockImplementation(() => globResults);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should throw an appropriate error for an invalid json config', async () => {
|
|
||||||
tempFs.appendFile('tsconfig.base.json', 'invalid');
|
|
||||||
try {
|
|
||||||
await buildProjectGraph();
|
|
||||||
fail('Invalid tsconfigs should cause project graph to throw error');
|
|
||||||
} catch (e) {
|
|
||||||
expect(e.message).toContain(`${tempFs.tempDir}/tsconfig.base.json`);
|
|
||||||
expect(e.message).toContain(`invalid`);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
it('should create nodes and dependencies with workspace projects', async () => {
|
it('should create nodes and dependencies with workspace projects', async () => {
|
||||||
const graph = await buildProjectGraph();
|
const graph = await buildProjectGraph();
|
||||||
|
|
||||||
|
|||||||
@ -11,13 +11,8 @@ import {
|
|||||||
shouldRecomputeWholeGraph,
|
shouldRecomputeWholeGraph,
|
||||||
writeCache,
|
writeCache,
|
||||||
} from './nx-deps-cache';
|
} from './nx-deps-cache';
|
||||||
import {
|
import { buildImplicitProjectDependencies } from './build-dependencies';
|
||||||
buildImplicitProjectDependencies,
|
|
||||||
ExplicitDependency,
|
|
||||||
} from './build-dependencies';
|
|
||||||
import { buildWorkspaceProjectNodes } from './build-nodes';
|
import { buildWorkspaceProjectNodes } from './build-nodes';
|
||||||
import * as os from 'os';
|
|
||||||
import { buildExplicitTypescriptAndPackageJsonDependencies } from './build-dependencies/build-explicit-typescript-and-package-json-dependencies';
|
|
||||||
import { loadNxPlugins } from '../utils/nx-plugin';
|
import { loadNxPlugins } from '../utils/nx-plugin';
|
||||||
import { defaultFileHasher } from '../hasher/file-hasher';
|
import { defaultFileHasher } from '../hasher/file-hasher';
|
||||||
import { createProjectFileMap } from './file-map-utils';
|
import { createProjectFileMap } from './file-map-utils';
|
||||||
@ -28,7 +23,7 @@ import {
|
|||||||
ProjectGraphProcessorContext,
|
ProjectGraphProcessorContext,
|
||||||
} from '../config/project-graph';
|
} from '../config/project-graph';
|
||||||
import { readJsonFile } from '../utils/fileutils';
|
import { readJsonFile } from '../utils/fileutils';
|
||||||
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../config/nx-json';
|
import { NxJsonConfiguration } from '../config/nx-json';
|
||||||
import { logger } from '../utils/logger';
|
import { logger } from '../utils/logger';
|
||||||
import { ProjectGraphBuilder } from './project-graph-builder';
|
import { ProjectGraphBuilder } from './project-graph-builder';
|
||||||
import {
|
import {
|
||||||
@ -36,11 +31,6 @@ import {
|
|||||||
ProjectsConfigurations,
|
ProjectsConfigurations,
|
||||||
} from '../config/workspace-json-project-json';
|
} from '../config/workspace-json-project-json';
|
||||||
import { readNxJson } from '../config/configuration';
|
import { readNxJson } from '../config/configuration';
|
||||||
import {
|
|
||||||
lockFileExists,
|
|
||||||
lockFileHash,
|
|
||||||
parseLockFile,
|
|
||||||
} from '../lock-file/lock-file';
|
|
||||||
import { Workspaces } from '../config/workspaces';
|
import { Workspaces } from '../config/workspaces';
|
||||||
import { existsSync } from 'fs';
|
import { existsSync } from 'fs';
|
||||||
import { PackageJson } from '../utils/package-json';
|
import { PackageJson } from '../utils/package-json';
|
||||||
@ -102,17 +92,7 @@ export async function buildProjectGraphUsingProjectFileMap(
|
|||||||
filesToProcess = projectFileMap;
|
filesToProcess = projectFileMap;
|
||||||
cachedFileData = {};
|
cachedFileData = {};
|
||||||
}
|
}
|
||||||
let partialGraph: ProjectGraph;
|
|
||||||
let lockHash = 'n/a';
|
|
||||||
// during the create-nx-workspace lock file might not exists yet
|
|
||||||
if (lockFileExists()) {
|
|
||||||
lockHash = lockFileHash();
|
|
||||||
if (cache && cache.lockFileHash === lockHash) {
|
|
||||||
partialGraph = isolatePartialGraphFromCache(cache);
|
|
||||||
} else {
|
|
||||||
partialGraph = parseLockFile();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const context = createContext(
|
const context = createContext(
|
||||||
projectsConfigurations,
|
projectsConfigurations,
|
||||||
nxJson,
|
nxJson,
|
||||||
@ -124,15 +104,13 @@ export async function buildProjectGraphUsingProjectFileMap(
|
|||||||
context,
|
context,
|
||||||
cachedFileData,
|
cachedFileData,
|
||||||
projectGraphVersion,
|
projectGraphVersion,
|
||||||
partialGraph,
|
cache
|
||||||
packageJsonDeps
|
|
||||||
);
|
);
|
||||||
const projectGraphCache = createCache(
|
const projectGraphCache = createCache(
|
||||||
nxJson,
|
nxJson,
|
||||||
packageJsonDeps,
|
packageJsonDeps,
|
||||||
projectGraph,
|
projectGraph,
|
||||||
rootTsConfig,
|
rootTsConfig
|
||||||
lockHash
|
|
||||||
);
|
);
|
||||||
if (shouldWriteCache) {
|
if (shouldWriteCache) {
|
||||||
writeCache(projectGraphCache);
|
writeCache(projectGraphCache);
|
||||||
@ -168,33 +146,24 @@ function readCombinedDeps() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// extract only external nodes and their dependencies
|
|
||||||
function isolatePartialGraphFromCache(cache: ProjectGraphCache): ProjectGraph {
|
|
||||||
const dependencies = {};
|
|
||||||
Object.keys(cache.dependencies).forEach((k) => {
|
|
||||||
if (cache.externalNodes[k]) {
|
|
||||||
dependencies[k] = cache.dependencies[k];
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
return {
|
|
||||||
nodes: {},
|
|
||||||
dependencies,
|
|
||||||
externalNodes: cache.externalNodes,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function buildProjectGraphUsingContext(
|
async function buildProjectGraphUsingContext(
|
||||||
nxJson: NxJsonConfiguration,
|
nxJson: NxJsonConfiguration,
|
||||||
ctx: ProjectGraphProcessorContext,
|
ctx: ProjectGraphProcessorContext,
|
||||||
cachedFileData: { [project: string]: { [file: string]: FileData } },
|
cachedFileData: { [project: string]: { [file: string]: FileData } },
|
||||||
projectGraphVersion: string,
|
projectGraphVersion: string,
|
||||||
partialGraph: ProjectGraph,
|
cache: ProjectGraphCache | null
|
||||||
packageJsonDeps: Record<string, string>
|
|
||||||
) {
|
) {
|
||||||
performance.mark('build project graph:start');
|
performance.mark('build project graph:start');
|
||||||
|
|
||||||
const builder = new ProjectGraphBuilder(partialGraph);
|
const builder = new ProjectGraphBuilder(
|
||||||
|
cache
|
||||||
|
? {
|
||||||
|
nodes: cache.nodes,
|
||||||
|
externalNodes: cache.externalNodes,
|
||||||
|
dependencies: cache.dependencies,
|
||||||
|
}
|
||||||
|
: null
|
||||||
|
);
|
||||||
builder.setVersion(projectGraphVersion);
|
builder.setVersion(projectGraphVersion);
|
||||||
|
|
||||||
await buildWorkspaceProjectNodes(ctx, builder, nxJson);
|
await buildWorkspaceProjectNodes(ctx, builder, nxJson);
|
||||||
@ -212,12 +181,6 @@ async function buildProjectGraphUsingContext(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
await buildExplicitDependencies(
|
|
||||||
jsPluginConfig(nxJson, packageJsonDeps),
|
|
||||||
ctx,
|
|
||||||
updatedBuilder
|
|
||||||
);
|
|
||||||
|
|
||||||
buildImplicitProjectDependencies(ctx, updatedBuilder);
|
buildImplicitProjectDependencies(ctx, updatedBuilder);
|
||||||
|
|
||||||
const finalGraph = updatedBuilder.getUpdatedProjectGraph();
|
const finalGraph = updatedBuilder.getUpdatedProjectGraph();
|
||||||
@ -232,216 +195,6 @@ async function buildProjectGraphUsingContext(
|
|||||||
return finalGraph;
|
return finalGraph;
|
||||||
}
|
}
|
||||||
|
|
||||||
function jsPluginConfig(
|
|
||||||
nxJson: NxJsonConfiguration,
|
|
||||||
packageJsonDeps: { [packageName: string]: string }
|
|
||||||
): NrwlJsPluginConfig {
|
|
||||||
if (nxJson?.pluginsConfig?.['@nrwl/js']) {
|
|
||||||
return nxJson?.pluginsConfig?.['@nrwl/js'];
|
|
||||||
}
|
|
||||||
if (
|
|
||||||
packageJsonDeps['@nrwl/workspace'] ||
|
|
||||||
packageJsonDeps['@nrwl/js'] ||
|
|
||||||
packageJsonDeps['@nrwl/node'] ||
|
|
||||||
packageJsonDeps['@nrwl/next'] ||
|
|
||||||
packageJsonDeps['@nrwl/react'] ||
|
|
||||||
packageJsonDeps['@nrwl/angular'] ||
|
|
||||||
packageJsonDeps['@nrwl/web']
|
|
||||||
) {
|
|
||||||
return { analyzePackageJson: true, analyzeSourceFiles: true };
|
|
||||||
} else {
|
|
||||||
return { analyzePackageJson: true, analyzeSourceFiles: false };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildExplicitDependencies(
|
|
||||||
jsPluginConfig: {
|
|
||||||
analyzeSourceFiles?: boolean;
|
|
||||||
analyzePackageJson?: boolean;
|
|
||||||
},
|
|
||||||
ctx: ProjectGraphProcessorContext,
|
|
||||||
builder: ProjectGraphBuilder
|
|
||||||
) {
|
|
||||||
let totalNumOfFilesToProcess = totalNumberOfFilesToProcess(ctx);
|
|
||||||
// using workers has an overhead, so we only do it when the number of
|
|
||||||
// files we need to process is >= 100 and there are more than 2 CPUs
|
|
||||||
// to be able to use at least 2 workers (1 worker per CPU and
|
|
||||||
// 1 CPU for the main thread)
|
|
||||||
if (totalNumOfFilesToProcess < 100 || getNumberOfWorkers() <= 2) {
|
|
||||||
return buildExplicitDependenciesWithoutWorkers(
|
|
||||||
jsPluginConfig,
|
|
||||||
ctx,
|
|
||||||
builder
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
return buildExplicitDependenciesUsingWorkers(
|
|
||||||
jsPluginConfig,
|
|
||||||
ctx,
|
|
||||||
totalNumOfFilesToProcess,
|
|
||||||
builder
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function totalNumberOfFilesToProcess(ctx: ProjectGraphProcessorContext) {
|
|
||||||
let totalNumOfFilesToProcess = 0;
|
|
||||||
Object.values(ctx.filesToProcess).forEach(
|
|
||||||
(t) => (totalNumOfFilesToProcess += t.length)
|
|
||||||
);
|
|
||||||
return totalNumOfFilesToProcess;
|
|
||||||
}
|
|
||||||
|
|
||||||
function splitFilesIntoBins(
|
|
||||||
ctx: ProjectGraphProcessorContext,
|
|
||||||
totalNumOfFilesToProcess: number,
|
|
||||||
numberOfWorkers: number
|
|
||||||
) {
|
|
||||||
// we want to have numberOfWorkers * 5 bins
|
|
||||||
const filesPerBin =
|
|
||||||
Math.round(totalNumOfFilesToProcess / numberOfWorkers / 5) + 1;
|
|
||||||
const bins: ProjectFileMap[] = [];
|
|
||||||
let currentProjectFileMap = {};
|
|
||||||
let currentNumberOfFiles = 0;
|
|
||||||
for (const source of Object.keys(ctx.filesToProcess)) {
|
|
||||||
for (const f of Object.values(ctx.filesToProcess[source])) {
|
|
||||||
if (!currentProjectFileMap[source]) currentProjectFileMap[source] = [];
|
|
||||||
currentProjectFileMap[source].push(f);
|
|
||||||
currentNumberOfFiles++;
|
|
||||||
|
|
||||||
if (currentNumberOfFiles >= filesPerBin) {
|
|
||||||
bins.push(currentProjectFileMap);
|
|
||||||
currentProjectFileMap = {};
|
|
||||||
currentNumberOfFiles = 0;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
bins.push(currentProjectFileMap);
|
|
||||||
return bins;
|
|
||||||
}
|
|
||||||
|
|
||||||
function createWorkerPool(numberOfWorkers: number) {
|
|
||||||
const res = [];
|
|
||||||
for (let i = 0; i < numberOfWorkers; ++i) {
|
|
||||||
res.push(
|
|
||||||
new (require('worker_threads').Worker)(
|
|
||||||
join(__dirname, './project-graph-worker.js'),
|
|
||||||
{
|
|
||||||
env: process.env,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildExplicitDependenciesWithoutWorkers(
|
|
||||||
jsPluginConfig: {
|
|
||||||
analyzeSourceFiles?: boolean;
|
|
||||||
analyzePackageJson?: boolean;
|
|
||||||
},
|
|
||||||
ctx: ProjectGraphProcessorContext,
|
|
||||||
builder: ProjectGraphBuilder
|
|
||||||
) {
|
|
||||||
buildExplicitTypescriptAndPackageJsonDependencies(
|
|
||||||
jsPluginConfig,
|
|
||||||
ctx.nxJsonConfiguration,
|
|
||||||
ctx.projectsConfigurations,
|
|
||||||
builder.graph,
|
|
||||||
ctx.filesToProcess
|
|
||||||
).forEach((r) => {
|
|
||||||
if (r.type === 'static') {
|
|
||||||
builder.addStaticDependency(
|
|
||||||
r.sourceProjectName,
|
|
||||||
r.targetProjectName,
|
|
||||||
r.sourceProjectFile
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
builder.addDynamicDependency(
|
|
||||||
r.sourceProjectName,
|
|
||||||
r.targetProjectName,
|
|
||||||
r.sourceProjectFile
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function buildExplicitDependenciesUsingWorkers(
|
|
||||||
jsPluginConfig: {
|
|
||||||
analyzeSourceFiles?: boolean;
|
|
||||||
analyzePackageJson?: boolean;
|
|
||||||
},
|
|
||||||
ctx: ProjectGraphProcessorContext,
|
|
||||||
totalNumOfFilesToProcess: number,
|
|
||||||
builder: ProjectGraphBuilder
|
|
||||||
) {
|
|
||||||
const numberOfWorkers = Math.min(
|
|
||||||
totalNumOfFilesToProcess,
|
|
||||||
getNumberOfWorkers()
|
|
||||||
);
|
|
||||||
const bins = splitFilesIntoBins(
|
|
||||||
ctx,
|
|
||||||
totalNumOfFilesToProcess,
|
|
||||||
numberOfWorkers
|
|
||||||
);
|
|
||||||
const workers = createWorkerPool(numberOfWorkers);
|
|
||||||
let numberOfExpectedResponses = bins.length;
|
|
||||||
|
|
||||||
return new Promise((res, reject) => {
|
|
||||||
for (let w of workers) {
|
|
||||||
w.on('message', (explicitDependencies) => {
|
|
||||||
explicitDependencies.forEach((r: ExplicitDependency) => {
|
|
||||||
if (r.type === 'static') {
|
|
||||||
builder.addStaticDependency(
|
|
||||||
r.sourceProjectName,
|
|
||||||
r.targetProjectName,
|
|
||||||
r.sourceProjectFile
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
builder.addDynamicDependency(
|
|
||||||
r.sourceProjectName,
|
|
||||||
r.targetProjectName,
|
|
||||||
r.sourceProjectFile
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
if (bins.length > 0) {
|
|
||||||
w.postMessage({ filesToProcess: bins.shift() });
|
|
||||||
}
|
|
||||||
// we processed all the bins
|
|
||||||
if (--numberOfExpectedResponses === 0) {
|
|
||||||
for (let w of workers) {
|
|
||||||
w.terminate();
|
|
||||||
}
|
|
||||||
res(null);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
w.on('error', reject);
|
|
||||||
w.on('exit', (code) => {
|
|
||||||
if (code !== 0) {
|
|
||||||
reject(
|
|
||||||
new Error(
|
|
||||||
`Unable to complete project graph creation. Worker stopped with exit code: ${code}`
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
w.postMessage({
|
|
||||||
nxJsonConfiguration: ctx.nxJsonConfiguration,
|
|
||||||
projectsConfigurations: ctx.projectsConfigurations,
|
|
||||||
projectGraph: builder.graph,
|
|
||||||
jsPluginConfig,
|
|
||||||
});
|
|
||||||
w.postMessage({ filesToProcess: bins.shift() });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
function getNumberOfWorkers(): number {
|
|
||||||
return process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
|
||||||
? +process.env.NX_PROJECT_GRAPH_MAX_WORKERS
|
|
||||||
: Math.min(os.cpus().length - 1, 8); // This is capped for cases in CI where `os.cpus()` returns way more CPUs than the resources that are allocated
|
|
||||||
}
|
|
||||||
|
|
||||||
function createContext(
|
function createContext(
|
||||||
projectsConfigurations: ProjectsConfigurations,
|
projectsConfigurations: ProjectsConfigurations,
|
||||||
nxJson: NxJsonConfiguration,
|
nxJson: NxJsonConfiguration,
|
||||||
|
|||||||
@ -19,8 +19,8 @@ export function createProjectFileMap(
|
|||||||
projectFileMap[projectName] ??= [];
|
projectFileMap[projectName] ??= [];
|
||||||
}
|
}
|
||||||
for (const f of allWorkspaceFiles) {
|
for (const f of allWorkspaceFiles) {
|
||||||
const matchingProjectFiles =
|
const projectFileMapKey = findProjectForPath(f.file, projectRootMappings);
|
||||||
projectFileMap[findProjectForPath(f.file, projectRootMappings)];
|
const matchingProjectFiles = projectFileMap[projectFileMapKey];
|
||||||
if (matchingProjectFiles) {
|
if (matchingProjectFiles) {
|
||||||
matchingProjectFiles.push(f);
|
matchingProjectFiles.push(f);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -298,8 +298,7 @@ describe('nx deps utils', () => {
|
|||||||
createNxJson({}),
|
createNxJson({}),
|
||||||
createPackageJsonDeps({}),
|
createPackageJsonDeps({}),
|
||||||
createCache({}) as ProjectGraph,
|
createCache({}) as ProjectGraph,
|
||||||
{},
|
{}
|
||||||
'abcd1234'
|
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -308,8 +307,7 @@ describe('nx deps utils', () => {
|
|||||||
createNxJson({}),
|
createNxJson({}),
|
||||||
createPackageJsonDeps({}),
|
createPackageJsonDeps({}),
|
||||||
createCache({}) as ProjectGraph,
|
createCache({}) as ProjectGraph,
|
||||||
undefined,
|
undefined
|
||||||
'abcd1234'
|
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(result).toBeDefined();
|
expect(result).toBeDefined();
|
||||||
@ -323,7 +321,6 @@ describe('nx deps utils', () => {
|
|||||||
'@nrwl/workspace': '12.0.0',
|
'@nrwl/workspace': '12.0.0',
|
||||||
plugin: '1.0.0',
|
plugin: '1.0.0',
|
||||||
},
|
},
|
||||||
lockFileHash: 'abcd1234',
|
|
||||||
pathMappings: {
|
pathMappings: {
|
||||||
mylib: ['libs/mylib/index.ts'],
|
mylib: ['libs/mylib/index.ts'],
|
||||||
},
|
},
|
||||||
|
|||||||
@ -23,7 +23,6 @@ import {
|
|||||||
export interface ProjectGraphCache {
|
export interface ProjectGraphCache {
|
||||||
version: string;
|
version: string;
|
||||||
deps: Record<string, string>;
|
deps: Record<string, string>;
|
||||||
lockFileHash: string;
|
|
||||||
pathMappings: Record<string, any>;
|
pathMappings: Record<string, any>;
|
||||||
nxJsonPlugins: { name: string; version: string }[];
|
nxJsonPlugins: { name: string; version: string }[];
|
||||||
pluginsConfig?: any;
|
pluginsConfig?: any;
|
||||||
@ -86,8 +85,7 @@ export function createCache(
|
|||||||
nxJson: NxJsonConfiguration<'*' | string[]>,
|
nxJson: NxJsonConfiguration<'*' | string[]>,
|
||||||
packageJsonDeps: Record<string, string>,
|
packageJsonDeps: Record<string, string>,
|
||||||
projectGraph: ProjectGraph,
|
projectGraph: ProjectGraph,
|
||||||
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } },
|
tsConfig: { compilerOptions?: { paths?: { [p: string]: any } } }
|
||||||
lockFileHash: string
|
|
||||||
) {
|
) {
|
||||||
const nxJsonPlugins = (nxJson.plugins || []).map((p) => ({
|
const nxJsonPlugins = (nxJson.plugins || []).map((p) => ({
|
||||||
name: p,
|
name: p,
|
||||||
@ -96,7 +94,6 @@ export function createCache(
|
|||||||
const newValue: ProjectGraphCache = {
|
const newValue: ProjectGraphCache = {
|
||||||
version: projectGraph.version || '5.1',
|
version: projectGraph.version || '5.1',
|
||||||
deps: packageJsonDeps,
|
deps: packageJsonDeps,
|
||||||
lockFileHash,
|
|
||||||
// compilerOptions may not exist, especially for repos converted through add-nx-to-monorepo
|
// compilerOptions may not exist, especially for repos converted through add-nx-to-monorepo
|
||||||
pathMappings: tsConfig?.compilerOptions?.paths || {},
|
pathMappings: tsConfig?.compilerOptions?.paths || {},
|
||||||
nxJsonPlugins,
|
nxJsonPlugins,
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
x
Reference in New Issue
Block a user