miel.truyen/nx
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

cleanup(core): move js lockfile parsing to v2 plugin (#18779)

Browse Source
This commit is contained in:
Craigory Coppola 2023-08-30 13:26:41 -05:00 committed by GitHub
parent b74f3671e1
commit 353d8d089d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1452 additions and 510 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/generated/devkit/CreateNodes.md
View File

@ -1,5 +1,5 @@
# Type alias: CreateNodes
Ƭ **CreateNodes**: [projectFilePattern: string, createNodesFunction: CreateNodesFunction]
Ƭ **CreateNodes**: readonly [projectFilePattern: string, createNodesFunction: CreateNodesFunction]
A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)

16
packages/js/src/utils/package-json/update-package-json.ts
View File

@ -7,6 +7,7 @@ import {
import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
import {
detectPackageManager,
ExecutorContext,
getOutputsForTargetAndConfiguration,
joinPathFragments,
@ -100,10 +101,19 @@ export function updatePackageJson(
writeJsonFile(`${options.outputPath}/package.json`, packageJson);
if (options.generateLockfile) {
const lockFile = createLockFile(packageJson);
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, {
const packageManager = detectPackageManager(context.root);
const lockFile = createLockFile(
packageJson,
context.projectGraph,
packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
});
}
);
}
}

16
packages/next/src/executors/build/build.impl.ts
View File

@ -1,4 +1,5 @@
import {
detectPackageManager,
ExecutorContext,
logger,
readJsonFile,
@ -83,10 +84,19 @@ export default async function buildExecutor(
writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson);
if (options.generateLockfile) {
const lockFile = createLockFile(builtPackageJson);
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, {
const packageManager = detectPackageManager(context.root);
const lockFile = createLockFile(
builtPackageJson,
context.projectGraph,
packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
});
}
);
}
// If output path is different from source path, then copy over the config and public files.

262
packages/nx/src/plugins/js/index.ts
View File

@ -1,98 +1,123 @@
import {
ProjectGraph,
ProjectGraphProcessor,
} from '../../config/project-graph';
import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../project-graph/project-graph-builder';
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
import { readNxJson } from '../../config/configuration';
import { fileExists, readJsonFile } from '../../utils/fileutils';
import { PackageJson } from '../../utils/package-json';
import {
lockFileExists,
lockFileHash,
parseLockFile,
} from './lock-file/lock-file';
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json';
import { dirname, join } from 'path';
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { workspaceRoot } from '../../utils/workspace-root';
import { readFileSync, writeFileSync } from 'fs';
import { ensureDirSync } from 'fs-extra';
import { dirname, join } from 'path';
import { performance } from 'perf_hooks';
import { ProjectGraph } from '../../config/project-graph';
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
import { combineGlobPatterns } from '../../utils/globs';
import {
CreateDependencies,
CreateDependenciesContext,
CreateNodes,
} from '../../utils/nx-plugin';
import {
getLockFileDependencies,
getLockFileName,
getLockFileNodes,
lockFileExists,
LOCKFILES,
} from './lock-file/lock-file';
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
import { jsPluginConfig } from './utils/config';
import { ProjectGraphDependencyWithFile } from '../../project-graph/project-graph-builder';
import { hashArray } from '../../hasher/file-hasher';
import { detectPackageManager } from '../../utils/package-manager';
import { workspaceRoot } from '../../utils/workspace-root';
const createDependencies: CreateDependencies = (context) => {
export const name = 'nx-js-graph-plugin';
interface ParsedLockFile {
externalNodes?: ProjectGraph['externalNodes'];
dependencies?: ProjectGraphDependencyWithFile[];
}
let parsedLockFile: ParsedLockFile = {};
export const createNodes: CreateNodes = [
// Look for all lockfiles
combineGlobPatterns(LOCKFILES),
(lockFile, context) => {
const pluginConfig = jsPluginConfig(context.nxJsonConfiguration);
if (!pluginConfig.analyzePackageJson) {
return {};
}
const packageManager = detectPackageManager(workspaceRoot);
// Only process the correct lockfile
if (lockFile !== getLockFileName(packageManager)) {
return {};
}
const lockFilePath = join(workspaceRoot, lockFile);
const lockFileContents = readFileSync(lockFilePath).toString();
const lockFileHash = hashArray([lockFileContents]);
if (!lockFileNeedsReprocessing(lockFileHash)) {
return {
externalNodes: readCachedParsedLockFile().externalNodes,
};
}
const externalNodes = getLockFileNodes(
packageManager,
lockFileContents,
lockFileHash
);
parsedLockFile.externalNodes = externalNodes;
return {
externalNodes,
};
},
];
export const createDependencies: CreateDependencies = (
ctx: CreateDependenciesContext
) => {
const pluginConfig = jsPluginConfig(ctx.nxJsonConfiguration);
const packageManager = detectPackageManager(workspaceRoot);
let lockfileDependencies: ProjectGraphDependencyWithFile[] = [];
// lockfile may not exist yet
if (
pluginConfig.analyzePackageJson &&
lockFileExists(packageManager) &&
parsedLockFile
) {
const lockFilePath = join(workspaceRoot, getLockFileName(packageManager));
const lockFileContents = readFileSync(lockFilePath).toString();
const lockFileHash = hashArray([lockFileContents]);
if (!lockFileNeedsReprocessing(lockFileHash)) {
lockfileDependencies = readCachedParsedLockFile().dependencies ?? [];
} else {
lockfileDependencies = getLockFileDependencies(
packageManager,
lockFileContents,
lockFileHash,
ctx.graph
);
parsedLockFile.dependencies = lockfileDependencies;
writeLastProcessedLockfileHash(lockFileHash, parsedLockFile);
}
}
performance.mark('build typescript dependencies - start');
const dependencies = buildExplicitDependencies(pluginConfig, context);
const explicitProjectDependencies = buildExplicitDependencies(
pluginConfig,
ctx
);
performance.mark('build typescript dependencies - end');
performance.measure(
'build typescript dependencies',
'build typescript dependencies - start',
'build typescript dependencies - end'
);
return dependencies;
return lockfileDependencies.concat(explicitProjectDependencies);
};
export const processProjectGraph: ProjectGraphProcessor = async (
graph,
context
) => {
const builder = new ProjectGraphBuilder(graph, context.fileMap);
const pluginConfig = jsPluginConfig(readNxJson());
if (pluginConfig.analyzePackageJson) {
if (
// during the create-nx-workspace lock file might not exists yet
lockFileExists() &&
pluginConfig.analyzeLockfile
) {
const lockHash = lockFileHash();
let parsedLockFile: ProjectGraph;
if (lockFileNeedsReprocessing(lockHash)) {
parsedLockFile = parseLockFile();
writeLastProcessedLockfileHash(lockHash, parsedLockFile);
} else {
parsedLockFile = readParsedLockFile();
}
builder.mergeProjectGraph(parsedLockFile);
}
}
const createDependenciesContext: CreateDependenciesContext = {
...context,
graph,
};
const dependencies = createDependencies(
createDependenciesContext
) as ProjectGraphDependencyWithFile[];
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
return builder.getUpdatedProjectGraph();
};
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
const parsedLockFile = join(
projectGraphCacheDirectory,
'parsed-lock-file.json'
);
function lockFileNeedsReprocessing(lockHash: string) {
try {
return readFileSync(lockFileHashFile).toString() !== lockHash;
@ -101,82 +126,21 @@ function lockFileNeedsReprocessing(lockHash: string) {
}
}
function writeLastProcessedLockfileHash(hash: string, lockFile: ProjectGraph) {
function writeLastProcessedLockfileHash(
hash: string,
lockFile: ParsedLockFile
) {
ensureDirSync(dirname(lockFileHashFile));
writeFileSync(parsedLockFile, JSON.stringify(lockFile, null, 2));
writeFileSync(cachedParsedLockFile, JSON.stringify(lockFile, null, 2));
writeFileSync(lockFileHashFile, hash);
}
function readParsedLockFile(): ProjectGraph {
return JSON.parse(readFileSync(parsedLockFile).toString());
function readCachedParsedLockFile(): ParsedLockFile {
return JSON.parse(readFileSync(cachedParsedLockFile).toString());
}
function jsPluginConfig(
nxJson: NxJsonConfiguration
): Required<NrwlJsPluginConfig> {
const nxJsonConfig: NrwlJsPluginConfig =
nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js'];
// using lerna _before_ installing deps is causing an issue when parsing lockfile.
// See: https://github.com/lerna/lerna/issues/3807
// Note that previous attempt to fix this caused issues with Nx itself, thus we're checking
// for Lerna explicitly.
// See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14
const analyzeLockfile =
!existsSync(join(workspaceRoot, 'lerna.json')) ||
existsSync(join(workspaceRoot, 'nx.json'));
if (nxJsonConfig) {
return {
analyzePackageJson: true,
analyzeSourceFiles: true,
analyzeLockfile,
...nxJsonConfig,
};
}
if (!fileExists(join(workspaceRoot, 'package.json'))) {
return {
analyzeLockfile: false,
analyzePackageJson: false,
analyzeSourceFiles: false,
};
}
const packageJson = readJsonFile<PackageJson>(
join(workspaceRoot, 'package.json')
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
const cachedParsedLockFile = join(
projectGraphCacheDirectory,
'parsed-lock-file.json'
);
const packageJsonDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
if (
packageJsonDeps['@nx/workspace'] ||
packageJsonDeps['@nx/js'] ||
packageJsonDeps['@nx/node'] ||
packageJsonDeps['@nx/next'] ||
packageJsonDeps['@nx/react'] ||
packageJsonDeps['@nx/angular'] ||
packageJsonDeps['@nx/web'] ||
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: true,
};
} else {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: false,
};
}
}

171
packages/nx/src/plugins/js/lock-file/lock-file.ts
View File

@ -11,15 +11,32 @@ import {
PackageManager,
} from '../../../utils/package-manager';
import { workspaceRoot } from '../../../utils/workspace-root';
import { ProjectGraph } from '../../../config/project-graph';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
ProjectGraph,
ProjectGraphExternalNode,
} from '../../../config/project-graph';
import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../../project-graph/project-graph-builder';
import { PackageJson } from '../../../utils/package-json';
import { hashArray } from '../../../hasher/file-hasher';
import { output } from '../../../utils/output';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
import {
getNpmLockfileNodes,
stringifyNpmLockfile,
getNpmLockfileDependencies,
} from './npm-parser';
import {
getPnpmLockfileDependencies,
getPnpmLockfileNodes,
stringifyPnpmLockfile,
} from './pnpm-parser';
import {
getYarnLockfileDependencies,
getYarnLockfileNodes,
stringifyYarnLockfile,
} from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { normalizePackageJson } from './utils/package-json';
import { readJsonFile } from '../../../utils/fileutils';
@ -27,17 +44,75 @@ import { readJsonFile } from '../../../utils/fileutils';
const YARN_LOCK_FILE = 'yarn.lock';
const NPM_LOCK_FILE = 'package-lock.json';
const PNPM_LOCK_FILE = 'pnpm-lock.yaml';
export const LOCKFILES = [YARN_LOCK_FILE, NPM_LOCK_FILE, PNPM_LOCK_FILE];
const YARN_LOCK_PATH = join(workspaceRoot, YARN_LOCK_FILE);
const NPM_LOCK_PATH = join(workspaceRoot, NPM_LOCK_FILE);
const PNPM_LOCK_PATH = join(workspaceRoot, PNPM_LOCK_FILE);
/**
* Check if lock file exists
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function lockFileExists(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): boolean {
export function getLockFileNodes(
packageManager: PackageManager,
contents: string,
lockFileHash: string
): Record<string, ProjectGraphExternalNode> {
try {
if (packageManager === 'yarn') {
const packageJson = readJsonFile('package.json');
return getYarnLockfileNodes(contents, lockFileHash, packageJson);
}
if (packageManager === 'pnpm') {
return getPnpmLockfileNodes(contents, lockFileHash);
}
if (packageManager === 'npm') {
return getNpmLockfileNodes(contents, lockFileHash);
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
/**
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function getLockFileDependencies(
packageManager: PackageManager,
contents: string,
lockFileHash: string,
projectGraph: ProjectGraph
): ProjectGraphDependencyWithFile[] {
try {
if (packageManager === 'yarn') {
return getYarnLockfileDependencies(contents, lockFileHash, projectGraph);
}
if (packageManager === 'pnpm') {
return getPnpmLockfileDependencies(contents, lockFileHash, projectGraph);
}
if (packageManager === 'npm') {
return getNpmLockfileDependencies(contents, lockFileHash, projectGraph);
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
export function lockFileExists(packageManager: PackageManager): boolean {
if (packageManager === 'yarn') {
return existsSync(YARN_LOCK_PATH);
}
@ -52,75 +127,12 @@ export function lockFileExists(
);
}
/**
* Hashes lock file content
*/
export function lockFileHash(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string {
let content: string;
if (packageManager === 'yarn') {
content = readFileSync(YARN_LOCK_PATH, 'utf8');
}
if (packageManager === 'pnpm') {
content = readFileSync(PNPM_LOCK_PATH, 'utf8');
}
if (packageManager === 'npm') {
content = readFileSync(NPM_LOCK_PATH, 'utf8');
}
if (content) {
return hashArray([content]);
} else {
throw new Error(
`Unknown package manager ${packageManager} or lock file missing`
);
}
}
/**
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function parseLockFile(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): ProjectGraph {
const builder = new ProjectGraphBuilder(null, null);
try {
if (packageManager === 'yarn') {
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
const packageJson = readJsonFile('package.json');
parseYarnLockfile(content, packageJson, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'pnpm') {
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
parsePnpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'npm') {
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
parseNpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
/**
* Returns lock file name based on the detected package manager in the root
* @param packageManager
* @returns
*/
export function getLockFileName(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string {
export function getLockFileName(packageManager: PackageManager): string {
if (packageManager === 'yarn') {
return YARN_LOCK_FILE;
}
@ -143,31 +155,22 @@ export function getLockFileName(
*/
export function createLockFile(
packageJson: PackageJson,
graph: ProjectGraph,
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string {
const normalizedPackageJson = normalizePackageJson(packageJson);
const content = readFileSync(getLockFileName(packageManager), 'utf8');
const rootPackageJson = readJsonFile('package.json');
const builder = new ProjectGraphBuilder();
try {
if (packageManager === 'yarn') {
parseYarnLockfile(content, rootPackageJson, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
}
if (packageManager === 'pnpm') {
parsePnpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
}
if (packageManager === 'npm') {
parseNpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
}

307
packages/nx/src/plugins/js/lock-file/npm-parser.spec.ts
View File

@ -1,5 +1,9 @@
import { joinPathFragments } from '../../../utils/path';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser';
import {
getNpmLockfileDependencies,
getNpmLockfileNodes,
stringifyNpmLockfile,
} from './npm-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs';
import { ProjectGraph } from '../../../config/project-graph';
@ -27,8 +31,31 @@ describe('NPM lock file utility', () => {
let graph: ProjectGraph;
beforeEach(() => {
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
@ -47,8 +74,31 @@ describe('NPM lock file utility', () => {
));
// this is original generated lock file
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(appLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(appLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(appLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const appGraph = builder.getUpdatedProjectGraph();
expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
@ -95,8 +145,31 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
@ -154,9 +227,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock-v2.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootV2LockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootV2LockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -252,9 +349,33 @@ describe('NPM lock file utility', () => {
cleanupTypes(prunedV2LockFile.packages);
cleanupTypes(prunedV2LockFile.dependencies, true);
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootV2LockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootV2LockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -339,9 +460,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock-v1.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369);
});
it('should parse v3', async () => {
@ -350,9 +495,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369);
});
});
@ -367,9 +536,31 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/optional/package.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(JSON.stringify(lockFile), hash);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(lockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8);
const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -392,9 +583,34 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/pruning/typescript/package.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -419,9 +635,34 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/pruning/devkit-yargs/package.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder);
const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyNpmLockfile(
prunedGraph,
@ -450,10 +691,13 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/workspaces/package-lock.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const result = builder.getUpdatedProjectGraph();
expect(Object.keys(result.externalNodes).length).toEqual(5);
const externalNodes = getNpmLockfileNodes(
JSON.stringify(lockFile),
uniq('mock-hash')
);
expect(Object.keys(externalNodes).length).toEqual(5);
});
it('should parse v1 lock file', async () => {
@ -461,10 +705,15 @@ describe('NPM lock file utility', () => {
__dirname,
'__fixtures__/workspaces/package-lock.v1.json'
));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder);
const result = builder.getUpdatedProjectGraph();
expect(Object.keys(result.externalNodes).length).toEqual(5);
const externalNodes = getNpmLockfileNodes(
JSON.stringify(lockFile),
uniq('mock')
);
expect(Object.keys(externalNodes).length).toEqual(5);
});
});
});
function uniq(str: string) {
return `str-${(Math.random() * 10000).toFixed(0)}`;
}

132
packages/nx/src/plugins/js/lock-file/npm-parser.ts
View File

@ -3,12 +3,16 @@ import { satisfies } from 'semver';
import { workspaceRoot } from '../../../utils/workspace-root';
import { reverse } from '../../../project-graph/operators';
import { NormalizedPackageJson } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import {
DependencyType,
ProjectGraph,
ProjectGraphExternalNode,
} from '../../../config/project-graph';
import { fileHasher, hashArray } from '../../../hasher/file-hasher';
import { hashArray } from '../../../hasher/file-hasher';
/**
* NPM
@ -50,21 +54,51 @@ type NpmLockFile = {
dependencies?: Record<string, NpmDependencyV1>;
};
export function parseNpmLockfile(
lockFileContent: string,
builder: ProjectGraphBuilder
) {
const data = JSON.parse(lockFileContent) as NpmLockFile;
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
addNodes(data, builder, keyMap);
addDependencies(data, builder, keyMap);
let keyMap = new Map<string, ProjectGraphExternalNode>();
let currentLockFileHash: string;
let parsedLockFile: NpmLockFile;
function parsePackageLockFile(lockFileContent: string, lockFileHash: string) {
if (lockFileHash === currentLockFileHash) {
return parsedLockFile;
}
function addNodes(
keyMap.clear();
const results = JSON.parse(lockFileContent) as NpmLockFile;
parsedLockFile = results;
currentLockFileHash = lockFileHash;
return results;
}
export function getNpmLockfileNodes(
lockFileContent: string,
lockFileHash: string
) {
const data = parsePackageLockFile(
lockFileContent,
lockFileHash
) as NpmLockFile;
// we use key => node map to avoid duplicate work when parsing keys
return getNodes(data, keyMap);
}
export function getNpmLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const data = parsePackageLockFile(
lockFileContent,
lockFileHash
) as NpmLockFile;
return getDependencies(data, keyMap, projectGraph);
}
function getNodes(
data: NpmLockFile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>
) {
const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();
@ -92,8 +126,7 @@ function addNodes(
depSnapshot,
`${snapshot.version.slice(5)}/node_modules/${depName}`,
nodes,
keyMap,
builder
keyMap
);
}
);
@ -104,13 +137,14 @@ function addNodes(
snapshot,
`node_modules/${packageName}`,
nodes,
keyMap,
builder
keyMap
);
}
});
}
const results: Record<string, ProjectGraphExternalNode> = {};
// some packages can be both hoisted and nested
// so we need to run this check once we have all the nodes and paths
for (const [packageName, versionMap] of nodes.entries()) {
@ -120,9 +154,10 @@ function addNodes(
}
versionMap.forEach((node) => {
builder.addExternalNode(node);
results[node.name] = node;
});
}
return results;
}
function addV1Node(
@ -130,8 +165,7 @@ function addV1Node(
snapshot: NpmDependencyV1,
path: string,
nodes: Map<string, Map<string, ProjectGraphExternalNode>>,
keyMap: Map<string, ProjectGraphExternalNode>,
builder: ProjectGraphBuilder
keyMap: Map<string, ProjectGraphExternalNode>
) {
createNode(packageName, snapshot.version, path, nodes, keyMap, snapshot);
@ -143,8 +177,7 @@ function addV1Node(
depSnapshot,
`${path}/node_modules/${depName}`,
nodes,
keyMap,
builder
keyMap
);
});
}
@ -210,11 +243,12 @@ function findV3Version(snapshot: NpmDependencyV3, packageName: string): string {
return version;
}
function addDependencies(
function getDependencies(
data: NpmLockFile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>
) {
keyMap: Map<string, ProjectGraphExternalNode>,
projectGraph: ProjectGraph
): ProjectGraphDependencyWithFile[] {
const dependencies: ProjectGraphDependencyWithFile[] = [];
if (data.lockfileVersion > 1) {
Object.entries(data.packages).forEach(([path, snapshot]) => {
// we are skipping workspaces packages
@ -231,7 +265,13 @@ function addDependencies(
Object.entries(section).forEach(([name, versionRange]) => {
const target = findTarget(path, keyMap, name, versionRange);
if (target) {
builder.addStaticDependency(sourceName, target.name);
const dep = {
source: sourceName,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
dependencies.push(dep);
}
});
}
@ -242,11 +282,13 @@ function addDependencies(
addV1NodeDependencies(
`node_modules/${packageName}`,
snapshot,
builder,
keyMap
dependencies,
keyMap,
projectGraph
);
});
}
return dependencies;
}
function findTarget(
@ -284,15 +326,22 @@ function findTarget(
function addV1NodeDependencies(
path: string,
snapshot: NpmDependencyV1,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>
dependencies: ProjectGraphDependencyWithFile[],
keyMap: Map<string, ProjectGraphExternalNode>,
projectGraph: ProjectGraph
) {
if (keyMap.has(path) && snapshot.requires) {
const source = keyMap.get(path).name;
Object.entries(snapshot.requires).forEach(([name, versionRange]) => {
const target = findTarget(path, keyMap, name, versionRange);
if (target) {
builder.addStaticDependency(source, target.name);
const dep = {
source: source,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
dependencies.push(dep);
}
});
}
@ -302,8 +351,9 @@ function addV1NodeDependencies(
addV1NodeDependencies(
`${path}/node_modules/${depName}`,
depSnapshot,
builder,
keyMap
dependencies,
keyMap,
projectGraph
);
});
}
@ -311,15 +361,15 @@ function addV1NodeDependencies(
if (peerDependencies) {
const node = keyMap.get(path);
Object.entries(peerDependencies).forEach(([depName, depSpec]) => {
if (
!builder.graph.dependencies[node.name]?.find(
(d) => d.target === depName
)
) {
const target = findTarget(path, keyMap, depName, depSpec);
if (target) {
builder.addStaticDependency(node.name, target.name);
}
const dep = {
source: node.name,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
dependencies.push(dep);
}
});
}

263
packages/nx/src/plugins/js/lock-file/pnpm-parser.spec.ts
View File

@ -1,9 +1,16 @@
import { joinPathFragments } from '../../../utils/path';
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser';
import {
getPnpmLockfileNodes,
getPnpmLockfileDependencies,
stringifyPnpmLockfile,
} from './pnpm-parser';
import { ProjectGraph } from '../../../config/project-graph';
import { vol } from 'memfs';
import { pruneProjectGraph } from './project-graph-pruning';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../../project-graph/project-graph-builder';
jest.mock('fs', () => {
const memFs = require('memfs').fs;
@ -119,8 +126,12 @@ describe('pnpm LockFile utility', () => {
vol.fromJSON(fileSys, '/root');
});
let externalNodes: ProjectGraph['externalNodes'];
let dependencies: ProjectGraphDependencyWithFile[];
let graph: ProjectGraph;
let lockFile: string;
let lockFileHash: string;
describe('v5.4', () => {
beforeEach(() => {
@ -128,13 +139,34 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/nextjs/pnpm-lock.yaml'
)).default;
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
lockFileHash = '__fixtures__/nextjs/pnpm-lock.yaml';
externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
it('should parse root lock file', async () => {
expect(Object.keys(graph.externalNodes).length).toEqual(1280);
expect(Object.keys(externalNodes).length).toEqual(1280);
});
it('should prune lock file', async () => {
@ -165,8 +197,28 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/nextjs/pnpm-lock-v6.yaml'
)).default;
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
lockFileHash = '__fixtures__/nextjs/pnpm-lock-v6.yaml';
externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
@ -184,10 +236,33 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/nextjs/app/pnpm-lock-v6.yaml'
)).default;
const appLockFileHash = '__fixtures__/nextjs/app/pnpm-lock-v6.yaml';
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(appLockFile, builder);
const appGraph = builder.getUpdatedProjectGraph();
const externalNodes = getPnpmLockfileNodes(
appLockFile,
appLockFileHash
);
let appGraph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
appLockFile,
appLockFileHash,
appGraph
);
const builder = new ProjectGraphBuilder(appGraph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
appGraph = builder.getUpdatedProjectGraph();
expect(Object.keys(appGraph.externalNodes).length).toEqual(864);
// this is our pruned lock file structure
@ -234,9 +309,31 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
)).default;
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml';
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(213);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -291,6 +388,7 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml'
)).default;
const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml';
const prunedLockFile: string = require(joinPathFragments(
__dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml.pruned'
@ -316,9 +414,29 @@ describe('pnpm LockFile utility', () => {
},
};
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
const result = stringifyPnpmLockfile(
prunedGraph,
@ -355,9 +473,31 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/duplicate-package/pnpm-lock.yaml'
)).default;
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const lockFileHash = '__fixtures__/duplicate-package/pnpm-lock.yaml';
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(370);
expect(Object.keys(graph.dependencies).length).toEqual(213);
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
@ -381,9 +521,29 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/optional/pnpm-lock.yaml'
)).default;
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
const lockFileHash = '__fixtures__/optional/pnpm-lock.yaml';
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8);
const packageJson = require(joinPathFragments(
@ -396,7 +556,7 @@ describe('pnpm LockFile utility', () => {
});
describe('pruning', () => {
let graph, lockFile;
let graph, lockFile, lockFileHash;
beforeEach(() => {
const fileSys = {
@ -420,9 +580,29 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/pruning/pnpm-lock.yaml'
)).default;
lockFileHash = '__fixtures__/pruning/pnpm-lock.yaml';
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
@ -471,9 +651,29 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/pruning/pnpm-lock-v6.yaml'
)).default;
lockFileHash = '__fixtures__/pruning/pnpm-lock-v6.yaml';
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
@ -518,7 +718,7 @@ describe('pnpm LockFile utility', () => {
});
describe('workspaces', () => {
let lockFile;
let lockFile, lockFileHash;
beforeAll(() => {
const fileSys = {
@ -534,13 +734,12 @@ describe('pnpm LockFile utility', () => {
__dirname,
'__fixtures__/workspaces/pnpm-lock.yaml'
)).default;
lockFileHash = '__fixtures__/workspaces/pnpm-lock.yaml';
});
it('should parse lock file', async () => {
const builder = new ProjectGraphBuilder();
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
expect(Object.keys(externalNodes).length).toEqual(5);
});
});
});

95
packages/nx/src/plugins/js/lock-file/pnpm-parser.ts
View File

@ -1,8 +1,8 @@
import type {
PackageSnapshot,
Lockfile,
ProjectSnapshot,
PackageSnapshot,
PackageSnapshots,
ProjectSnapshot,
} from '@pnpm/lockfile-types';
import {
isV6Lockfile,
@ -10,36 +10,65 @@ import {
parseAndNormalizePnpmLockfile,
stringifyToPnpmYaml,
} from './utils/pnpm-normalizer';
import { getHoistedPackageVersion } from './utils/package-json';
import { NormalizedPackageJson } from './utils/package-json';
import { sortObjectByKeys } from '../../../utils/object-sort';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import {
getHoistedPackageVersion,
NormalizedPackageJson,
} from './utils/package-json';
import { sortObjectByKeys } from '../../../utils/object-sort';
import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import {
DependencyType,
ProjectGraph,
ProjectGraphExternalNode,
} from '../../../config/project-graph';
import { hashArray } from '../../../hasher/file-hasher';
export function parsePnpmLockfile(
lockFileContent: string,
builder: ProjectGraphBuilder
): void {
const data = parseAndNormalizePnpmLockfile(lockFileContent);
const isV6 = isV6Lockfile(data);
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
let keyMap = new Map<string, ProjectGraphExternalNode>();
let currentLockFileHash: string;
addNodes(data, builder, keyMap, isV6);
addDependencies(data, builder, keyMap, isV6);
let parsedLockFile: Lockfile;
function parsePnpmLockFile(lockFileContent: string, lockFileHash: string) {
if (lockFileHash === currentLockFileHash) {
return parsedLockFile;
}
function addNodes(
keyMap.clear();
const results = parseAndNormalizePnpmLockfile(lockFileContent);
parsedLockFile = results;
currentLockFileHash = lockFileHash;
return results;
}
export function getPnpmLockfileNodes(
lockFileContent: string,
lockFileHash: string
) {
const data = parsePnpmLockFile(lockFileContent, lockFileHash);
const isV6 = isV6Lockfile(data);
return getNodes(data, keyMap, isV6);
}
export function getPnpmLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const data = parsePnpmLockFile(lockFileContent, lockFileHash);
const isV6 = isV6Lockfile(data);
return getDependencies(data, keyMap, isV6, projectGraph);
}
function getNodes(
data: Lockfile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>,
isV6: boolean
) {
): Record<string, ProjectGraphExternalNode> {
const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();
Object.entries(data.packages).forEach(([key, snapshot]) => {
@ -80,6 +109,8 @@ function addNodes(
});
const hoistedDeps = loadPnpmHoistedDepsDefinition();
const results: Record<string, ProjectGraphExternalNode> = {};
for (const [packageName, versionMap] of nodes.entries()) {
let hoistedNode: ProjectGraphExternalNode;
if (versionMap.size === 1) {
@ -93,9 +124,10 @@ function addNodes(
}
versionMap.forEach((node) => {
builder.addExternalNode(node);
results[node.name] = node;
});
}
return results;
}
function getHoistedVersion(
@ -121,12 +153,13 @@ function getHoistedVersion(
return version;
}
function addDependencies(
function getDependencies(
data: Lockfile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>,
isV6: boolean
) {
isV6: boolean,
projectGraph: ProjectGraph
): ProjectGraphDependencyWithFile[] {
const results: ProjectGraphDependencyWithFile[] = [];
Object.entries(data.packages).forEach(([key, snapshot]) => {
const node = keyMap.get(key);
[snapshot.dependencies, snapshot.optionalDependencies].forEach(
@ -138,16 +171,24 @@ function addDependencies(
isV6
);
const target =
builder.graph.externalNodes[`npm:${name}@${version}`] ||
builder.graph.externalNodes[`npm:${name}`];
projectGraph.externalNodes[`npm:${name}@${version}`] ||
projectGraph.externalNodes[`npm:${name}`];
if (target) {
builder.addStaticDependency(node.name, target.name);
const dep = {
source: node.name,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
results.push(dep);
}
});
}
}
);
});
return results;
}
function parseBaseVersion(rawVersion: string, isV6: boolean): string {

430
packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts
View File

@ -1,5 +1,9 @@
import { joinPathFragments } from '../../../utils/path';
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser';
import {
getYarnLockfileNodes,
getYarnLockfileDependencies,
stringifyYarnLockfile,
} from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs';
import { ProjectGraph } from '../../../config/project-graph';
@ -168,7 +172,6 @@ describe('yarn LockFile utility', () => {
let graph: ProjectGraph;
beforeEach(() => {
const builder = new ProjectGraphBuilder();
lockFile = require(joinPathFragments(
__dirname,
'__fixtures__/nextjs/yarn.lock'
@ -177,7 +180,25 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/nextjs/package.json'
));
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
});
@ -383,12 +404,17 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(classicLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(127);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
classicLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(127);
expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
@ -399,7 +425,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==",
@ -410,7 +436,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{
"data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -421,7 +447,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix'])
expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(`
{
"data": {
@ -465,9 +491,26 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default;
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -503,9 +546,30 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default;
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, normalizedPackageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
lockFile,
hash,
normalizedPackageJson
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -529,12 +593,17 @@ describe('yarn LockFile utility', () => {
__dirname,
'__fixtures__/auxiliary-packages/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(berryLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(129);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
berryLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(129);
expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
{
"data": {
"hash": "c154e566406683e7bcb746e000b84d74465b3a832c45d59912b9b55cd50dee66e5c4b1e5566dba26154040e51672f9aa450a9aef0c97cfc7336b78b7afb9540a",
@ -545,7 +614,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
{
"data": {
"hash": "215edd0978320a3354188f84a537d45841f2449af4df4379f79b9b777e71aa4f5722cc9d1717eabd2a70d38ef76ab7b708d24d83ea6a6c909dfd8833de98b437",
@ -556,7 +625,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{
"data": {
"hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a",
@ -567,7 +636,7 @@ describe('yarn LockFile utility', () => {
"type": "npm",
}
`);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix'])
expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(`
{
"data": {
@ -612,9 +681,26 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -664,9 +750,26 @@ __metadata:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(`
{
"npm:@docusaurus/core": {
@ -728,10 +831,10 @@ __metadata:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(externalNodes).toMatchInlineSnapshot(`
{
"npm:@docusaurus/core": {
"data": {
@ -808,10 +911,10 @@ postgres@charsleysa/postgres#fix-errors-compiled:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
@ -822,7 +925,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -833,7 +936,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{
"data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -878,10 +981,10 @@ postgres@charsleysa/postgres#fix-errors-compiled:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
@ -892,7 +995,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -903,7 +1006,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{
"data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -934,10 +1037,10 @@ nx-cloud@latest:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{
"data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -961,12 +1064,17 @@ nx-cloud@latest:
__dirname,
'__fixtures__/auxiliary-packages/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(berryLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(129);
expect(graph.externalNodes['npm:react']).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
berryLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(129);
expect(externalNodes['npm:react']).toMatchInlineSnapshot(`
{
"data": {
"hash": "88e38092da8839b830cda6feef2e8505dec8ace60579e46aa5490fc3dc9bba0bd50336507dc166f43e3afc1c42939c09fe33b25fae889d6f402721dcd78fca1b",
@ -978,7 +1086,7 @@ nx-cloud@latest:
}
`);
expect(graph.externalNodes['npm:typescript']).toMatchInlineSnapshot(`
expect(externalNodes['npm:typescript']).toMatchInlineSnapshot(`
{
"data": {
"hash": "ee000bc26848147ad423b581bd250075662a354d84f0e06eb76d3b892328d8d4440b7487b5a83e851b12b255f55d71835b008a66cbf8f255a11e4400159237db",
@ -989,7 +1097,7 @@ nx-cloud@latest:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(`
expect(externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(`
{
"data": {
"hash": "7dcc3600998448c496228e062d7edd8ecf959fa1ddb9721e91bb1f60f1a2284fd0e12e09edc022170988e2fb54acf101c79dc09fe9c54a21c9941e682eb73b92",
@ -1000,7 +1108,7 @@ nx-cloud@latest:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{
"data": {
"hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a",
@ -1011,7 +1119,7 @@ nx-cloud@latest:
"type": "npm",
}
`);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix'])
expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(`
{
"data": {
@ -1073,10 +1181,14 @@ nx-cloud@latest:
__dirname,
'__fixtures__/duplicate-package/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(classicLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(371);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
classicLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(371);
});
});
@ -1103,9 +1215,27 @@ nx-cloud@latest:
__dirname,
'__fixtures__/optional/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(103);
const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -1286,9 +1416,27 @@ nx-cloud@latest:
__dirname,
'__fixtures__/pruning/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -1317,9 +1465,27 @@ nx-cloud@latest:
__dirname,
'__fixtures__/pruning/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyYarnLockfile(
prunedGraph,
@ -1354,10 +1520,10 @@ nx-cloud@latest:
__dirname,
'__fixtures__/workspaces/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(Object.keys(externalNodes).length).toEqual(5);
});
it('should parse berry lock file', async () => {
@ -1369,10 +1535,10 @@ nx-cloud@latest:
__dirname,
'__fixtures__/workspaces/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(Object.keys(externalNodes).length).toEqual(5);
});
});
@ -1435,9 +1601,24 @@ type-fest@^0.20.2:
tslib: '^2.4.0',
},
};
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
{
@ -1525,9 +1706,26 @@ __metadata:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
{
"data": {
@ -1623,9 +1821,26 @@ __metadata:
'__fixtures__/mixed-keys/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(`
{
"npm:@isaacs/cliui": {
@ -1835,9 +2050,26 @@ __metadata:
'__fixtures__/mixed-keys/package.json'
));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(`
{
"npm:@isaacs/cliui": {
@ -2099,10 +2331,10 @@ __metadata:
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(`
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
expect(externalNodes).toMatchInlineSnapshot(`
{
"npm:@octokit/request-error": {
"data": {
@ -2196,9 +2428,27 @@ __metadata:
resolve: '^1.12.0',
},
};
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson);
const result = stringifyYarnLockfile(prunedGraph, lockFile, packageJson);
expect(result).toMatchInlineSnapshot(`
@ -2247,3 +2497,7 @@ __metadata:
});
});
});
function uniq(str: string) {
return `str-${(Math.random() * 10000).toFixed(0)}`;
}

98
packages/nx/src/plugins/js/lock-file/yarn-parser.ts
View File

@ -1,8 +1,14 @@
import { getHoistedPackageVersion } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { satisfies, Range, gt } from 'semver';
import { NormalizedPackageJson } from './utils/package-json';
import {
getHoistedPackageVersion,
NormalizedPackageJson,
} from './utils/package-json';
import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import { gt, Range, satisfies } from 'semver';
import {
DependencyType,
ProjectGraph,
ProjectGraphExternalNode,
} from '../../../config/project-graph';
@ -31,23 +37,61 @@ type YarnDependency = {
linkType?: 'soft' | 'hard';
};
export function parseYarnLockfile(
lockFileContent: string,
packageJson: NormalizedPackageJson,
builder: ProjectGraphBuilder
) {
const { parseSyml } = require('@yarnpkg/parsers');
const { __metadata, ...dependencies } = parseSyml(lockFileContent);
const isBerry = !!__metadata;
let currentLockFileHash: string;
let cachedParsedLockFile;
// we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>();
let keyMap = new Map<string, ProjectGraphExternalNode>();
function parseLockFile(lockFileContent: string, lockFileHash: string) {
if (currentLockFileHash === lockFileHash) {
return cachedParsedLockFile;
}
const { parseSyml } =
require('@yarnpkg/parsers') as typeof import('@yarnpkg/parsers');
keyMap.clear();
const result = parseSyml(lockFileContent);
cachedParsedLockFile = result;
currentLockFileHash = lockFileHash;
return result;
}
export function getYarnLockfileNodes(
lockFileContent: string,
lockFileHash: string,
packageJson: NormalizedPackageJson
) {
const { __metadata, ...dependencies } = parseLockFile(
lockFileContent,
lockFileHash
);
const isBerry = !!__metadata;
// yarn classic splits keys when parsing so we need to stich them back together
const groupedDependencies = groupDependencies(dependencies, isBerry);
addNodes(groupedDependencies, packageJson, builder, keyMap, isBerry);
addDependencies(groupedDependencies, builder, keyMap);
return getNodes(groupedDependencies, packageJson, keyMap, isBerry);
}
export function getYarnLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const { __metadata, ...dependencies } = parseLockFile(
lockFileContent,
lockFileHash
);
const isBerry = !!__metadata;
// yarn classic splits keys when parsing so we need to stich them back together
const groupedDependencies = groupDependencies(dependencies, isBerry);
return getDependencies(groupedDependencies, keyMap, projectGraph);
}
function getPackageNameKeyPairs(keys: string): Map<string, Set<string>> {
@ -63,10 +107,9 @@ function getPackageNameKeyPairs(keys: string): Map<string, Set<string>> {
return result;
}
function addNodes(
function getNodes(
dependencies: Record<string, YarnDependency>,
packageJson: NormalizedPackageJson,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>,
isBerry: boolean
) {
@ -128,6 +171,7 @@ function addNodes(
});
});
const externalNodes: Record<string, ProjectGraphExternalNode> = {};
for (const [packageName, versionMap] of nodes.entries()) {
const hoistedNode = findHoistedNode(packageName, versionMap, combinedDeps);
if (hoistedNode) {
@ -135,9 +179,10 @@ function addNodes(
}
versionMap.forEach((node) => {
builder.addExternalNode(node);
externalNodes[node.name] = node;
});
}
return externalNodes;
}
function findHoistedNode(
@ -241,11 +286,12 @@ function getHoistedVersion(packageName: string): string {
}
}
function addDependencies(
function getDependencies(
dependencies: Record<string, YarnDependency>,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>
keyMap: Map<string, ProjectGraphExternalNode>,
projectGraph: ProjectGraph
) {
const projectGraphDependencies: ProjectGraphDependencyWithFile[] = [];
Object.keys(dependencies).forEach((keys) => {
const snapshot = dependencies[keys];
keys.split(', ').forEach((key) => {
@ -259,7 +305,13 @@ function addDependencies(
keyMap.get(`${name}@npm:${versionRange}`) ||
keyMap.get(`${name}@${versionRange}`);
if (target) {
builder.addStaticDependency(node.name, target.name);
const dep = {
source: node.name,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
projectGraphDependencies.push(dep);
}
});
}
@ -268,6 +320,8 @@ function addDependencies(
}
});
});
return projectGraphDependencies;
}
export function stringifyYarnLockfile(

80
packages/nx/src/plugins/js/utils/config.ts Normal file
View File

@ -0,0 +1,80 @@
import { join } from 'node:path';
import {
NrwlJsPluginConfig,
NxJsonConfiguration,
} from '../../../config/nx-json';
import { fileExists, readJsonFile } from '../../../utils/fileutils';
import { PackageJson } from '../../../utils/package-json';
import { workspaceRoot } from '../../../utils/workspace-root';
import { existsSync } from 'fs';
export function jsPluginConfig(
nxJson: NxJsonConfiguration
): Required<NrwlJsPluginConfig> {
const nxJsonConfig: NrwlJsPluginConfig =
nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js'];
// using lerna _before_ installing deps is causing an issue when parsing lockfile.
// See: https://github.com/lerna/lerna/issues/3807
// Note that previous attempt to fix this caused issues with Nx itself, thus we're checking
// for Lerna explicitly.
// See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14
const analyzeLockfile =
!existsSync(join(workspaceRoot, 'lerna.json')) ||
existsSync(join(workspaceRoot, 'nx.json'));
if (nxJsonConfig) {
return {
analyzePackageJson: true,
analyzeSourceFiles: true,
analyzeLockfile,
...nxJsonConfig,
};
}
if (!fileExists(join(workspaceRoot, 'package.json'))) {
return {
analyzeLockfile: false,
analyzePackageJson: false,
analyzeSourceFiles: false,
};
}
const packageJson = readJsonFile<PackageJson>(
join(workspaceRoot, 'package.json')
);
const packageJsonDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
if (
packageJsonDeps['@nx/workspace'] ||
packageJsonDeps['@nx/js'] ||
packageJsonDeps['@nx/node'] ||
packageJsonDeps['@nx/next'] ||
packageJsonDeps['@nx/react'] ||
packageJsonDeps['@nx/angular'] ||
packageJsonDeps['@nx/web'] ||
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: true,
};
} else {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: false,
};
}
}

44
packages/nx/src/utils/nx-plugin.ts
View File

@ -67,7 +67,7 @@ export type CreateNodesFunction = (
/**
* A pair of file patterns and {@link CreateNodesFunction}
*/
export type CreateNodes = [
export type CreateNodes = readonly [
projectFilePattern: string,
createNodesFunction: CreateNodesFunction
];
@ -192,7 +192,7 @@ export async function loadNxPluginAsync(
let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root);
const plugin = (await import(pluginPath)) as NxPlugin;
plugin.name = name;
plugin.name ??= name;
nxPluginCache.set(moduleName, plugin);
return plugin;
}
@ -205,7 +205,7 @@ function loadNxPluginSync(moduleName: string, paths: string[], root: string) {
let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root);
const plugin = require(pluginPath) as NxPlugin;
plugin.name = name;
plugin.name ??= name;
nxPluginCache.set(moduleName, plugin);
return plugin;
}
@ -218,14 +218,10 @@ export function loadNxPluginsSync(
paths = getNxRequirePaths(),
root = workspaceRoot
): (NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[] {
const result: NxPlugin[] = [];
// TODO: This should be specified in nx.json
// Temporarily load js as if it were a plugin which is built into nx
// In the future, this will be optional and need to be specified in nx.json
const jsPlugin: any = require('../plugins/js');
jsPlugin.name = 'nx-js-graph-plugin';
result.push(jsPlugin as NxPlugin);
const result: NxPlugin[] = [...getDefaultPluginsSync(root)];
if (shouldMergeAngularProjects(root, false)) {
result.push(NxAngularJsonPlugin);
@ -259,18 +255,12 @@ export async function loadNxPlugins(
paths = getNxRequirePaths(),
root = workspaceRoot
): Promise<(NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[]> {
const result: NxPlugin[] = [];
const result: NxPlugin[] = [...(await getDefaultPlugins(root))];
// TODO: This should be specified in nx.json
// TODO: These should be specified in nx.json
// Temporarily load js as if it were a plugin which is built into nx
// In the future, this will be optional and need to be specified in nx.json
const jsPlugin: any = await import('../plugins/js');
jsPlugin.name = 'nx-js-graph-plugin';
result.push(jsPlugin as NxPlugin);
if (shouldMergeAngularProjects(root, false)) {
result.push(NxAngularJsonPlugin);
}
result.push();
plugins ??= [];
for (const plugin of plugins) {
@ -484,3 +474,23 @@ function readPluginMainFromProjectConfiguration(
{};
return main;
}
async function getDefaultPlugins(root: string) {
const plugins: NxPlugin[] = [await import('../plugins/js')];
if (shouldMergeAngularProjects(root, false)) {
plugins.push(
await import('../adapter/angular-json').then((m) => m.NxAngularJsonPlugin)
);
}
return plugins;
}
function getDefaultPluginsSync(root: string) {
const plugins: NxPlugin[] = [require('../plugins/js')];
if (shouldMergeAngularProjects(root, false)) {
plugins.push(require('../adapter/angular-json').NxAngularJsonPlugin);
}
return plugins;
}

16
packages/vite/src/executors/build/build.impl.ts
View File

@ -1,4 +1,5 @@
import {
detectPackageManager,
ExecutorContext,
logger,
stripIndents,
@ -79,11 +80,20 @@ export async function* viteBuildExecutor(
builtPackageJson.type = 'module';
writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson);
const packageManager = detectPackageManager(context.root);
const lockFile = createLockFile(builtPackageJson);
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, {
const lockFile = createLockFile(
builtPackageJson,
context.projectGraph,
packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
});
}
);
}
// For buildable libs, copy package.json if it exists.
else if (

14
packages/webpack/src/plugins/generate-package-json-plugin.ts
View File

@ -1,6 +1,11 @@
import { type Compiler, sources, type WebpackPluginInstance } from 'webpack';
import { createLockFile, createPackageJson } from '@nx/js';
import { ExecutorContext, type ProjectGraph, serializeJson } from '@nx/devkit';
import {
detectPackageManager,
ExecutorContext,
type ProjectGraph,
serializeJson,
} from '@nx/devkit';
import {
getHelperDependenciesFromProjectGraph,
getLockFileName,
@ -66,9 +71,12 @@ export class GeneratePackageJsonPlugin implements WebpackPluginInstance {
'package.json',
new sources.RawSource(serializeJson(packageJson))
);
const packageManager = detectPackageManager(this.context.root);
compilation.emitAsset(
getLockFileName(),
new sources.RawSource(createLockFile(packageJson))
getLockFileName(packageManager),
new sources.RawSource(
createLockFile(packageJson, this.projectGraph, packageManager)
)
);
}
);