cleanup(core): move js lockfile parsing to v2 plugin (#18779)

This commit is contained in:
Craigory Coppola 2023-08-30 13:26:41 -05:00 committed by GitHub
parent b74f3671e1
commit 353d8d089d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1452 additions and 510 deletions

View File

@ -1,5 +1,5 @@
# Type alias: CreateNodes # Type alias: CreateNodes
Ƭ **CreateNodes**: [projectFilePattern: string, createNodesFunction: CreateNodesFunction] Ƭ **CreateNodes**: readonly [projectFilePattern: string, createNodesFunction: CreateNodesFunction]
A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction)

View File

@ -7,6 +7,7 @@ import {
import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json'; import { createPackageJson } from 'nx/src/plugins/js/package-json/create-package-json';
import { import {
detectPackageManager,
ExecutorContext, ExecutorContext,
getOutputsForTargetAndConfiguration, getOutputsForTargetAndConfiguration,
joinPathFragments, joinPathFragments,
@ -100,10 +101,19 @@ export function updatePackageJson(
writeJsonFile(`${options.outputPath}/package.json`, packageJson); writeJsonFile(`${options.outputPath}/package.json`, packageJson);
if (options.generateLockfile) { if (options.generateLockfile) {
const lockFile = createLockFile(packageJson); const packageManager = detectPackageManager(context.root);
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { const lockFile = createLockFile(
encoding: 'utf-8', packageJson,
}); context.projectGraph,
packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
}
);
} }
} }

View File

@ -1,4 +1,5 @@
import { import {
detectPackageManager,
ExecutorContext, ExecutorContext,
logger, logger,
readJsonFile, readJsonFile,
@ -83,10 +84,19 @@ export default async function buildExecutor(
writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson); writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson);
if (options.generateLockfile) { if (options.generateLockfile) {
const lockFile = createLockFile(builtPackageJson); const packageManager = detectPackageManager(context.root);
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { const lockFile = createLockFile(
encoding: 'utf-8', builtPackageJson,
}); context.projectGraph,
packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
}
);
} }
// If output path is different from source path, then copy over the config and public files. // If output path is different from source path, then copy over the config and public files.

View File

@ -1,98 +1,123 @@
import { import { readFileSync, writeFileSync } from 'fs';
ProjectGraph,
ProjectGraphProcessor,
} from '../../config/project-graph';
import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../project-graph/project-graph-builder';
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
import { readNxJson } from '../../config/configuration';
import { fileExists, readJsonFile } from '../../utils/fileutils';
import { PackageJson } from '../../utils/package-json';
import {
lockFileExists,
lockFileHash,
parseLockFile,
} from './lock-file/lock-file';
import { NrwlJsPluginConfig, NxJsonConfiguration } from '../../config/nx-json';
import { dirname, join } from 'path';
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
import { existsSync, readFileSync, writeFileSync } from 'fs';
import { workspaceRoot } from '../../utils/workspace-root';
import { ensureDirSync } from 'fs-extra'; import { ensureDirSync } from 'fs-extra';
import { dirname, join } from 'path';
import { performance } from 'perf_hooks'; import { performance } from 'perf_hooks';
import { ProjectGraph } from '../../config/project-graph';
import { projectGraphCacheDirectory } from '../../utils/cache-directory';
import { combineGlobPatterns } from '../../utils/globs';
import { import {
CreateDependencies, CreateDependencies,
CreateDependenciesContext, CreateDependenciesContext,
CreateNodes,
} from '../../utils/nx-plugin'; } from '../../utils/nx-plugin';
import {
getLockFileDependencies,
getLockFileName,
getLockFileNodes,
lockFileExists,
LOCKFILES,
} from './lock-file/lock-file';
import { buildExplicitDependencies } from './project-graph/build-dependencies/build-dependencies';
import { jsPluginConfig } from './utils/config';
import { ProjectGraphDependencyWithFile } from '../../project-graph/project-graph-builder';
import { hashArray } from '../../hasher/file-hasher';
import { detectPackageManager } from '../../utils/package-manager';
import { workspaceRoot } from '../../utils/workspace-root';
const createDependencies: CreateDependencies = (context) => { export const name = 'nx-js-graph-plugin';
const pluginConfig = jsPluginConfig(context.nxJsonConfiguration);
interface ParsedLockFile {
externalNodes?: ProjectGraph['externalNodes'];
dependencies?: ProjectGraphDependencyWithFile[];
}
let parsedLockFile: ParsedLockFile = {};
export const createNodes: CreateNodes = [
// Look for all lockfiles
combineGlobPatterns(LOCKFILES),
(lockFile, context) => {
const pluginConfig = jsPluginConfig(context.nxJsonConfiguration);
if (!pluginConfig.analyzePackageJson) {
return {};
}
const packageManager = detectPackageManager(workspaceRoot);
// Only process the correct lockfile
if (lockFile !== getLockFileName(packageManager)) {
return {};
}
const lockFilePath = join(workspaceRoot, lockFile);
const lockFileContents = readFileSync(lockFilePath).toString();
const lockFileHash = hashArray([lockFileContents]);
if (!lockFileNeedsReprocessing(lockFileHash)) {
return {
externalNodes: readCachedParsedLockFile().externalNodes,
};
}
const externalNodes = getLockFileNodes(
packageManager,
lockFileContents,
lockFileHash
);
parsedLockFile.externalNodes = externalNodes;
return {
externalNodes,
};
},
];
export const createDependencies: CreateDependencies = (
ctx: CreateDependenciesContext
) => {
const pluginConfig = jsPluginConfig(ctx.nxJsonConfiguration);
const packageManager = detectPackageManager(workspaceRoot);
let lockfileDependencies: ProjectGraphDependencyWithFile[] = [];
// lockfile may not exist yet
if (
pluginConfig.analyzePackageJson &&
lockFileExists(packageManager) &&
parsedLockFile
) {
const lockFilePath = join(workspaceRoot, getLockFileName(packageManager));
const lockFileContents = readFileSync(lockFilePath).toString();
const lockFileHash = hashArray([lockFileContents]);
if (!lockFileNeedsReprocessing(lockFileHash)) {
lockfileDependencies = readCachedParsedLockFile().dependencies ?? [];
} else {
lockfileDependencies = getLockFileDependencies(
packageManager,
lockFileContents,
lockFileHash,
ctx.graph
);
parsedLockFile.dependencies = lockfileDependencies;
writeLastProcessedLockfileHash(lockFileHash, parsedLockFile);
}
}
performance.mark('build typescript dependencies - start'); performance.mark('build typescript dependencies - start');
const dependencies = buildExplicitDependencies(pluginConfig, context); const explicitProjectDependencies = buildExplicitDependencies(
pluginConfig,
ctx
);
performance.mark('build typescript dependencies - end'); performance.mark('build typescript dependencies - end');
performance.measure( performance.measure(
'build typescript dependencies', 'build typescript dependencies',
'build typescript dependencies - start', 'build typescript dependencies - start',
'build typescript dependencies - end' 'build typescript dependencies - end'
); );
return dependencies; return lockfileDependencies.concat(explicitProjectDependencies);
}; };
export const processProjectGraph: ProjectGraphProcessor = async (
graph,
context
) => {
const builder = new ProjectGraphBuilder(graph, context.fileMap);
const pluginConfig = jsPluginConfig(readNxJson());
if (pluginConfig.analyzePackageJson) {
if (
// during the create-nx-workspace lock file might not exists yet
lockFileExists() &&
pluginConfig.analyzeLockfile
) {
const lockHash = lockFileHash();
let parsedLockFile: ProjectGraph;
if (lockFileNeedsReprocessing(lockHash)) {
parsedLockFile = parseLockFile();
writeLastProcessedLockfileHash(lockHash, parsedLockFile);
} else {
parsedLockFile = readParsedLockFile();
}
builder.mergeProjectGraph(parsedLockFile);
}
}
const createDependenciesContext: CreateDependenciesContext = {
...context,
graph,
};
const dependencies = createDependencies(
createDependenciesContext
) as ProjectGraphDependencyWithFile[];
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
return builder.getUpdatedProjectGraph();
};
const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
const parsedLockFile = join(
projectGraphCacheDirectory,
'parsed-lock-file.json'
);
function lockFileNeedsReprocessing(lockHash: string) { function lockFileNeedsReprocessing(lockHash: string) {
try { try {
return readFileSync(lockFileHashFile).toString() !== lockHash; return readFileSync(lockFileHashFile).toString() !== lockHash;
@ -101,82 +126,21 @@ function lockFileNeedsReprocessing(lockHash: string) {
} }
} }
function writeLastProcessedLockfileHash(hash: string, lockFile: ProjectGraph) { function writeLastProcessedLockfileHash(
hash: string,
lockFile: ParsedLockFile
) {
ensureDirSync(dirname(lockFileHashFile)); ensureDirSync(dirname(lockFileHashFile));
writeFileSync(parsedLockFile, JSON.stringify(lockFile, null, 2)); writeFileSync(cachedParsedLockFile, JSON.stringify(lockFile, null, 2));
writeFileSync(lockFileHashFile, hash); writeFileSync(lockFileHashFile, hash);
} }
function readParsedLockFile(): ProjectGraph { function readCachedParsedLockFile(): ParsedLockFile {
return JSON.parse(readFileSync(parsedLockFile).toString()); return JSON.parse(readFileSync(cachedParsedLockFile).toString());
} }
function jsPluginConfig( const lockFileHashFile = join(projectGraphCacheDirectory, 'lockfile.hash');
nxJson: NxJsonConfiguration const cachedParsedLockFile = join(
): Required<NrwlJsPluginConfig> { projectGraphCacheDirectory,
const nxJsonConfig: NrwlJsPluginConfig = 'parsed-lock-file.json'
nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js']; );
// using lerna _before_ installing deps is causing an issue when parsing lockfile.
// See: https://github.com/lerna/lerna/issues/3807
// Note that previous attempt to fix this caused issues with Nx itself, thus we're checking
// for Lerna explicitly.
// See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14
const analyzeLockfile =
!existsSync(join(workspaceRoot, 'lerna.json')) ||
existsSync(join(workspaceRoot, 'nx.json'));
if (nxJsonConfig) {
return {
analyzePackageJson: true,
analyzeSourceFiles: true,
analyzeLockfile,
...nxJsonConfig,
};
}
if (!fileExists(join(workspaceRoot, 'package.json'))) {
return {
analyzeLockfile: false,
analyzePackageJson: false,
analyzeSourceFiles: false,
};
}
const packageJson = readJsonFile<PackageJson>(
join(workspaceRoot, 'package.json')
);
const packageJsonDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
if (
packageJsonDeps['@nx/workspace'] ||
packageJsonDeps['@nx/js'] ||
packageJsonDeps['@nx/node'] ||
packageJsonDeps['@nx/next'] ||
packageJsonDeps['@nx/react'] ||
packageJsonDeps['@nx/angular'] ||
packageJsonDeps['@nx/web'] ||
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: true,
};
} else {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: false,
};
}
}

View File

@ -11,15 +11,32 @@ import {
PackageManager, PackageManager,
} from '../../../utils/package-manager'; } from '../../../utils/package-manager';
import { workspaceRoot } from '../../../utils/workspace-root'; import { workspaceRoot } from '../../../utils/workspace-root';
import { ProjectGraph } from '../../../config/project-graph'; import {
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; ProjectGraph,
ProjectGraphExternalNode,
} from '../../../config/project-graph';
import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../../project-graph/project-graph-builder';
import { PackageJson } from '../../../utils/package-json'; import { PackageJson } from '../../../utils/package-json';
import { hashArray } from '../../../hasher/file-hasher';
import { output } from '../../../utils/output'; import { output } from '../../../utils/output';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser'; import {
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser'; getNpmLockfileNodes,
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser'; stringifyNpmLockfile,
getNpmLockfileDependencies,
} from './npm-parser';
import {
getPnpmLockfileDependencies,
getPnpmLockfileNodes,
stringifyPnpmLockfile,
} from './pnpm-parser';
import {
getYarnLockfileDependencies,
getYarnLockfileNodes,
stringifyYarnLockfile,
} from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning'; import { pruneProjectGraph } from './project-graph-pruning';
import { normalizePackageJson } from './utils/package-json'; import { normalizePackageJson } from './utils/package-json';
import { readJsonFile } from '../../../utils/fileutils'; import { readJsonFile } from '../../../utils/fileutils';
@ -27,17 +44,75 @@ import { readJsonFile } from '../../../utils/fileutils';
const YARN_LOCK_FILE = 'yarn.lock'; const YARN_LOCK_FILE = 'yarn.lock';
const NPM_LOCK_FILE = 'package-lock.json'; const NPM_LOCK_FILE = 'package-lock.json';
const PNPM_LOCK_FILE = 'pnpm-lock.yaml'; const PNPM_LOCK_FILE = 'pnpm-lock.yaml';
export const LOCKFILES = [YARN_LOCK_FILE, NPM_LOCK_FILE, PNPM_LOCK_FILE];
const YARN_LOCK_PATH = join(workspaceRoot, YARN_LOCK_FILE); const YARN_LOCK_PATH = join(workspaceRoot, YARN_LOCK_FILE);
const NPM_LOCK_PATH = join(workspaceRoot, NPM_LOCK_FILE); const NPM_LOCK_PATH = join(workspaceRoot, NPM_LOCK_FILE);
const PNPM_LOCK_PATH = join(workspaceRoot, PNPM_LOCK_FILE); const PNPM_LOCK_PATH = join(workspaceRoot, PNPM_LOCK_FILE);
/** /**
* Check if lock file exists * Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/ */
export function lockFileExists( export function getLockFileNodes(
packageManager: PackageManager = detectPackageManager(workspaceRoot) packageManager: PackageManager,
): boolean { contents: string,
lockFileHash: string
): Record<string, ProjectGraphExternalNode> {
try {
if (packageManager === 'yarn') {
const packageJson = readJsonFile('package.json');
return getYarnLockfileNodes(contents, lockFileHash, packageJson);
}
if (packageManager === 'pnpm') {
return getPnpmLockfileNodes(contents, lockFileHash);
}
if (packageManager === 'npm') {
return getNpmLockfileNodes(contents, lockFileHash);
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
/**
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function getLockFileDependencies(
packageManager: PackageManager,
contents: string,
lockFileHash: string,
projectGraph: ProjectGraph
): ProjectGraphDependencyWithFile[] {
try {
if (packageManager === 'yarn') {
return getYarnLockfileDependencies(contents, lockFileHash, projectGraph);
}
if (packageManager === 'pnpm') {
return getPnpmLockfileDependencies(contents, lockFileHash, projectGraph);
}
if (packageManager === 'npm') {
return getNpmLockfileDependencies(contents, lockFileHash, projectGraph);
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
export function lockFileExists(packageManager: PackageManager): boolean {
if (packageManager === 'yarn') { if (packageManager === 'yarn') {
return existsSync(YARN_LOCK_PATH); return existsSync(YARN_LOCK_PATH);
} }
@ -52,75 +127,12 @@ export function lockFileExists(
); );
} }
/**
* Hashes lock file content
*/
export function lockFileHash(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string {
let content: string;
if (packageManager === 'yarn') {
content = readFileSync(YARN_LOCK_PATH, 'utf8');
}
if (packageManager === 'pnpm') {
content = readFileSync(PNPM_LOCK_PATH, 'utf8');
}
if (packageManager === 'npm') {
content = readFileSync(NPM_LOCK_PATH, 'utf8');
}
if (content) {
return hashArray([content]);
} else {
throw new Error(
`Unknown package manager ${packageManager} or lock file missing`
);
}
}
/**
* Parses lock file and maps dependencies and metadata to {@link LockFileGraph}
*/
export function parseLockFile(
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): ProjectGraph {
const builder = new ProjectGraphBuilder(null, null);
try {
if (packageManager === 'yarn') {
const content = readFileSync(YARN_LOCK_PATH, 'utf8');
const packageJson = readJsonFile('package.json');
parseYarnLockfile(content, packageJson, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'pnpm') {
const content = readFileSync(PNPM_LOCK_PATH, 'utf8');
parsePnpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
if (packageManager === 'npm') {
const content = readFileSync(NPM_LOCK_PATH, 'utf8');
parseNpmLockfile(content, builder);
return builder.getUpdatedProjectGraph();
}
} catch (e) {
if (!isPostInstallProcess()) {
output.error({
title: `Failed to parse ${packageManager} lockfile`,
bodyLines: errorBodyLines(e),
});
}
return;
}
throw new Error(`Unknown package manager: ${packageManager}`);
}
/** /**
* Returns lock file name based on the detected package manager in the root * Returns lock file name based on the detected package manager in the root
* @param packageManager * @param packageManager
* @returns * @returns
*/ */
export function getLockFileName( export function getLockFileName(packageManager: PackageManager): string {
packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string {
if (packageManager === 'yarn') { if (packageManager === 'yarn') {
return YARN_LOCK_FILE; return YARN_LOCK_FILE;
} }
@ -143,31 +155,22 @@ export function getLockFileName(
*/ */
export function createLockFile( export function createLockFile(
packageJson: PackageJson, packageJson: PackageJson,
graph: ProjectGraph,
packageManager: PackageManager = detectPackageManager(workspaceRoot) packageManager: PackageManager = detectPackageManager(workspaceRoot)
): string { ): string {
const normalizedPackageJson = normalizePackageJson(packageJson); const normalizedPackageJson = normalizePackageJson(packageJson);
const content = readFileSync(getLockFileName(packageManager), 'utf8'); const content = readFileSync(getLockFileName(packageManager), 'utf8');
const rootPackageJson = readJsonFile('package.json');
const builder = new ProjectGraphBuilder();
try { try {
if (packageManager === 'yarn') { if (packageManager === 'yarn') {
parseYarnLockfile(content, rootPackageJson, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson); return stringifyYarnLockfile(prunedGraph, content, normalizedPackageJson);
} }
if (packageManager === 'pnpm') { if (packageManager === 'pnpm') {
parsePnpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson); return stringifyPnpmLockfile(prunedGraph, content, normalizedPackageJson);
} }
if (packageManager === 'npm') { if (packageManager === 'npm') {
parseNpmLockfile(content, builder);
const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson); return stringifyNpmLockfile(prunedGraph, content, normalizedPackageJson);
} }

View File

@ -1,5 +1,9 @@
import { joinPathFragments } from '../../../utils/path'; import { joinPathFragments } from '../../../utils/path';
import { parseNpmLockfile, stringifyNpmLockfile } from './npm-parser'; import {
getNpmLockfileDependencies,
getNpmLockfileNodes,
stringifyNpmLockfile,
} from './npm-parser';
import { pruneProjectGraph } from './project-graph-pruning'; import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs'; import { vol } from 'memfs';
import { ProjectGraph } from '../../../config/project-graph'; import { ProjectGraph } from '../../../config/project-graph';
@ -27,8 +31,31 @@ describe('NPM lock file utility', () => {
let graph: ProjectGraph; let graph: ProjectGraph;
beforeEach(() => { beforeEach(() => {
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
@ -47,8 +74,31 @@ describe('NPM lock file utility', () => {
)); ));
// this is original generated lock file // this is original generated lock file
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(appLockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(appLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(appLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const appGraph = builder.getUpdatedProjectGraph(); const appGraph = builder.getUpdatedProjectGraph();
expect(Object.keys(appGraph.externalNodes).length).toEqual(984); expect(Object.keys(appGraph.externalNodes).length).toEqual(984);
@ -95,8 +145,31 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock.json' '__fixtures__/auxiliary-packages/package-lock.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202 expect(Object.keys(graph.externalNodes).length).toEqual(212); // 202
@ -154,9 +227,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/auxiliary-packages/package-lock-v2.json' '__fixtures__/auxiliary-packages/package-lock-v2.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootV2LockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootV2LockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(212); expect(Object.keys(graph.externalNodes).length).toEqual(212);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -252,9 +349,33 @@ describe('NPM lock file utility', () => {
cleanupTypes(prunedV2LockFile.packages); cleanupTypes(prunedV2LockFile.packages);
cleanupTypes(prunedV2LockFile.dependencies, true); cleanupTypes(prunedV2LockFile.dependencies, true);
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootV2LockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootV2LockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootV2LockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyNpmLockfile( const result = stringifyNpmLockfile(
prunedGraph, prunedGraph,
@ -339,9 +460,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock-v1.json' '__fixtures__/duplicate-package/package-lock-v1.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369); expect(Object.keys(graph.externalNodes).length).toEqual(369);
}); });
it('should parse v3', async () => { it('should parse v3', async () => {
@ -350,9 +495,33 @@ describe('NPM lock file utility', () => {
'__fixtures__/duplicate-package/package-lock.json' '__fixtures__/duplicate-package/package-lock.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(369); expect(Object.keys(graph.externalNodes).length).toEqual(369);
}); });
}); });
@ -367,9 +536,31 @@ describe('NPM lock file utility', () => {
__dirname, __dirname,
'__fixtures__/optional/package.json' '__fixtures__/optional/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder); const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(JSON.stringify(lockFile), hash);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(lockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8); expect(Object.keys(graph.externalNodes).length).toEqual(8);
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -392,9 +583,34 @@ describe('NPM lock file utility', () => {
__dirname, __dirname,
'__fixtures__/pruning/typescript/package.json' '__fixtures__/pruning/typescript/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson); const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyNpmLockfile( const result = stringifyNpmLockfile(
prunedGraph, prunedGraph,
@ -419,9 +635,34 @@ describe('NPM lock file utility', () => {
__dirname, __dirname,
'__fixtures__/pruning/devkit-yargs/package.json' '__fixtures__/pruning/devkit-yargs/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(rootLockFile), builder); const hash = uniq('mock-hash');
const externalNodes = getNpmLockfileNodes(
JSON.stringify(rootLockFile),
hash
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getNpmLockfileDependencies(
JSON.stringify(rootLockFile),
hash,
pg
);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson); const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyNpmLockfile( const result = stringifyNpmLockfile(
prunedGraph, prunedGraph,
@ -450,10 +691,13 @@ describe('NPM lock file utility', () => {
__dirname, __dirname,
'__fixtures__/workspaces/package-lock.json' '__fixtures__/workspaces/package-lock.json'
)); ));
const builder = new ProjectGraphBuilder();
parseNpmLockfile(JSON.stringify(lockFile), builder); const externalNodes = getNpmLockfileNodes(
const result = builder.getUpdatedProjectGraph(); JSON.stringify(lockFile),
expect(Object.keys(result.externalNodes).length).toEqual(5); uniq('mock-hash')
);
expect(Object.keys(externalNodes).length).toEqual(5);
}); });
it('should parse v1 lock file', async () => { it('should parse v1 lock file', async () => {
@ -461,10 +705,15 @@ describe('NPM lock file utility', () => {
__dirname, __dirname,
'__fixtures__/workspaces/package-lock.v1.json' '__fixtures__/workspaces/package-lock.v1.json'
)); ));
const builder = new ProjectGraphBuilder(); const externalNodes = getNpmLockfileNodes(
parseNpmLockfile(JSON.stringify(lockFile), builder); JSON.stringify(lockFile),
const result = builder.getUpdatedProjectGraph(); uniq('mock')
expect(Object.keys(result.externalNodes).length).toEqual(5); );
expect(Object.keys(externalNodes).length).toEqual(5);
}); });
}); });
}); });
function uniq(str: string) {
return `str-${(Math.random() * 10000).toFixed(0)}`;
}

View File

@ -3,12 +3,16 @@ import { satisfies } from 'semver';
import { workspaceRoot } from '../../../utils/workspace-root'; import { workspaceRoot } from '../../../utils/workspace-root';
import { reverse } from '../../../project-graph/operators'; import { reverse } from '../../../project-graph/operators';
import { NormalizedPackageJson } from './utils/package-json'; import { NormalizedPackageJson } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import {
DependencyType,
ProjectGraph, ProjectGraph,
ProjectGraphExternalNode, ProjectGraphExternalNode,
} from '../../../config/project-graph'; } from '../../../config/project-graph';
import { fileHasher, hashArray } from '../../../hasher/file-hasher'; import { hashArray } from '../../../hasher/file-hasher';
/** /**
* NPM * NPM
@ -50,21 +54,51 @@ type NpmLockFile = {
dependencies?: Record<string, NpmDependencyV1>; dependencies?: Record<string, NpmDependencyV1>;
}; };
export function parseNpmLockfile( // we use key => node map to avoid duplicate work when parsing keys
lockFileContent: string, let keyMap = new Map<string, ProjectGraphExternalNode>();
builder: ProjectGraphBuilder let currentLockFileHash: string;
) {
const data = JSON.parse(lockFileContent) as NpmLockFile;
// we use key => node map to avoid duplicate work when parsing keys let parsedLockFile: NpmLockFile;
const keyMap = new Map<string, ProjectGraphExternalNode>(); function parsePackageLockFile(lockFileContent: string, lockFileHash: string) {
addNodes(data, builder, keyMap); if (lockFileHash === currentLockFileHash) {
addDependencies(data, builder, keyMap); return parsedLockFile;
}
keyMap.clear();
const results = JSON.parse(lockFileContent) as NpmLockFile;
parsedLockFile = results;
currentLockFileHash = lockFileHash;
return results;
} }
function addNodes( export function getNpmLockfileNodes(
lockFileContent: string,
lockFileHash: string
) {
const data = parsePackageLockFile(
lockFileContent,
lockFileHash
) as NpmLockFile;
// we use key => node map to avoid duplicate work when parsing keys
return getNodes(data, keyMap);
}
export function getNpmLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const data = parsePackageLockFile(
lockFileContent,
lockFileHash
) as NpmLockFile;
return getDependencies(data, keyMap, projectGraph);
}
function getNodes(
data: NpmLockFile, data: NpmLockFile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode> keyMap: Map<string, ProjectGraphExternalNode>
) { ) {
const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map(); const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();
@ -92,8 +126,7 @@ function addNodes(
depSnapshot, depSnapshot,
`${snapshot.version.slice(5)}/node_modules/${depName}`, `${snapshot.version.slice(5)}/node_modules/${depName}`,
nodes, nodes,
keyMap, keyMap
builder
); );
} }
); );
@ -104,13 +137,14 @@ function addNodes(
snapshot, snapshot,
`node_modules/${packageName}`, `node_modules/${packageName}`,
nodes, nodes,
keyMap, keyMap
builder
); );
} }
}); });
} }
const results: Record<string, ProjectGraphExternalNode> = {};
// some packages can be both hoisted and nested // some packages can be both hoisted and nested
// so we need to run this check once we have all the nodes and paths // so we need to run this check once we have all the nodes and paths
for (const [packageName, versionMap] of nodes.entries()) { for (const [packageName, versionMap] of nodes.entries()) {
@ -120,9 +154,10 @@ function addNodes(
} }
versionMap.forEach((node) => { versionMap.forEach((node) => {
builder.addExternalNode(node); results[node.name] = node;
}); });
} }
return results;
} }
function addV1Node( function addV1Node(
@ -130,8 +165,7 @@ function addV1Node(
snapshot: NpmDependencyV1, snapshot: NpmDependencyV1,
path: string, path: string,
nodes: Map<string, Map<string, ProjectGraphExternalNode>>, nodes: Map<string, Map<string, ProjectGraphExternalNode>>,
keyMap: Map<string, ProjectGraphExternalNode>, keyMap: Map<string, ProjectGraphExternalNode>
builder: ProjectGraphBuilder
) { ) {
createNode(packageName, snapshot.version, path, nodes, keyMap, snapshot); createNode(packageName, snapshot.version, path, nodes, keyMap, snapshot);
@ -143,8 +177,7 @@ function addV1Node(
depSnapshot, depSnapshot,
`${path}/node_modules/${depName}`, `${path}/node_modules/${depName}`,
nodes, nodes,
keyMap, keyMap
builder
); );
}); });
} }
@ -210,11 +243,12 @@ function findV3Version(snapshot: NpmDependencyV3, packageName: string): string {
return version; return version;
} }
function addDependencies( function getDependencies(
data: NpmLockFile, data: NpmLockFile,
builder: ProjectGraphBuilder, keyMap: Map<string, ProjectGraphExternalNode>,
keyMap: Map<string, ProjectGraphExternalNode> projectGraph: ProjectGraph
) { ): ProjectGraphDependencyWithFile[] {
const dependencies: ProjectGraphDependencyWithFile[] = [];
if (data.lockfileVersion > 1) { if (data.lockfileVersion > 1) {
Object.entries(data.packages).forEach(([path, snapshot]) => { Object.entries(data.packages).forEach(([path, snapshot]) => {
// we are skipping workspaces packages // we are skipping workspaces packages
@ -231,7 +265,13 @@ function addDependencies(
Object.entries(section).forEach(([name, versionRange]) => { Object.entries(section).forEach(([name, versionRange]) => {
const target = findTarget(path, keyMap, name, versionRange); const target = findTarget(path, keyMap, name, versionRange);
if (target) { if (target) {
builder.addStaticDependency(sourceName, target.name); const dep = {
source: sourceName,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
dependencies.push(dep);
} }
}); });
} }
@ -242,11 +282,13 @@ function addDependencies(
addV1NodeDependencies( addV1NodeDependencies(
`node_modules/${packageName}`, `node_modules/${packageName}`,
snapshot, snapshot,
builder, dependencies,
keyMap keyMap,
projectGraph
); );
}); });
} }
return dependencies;
} }
function findTarget( function findTarget(
@ -284,15 +326,22 @@ function findTarget(
function addV1NodeDependencies( function addV1NodeDependencies(
path: string, path: string,
snapshot: NpmDependencyV1, snapshot: NpmDependencyV1,
builder: ProjectGraphBuilder, dependencies: ProjectGraphDependencyWithFile[],
keyMap: Map<string, ProjectGraphExternalNode> keyMap: Map<string, ProjectGraphExternalNode>,
projectGraph: ProjectGraph
) { ) {
if (keyMap.has(path) && snapshot.requires) { if (keyMap.has(path) && snapshot.requires) {
const source = keyMap.get(path).name; const source = keyMap.get(path).name;
Object.entries(snapshot.requires).forEach(([name, versionRange]) => { Object.entries(snapshot.requires).forEach(([name, versionRange]) => {
const target = findTarget(path, keyMap, name, versionRange); const target = findTarget(path, keyMap, name, versionRange);
if (target) { if (target) {
builder.addStaticDependency(source, target.name); const dep = {
source: source,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
dependencies.push(dep);
} }
}); });
} }
@ -302,8 +351,9 @@ function addV1NodeDependencies(
addV1NodeDependencies( addV1NodeDependencies(
`${path}/node_modules/${depName}`, `${path}/node_modules/${depName}`,
depSnapshot, depSnapshot,
builder, dependencies,
keyMap keyMap,
projectGraph
); );
}); });
} }
@ -311,15 +361,15 @@ function addV1NodeDependencies(
if (peerDependencies) { if (peerDependencies) {
const node = keyMap.get(path); const node = keyMap.get(path);
Object.entries(peerDependencies).forEach(([depName, depSpec]) => { Object.entries(peerDependencies).forEach(([depName, depSpec]) => {
if ( const target = findTarget(path, keyMap, depName, depSpec);
!builder.graph.dependencies[node.name]?.find( if (target) {
(d) => d.target === depName const dep = {
) source: node.name,
) { target: target.name,
const target = findTarget(path, keyMap, depName, depSpec); dependencyType: DependencyType.static,
if (target) { };
builder.addStaticDependency(node.name, target.name); validateDependency(projectGraph, dep);
} dependencies.push(dep);
} }
}); });
} }

View File

@ -1,9 +1,16 @@
import { joinPathFragments } from '../../../utils/path'; import { joinPathFragments } from '../../../utils/path';
import { parsePnpmLockfile, stringifyPnpmLockfile } from './pnpm-parser'; import {
getPnpmLockfileNodes,
getPnpmLockfileDependencies,
stringifyPnpmLockfile,
} from './pnpm-parser';
import { ProjectGraph } from '../../../config/project-graph'; import { ProjectGraph } from '../../../config/project-graph';
import { vol } from 'memfs'; import { vol } from 'memfs';
import { pruneProjectGraph } from './project-graph-pruning'; import { pruneProjectGraph } from './project-graph-pruning';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder'; import {
ProjectGraphBuilder,
ProjectGraphDependencyWithFile,
} from '../../../project-graph/project-graph-builder';
jest.mock('fs', () => { jest.mock('fs', () => {
const memFs = require('memfs').fs; const memFs = require('memfs').fs;
@ -119,8 +126,12 @@ describe('pnpm LockFile utility', () => {
vol.fromJSON(fileSys, '/root'); vol.fromJSON(fileSys, '/root');
}); });
let externalNodes: ProjectGraph['externalNodes'];
let dependencies: ProjectGraphDependencyWithFile[];
let graph: ProjectGraph; let graph: ProjectGraph;
let lockFile: string; let lockFile: string;
let lockFileHash: string;
describe('v5.4', () => { describe('v5.4', () => {
beforeEach(() => { beforeEach(() => {
@ -128,13 +139,34 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/nextjs/pnpm-lock.yaml' '__fixtures__/nextjs/pnpm-lock.yaml'
)).default; )).default;
const builder = new ProjectGraphBuilder(); lockFileHash = '__fixtures__/nextjs/pnpm-lock.yaml';
parsePnpmLockfile(lockFile, builder);
externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
it('should parse root lock file', async () => { it('should parse root lock file', async () => {
expect(Object.keys(graph.externalNodes).length).toEqual(1280); expect(Object.keys(externalNodes).length).toEqual(1280);
}); });
it('should prune lock file', async () => { it('should prune lock file', async () => {
@ -165,8 +197,28 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/nextjs/pnpm-lock-v6.yaml' '__fixtures__/nextjs/pnpm-lock-v6.yaml'
)).default; )).default;
const builder = new ProjectGraphBuilder(); lockFileHash = '__fixtures__/nextjs/pnpm-lock-v6.yaml';
parsePnpmLockfile(lockFile, builder); externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
graph = {
nodes: {},
dependencies: {},
externalNodes,
};
dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
@ -184,10 +236,33 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/nextjs/app/pnpm-lock-v6.yaml' '__fixtures__/nextjs/app/pnpm-lock-v6.yaml'
)).default; )).default;
const appLockFileHash = '__fixtures__/nextjs/app/pnpm-lock-v6.yaml';
const builder = new ProjectGraphBuilder(); const externalNodes = getPnpmLockfileNodes(
parsePnpmLockfile(appLockFile, builder); appLockFile,
const appGraph = builder.getUpdatedProjectGraph(); appLockFileHash
);
let appGraph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
appLockFile,
appLockFileHash,
appGraph
);
const builder = new ProjectGraphBuilder(appGraph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
appGraph = builder.getUpdatedProjectGraph();
expect(Object.keys(appGraph.externalNodes).length).toEqual(864); expect(Object.keys(appGraph.externalNodes).length).toEqual(864);
// this is our pruned lock file structure // this is our pruned lock file structure
@ -234,9 +309,31 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml' '__fixtures__/auxiliary-packages/pnpm-lock.yaml'
)).default; )).default;
const builder = new ProjectGraphBuilder(); const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml';
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(213); expect(Object.keys(graph.externalNodes).length).toEqual(213);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
@ -291,6 +388,7 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml' '__fixtures__/auxiliary-packages/pnpm-lock.yaml'
)).default; )).default;
const lockFileHash = '__fixtures__/auxiliary-packages/pnpm-lock.yaml';
const prunedLockFile: string = require(joinPathFragments( const prunedLockFile: string = require(joinPathFragments(
__dirname, __dirname,
'__fixtures__/auxiliary-packages/pnpm-lock.yaml.pruned' '__fixtures__/auxiliary-packages/pnpm-lock.yaml.pruned'
@ -316,9 +414,29 @@ describe('pnpm LockFile utility', () => {
}, },
}; };
const builder = new ProjectGraphBuilder(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
parsePnpmLockfile(lockFile, builder); let graph: ProjectGraph = {
const graph = builder.getUpdatedProjectGraph(); nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, prunedPackageJson); const prunedGraph = pruneProjectGraph(graph, prunedPackageJson);
const result = stringifyPnpmLockfile( const result = stringifyPnpmLockfile(
prunedGraph, prunedGraph,
@ -355,9 +473,31 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/duplicate-package/pnpm-lock.yaml' '__fixtures__/duplicate-package/pnpm-lock.yaml'
)).default; )).default;
const builder = new ProjectGraphBuilder(); const lockFileHash = '__fixtures__/duplicate-package/pnpm-lock.yaml';
parsePnpmLockfile(lockFile, builder);
const graph = builder.getUpdatedProjectGraph(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(370); expect(Object.keys(graph.externalNodes).length).toEqual(370);
expect(Object.keys(graph.dependencies).length).toEqual(213); expect(Object.keys(graph.dependencies).length).toEqual(213);
expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6); expect(graph.dependencies['npm:@nrwl/devkit'].length).toEqual(6);
@ -381,9 +521,29 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/optional/pnpm-lock.yaml' '__fixtures__/optional/pnpm-lock.yaml'
)).default; )).default;
const builder = new ProjectGraphBuilder(); const lockFileHash = '__fixtures__/optional/pnpm-lock.yaml';
parsePnpmLockfile(lockFile, builder); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
const graph = builder.getUpdatedProjectGraph(); let graph: ProjectGraph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(8); expect(Object.keys(graph.externalNodes).length).toEqual(8);
const packageJson = require(joinPathFragments( const packageJson = require(joinPathFragments(
@ -396,7 +556,7 @@ describe('pnpm LockFile utility', () => {
}); });
describe('pruning', () => { describe('pruning', () => {
let graph, lockFile; let graph, lockFile, lockFileHash;
beforeEach(() => { beforeEach(() => {
const fileSys = { const fileSys = {
@ -420,9 +580,29 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/pruning/pnpm-lock.yaml' '__fixtures__/pruning/pnpm-lock.yaml'
)).default; )).default;
lockFileHash = '__fixtures__/pruning/pnpm-lock.yaml';
const builder = new ProjectGraphBuilder(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
parsePnpmLockfile(lockFile, builder); graph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
@ -471,9 +651,29 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/pruning/pnpm-lock-v6.yaml' '__fixtures__/pruning/pnpm-lock-v6.yaml'
)).default; )).default;
lockFileHash = '__fixtures__/pruning/pnpm-lock-v6.yaml';
const builder = new ProjectGraphBuilder(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
parsePnpmLockfile(lockFile, builder); graph = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getPnpmLockfileDependencies(
lockFile,
lockFileHash,
graph
);
const builder = new ProjectGraphBuilder(graph);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
@ -518,7 +718,7 @@ describe('pnpm LockFile utility', () => {
}); });
describe('workspaces', () => { describe('workspaces', () => {
let lockFile; let lockFile, lockFileHash;
beforeAll(() => { beforeAll(() => {
const fileSys = { const fileSys = {
@ -534,13 +734,12 @@ describe('pnpm LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/workspaces/pnpm-lock.yaml' '__fixtures__/workspaces/pnpm-lock.yaml'
)).default; )).default;
lockFileHash = '__fixtures__/workspaces/pnpm-lock.yaml';
}); });
it('should parse lock file', async () => { it('should parse lock file', async () => {
const builder = new ProjectGraphBuilder(); const externalNodes = getPnpmLockfileNodes(lockFile, lockFileHash);
parsePnpmLockfile(lockFile, builder); expect(Object.keys(externalNodes).length).toEqual(5);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5);
}); });
}); });
}); });

View File

@ -1,8 +1,8 @@
import type { import type {
PackageSnapshot,
Lockfile, Lockfile,
ProjectSnapshot, PackageSnapshot,
PackageSnapshots, PackageSnapshots,
ProjectSnapshot,
} from '@pnpm/lockfile-types'; } from '@pnpm/lockfile-types';
import { import {
isV6Lockfile, isV6Lockfile,
@ -10,36 +10,65 @@ import {
parseAndNormalizePnpmLockfile, parseAndNormalizePnpmLockfile,
stringifyToPnpmYaml, stringifyToPnpmYaml,
} from './utils/pnpm-normalizer'; } from './utils/pnpm-normalizer';
import { getHoistedPackageVersion } from './utils/package-json';
import { NormalizedPackageJson } from './utils/package-json';
import { sortObjectByKeys } from '../../../utils/object-sort';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { import {
getHoistedPackageVersion,
NormalizedPackageJson,
} from './utils/package-json';
import { sortObjectByKeys } from '../../../utils/object-sort';
import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import {
DependencyType,
ProjectGraph, ProjectGraph,
ProjectGraphExternalNode, ProjectGraphExternalNode,
} from '../../../config/project-graph'; } from '../../../config/project-graph';
import { hashArray } from '../../../hasher/file-hasher'; import { hashArray } from '../../../hasher/file-hasher';
export function parsePnpmLockfile( // we use key => node map to avoid duplicate work when parsing keys
lockFileContent: string, let keyMap = new Map<string, ProjectGraphExternalNode>();
builder: ProjectGraphBuilder let currentLockFileHash: string;
): void {
const data = parseAndNormalizePnpmLockfile(lockFileContent);
const isV6 = isV6Lockfile(data);
// we use key => node map to avoid duplicate work when parsing keys let parsedLockFile: Lockfile;
const keyMap = new Map<string, ProjectGraphExternalNode>(); function parsePnpmLockFile(lockFileContent: string, lockFileHash: string) {
if (lockFileHash === currentLockFileHash) {
return parsedLockFile;
}
addNodes(data, builder, keyMap, isV6); keyMap.clear();
addDependencies(data, builder, keyMap, isV6); const results = parseAndNormalizePnpmLockfile(lockFileContent);
parsedLockFile = results;
currentLockFileHash = lockFileHash;
return results;
} }
function addNodes( export function getPnpmLockfileNodes(
lockFileContent: string,
lockFileHash: string
) {
const data = parsePnpmLockFile(lockFileContent, lockFileHash);
const isV6 = isV6Lockfile(data);
return getNodes(data, keyMap, isV6);
}
export function getPnpmLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const data = parsePnpmLockFile(lockFileContent, lockFileHash);
const isV6 = isV6Lockfile(data);
return getDependencies(data, keyMap, isV6, projectGraph);
}
function getNodes(
data: Lockfile, data: Lockfile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>, keyMap: Map<string, ProjectGraphExternalNode>,
isV6: boolean isV6: boolean
) { ): Record<string, ProjectGraphExternalNode> {
const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map(); const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();
Object.entries(data.packages).forEach(([key, snapshot]) => { Object.entries(data.packages).forEach(([key, snapshot]) => {
@ -80,6 +109,8 @@ function addNodes(
}); });
const hoistedDeps = loadPnpmHoistedDepsDefinition(); const hoistedDeps = loadPnpmHoistedDepsDefinition();
const results: Record<string, ProjectGraphExternalNode> = {};
for (const [packageName, versionMap] of nodes.entries()) { for (const [packageName, versionMap] of nodes.entries()) {
let hoistedNode: ProjectGraphExternalNode; let hoistedNode: ProjectGraphExternalNode;
if (versionMap.size === 1) { if (versionMap.size === 1) {
@ -93,9 +124,10 @@ function addNodes(
} }
versionMap.forEach((node) => { versionMap.forEach((node) => {
builder.addExternalNode(node); results[node.name] = node;
}); });
} }
return results;
} }
function getHoistedVersion( function getHoistedVersion(
@ -121,12 +153,13 @@ function getHoistedVersion(
return version; return version;
} }
function addDependencies( function getDependencies(
data: Lockfile, data: Lockfile,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>, keyMap: Map<string, ProjectGraphExternalNode>,
isV6: boolean isV6: boolean,
) { projectGraph: ProjectGraph
): ProjectGraphDependencyWithFile[] {
const results: ProjectGraphDependencyWithFile[] = [];
Object.entries(data.packages).forEach(([key, snapshot]) => { Object.entries(data.packages).forEach(([key, snapshot]) => {
const node = keyMap.get(key); const node = keyMap.get(key);
[snapshot.dependencies, snapshot.optionalDependencies].forEach( [snapshot.dependencies, snapshot.optionalDependencies].forEach(
@ -138,16 +171,24 @@ function addDependencies(
isV6 isV6
); );
const target = const target =
builder.graph.externalNodes[`npm:${name}@${version}`] || projectGraph.externalNodes[`npm:${name}@${version}`] ||
builder.graph.externalNodes[`npm:${name}`]; projectGraph.externalNodes[`npm:${name}`];
if (target) { if (target) {
builder.addStaticDependency(node.name, target.name); const dep = {
source: node.name,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
results.push(dep);
} }
}); });
} }
} }
); );
}); });
return results;
} }
function parseBaseVersion(rawVersion: string, isV6: boolean): string { function parseBaseVersion(rawVersion: string, isV6: boolean): string {

View File

@ -1,5 +1,9 @@
import { joinPathFragments } from '../../../utils/path'; import { joinPathFragments } from '../../../utils/path';
import { parseYarnLockfile, stringifyYarnLockfile } from './yarn-parser'; import {
getYarnLockfileNodes,
getYarnLockfileDependencies,
stringifyYarnLockfile,
} from './yarn-parser';
import { pruneProjectGraph } from './project-graph-pruning'; import { pruneProjectGraph } from './project-graph-pruning';
import { vol } from 'memfs'; import { vol } from 'memfs';
import { ProjectGraph } from '../../../config/project-graph'; import { ProjectGraph } from '../../../config/project-graph';
@ -168,7 +172,6 @@ describe('yarn LockFile utility', () => {
let graph: ProjectGraph; let graph: ProjectGraph;
beforeEach(() => { beforeEach(() => {
const builder = new ProjectGraphBuilder();
lockFile = require(joinPathFragments( lockFile = require(joinPathFragments(
__dirname, __dirname,
'__fixtures__/nextjs/yarn.lock' '__fixtures__/nextjs/yarn.lock'
@ -177,7 +180,25 @@ describe('yarn LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/nextjs/package.json' '__fixtures__/nextjs/package.json'
)); ));
parseYarnLockfile(lockFile, packageJson, builder);
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
graph = builder.getUpdatedProjectGraph(); graph = builder.getUpdatedProjectGraph();
}); });
@ -383,12 +404,17 @@ describe('yarn LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/auxiliary-packages/package.json' '__fixtures__/auxiliary-packages/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(classicLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(127);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
classicLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(127);
expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "hash": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
@ -399,7 +425,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==", "hash": "sha512-362NP+zlprccbEt/SkxKfRMHnNY85V74mVnpUpNyr3F35covl09Kec7/sEFLt3RA4oXmewtoaanoIf67SE5Y5g==",
@ -410,7 +436,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -421,7 +447,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(` .toMatchInlineSnapshot(`
{ {
"data": { "data": {
@ -465,9 +491,26 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned' '__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default; )).default;
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile( const result = stringifyYarnLockfile(
prunedGraph, prunedGraph,
@ -503,9 +546,30 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/yarn.lock.pruned' '__fixtures__/auxiliary-packages/yarn.lock.pruned'
)).default; )).default;
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, normalizedPackageJson, builder); const externalNodes = getYarnLockfileNodes(
lockFile,
hash,
normalizedPackageJson
);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile( const result = stringifyYarnLockfile(
prunedGraph, prunedGraph,
@ -529,12 +593,17 @@ describe('yarn LockFile utility', () => {
__dirname, __dirname,
'__fixtures__/auxiliary-packages/package.json' '__fixtures__/auxiliary-packages/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(berryLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(129);
expect(graph.externalNodes['npm:minimatch']).toMatchInlineSnapshot(` const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
berryLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(129);
expect(externalNodes['npm:minimatch']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "c154e566406683e7bcb746e000b84d74465b3a832c45d59912b9b55cd50dee66e5c4b1e5566dba26154040e51672f9aa450a9aef0c97cfc7336b78b7afb9540a", "hash": "c154e566406683e7bcb746e000b84d74465b3a832c45d59912b9b55cd50dee66e5c4b1e5566dba26154040e51672f9aa450a9aef0c97cfc7336b78b7afb9540a",
@ -545,7 +614,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(` expect(externalNodes['npm:minimatch@5.1.1']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "215edd0978320a3354188f84a537d45841f2449af4df4379f79b9b777e71aa4f5722cc9d1717eabd2a70d38ef76ab7b708d24d83ea6a6c909dfd8833de98b437", "hash": "215edd0978320a3354188f84a537d45841f2449af4df4379f79b9b777e71aa4f5722cc9d1717eabd2a70d38ef76ab7b708d24d83ea6a6c909dfd8833de98b437",
@ -556,7 +625,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a", "hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a",
@ -567,7 +636,7 @@ describe('yarn LockFile utility', () => {
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(` .toMatchInlineSnapshot(`
{ {
"data": { "data": {
@ -612,9 +681,26 @@ describe('yarn LockFile utility', () => {
'__fixtures__/auxiliary-packages/package.json' '__fixtures__/auxiliary-packages/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson); const prunedGraph = pruneProjectGraph(graph, normalizedPackageJson);
const result = stringifyYarnLockfile( const result = stringifyYarnLockfile(
prunedGraph, prunedGraph,
@ -664,9 +750,26 @@ __metadata:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(` expect(graph.externalNodes).toMatchInlineSnapshot(`
{ {
"npm:@docusaurus/core": { "npm:@docusaurus/core": {
@ -728,10 +831,10 @@ __metadata:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(` expect(externalNodes).toMatchInlineSnapshot(`
{ {
"npm:@docusaurus/core": { "npm:@docusaurus/core": {
"data": { "data": {
@ -808,10 +911,10 @@ postgres@charsleysa/postgres#fix-errors-compiled:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==", "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
@ -822,7 +925,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -833,7 +936,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -878,10 +981,10 @@ postgres@charsleysa/postgres#fix-errors-compiled:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(` expect(externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==", "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
@ -892,7 +995,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -903,7 +1006,7 @@ postgres@charsleysa/postgres#fix-errors-compiled:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb", "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
@ -934,10 +1037,10 @@ nx-cloud@latest:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(` expect(externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==", "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
@ -961,12 +1064,17 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/auxiliary-packages/package.json' '__fixtures__/auxiliary-packages/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(berryLockFile, packageJson, builder);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(129);
expect(graph.externalNodes['npm:react']).toMatchInlineSnapshot(` const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(
berryLockFile,
hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(129);
expect(externalNodes['npm:react']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "88e38092da8839b830cda6feef2e8505dec8ace60579e46aa5490fc3dc9bba0bd50336507dc166f43e3afc1c42939c09fe33b25fae889d6f402721dcd78fca1b", "hash": "88e38092da8839b830cda6feef2e8505dec8ace60579e46aa5490fc3dc9bba0bd50336507dc166f43e3afc1c42939c09fe33b25fae889d6f402721dcd78fca1b",
@ -978,7 +1086,7 @@ nx-cloud@latest:
} }
`); `);
expect(graph.externalNodes['npm:typescript']).toMatchInlineSnapshot(` expect(externalNodes['npm:typescript']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "ee000bc26848147ad423b581bd250075662a354d84f0e06eb76d3b892328d8d4440b7487b5a83e851b12b255f55d71835b008a66cbf8f255a11e4400159237db", "hash": "ee000bc26848147ad423b581bd250075662a354d84f0e06eb76d3b892328d8d4440b7487b5a83e851b12b255f55d71835b008a66cbf8f255a11e4400159237db",
@ -989,7 +1097,7 @@ nx-cloud@latest:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(` expect(externalNodes['npm:@nrwl/devkit']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "7dcc3600998448c496228e062d7edd8ecf959fa1ddb9721e91bb1f60f1a2284fd0e12e09edc022170988e2fb54acf101c79dc09fe9c54a21c9941e682eb73b92", "hash": "7dcc3600998448c496228e062d7edd8ecf959fa1ddb9721e91bb1f60f1a2284fd0e12e09edc022170988e2fb54acf101c79dc09fe9c54a21c9941e682eb73b92",
@ -1000,7 +1108,7 @@ nx-cloud@latest:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(` expect(externalNodes['npm:postgres']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
"hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a", "hash": "521660853e0c9f1c604cf43d32c75e2b4675e2d912eaec7bb6749716539dd53f1dfaf575a422087f6a53362f5162f9a4b8a88cc1dadf9d7580423fc05137767a",
@ -1011,7 +1119,7 @@ nx-cloud@latest:
"type": "npm", "type": "npm",
} }
`); `);
expect(graph.externalNodes['npm:eslint-plugin-disable-autofix']) expect(externalNodes['npm:eslint-plugin-disable-autofix'])
.toMatchInlineSnapshot(` .toMatchInlineSnapshot(`
{ {
"data": { "data": {
@ -1073,10 +1181,14 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/duplicate-package/package.json' '__fixtures__/duplicate-package/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(classicLockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(
const graph = builder.getUpdatedProjectGraph(); classicLockFile,
expect(Object.keys(graph.externalNodes).length).toEqual(371); hash,
packageJson
);
expect(Object.keys(externalNodes).length).toEqual(371);
}); });
}); });
@ -1103,9 +1215,27 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/optional/package.json' '__fixtures__/optional/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder); const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(103); expect(Object.keys(graph.externalNodes).length).toEqual(103);
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
@ -1286,9 +1416,27 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/pruning/package.json' '__fixtures__/pruning/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder); const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson); const prunedGraph = pruneProjectGraph(graph, typescriptPackageJson);
const result = stringifyYarnLockfile( const result = stringifyYarnLockfile(
prunedGraph, prunedGraph,
@ -1317,9 +1465,27 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/pruning/package.json' '__fixtures__/pruning/package.json'
)); ));
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder); const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, multiPackageJson); const prunedGraph = pruneProjectGraph(graph, multiPackageJson);
const result = stringifyYarnLockfile( const result = stringifyYarnLockfile(
prunedGraph, prunedGraph,
@ -1354,10 +1520,10 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/workspaces/package.json' '__fixtures__/workspaces/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5); expect(Object.keys(externalNodes).length).toEqual(5);
}); });
it('should parse berry lock file', async () => { it('should parse berry lock file', async () => {
@ -1369,10 +1535,10 @@ nx-cloud@latest:
__dirname, __dirname,
'__fixtures__/workspaces/package.json' '__fixtures__/workspaces/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(Object.keys(graph.externalNodes).length).toEqual(5); expect(Object.keys(externalNodes).length).toEqual(5);
}); });
}); });
@ -1435,9 +1601,24 @@ type-fest@^0.20.2:
tslib: '^2.4.0', tslib: '^2.4.0',
}, },
}; };
const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(); const builder = new ProjectGraphBuilder(pg);
parseYarnLockfile(lockFile, packageJson, builder); for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(` expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
{ {
@ -1525,9 +1706,26 @@ __metadata:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(` expect(graph.externalNodes['npm:tslib']).toMatchInlineSnapshot(`
{ {
"data": { "data": {
@ -1623,9 +1821,26 @@ __metadata:
'__fixtures__/mixed-keys/package.json' '__fixtures__/mixed-keys/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(` expect(graph.externalNodes).toMatchInlineSnapshot(`
{ {
"npm:@isaacs/cliui": { "npm:@isaacs/cliui": {
@ -1835,9 +2050,26 @@ __metadata:
'__fixtures__/mixed-keys/package.json' '__fixtures__/mixed-keys/package.json'
)); ));
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(` expect(graph.externalNodes).toMatchInlineSnapshot(`
{ {
"npm:@isaacs/cliui": { "npm:@isaacs/cliui": {
@ -2099,10 +2331,10 @@ __metadata:
}, },
}; };
const builder = new ProjectGraphBuilder(); const hash = uniq('mock-hash');
parseYarnLockfile(lockFile, packageJson, builder); const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const graph = builder.getUpdatedProjectGraph();
expect(graph.externalNodes).toMatchInlineSnapshot(` expect(externalNodes).toMatchInlineSnapshot(`
{ {
"npm:@octokit/request-error": { "npm:@octokit/request-error": {
"data": { "data": {
@ -2196,9 +2428,27 @@ __metadata:
resolve: '^1.12.0', resolve: '^1.12.0',
}, },
}; };
const builder = new ProjectGraphBuilder();
parseYarnLockfile(lockFile, packageJson, builder); const hash = uniq('mock-hash');
const externalNodes = getYarnLockfileNodes(lockFile, hash, packageJson);
const pg = {
nodes: {},
dependencies: {},
externalNodes,
};
const dependencies = getYarnLockfileDependencies(lockFile, hash, pg);
const builder = new ProjectGraphBuilder(pg);
for (const dep of dependencies) {
builder.addDependency(
dep.source,
dep.target,
dep.dependencyType,
dep.sourceFile
);
}
const graph = builder.getUpdatedProjectGraph(); const graph = builder.getUpdatedProjectGraph();
const prunedGraph = pruneProjectGraph(graph, packageJson); const prunedGraph = pruneProjectGraph(graph, packageJson);
const result = stringifyYarnLockfile(prunedGraph, lockFile, packageJson); const result = stringifyYarnLockfile(prunedGraph, lockFile, packageJson);
expect(result).toMatchInlineSnapshot(` expect(result).toMatchInlineSnapshot(`
@ -2247,3 +2497,7 @@ __metadata:
}); });
}); });
}); });
function uniq(str: string) {
return `str-${(Math.random() * 10000).toFixed(0)}`;
}

View File

@ -1,8 +1,14 @@
import { getHoistedPackageVersion } from './utils/package-json';
import { ProjectGraphBuilder } from '../../../project-graph/project-graph-builder';
import { satisfies, Range, gt } from 'semver';
import { NormalizedPackageJson } from './utils/package-json';
import { import {
getHoistedPackageVersion,
NormalizedPackageJson,
} from './utils/package-json';
import {
ProjectGraphDependencyWithFile,
validateDependency,
} from '../../../project-graph/project-graph-builder';
import { gt, Range, satisfies } from 'semver';
import {
DependencyType,
ProjectGraph, ProjectGraph,
ProjectGraphExternalNode, ProjectGraphExternalNode,
} from '../../../config/project-graph'; } from '../../../config/project-graph';
@ -31,23 +37,61 @@ type YarnDependency = {
linkType?: 'soft' | 'hard'; linkType?: 'soft' | 'hard';
}; };
export function parseYarnLockfile( let currentLockFileHash: string;
lockFileContent: string, let cachedParsedLockFile;
packageJson: NormalizedPackageJson,
builder: ProjectGraphBuilder
) {
const { parseSyml } = require('@yarnpkg/parsers');
const { __metadata, ...dependencies } = parseSyml(lockFileContent);
const isBerry = !!__metadata;
// we use key => node map to avoid duplicate work when parsing keys // we use key => node map to avoid duplicate work when parsing keys
const keyMap = new Map<string, ProjectGraphExternalNode>(); let keyMap = new Map<string, ProjectGraphExternalNode>();
function parseLockFile(lockFileContent: string, lockFileHash: string) {
if (currentLockFileHash === lockFileHash) {
return cachedParsedLockFile;
}
const { parseSyml } =
require('@yarnpkg/parsers') as typeof import('@yarnpkg/parsers');
keyMap.clear();
const result = parseSyml(lockFileContent);
cachedParsedLockFile = result;
currentLockFileHash = lockFileHash;
return result;
}
export function getYarnLockfileNodes(
lockFileContent: string,
lockFileHash: string,
packageJson: NormalizedPackageJson
) {
const { __metadata, ...dependencies } = parseLockFile(
lockFileContent,
lockFileHash
);
const isBerry = !!__metadata;
// yarn classic splits keys when parsing so we need to stich them back together // yarn classic splits keys when parsing so we need to stich them back together
const groupedDependencies = groupDependencies(dependencies, isBerry); const groupedDependencies = groupDependencies(dependencies, isBerry);
addNodes(groupedDependencies, packageJson, builder, keyMap, isBerry); return getNodes(groupedDependencies, packageJson, keyMap, isBerry);
addDependencies(groupedDependencies, builder, keyMap); }
export function getYarnLockfileDependencies(
lockFileContent: string,
lockFileHash: string,
projectGraph: ProjectGraph
) {
const { __metadata, ...dependencies } = parseLockFile(
lockFileContent,
lockFileHash
);
const isBerry = !!__metadata;
// yarn classic splits keys when parsing so we need to stich them back together
const groupedDependencies = groupDependencies(dependencies, isBerry);
return getDependencies(groupedDependencies, keyMap, projectGraph);
} }
function getPackageNameKeyPairs(keys: string): Map<string, Set<string>> { function getPackageNameKeyPairs(keys: string): Map<string, Set<string>> {
@ -63,10 +107,9 @@ function getPackageNameKeyPairs(keys: string): Map<string, Set<string>> {
return result; return result;
} }
function addNodes( function getNodes(
dependencies: Record<string, YarnDependency>, dependencies: Record<string, YarnDependency>,
packageJson: NormalizedPackageJson, packageJson: NormalizedPackageJson,
builder: ProjectGraphBuilder,
keyMap: Map<string, ProjectGraphExternalNode>, keyMap: Map<string, ProjectGraphExternalNode>,
isBerry: boolean isBerry: boolean
) { ) {
@ -128,6 +171,7 @@ function addNodes(
}); });
}); });
const externalNodes: Record<string, ProjectGraphExternalNode> = {};
for (const [packageName, versionMap] of nodes.entries()) { for (const [packageName, versionMap] of nodes.entries()) {
const hoistedNode = findHoistedNode(packageName, versionMap, combinedDeps); const hoistedNode = findHoistedNode(packageName, versionMap, combinedDeps);
if (hoistedNode) { if (hoistedNode) {
@ -135,9 +179,10 @@ function addNodes(
} }
versionMap.forEach((node) => { versionMap.forEach((node) => {
builder.addExternalNode(node); externalNodes[node.name] = node;
}); });
} }
return externalNodes;
} }
function findHoistedNode( function findHoistedNode(
@ -241,11 +286,12 @@ function getHoistedVersion(packageName: string): string {
} }
} }
function addDependencies( function getDependencies(
dependencies: Record<string, YarnDependency>, dependencies: Record<string, YarnDependency>,
builder: ProjectGraphBuilder, keyMap: Map<string, ProjectGraphExternalNode>,
keyMap: Map<string, ProjectGraphExternalNode> projectGraph: ProjectGraph
) { ) {
const projectGraphDependencies: ProjectGraphDependencyWithFile[] = [];
Object.keys(dependencies).forEach((keys) => { Object.keys(dependencies).forEach((keys) => {
const snapshot = dependencies[keys]; const snapshot = dependencies[keys];
keys.split(', ').forEach((key) => { keys.split(', ').forEach((key) => {
@ -259,7 +305,13 @@ function addDependencies(
keyMap.get(`${name}@npm:${versionRange}`) || keyMap.get(`${name}@npm:${versionRange}`) ||
keyMap.get(`${name}@${versionRange}`); keyMap.get(`${name}@${versionRange}`);
if (target) { if (target) {
builder.addStaticDependency(node.name, target.name); const dep = {
source: node.name,
target: target.name,
dependencyType: DependencyType.static,
};
validateDependency(projectGraph, dep);
projectGraphDependencies.push(dep);
} }
}); });
} }
@ -268,6 +320,8 @@ function addDependencies(
} }
}); });
}); });
return projectGraphDependencies;
} }
export function stringifyYarnLockfile( export function stringifyYarnLockfile(

View File

@ -0,0 +1,80 @@
import { join } from 'node:path';
import {
NrwlJsPluginConfig,
NxJsonConfiguration,
} from '../../../config/nx-json';
import { fileExists, readJsonFile } from '../../../utils/fileutils';
import { PackageJson } from '../../../utils/package-json';
import { workspaceRoot } from '../../../utils/workspace-root';
import { existsSync } from 'fs';
export function jsPluginConfig(
nxJson: NxJsonConfiguration
): Required<NrwlJsPluginConfig> {
const nxJsonConfig: NrwlJsPluginConfig =
nxJson?.pluginsConfig?.['@nx/js'] ?? nxJson?.pluginsConfig?.['@nrwl/js'];
// using lerna _before_ installing deps is causing an issue when parsing lockfile.
// See: https://github.com/lerna/lerna/issues/3807
// Note that previous attempt to fix this caused issues with Nx itself, thus we're checking
// for Lerna explicitly.
// See: https://github.com/nrwl/nx/pull/18784/commits/5416138e1ddc1945d5b289672dfb468e8c544e14
const analyzeLockfile =
!existsSync(join(workspaceRoot, 'lerna.json')) ||
existsSync(join(workspaceRoot, 'nx.json'));
if (nxJsonConfig) {
return {
analyzePackageJson: true,
analyzeSourceFiles: true,
analyzeLockfile,
...nxJsonConfig,
};
}
if (!fileExists(join(workspaceRoot, 'package.json'))) {
return {
analyzeLockfile: false,
analyzePackageJson: false,
analyzeSourceFiles: false,
};
}
const packageJson = readJsonFile<PackageJson>(
join(workspaceRoot, 'package.json')
);
const packageJsonDeps = {
...packageJson.dependencies,
...packageJson.devDependencies,
};
if (
packageJsonDeps['@nx/workspace'] ||
packageJsonDeps['@nx/js'] ||
packageJsonDeps['@nx/node'] ||
packageJsonDeps['@nx/next'] ||
packageJsonDeps['@nx/react'] ||
packageJsonDeps['@nx/angular'] ||
packageJsonDeps['@nx/web'] ||
packageJsonDeps['@nrwl/workspace'] ||
packageJsonDeps['@nrwl/js'] ||
packageJsonDeps['@nrwl/node'] ||
packageJsonDeps['@nrwl/next'] ||
packageJsonDeps['@nrwl/react'] ||
packageJsonDeps['@nrwl/angular'] ||
packageJsonDeps['@nrwl/web']
) {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: true,
};
} else {
return {
analyzePackageJson: true,
analyzeLockfile,
analyzeSourceFiles: false,
};
}
}

View File

@ -67,7 +67,7 @@ export type CreateNodesFunction = (
/** /**
* A pair of file patterns and {@link CreateNodesFunction} * A pair of file patterns and {@link CreateNodesFunction}
*/ */
export type CreateNodes = [ export type CreateNodes = readonly [
projectFilePattern: string, projectFilePattern: string,
createNodesFunction: CreateNodesFunction createNodesFunction: CreateNodesFunction
]; ];
@ -192,7 +192,7 @@ export async function loadNxPluginAsync(
let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root); let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root);
const plugin = (await import(pluginPath)) as NxPlugin; const plugin = (await import(pluginPath)) as NxPlugin;
plugin.name = name; plugin.name ??= name;
nxPluginCache.set(moduleName, plugin); nxPluginCache.set(moduleName, plugin);
return plugin; return plugin;
} }
@ -205,7 +205,7 @@ function loadNxPluginSync(moduleName: string, paths: string[], root: string) {
let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root); let { pluginPath, name } = getPluginPathAndName(moduleName, paths, root);
const plugin = require(pluginPath) as NxPlugin; const plugin = require(pluginPath) as NxPlugin;
plugin.name = name; plugin.name ??= name;
nxPluginCache.set(moduleName, plugin); nxPluginCache.set(moduleName, plugin);
return plugin; return plugin;
} }
@ -218,14 +218,10 @@ export function loadNxPluginsSync(
paths = getNxRequirePaths(), paths = getNxRequirePaths(),
root = workspaceRoot root = workspaceRoot
): (NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[] { ): (NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[] {
const result: NxPlugin[] = [];
// TODO: This should be specified in nx.json // TODO: This should be specified in nx.json
// Temporarily load js as if it were a plugin which is built into nx // Temporarily load js as if it were a plugin which is built into nx
// In the future, this will be optional and need to be specified in nx.json // In the future, this will be optional and need to be specified in nx.json
const jsPlugin: any = require('../plugins/js'); const result: NxPlugin[] = [...getDefaultPluginsSync(root)];
jsPlugin.name = 'nx-js-graph-plugin';
result.push(jsPlugin as NxPlugin);
if (shouldMergeAngularProjects(root, false)) { if (shouldMergeAngularProjects(root, false)) {
result.push(NxAngularJsonPlugin); result.push(NxAngularJsonPlugin);
@ -259,18 +255,12 @@ export async function loadNxPlugins(
paths = getNxRequirePaths(), paths = getNxRequirePaths(),
root = workspaceRoot root = workspaceRoot
): Promise<(NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[]> { ): Promise<(NxPluginV2 & Pick<NxPluginV1, 'processProjectGraph'>)[]> {
const result: NxPlugin[] = []; const result: NxPlugin[] = [...(await getDefaultPlugins(root))];
// TODO: This should be specified in nx.json // TODO: These should be specified in nx.json
// Temporarily load js as if it were a plugin which is built into nx // Temporarily load js as if it were a plugin which is built into nx
// In the future, this will be optional and need to be specified in nx.json // In the future, this will be optional and need to be specified in nx.json
const jsPlugin: any = await import('../plugins/js'); result.push();
jsPlugin.name = 'nx-js-graph-plugin';
result.push(jsPlugin as NxPlugin);
if (shouldMergeAngularProjects(root, false)) {
result.push(NxAngularJsonPlugin);
}
plugins ??= []; plugins ??= [];
for (const plugin of plugins) { for (const plugin of plugins) {
@ -484,3 +474,23 @@ function readPluginMainFromProjectConfiguration(
{}; {};
return main; return main;
} }
async function getDefaultPlugins(root: string) {
const plugins: NxPlugin[] = [await import('../plugins/js')];
if (shouldMergeAngularProjects(root, false)) {
plugins.push(
await import('../adapter/angular-json').then((m) => m.NxAngularJsonPlugin)
);
}
return plugins;
}
function getDefaultPluginsSync(root: string) {
const plugins: NxPlugin[] = [require('../plugins/js')];
if (shouldMergeAngularProjects(root, false)) {
plugins.push(require('../adapter/angular-json').NxAngularJsonPlugin);
}
return plugins;
}

View File

@ -1,4 +1,5 @@
import { import {
detectPackageManager,
ExecutorContext, ExecutorContext,
logger, logger,
stripIndents, stripIndents,
@ -79,11 +80,20 @@ export async function* viteBuildExecutor(
builtPackageJson.type = 'module'; builtPackageJson.type = 'module';
writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson); writeJsonFile(`${options.outputPath}/package.json`, builtPackageJson);
const packageManager = detectPackageManager(context.root);
const lockFile = createLockFile(builtPackageJson); const lockFile = createLockFile(
writeFileSync(`${options.outputPath}/${getLockFileName()}`, lockFile, { builtPackageJson,
encoding: 'utf-8', context.projectGraph,
}); packageManager
);
writeFileSync(
`${options.outputPath}/${getLockFileName(packageManager)}`,
lockFile,
{
encoding: 'utf-8',
}
);
} }
// For buildable libs, copy package.json if it exists. // For buildable libs, copy package.json if it exists.
else if ( else if (

View File

@ -1,6 +1,11 @@
import { type Compiler, sources, type WebpackPluginInstance } from 'webpack'; import { type Compiler, sources, type WebpackPluginInstance } from 'webpack';
import { createLockFile, createPackageJson } from '@nx/js'; import { createLockFile, createPackageJson } from '@nx/js';
import { ExecutorContext, type ProjectGraph, serializeJson } from '@nx/devkit'; import {
detectPackageManager,
ExecutorContext,
type ProjectGraph,
serializeJson,
} from '@nx/devkit';
import { import {
getHelperDependenciesFromProjectGraph, getHelperDependenciesFromProjectGraph,
getLockFileName, getLockFileName,
@ -66,9 +71,12 @@ export class GeneratePackageJsonPlugin implements WebpackPluginInstance {
'package.json', 'package.json',
new sources.RawSource(serializeJson(packageJson)) new sources.RawSource(serializeJson(packageJson))
); );
const packageManager = detectPackageManager(this.context.root);
compilation.emitAsset( compilation.emitAsset(
getLockFileName(), getLockFileName(packageManager),
new sources.RawSource(createLockFile(packageJson)) new sources.RawSource(
createLockFile(packageJson, this.projectGraph, packageManager)
)
); );
} }
); );