chore(core): remove globForProjectFiles (#18288)

This commit is contained in:
Jason Jean 2023-07-25 12:47:26 -04:00 committed by GitHub
parent b2763e1cc9
commit 2dda1914b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 157 additions and 248 deletions

View File

@ -4,13 +4,8 @@ import {
toProjectName,
Workspaces,
} from './workspaces';
import { NxJsonConfiguration } from './nx-json';
import { vol } from 'memfs';
import * as fastGlob from 'fast-glob';
import { TargetConfiguration } from './workspace-json-project-json';
jest.mock('fs', () => require('memfs').fs);
import { TempFs } from '../utils/testing/temp-fs';
const libConfig = (name) => ({
root: `libs/${name}`,
@ -24,26 +19,16 @@ const packageLibConfig = (root) => ({
});
describe('Workspaces', () => {
let globResults: string[];
let fs: TempFs;
beforeEach(() => {
globResults = [
'libs/lib1/package.json',
'libs/lib1/project.json',
'libs/lib2/package.json',
'libs/domain/lib3/package.json',
'libs/domain/lib4/project.json',
'libs/domain/lib4/package.json',
];
jest.spyOn(fastGlob, 'sync').mockImplementation(() => globResults);
fs = new TempFs('Workspaces');
});
afterEach(() => {
jest.resetAllMocks();
vol.reset();
fs.cleanup();
});
describe('readWorkspaceConfiguration', () => {
it('should be able to inline project configurations', () => {
it('should be able to inline project configurations', async () => {
const standaloneConfig = libConfig('lib1');
const config = {
@ -53,46 +38,41 @@ describe('Workspaces', () => {
lib2: libConfig('lib2'),
},
};
vol.fromJSON(
{
'libs/lib1/project.json': JSON.stringify(standaloneConfig),
'libs/lib2/package.json': JSON.stringify({}),
'libs/domain/lib3/package.json': JSON.stringify({}),
'libs/domain/lib4/project.json': JSON.stringify({}),
'workspace.json': JSON.stringify(config),
},
'/root'
);
await fs.createFiles({
'libs/lib1/project.json': JSON.stringify(standaloneConfig),
'libs/lib2/package.json': JSON.stringify({}),
'libs/domain/lib3/package.json': JSON.stringify({}),
'libs/domain/lib4/project.json': JSON.stringify({}),
'workspace.json': JSON.stringify(config),
});
const workspaces = new Workspaces('/root');
const workspaces = new Workspaces(fs.tempDir);
const resolved = workspaces.readProjectsConfigurations();
expect(resolved.projects.lib1).toEqual(standaloneConfig);
});
it('should build project configurations from glob', () => {
it('should build project configurations from glob', async () => {
const lib1Config = libConfig('lib1');
const lib2Config = packageLibConfig('libs/lib2');
const domainPackageConfig = packageLibConfig('libs/domain/lib3');
const domainLibConfig = libConfig('domain/lib4');
vol.fromJSON(
{
'libs/lib1/project.json': JSON.stringify(lib1Config),
'libs/lib1/package.json': JSON.stringify({ name: 'some-other-name' }),
'libs/lib2/package.json': JSON.stringify({ name: 'lib2' }),
'libs/domain/lib3/package.json': JSON.stringify({
name: 'domain-lib3',
}),
'libs/domain/lib4/project.json': JSON.stringify(domainLibConfig),
'libs/domain/lib4/package.json': JSON.stringify({}),
'package.json': JSON.stringify({
workspaces: ['**/package.json'],
}),
},
'/root'
);
await fs.createFiles({
'libs/lib1/project.json': JSON.stringify(lib1Config),
'libs/lib1/package.json': JSON.stringify({ name: 'some-other-name' }),
'libs/lib2/package.json': JSON.stringify({ name: 'lib2' }),
'libs/domain/lib3/package.json': JSON.stringify({
name: 'domain-lib3',
}),
'libs/domain/lib4/project.json': JSON.stringify(domainLibConfig),
'libs/domain/lib4/package.json': JSON.stringify({}),
'package.json': JSON.stringify({
name: 'package-name',
workspaces: ['**/package.json'],
}),
});
const workspaces = new Workspaces('/root');
const workspaces = new Workspaces(fs.tempDir);
const { projects } = workspaces.readProjectsConfigurations();
// projects got deduped so the workspace one remained
expect(projects['lib1']).toEqual({
@ -107,34 +87,24 @@ describe('Workspaces', () => {
describe('to project name', () => {
it('should lowercase names', () => {
const nxJson: NxJsonConfiguration = {
npmScope: '',
workspaceLayout: {
appsDir: 'my-apps',
libsDir: 'packages',
},
};
const appResults = toProjectName('my-apps/directory/my-app/package.json');
const libResults = toProjectName('packages/directory/MyLib/package.json');
expect(appResults).toEqual('my-app');
expect(libResults).toEqual('mylib');
});
it('should use the workspace globs in package.json', () => {
globResults = ['packages/my-package/package.json'];
vol.fromJSON(
{
'packages/my-package/package.json': JSON.stringify({
name: 'my-package',
}),
'package.json': JSON.stringify({
workspaces: ['packages/**'],
}),
},
'/root2'
);
it('should use the workspace globs in package.json', async () => {
await fs.createFiles({
'packages/my-package/package.json': JSON.stringify({
name: 'my-package',
}),
'package.json': JSON.stringify({
name: 'package-name',
workspaces: ['packages/**'],
}),
});
const workspaces = new Workspaces('/root2');
const workspaces = new Workspaces(fs.tempDir);
const resolved = workspaces.readProjectsConfigurations();
expect(resolved.projects['my-package']).toEqual({
root: 'packages/my-package',

View File

@ -1,14 +1,13 @@
import { sync as globSync } from 'fast-glob';
import { existsSync } from 'fs';
import * as path from 'path';
import { basename, dirname, join } from 'path';
import { performance } from 'perf_hooks';
import { workspaceRoot } from '../utils/workspace-root';
import { readJsonFile, readYamlFile } from '../utils/fileutils';
import { logger, NX_PREFIX } from '../utils/logger';
import { loadNxPlugins, loadNxPluginsSync } from '../utils/nx-plugin';
import type { NxJsonConfiguration, TargetDefaults } from './nx-json';
import { readNxJson } from './nx-json';
import {
ProjectConfiguration,
ProjectsConfigurations,
@ -21,9 +20,7 @@ import {
mergeAngularJsonAndProjects,
shouldMergeAngularProjects,
} from '../adapter/angular-json';
import { getNxRequirePaths } from '../utils/installation-directory';
import { getIgnoredGlobs } from '../utils/ignore';
import { readNxJson } from './nx-json';
import { retrieveProjectConfigurationPaths } from '../project-graph/utils/retrieve-workspace-files';
export class Workspaces {
private cachedProjectsConfig: ProjectsConfigurations;
@ -38,7 +35,6 @@ export class Workspaces {
* @deprecated
*/
readProjectsConfigurations(opts?: {
_ignorePluginInference?: boolean;
_includeProjectsFromAngularJson?: boolean;
}): ProjectsConfigurations {
if (
@ -48,19 +44,10 @@ export class Workspaces {
return this.cachedProjectsConfig;
}
const nxJson = readNxJson(this.root);
const projectPaths = retrieveProjectConfigurationPaths(this.root, nxJson);
let projectsConfigurations = buildProjectsConfigurationsFromProjectPaths(
nxJson,
globForProjectFiles(
this.root,
opts?._ignorePluginInference
? []
: getGlobPatternsFromPlugins(
nxJson,
getNxRequirePaths(this.root),
this.root
),
nxJson
),
projectPaths,
(path) => readJsonFile(join(this.root, path))
);
if (
@ -132,9 +119,6 @@ export function toProjectName(fileName: string): string {
return parts[parts.length - 1].toLowerCase();
}
let projectGlobCache: string[];
let projectGlobCacheKey: string;
/**
* @deprecated Use getGlobPatternsFromPluginsAsync instead.
*/
@ -247,103 +231,6 @@ function removeRelativePath(pattern: string): string {
return pattern.startsWith('./') ? pattern.substring(2) : pattern;
}
export function globForProjectFiles(
root: string,
pluginsGlobPatterns: string[],
nxJson?: NxJsonConfiguration
) {
// Deal w/ Caching
const cacheKey = [root, ...pluginsGlobPatterns].join(',');
if (
process.env.NX_PROJECT_GLOB_CACHE !== 'false' &&
projectGlobCache &&
cacheKey === projectGlobCacheKey
) {
return projectGlobCache;
}
projectGlobCacheKey = cacheKey;
const _globPatternsFromPackageManagerWorkspaces =
getGlobPatternsFromPackageManagerWorkspaces(root);
const globPatternsFromPackageManagerWorkspaces =
_globPatternsFromPackageManagerWorkspaces ?? [];
const globsToInclude = globPatternsFromPackageManagerWorkspaces.filter(
(glob) => !glob.startsWith('!')
);
const globsToExclude = globPatternsFromPackageManagerWorkspaces
.filter((glob) => glob.startsWith('!'))
.map((glob) => glob.substring(1))
.map((glob) => (glob.startsWith('/') ? glob.substring(1) : glob));
const projectGlobPatterns: string[] = [
'project.json',
'**/project.json',
...globsToInclude,
];
projectGlobPatterns.push(...pluginsGlobPatterns);
const combinedProjectGlobPattern = '{' + projectGlobPatterns.join(',') + '}';
performance.mark('start-glob-for-projects');
/**
* This configures the files and directories which we always want to ignore as part of file watching
* and which we know the location of statically (meaning irrespective of user configuration files).
* This has the advantage of being ignored directly within globSync
*
* Other ignored entries will need to be determined dynamically by reading and evaluating the user's
* .gitignore and .nxignore files below.
*/
const staticIgnores = [
'node_modules',
'**/node_modules',
'dist',
'.git',
...globsToExclude,
...getIgnoredGlobs(root, false),
];
/**
* TODO: This utility has been implemented multiple times across the Nx codebase,
* discuss whether it should be moved to a shared location.
*/
const opts = {
ignore: staticIgnores,
absolute: false,
cwd: root,
dot: true,
suppressErrors: true,
};
const globResults = globSync(combinedProjectGlobPattern, opts);
projectGlobCache = deduplicateProjectFiles(globResults);
// TODO @vsavkin remove after Nx 16
if (
projectGlobCache.length === 0 &&
_globPatternsFromPackageManagerWorkspaces === undefined &&
nxJson?.extends === 'nx/presets/npm.json'
) {
output.warn({
title:
'Nx could not find any projects. Check if you need to configure workspaces in package.json or pnpm-workspace.yaml',
});
}
performance.mark('finish-glob-for-projects');
performance.measure(
'glob-for-project-files',
'start-glob-for-projects',
'finish-glob-for-projects'
);
return projectGlobCache;
}
/**
* @description Loops through files and reduces them to 1 file per project.
* @param files Array of files that may represent projects

View File

@ -6,8 +6,6 @@ import {
import {
buildProjectsConfigurationsFromProjectPaths,
deduplicateProjectFiles,
getGlobPatternsFromPlugins,
globForProjectFiles,
renamePropertyWithStableKeys,
} from '../../config/workspaces';
import { joinPathFragments, normalizePath } from '../../utils/path';
@ -18,7 +16,7 @@ import { readJson, writeJson } from './json';
import { PackageJson } from '../../utils/package-json';
import { readNxJson } from './nx-json';
import { output } from '../../utils/output';
import { getNxRequirePaths } from '../../utils/installation-directory';
import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files';
export { readNxJson, updateNxJson } from './nx-json';
export {
@ -183,11 +181,7 @@ function readAndCombineAllProjectConfigurations(tree: Tree): {
} {
const nxJson = readNxJson(tree);
const globbedFiles = globForProjectFiles(
tree.root,
getGlobPatternsFromPlugins(nxJson, getNxRequirePaths(tree.root), tree.root),
nxJson
).map(normalizePath);
const globbedFiles = retrieveProjectConfigurationPaths(tree.root, nxJson);
const createdFiles = findCreatedProjectFiles(tree);
const deletedFiles = findDeletedProjectFiles(tree);
const projectFiles = [...globbedFiles, ...createdFiles].filter(

View File

@ -1,25 +1,13 @@
import { Tree } from '../../generators/tree';
import { readNxJson } from '../../generators/utils/nx-json';
import {
getGlobPatternsFromPluginsAsync,
globForProjectFiles,
} from '../../config/workspaces';
import { dirname } from 'path';
import { readJson, writeJson } from '../../generators/utils/json';
import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available';
import { getNxRequirePaths } from '../../utils/installation-directory';
import { retrieveProjectConfigurationPaths } from '../../project-graph/utils/retrieve-workspace-files';
export default async function (tree: Tree) {
const nxJson = readNxJson(tree);
const projectFiles = globForProjectFiles(
tree.root,
await getGlobPatternsFromPluginsAsync(
nxJson,
getNxRequirePaths(tree.root),
tree.root
),
nxJson
);
const projectFiles = retrieveProjectConfigurationPaths(tree.root, nxJson);
const projectJsons = projectFiles.filter((f) => f.endsWith('project.json'));
for (let f of projectJsons) {

View File

@ -38,6 +38,8 @@ export const enum WorkspaceErrors {
Generic = 'Generic'
}
/** Get workspace config files based on provided globs */
export function getProjectConfigurationFiles(workspaceRoot: string, globs: Array<string>): Array<string>
/** Get workspace config files based on provided globs */
export function getProjectConfigurations(workspaceRoot: string, globs: Array<string>, parseConfigurations: (arg0: Array<string>) => Record<string, object>): Record<string, object>
export interface NxWorkspaceFiles {
projectFileMap: Record<string, Array<FileData>>

View File

@ -246,7 +246,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, ImportResult, findImports, EventType, Watcher, WorkspaceErrors, getProjectConfigurations, getWorkspaceFilesNative } = nativeBinding
const { expandOutputs, remove, copy, hashArray, hashFile, hashFiles, hashFilesMatchingGlobs, ImportResult, findImports, EventType, Watcher, WorkspaceErrors, getProjectConfigurationFiles, getProjectConfigurations, getWorkspaceFilesNative } = nativeBinding
module.exports.expandOutputs = expandOutputs
module.exports.remove = remove
@ -260,5 +260,6 @@ module.exports.findImports = findImports
module.exports.EventType = EventType
module.exports.Watcher = Watcher
module.exports.WorkspaceErrors = WorkspaceErrors
module.exports.getProjectConfigurationFiles = getProjectConfigurationFiles
module.exports.getProjectConfigurations = getProjectConfigurations
module.exports.getWorkspaceFilesNative = getWorkspaceFilesNative

View File

@ -7,6 +7,28 @@ use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::path::{Path, PathBuf};
#[napi]
/// Get workspace config files based on provided globs
pub fn get_project_configuration_files(
workspace_root: String,
globs: Vec<String>,
) -> napi::Result<Vec<String>> {
let globs = build_glob_set(&globs)?;
let config_paths: Vec<String> = nx_walker(workspace_root, move |rec| {
let mut config_paths: HashMap<PathBuf, PathBuf> = HashMap::new();
for (path, _) in rec {
insert_config_file_into_map(path, &mut config_paths, &globs);
}
config_paths
.into_values()
.map(|p| p.to_normalized_string())
.collect()
});
Ok(config_paths)
}
#[napi]
/// Get workspace config files based on provided globs
pub fn get_project_configurations<ConfigurationParser>(

View File

@ -1,10 +1,10 @@
import { TempFs } from '../utils/testing/temp-fs';
import { globForProjectFiles } from './workspaces';
import { TempFs } from '../../utils/testing/temp-fs';
import { retrieveProjectConfigurationPaths } from './retrieve-workspace-files';
describe('globForProjectFiles', () => {
describe('retrieveProjectConfigurationPaths', () => {
let fs: TempFs;
beforeAll(() => {
fs = new TempFs('glob-for-project-files');
fs = new TempFs('retrieveProjectConfigurationPaths');
});
afterAll(() => {
fs.cleanup();
@ -24,10 +24,10 @@ describe('globForProjectFiles', () => {
name: 'project-1',
})
);
expect(globForProjectFiles(fs.tempDir, [])).not.toContain(
expect(retrieveProjectConfigurationPaths(fs.tempDir, {})).not.toContain(
'not-projects/project.json'
);
expect(globForProjectFiles(fs.tempDir, [])).toContain(
expect(retrieveProjectConfigurationPaths(fs.tempDir, {})).toContain(
'projects/project.json'
);
});

View File

@ -2,6 +2,7 @@ import { performance } from 'perf_hooks';
import {
buildProjectsConfigurationsFromProjectPaths,
getGlobPatternsFromPackageManagerWorkspaces,
getGlobPatternsFromPlugins,
getGlobPatternsFromPluginsAsync,
mergeTargetConfigurations,
readTargetDefaultsForTarget,
@ -17,7 +18,7 @@ import {
mergeAngularJsonAndProjects,
shouldMergeAngularProjects,
} from '../../adapter/angular-json';
import { NxJsonConfiguration } from '../../config/nx-json';
import { NxJsonConfiguration, readNxJson } from '../../config/nx-json';
import { FileData, ProjectFileMap } from '../../config/project-graph';
import type { NxWorkspaceFiles } from '../../native';
@ -49,13 +50,7 @@ export async function retrieveWorkspaceFiles(
workspaceRoot,
globs,
(configs: string[]): Record<string, ProjectConfiguration> => {
const projectConfigurations = createProjectConfigurations(
workspaceRoot,
nxJson,
configs
);
return projectConfigurations.projects;
return createProjectConfigurations(workspaceRoot, nxJson, configs);
}
) as NxWorkspaceFiles;
performance.mark('get-workspace-files:end');
@ -85,21 +80,58 @@ export async function retrieveProjectConfigurations(
workspaceRoot: string,
nxJson: NxJsonConfiguration
): Promise<Record<string, ProjectConfiguration>> {
const { getProjectConfigurations } = require('../../native');
const { getProjectConfigurations } =
require('../../native') as typeof import('../../native');
const globs = await configurationGlobs(workspaceRoot, nxJson);
return getProjectConfigurations(
workspaceRoot,
globs,
(configs: string[]): Record<string, ProjectConfiguration> => {
const projectConfigurations = createProjectConfigurations(
workspaceRoot,
nxJson,
configs
);
return projectConfigurations.projects;
return createProjectConfigurations(workspaceRoot, nxJson, configs);
}
);
) as Record<string, ProjectConfiguration>;
}
export function retrieveProjectConfigurationPaths(
root: string,
nxJson: NxJsonConfiguration
): string[] {
const projectGlobPatterns = configurationGlobsSync(root, nxJson);
const { getProjectConfigurationFiles } =
require('../../native') as typeof import('../../native');
return getProjectConfigurationFiles(root, projectGlobPatterns);
}
const projectsWithoutPluginCache = new Map<
string,
Record<string, ProjectConfiguration>
>();
// TODO: This function is called way too often, it should be optimized without this cache
export function retrieveProjectConfigurationsWithoutPluginInference(
root: string
): Record<string, ProjectConfiguration> {
const nxJson = readNxJson(root);
const projectGlobPatterns = configurationGlobsWithoutPlugins(root);
const cacheKey = root + ',' + projectGlobPatterns.join(',');
if (projectsWithoutPluginCache.has(cacheKey)) {
return projectsWithoutPluginCache.get(cacheKey);
}
const { getProjectConfigurations } =
require('../../native') as typeof import('../../native');
const projectConfigurations = getProjectConfigurations(
root,
projectGlobPatterns,
(configs: string[]): Record<string, ProjectConfiguration> => {
return createProjectConfigurations(root, nxJson, configs);
}
) as Record<string, ProjectConfiguration>;
projectsWithoutPluginCache.set(cacheKey, projectConfigurations);
return projectConfigurations;
}
function buildAllWorkspaceFiles(
@ -124,7 +156,7 @@ function createProjectConfigurations(
workspaceRoot: string,
nxJson: NxJsonConfiguration,
configFiles: string[]
): ProjectsConfigurations {
): Record<string, ProjectConfiguration> {
performance.mark('build-project-configs:start');
let projectConfigurations = mergeTargetDefaultsIntoProjectDescriptions(
@ -147,10 +179,7 @@ function createProjectConfigurations(
'build-project-configs:end'
);
return {
version: 2,
projects: projectConfigurations,
};
return projectConfigurations;
}
function mergeTargetDefaultsIntoProjectDescriptions(
@ -190,10 +219,29 @@ async function configurationGlobs(
workspaceRoot
);
return [...configurationGlobsWithoutPlugins(workspaceRoot), ...pluginGlobs];
}
/**
* @deprecated Use {@link configurationGlobs} instead.
*/
function configurationGlobsSync(
workspaceRoot: string,
nxJson: NxJsonConfiguration
): string[] {
let pluginGlobs = getGlobPatternsFromPlugins(
nxJson,
getNxRequirePaths(workspaceRoot),
workspaceRoot
);
return [...configurationGlobsWithoutPlugins(workspaceRoot), ...pluginGlobs];
}
function configurationGlobsWithoutPlugins(workspaceRoot: string): string[] {
return [
'project.json',
'**/project.json',
...pluginGlobs,
...getGlobPatternsFromPackageManagerWorkspaces(workspaceRoot),
];
}

View File

@ -2,7 +2,6 @@ import { sync } from 'fast-glob';
import { existsSync } from 'fs';
import * as path from 'path';
import { ProjectGraphProcessor } from '../config/project-graph';
import { Workspaces } from '../config/workspaces';
import { workspaceRoot } from './workspace-root';
import { readJsonFile } from '../utils/fileutils';
@ -16,7 +15,6 @@ import {
} from '../plugins/js/utils/register';
import {
ProjectConfiguration,
ProjectsConfigurations,
TargetConfiguration,
} from '../config/workspace-json-project-json';
import { logger } from './logger';
@ -30,6 +28,7 @@ import { getNxRequirePaths } from './installation-directory';
import { readTsConfig } from '../plugins/js/utils/typescript';
import type * as ts from 'typescript';
import { retrieveProjectConfigurationsWithoutPluginInference } from '../project-graph/utils/retrieve-workspace-files';
export type ProjectTargetConfigurator = (
file: string
@ -287,10 +286,8 @@ export function registerPluginTSTranspiler() {
}
function lookupLocalPlugin(importPath: string, root = workspaceRoot) {
const workspace = new Workspaces(root).readProjectsConfigurations({
_ignorePluginInference: true,
});
const plugin = findNxProjectForImportPath(importPath, workspace, root);
const projects = retrieveProjectConfigurationsWithoutPluginInference(root);
const plugin = findNxProjectForImportPath(importPath, projects, root);
if (!plugin) {
return null;
}
@ -299,13 +296,13 @@ function lookupLocalPlugin(importPath: string, root = workspaceRoot) {
registerPluginTSTranspiler();
}
const projectConfig = workspace.projects[plugin];
const projectConfig: ProjectConfiguration = projects[plugin];
return { path: path.join(root, projectConfig.root), projectConfig };
}
function findNxProjectForImportPath(
importPath: string,
projects: ProjectsConfigurations,
projects: Record<string, ProjectConfiguration>,
root = workspaceRoot
): string | null {
const tsConfigPaths: Record<string, string[]> = readTsConfigPaths(root);
@ -314,7 +311,7 @@ function findNxProjectForImportPath(
);
if (possiblePaths?.length) {
const projectRootMappings =
createProjectRootMappingsFromProjectConfigurations(projects.projects);
createProjectRootMappingsFromProjectConfigurations(projects);
for (const tsConfigPath of possiblePaths) {
const nxProject = findProjectForPath(tsConfigPath, projectRootMappings);
if (nxProject) {