feat(node): add support for workspace libs when not bundling (#15069)

e2e/node/src/node-server.test.ts

@@ -5,9 +5,11 @@ import {
   killPort,
   newProject,
   promisifiedTreeKill,
+  readFile,
   runCLI,
   runCommandUntil,
   uniq,
+  updateFile,
 } from '@nrwl/e2e/utils';
 
 describe('Node Applications + webpack', () => {
@@ -15,12 +17,24 @@ describe('Node Applications + webpack', () => {
 
   afterEach(() => cleanupProject());
 
+  function addLibImport(appName: string, libName: string) {
+    const content = readFile(`apps/${appName}/src/main.ts`);
+    updateFile(
+      `apps/${appName}/src/main.ts`,
+      `
+      import { ${libName} } from '@proj/${libName}';
+      ${content}
+      console.log(${libName}());
+      `
+    );
+  }
+
   async function runE2eTests(appName: string) {
     process.env.PORT = '5000';
     const childProcess = await runCommandUntil(`serve ${appName}`, (output) => {
       return output.includes('http://localhost:5000');
     });
-    const result = runCLI(`e2e ${appName}-e2e`);
+    const result = runCLI(`e2e ${appName}-e2e --verbose`);
     expect(result).toContain('Setting up...');
     expect(result).toContain('Tearing down..');
     expect(result).toContain('Successfully ran target e2e');
@@ -31,10 +45,12 @@ describe('Node Applications + webpack', () => {
   }
 
   it('should generate an app using webpack', async () => {
+    const utilLib = uniq('util');
     const expressApp = uniq('expressapp');
     const fastifyApp = uniq('fastifyapp');
     const koaApp = uniq('koaapp');
 
+    runCLI(`generate @nrwl/node:lib ${utilLib}`);
     runCLI(
       `generate @nrwl/node:app ${expressApp} --framework=express --no-interactive`
     );

packages/esbuild/src/executors/esbuild/lib/build-esbuild-options.spec.ts

@@ -13,6 +13,16 @@ describe('buildEsbuildOptions', () => {
         },
       },
     },
+    projectGraph: {
+      nodes: {
+        myapp: {
+          type: 'app',
+          name: 'myapp',
+          data: { root: 'apps/myapp', files: [] },
+        },
+      },
+      dependencies: { myapp: [] },
+    },
     nxJsonConfiguration: {},
     isVerbose: false,
     root: path.join(__dirname, 'fixtures'),

packages/esbuild/src/executors/esbuild/lib/build-esbuild-options.ts

@@ -1,15 +1,19 @@
 import * as esbuild from 'esbuild';
 import * as path from 'path';
 import { parse } from 'path';
-import * as glob from 'fast-glob';
+import {
+  ExecutorContext,
+  getImportPath,
+  joinPathFragments,
+} from '@nrwl/devkit';
+import { mkdirSync, writeFileSync } from 'fs';
+
 import { getClientEnvironment } from '../../../utils/environment-variables';
 import {
   EsBuildExecutorOptions,
   NormalizedEsBuildExecutorOptions,
 } from '../schema';
-import { ExecutorContext } from 'nx/src/config/misc-interfaces';
-import { joinPathFragments } from 'nx/src/utils/path';
-import { readJsonFile } from 'nx/src/utils/fileutils';
+import { getEntryPoints } from '../../../utils/get-entry-points';
 
 const ESM_FILE_EXTENSION = '.js';
 const CJS_FILE_EXTENSION = '.cjs';
@@ -19,6 +23,7 @@ export function buildEsbuildOptions(
   options: NormalizedEsBuildExecutorOptions,
   context: ExecutorContext
 ): esbuild.BuildOptions {
+  const outExtension = getOutExtension(format, options);
   const esbuildOptions: esbuild.BuildOptions = {
     ...options.esbuildOptions,
     entryNames:
@@ -35,7 +40,7 @@ export function buildEsbuildOptions(
     tsconfig: options.tsConfig,
     format,
     outExtension: {
-      '.js': getOutExtension(format, options),
+      '.js': outExtension,
     },
   };
 
@@ -52,20 +57,69 @@ export function buildEsbuildOptions(
   const entryPoints = options.additionalEntryPoints
     ? [options.main, ...options.additionalEntryPoints]
     : [options.main];
-  if (!options.bundle) {
-    const projectRoot =
-      context.projectsConfigurations.projects[context.projectName].root;
-    const tsconfig = readJsonFile(path.join(context.root, options.tsConfig));
-    const matchedFiles = glob
-      .sync(tsconfig.include ?? [], {
-        cwd: projectRoot,
-        ignore: (tsconfig.exclude ?? []).concat([options.main]),
-      })
-      .map((f) => path.join(projectRoot, f))
-      .filter((f) => !entryPoints.includes(f));
-    entryPoints.push(...matchedFiles);
+
+  if (options.bundle) {
+    esbuildOptions.entryPoints = entryPoints;
+  } else if (options.platform === 'node' && format === 'cjs') {
+    // When target platform Node and target format is CJS, then also transpile workspace libs used by the app.
+    // Provide a `require` override in the main entry file so workspace libs can be loaded when running the app.
+    const manifest: Array<{ module: string; root: string }> = []; // Manifest allows the built app to load compiled workspace libs.
+    const entryPointsFromProjects = getEntryPoints(
+      context.projectName,
+      context,
+      {
+        initialEntryPoints: entryPoints,
+        recursive: true,
+        onProjectFilesMatched: (currProjectName) => {
+          manifest.push({
+            module: getImportPath(
+              context.nxJsonConfiguration.npmScope,
+              currProjectName
+            ),
+            root: context.projectGraph.nodes[currProjectName].data.root,
+          });
+        },
+      }
+    );
+
+    esbuildOptions.entryPoints = [
+      // Write a main entry file that registers workspace libs and then calls the user-defined main.
+      writeTmpEntryWithRequireOverrides(
+        manifest,
+        outExtension,
+        options,
+        context
+      ),
+      ...entryPointsFromProjects.map((f) => {
+        /**
+         * Maintain same directory structure as the workspace, so that other workspace libs may be used by the project.
+         * dist
+         * └── apps
+         *     └── demo
+         *         ├── apps
+         *         │   └── demo
+         *         │       └── src
+         *         │           └── main.js (requires '@acme/utils' which is mapped to libs/utils/src/index.js)
+         *         ├── libs
+         *         │   └── utils
+         *         │       └── src
+         *         │           └── index.js
+         *         └── main.js (entry with require overrides)
+         */
+        const { dir, name } = path.parse(f);
+        return {
+          in: f,
+          out: path.join(dir, name),
+        };
+      }),
+    ];
+  } else {
+    // Otherwise, just transpile the project source files. Any workspace lib will need to be published separately.
+    esbuildOptions.entryPoints = getEntryPoints(context.projectName, context, {
+      initialEntryPoints: entryPoints,
+      recursive: false,
+    });
   }
-  esbuildOptions.entryPoints = entryPoints;
 
   return esbuildOptions;
 }
@@ -100,3 +154,108 @@ export function getOutfile(
   const { dir, name } = parse(candidate);
   return `${dir}/${name}${ext}`;
 }
+
+function writeTmpEntryWithRequireOverrides(
+  manifest: Array<{ module: string; root: string }>,
+  outExtension: '.cjs' | '.js' | '.mjs',
+  options: NormalizedEsBuildExecutorOptions,
+  context: ExecutorContext
+): { in: string; out: string } {
+  const project = context.projectGraph?.nodes[context.projectName];
+  // Write a temp main entry source that registers workspace libs.
+  const tmpPath = path.join(
+    context.root,
+    'tmp',
+    context.projectGraph?.nodes[context.projectName].name
+  );
+  mkdirSync(tmpPath, { recursive: true });
+
+  const { name: mainFileName, dir: mainPathRelativeToDist } = path.parse(
+    options.main
+  );
+  const mainWithRequireOverridesInPath = path.join(
+    tmpPath,
+    `main-with-require-overrides.js`
+  );
+  writeFileSync(
+    mainWithRequireOverridesInPath,
+    getRegisterFileContent(
+      manifest,
+      `./${path.join(
+        mainPathRelativeToDist,
+        `${mainFileName}${outExtension}`
+      )}`,
+      outExtension
+    )
+  );
+
+  let mainWithRequireOverridesOutPath: string;
+  if (options.outputFileName) {
+    mainWithRequireOverridesOutPath = path.parse(options.outputFileName).name;
+  } else if (mainPathRelativeToDist === '' || mainPathRelativeToDist === '.') {
+    // If the user customized their entry such that it is not inside `src/` folder
+    // then they have to provide the outputFileName
+    throw new Error(
+      `There is a conflict between Nx-generated main file and the project's main file. Set --outputFileName=nx-main.js to fix this error.`
+    );
+  } else {
+    mainWithRequireOverridesOutPath = path.parse(mainFileName).name;
+  }
+
+  return {
+    in: mainWithRequireOverridesInPath,
+    out: mainWithRequireOverridesOutPath,
+  };
+}
+
+function getRegisterFileContent(
+  manifest: Array<{ module: string; root: string }>,
+  mainFile: string,
+  outExtension = '.js'
+) {
+  return `
+/**
+ * IMPORTANT: Do not modify this file.
+ * This file allows the app to run without bundling in workspace libraries.
+ * Must be contained in the ".nx" folder inside the output path.
+ */
+const Module = require('module');
+const path = require('path');
+const fs = require('fs');
+const originalResolveFilename = Module._resolveFilename;
+const distPath = __dirname;
+const manifest = ${JSON.stringify(manifest)};
+
+Module._resolveFilename = function(request, parent) {
+  const entry = manifest.find(x => request === x.module || request.startsWith(x.module + '/'));
+  let found;
+  if (entry) {
+    if (request === entry.module) {
+      // Known entry paths for libraries. Add more if missing.
+      const candidates = [
+        path.join(distPath, entry.root, 'src/index' + '${outExtension}'),
+        path.join(distPath, entry.root, 'src/main' + '${outExtension}'),
+        path.join(distPath, entry.root, 'index' + '${outExtension}'),
+        path.join(distPath, entry.root, 'main' + '${outExtension}')
+      ];
+      found = candidates.find(f => fs.statSync(f).isFile());
+    } else {
+      const candidate = path.join(distPath, entry.root, request.replace(entry.module, '') + '${outExtension}');
+      if (fs.statSync(candidate).isFile()) {
+        found = candidate;
+      }
+    }
+  }
+  
+  if (found) {
+    const modifiedArguments = [found, ...[].slice.call(arguments, 1)];
+    return originalResolveFilename.apply(this, modifiedArguments);
+  } else {
+    return originalResolveFilename.apply(this, arguments);
+  }
+};
+
+// Call the user-defined main.
+require('${mainFile}');
+`;
+}

packages/esbuild/src/utils/get-entry-points.ts

@@ -0,0 +1,71 @@
+import { ExecutorContext, readJsonFile } from '@nrwl/devkit';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as glob from 'fast-glob';
+
+export interface GetEntryPointsOptions {
+  recursive?: boolean;
+  initialEntryPoints?: string[];
+  onProjectFilesMatched?: (projectName: string, files: string[]) => void;
+}
+
+export function getEntryPoints(
+  projectName: string,
+  context: ExecutorContext,
+  options: GetEntryPointsOptions = {}
+): string[] {
+  const tsconfigCandidates = [
+    'tsconfig.app.json',
+    'tsconfig.lib.json',
+    'tsconfig.json',
+    'tsconfig.base.json',
+  ];
+  const entryPoints = options.initialEntryPoints
+    ? new Set(options.initialEntryPoints)
+    : new Set<string>();
+  const seenProjects = new Set<string>();
+
+  const findEntryPoints = (projectName: string): void => {
+    if (seenProjects.has(projectName)) return;
+    seenProjects.add(projectName);
+
+    const project = context.projectGraph?.nodes[projectName];
+    if (!project) return;
+
+    const tsconfigFileName = tsconfigCandidates.find((f) => {
+      try {
+        return fs.statSync(path.join(project.data.root, f)).isFile();
+      } catch {
+        return false;
+      }
+    });
+    // Workspace projects may not be a TS project, so skip reading source files if tsconfig is not found.
+    if (tsconfigFileName) {
+      const tsconfig = readJsonFile(
+        path.join(project.data.root, tsconfigFileName)
+      );
+      const projectFiles = glob
+        .sync(tsconfig.include ?? [], {
+          cwd: project.data.root,
+          ignore: tsconfig.exclude ?? [],
+        })
+        .map((f) => path.join(project.data.root, f));
+
+      projectFiles.forEach((f) => entryPoints.add(f));
+      options?.onProjectFilesMatched?.(projectName, projectFiles);
+    }
+
+    if (options.recursive) {
+      const deps = context.projectGraph.dependencies[projectName];
+      deps.forEach((dep) => {
+        if (context.projectGraph.nodes[dep.target]) {
+          findEntryPoints(dep.target);
+        }
+      });
+    }
+  };
+
+  findEntryPoints(projectName);
+
+  return Array.from(entryPoints);
+}

packages/node/src/generators/application/application.ts

@@ -92,6 +92,7 @@ function getEsBuildConfig(
     executor: '@nrwl/esbuild:esbuild',
     outputs: ['{options.outputPath}'],
     options: {
+      platform: 'node',
       outputPath: joinPathFragments(
         'dist',
         options.rootProject ? options.name : options.appProjectRoot