feat(core): update dotenv and load root env variables earlier (#18456)

Co-authored-by: FrozenPandaz <jasonjean1993@gmail.com>

package.json

@@ -155,7 +155,7 @@
     "cz-git": "^1.4.0",
     "czg": "^1.4.0",
     "detect-port": "^1.5.1",
-    "dotenv": "~10.0.0",
+    "dotenv": "~16.3.1",
     "ejs": "^3.1.7",
     "enhanced-resolve": "^5.8.3",
     "esbuild": "^0.17.5",

packages/nx/bin/init-local.ts

@@ -19,8 +19,6 @@ export function initLocal(workspace: WorkspaceTypeAndRoot) {
 
   try {
     performance.mark('init-local');
-    require('nx/src/utils/perf-logging');
-
     monkeyPatchRequire();
 
     if (workspace.type !== 'nx' && shouldDelegateToAngularCLI()) {

packages/nx/bin/nx.ts

@@ -4,6 +4,7 @@ import {
   WorkspaceTypeAndRoot,
 } from '../src/utils/find-workspace-root';
 import * as chalk from 'chalk';
+import { config as loadDotEnvFile } from 'dotenv';
 import { initLocal } from './init-local';
 import { output } from '../src/utils/output';
 import {
@@ -16,6 +17,7 @@ import { readModulePackageJson } from '../src/utils/package-json';
 import { execSync } from 'child_process';
 import { join } from 'path';
 import { assertSupportedPlatform } from '../src/native/assert-supported-platform';
+import { performance } from 'perf_hooks';
 
 function main() {
   if (
@@ -26,6 +28,17 @@ function main() {
     assertSupportedPlatform();
   }
 
+  require('nx/src/utils/perf-logging');
+
+  performance.mark('loading dotenv files:start');
+  loadDotEnvFiles();
+  performance.mark('loading dotenv files:end');
+  performance.measure(
+    'loading dotenv files',
+    'loading dotenv files:start',
+    'loading dotenv files:end'
+  );
+
   const workspace = findWorkspaceRoot(process.cwd());
   // new is a special case because there is no local workspace to load
   if (
@@ -88,6 +101,20 @@ function main() {
   }
 }
 
+/**
+ * This loads dotenv files from:
+ * - .env
+ * - .local.env
+ * - .env.local
+ */
+function loadDotEnvFiles() {
+  for (const file of ['.env', '.local.env', '.env.local']) {
+    loadDotEnvFile({
+      path: file,
+    });
+  }
+}
+
 function handleNoWorkspace(globalNxVersion?: string) {
   output.log({
     title: `The current directory isn't part of an Nx workspace.`,

packages/nx/package.json

@@ -41,7 +41,7 @@
     "cli-cursor": "3.1.0",
     "cli-spinners": "2.6.1",
     "cliui": "^7.0.2",
-    "dotenv": "~10.0.0",
+    "dotenv": "~16.3.1",
     "enquirer": "~2.3.6",
     "fast-glob": "3.2.7",
     "figures": "3.2.0",

packages/nx/src/tasks-runner/forked-process-task-runner.ts

@@ -1,5 +1,5 @@
 import { readFileSync, writeFileSync } from 'fs';
-import * as dotenv from 'dotenv';
+import { config as loadDotEnvFile } from 'dotenv';
 import { ChildProcess, fork, Serializable } from 'child_process';
 import * as chalk from 'chalk';
 import * as logTransformer from 'strong-log-transformer';
@@ -301,8 +301,6 @@ export class ForkedProcessTaskRunner {
   // region Environment Variables
   private getEnvVariablesForProcess() {
     return {
-      // Start With Dotenv Variables
-      ...this.getDotenvVariablesForForkedProcess(),
       // User Process Env Variables override Dotenv Variables
       ...process.env,
       // Nx Env Variables overrides everything
@@ -318,11 +316,15 @@ export class ForkedProcessTaskRunner {
     outputPath: string,
     streamOutput: boolean
   ) {
+    // Unload any dot env files at the root of the workspace that were loaded on init of Nx.
+    const taskEnv = this.unloadDotEnvFiles({ ...process.env });
+
     const res = {
       // Start With Dotenv Variables
-      ...this.getDotenvVariablesForTask(task),
+      ...(process.env.NX_LOAD_DOT_ENV_FILES === 'true'
-      // User Process Env Variables override Dotenv Variables
+        ? this.loadDotEnvFilesForTask(task, taskEnv)
-      ...process.env,
+        : // If not loading dot env files, ensure env vars created by system are still loaded
+          taskEnv),
       // Nx Env Variables overrides everything
       ...this.getNxEnvVariablesForTask(
         task,
@@ -397,57 +399,76 @@ export class ForkedProcessTaskRunner {
     };
   }
 
-  private getDotenvVariablesForForkedProcess() {
+  private loadDotEnvFilesForTask(
-    return {
+    task: Task,
-      ...parseEnv('.env'),
+    environmentVariables: NodeJS.ProcessEnv
-      ...parseEnv('.local.env'),
+  ) {
-      ...parseEnv('.env.local'),
+    // Collect dot env files that may pertain to a task
-    };
+    const dotEnvFiles = [
+      // Load DotEnv Files for a configuration in the project root
+      ...(task.target.configuration
+        ? [
+            `${task.projectRoot}/.env.${task.target.target}.${task.target.configuration}`,
+            `${task.projectRoot}/.env.${task.target.configuration}`,
+            `${task.projectRoot}/.${task.target.target}.${task.target.configuration}.env`,
+            `${task.projectRoot}/.${task.target.configuration}.env`,
+          ]
+        : []),
+
+      // Load DotEnv Files for a target in the project root
+      `${task.projectRoot}/.env.${task.target.target}`,
+      `${task.projectRoot}/.${task.target.target}.env`,
+      `${task.projectRoot}/.env.local`,
+      `${task.projectRoot}/.local.env`,
+      `${task.projectRoot}/.env`,
+
+      // Load DotEnv Files for a configuration in the workspace root
+      ...(task.target.configuration
+        ? [
+            `.env.${task.target.target}.${task.target.configuration}`,
+            `.env.${task.target.configuration}`,
+            `.${task.target.target}.${task.target.configuration}.env`,
+            `.${task.target.configuration}.env`,
+          ]
+        : []),
+
+      // Load DotEnv Files for a target in the workspace root
+      `.env.${task.target.target}`,
+      `.${task.target.target}.env`,
+
+      // Load base DotEnv Files at workspace root
+      `.env`,
+      `.local.env`,
+      `.env.local`,
+    ];
+
+    for (const file of dotEnvFiles) {
+      loadDotEnvFile({
+        path: file,
+        processEnv: environmentVariables,
+        // Do not override existing env variables as we load
+        override: false,
+      });
+    }
+
+    return environmentVariables;
   }
 
-  private getDotenvVariablesForTask(task: Task) {
+  private unloadDotEnvFiles(environmentVariables: NodeJS.ProcessEnv) {
-    if (process.env.NX_LOAD_DOT_ENV_FILES == 'true') {
+    const unloadDotEnvFile = (filename: string) => {
-      return {
+      let parsedDotEnvFile: NodeJS.ProcessEnv = {};
-        ...this.getDotenvVariablesForForkedProcess(),
+      loadDotEnvFile({ path: filename, processEnv: parsedDotEnvFile });
-        ...parseEnv(`.${task.target.target}.env`),
+      Object.keys(parsedDotEnvFile).forEach((envVarKey) => {
-        ...parseEnv(`.env.${task.target.target}`),
+        if (environmentVariables[envVarKey] === parsedDotEnvFile[envVarKey]) {
-        ...(task.target.configuration
+          delete environmentVariables[envVarKey];
-          ? {
+        }
-              ...parseEnv(`.${task.target.configuration}.env`),
+      });
-              ...parseEnv(
+    };
-                `.${task.target.target}.${task.target.configuration}.env`
+
-              ),
+    for (const file of ['.env', '.local.env', '.env.local']) {
-              ...parseEnv(`.env.${task.target.configuration}`),
+      unloadDotEnvFile(file);
-              ...parseEnv(
-                `.env.${task.target.target}.${task.target.configuration}`
-              ),
-            }
-          : {}),
-        ...parseEnv(`${task.projectRoot}/.env`),
-        ...parseEnv(`${task.projectRoot}/.local.env`),
-        ...parseEnv(`${task.projectRoot}/.env.local`),
-        ...parseEnv(`${task.projectRoot}/.${task.target.target}.env`),
-        ...parseEnv(`${task.projectRoot}/.env.${task.target.target}`),
-        ...(task.target.configuration
-          ? {
-              ...parseEnv(
-                `${task.projectRoot}/.${task.target.configuration}.env`
-              ),
-              ...parseEnv(
-                `${task.projectRoot}/.${task.target.target}.${task.target.configuration}.env`
-              ),
-              ...parseEnv(
-                `${task.projectRoot}/.env.${task.target.configuration}`
-              ),
-              ...parseEnv(
-                `${task.projectRoot}/.env.${task.target.target}.${task.target.configuration}`
-              ),
-            }
-          : {}),
-      };
-    } else {
-      return {};
     }
+    return environmentVariables;
   }
 
   // endregion Environment Variables
@@ -507,13 +528,6 @@ export class ForkedProcessTaskRunner {
   }
 }
 
-function parseEnv(path: string) {
-  try {
-    const envContents = readFileSync(path);
-    return dotenv.parse(envContents);
-  } catch (e) {}
-}
-
 const colors = [
   chalk.green,
   chalk.greenBright,

pnpm-lock.yaml

@@ -541,8 +541,8 @@ devDependencies:
     specifier: ^1.5.1
     version: 1.5.1
   dotenv:
-    specifier: ~10.0.0
+    specifier: ~16.3.1
-    version: 10.0.0
+    version: 16.3.1
   ejs:
     specifier: ^3.1.7
     version: 3.1.8
@@ -13732,8 +13732,8 @@ packages:
     engines: {node: '>=10'}
     dev: true
 
-  /dotenv@16.0.3:
+  /dotenv@16.3.1:
-    resolution: {integrity: sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==}
+    resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==}
     engines: {node: '>=12'}
     dev: true
 
@@ -17989,7 +17989,7 @@ packages:
     engines: {node: '>=14.0.0'}
     dependencies:
       app-root-dir: 1.0.2
-      dotenv: 16.0.3
+      dotenv: 16.3.1
       dotenv-expand: 10.0.0
     dev: true