chore(repo): dogfood nx release commands (#19237)

.gitignore

@@ -24,7 +24,7 @@ jest.debug.config.js
 # Issues scraper creates these files, stored by github's cache
 /scripts/issues-scraper/cached
 
-# Lerna creates this
+# We don't commit a CHANELGELOG.md file to the repo, we only create Github releases
 CHANGELOG.md
 
 # Next.js

CONTRIBUTING.md

@@ -112,9 +112,7 @@ Yarn Berry operates slightly differently than Yarn Classic. In order to publish
     - localhost
   ```
 
-- Run `pnpm nx-release --local` in Terminal 2 to publish next minor version. If this version already exists, you can
-  bump the minor version in `lerna.json` to toggle the next minor. The output will report the version of published
-  packages.
+- Run `pnpm nx-release minor --local` in Terminal 2 to publish next minor version. The output will report the version of published packages.
 - Go to your target folder (e.g. `cd ./tmp`) in Terminal 2
 - Run `yarn dlx create-nx-workspace@123.4.5` in Terminal 2 (replace `123.4.5` with the version that got published).
 

e2e/utils/global-setup.ts

@@ -8,6 +8,8 @@ import { existsSync, removeSync } from 'fs-extra';
 import { Config } from '@jest/types';
 import * as isCI from 'is-ci';
 
+const LARGE_BUFFER = 1024 * 1000000;
+
 export default async function (globalConfig: Config.ConfigGlobals) {
   const isVerbose: boolean =
     process.env.NX_VERBOSE_LOGGING === 'true' || !!globalConfig.verbose;
@@ -33,6 +35,7 @@ export default async function (globalConfig: Config.ConfigGlobals) {
     await new Promise<void>((res, rej) => {
       const publishProcess = exec(`pnpm nx-release --local ${publishVersion}`, {
         env: process.env,
+        maxBuffer: LARGE_BUFFER,
       });
       let logs = Buffer.from('');
       if (isVerbose) {

lerna.json

@@ -1,10 +0,0 @@
-{
-  "packages": ["build/packages/*", "build/packages/nx/native-packages/*"],
-  "version": "16.9.0-beta.3",
-  "granularPathspec": false,
-  "command": {
-    "publish": {
-      "graphType": "all"
-    }
-  }
-}

nx.json

@@ -56,7 +56,25 @@
       }
     ]
   },
+  "release": {
+    "groups": {
+      "npm": {
+        "projects": ["packages/*", "packages-legacy/*"],
+        "version": {
+          "generatorOptions": {
+            "packageRoot": "build/packages/{projectName}",
+            "currentVersionResolver": "registry"
+          }
+        }
+      }
+    }
+  },
   "targetDefaults": {
+    "nx-release-publish": {
+      "options": {
+        "packageRoot": "build/packages/{projectName}"
+      }
+    },
     "build": {
       "dependsOn": ["build-base", "build-native"],
       "inputs": [

package.json

@@ -14,7 +14,6 @@
     "check-codeowners": "ts-node -P ./scripts/tsconfig.scripts.json ./scripts/check-codeowners.ts",
     "nx-release": "ts-node -P ./scripts/tsconfig.release.json ./scripts/nx-release",
     "prepublishOnly": "node ./scripts/update-package-group.js",
-    "version": "pnpm prettier lerna.json --write",
     "local-registry": "nx local-registry @nx/nx-source",
     "documentation": "ts-node -P scripts/tsconfig.scripts.json ./scripts/documentation/generators/main.ts && pnpm check-documentation-map",
     "submit-plugin": "node ./scripts/submit-plugin.js",
@@ -209,7 +208,6 @@
     "jsonc-eslint-parser": "^2.1.0",
     "jsonc-parser": "3.2.0",
     "kill-port": "^1.6.1",
-    "lerna": "6.6.2",
     "less": "4.1.3",
     "less-loader": "11.1.0",
     "license-webpack-plugin": "^4.0.2",
@@ -371,10 +369,6 @@
       "check-codeowners",
       "documentation"
     ]
-  },
-  "pnpm": {
-    "patchedDependencies": {
-      "lerna@6.6.2": "patches/lerna@6.6.2.patch"
-    }
   }
 }
+

packages/nx/native-packages/darwin-arm64/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/darwin-x64/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/freebsd-x64/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/linux-arm-gnueabihf/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/linux-arm64-gnu/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/linux-arm64-musl/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/linux-x64-gnu/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/linux-x64-musl/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/win32-arm64-msvc/project.json

@@ -0,0 +1 @@
+{}

packages/nx/native-packages/win32-x64-msvc/project.json

@@ -0,0 +1 @@
+{}

packages/nx/project.json

@@ -19,8 +19,12 @@
         }
       }
     },
+    "copy-native-package-directories": {
+      "command": "cp -R build/packages/nx/native-packages/* build/packages"
+    },
     "artifacts": {
-      "command": "pnpm napi artifacts -c build/packages/nx/package.json -d ./artifacts --dist build/packages/nx/native-packages"
+      "dependsOn": ["copy-native-package-directories"],
+      "command": "pnpm napi artifacts -c build/packages/nx/package.json -d ./artifacts --dist build/packages"
     },
     "build-base": {
       "executor": "@nx/js:tsc",

patches/lerna@6.6.2.patch

@@ -1,48 +0,0 @@
-diff --git a/dist/cli.js b/dist/cli.js
-index 8cd49ff1e99dca8241a75bc412c669a85475fce2..8f081b7ee0769a493b11f22c5d5d0752e8285629 100755
---- a/dist/cli.js
-+++ b/dist/cli.js
-@@ -7933,7 +7933,10 @@ var require_src10 = __commonJS({
-                 logPacked(pkg.packed);
-                 return pkg;
-               }).catch((err) => {
--                if (err.code === "EPUBLISHCONFLICT") {
-+                if (
-+                  err.code === "EPUBLISHCONFLICT" ||
-+                  (err.code === "E403" &&
-+                    err.body?.error?.includes("You cannot publish over the previously published versions"))) {
-                   tracker.warn("publish", `Package is already published: ${pkg.name}@${pkg.version}`);
-                   tracker.completeWork(1);
-                   return pkg;
-diff --git a/dist/commands/publish/index.js b/dist/commands/publish/index.js
-index 34fe8f496af5e7adaf58a7f9c4c3a11d812608c3..bbab4d70c044729b174208b2f1250f54ab69e56d 100644
---- a/dist/commands/publish/index.js
-+++ b/dist/commands/publish/index.js
-@@ -6094,7 +6094,10 @@ var require_src2 = __commonJS({
-                 logPacked(pkg.packed);
-                 return pkg;
-               }).catch((err) => {
--                if (err.code === "EPUBLISHCONFLICT") {
-+                if (
-+                  err.code === "EPUBLISHCONFLICT" ||
-+                  (err.code === "E403" &&
-+                    err.body?.error?.includes("You cannot publish over the previously published versions"))) {
-                   tracker.warn("publish", `Package is already published: ${pkg.name}@${pkg.version}`);
-                   tracker.completeWork(1);
-                   return pkg;
-diff --git a/dist/index.js b/dist/index.js
-index a1f83a569598efc7f129d8de7d004f45177514f4..86364db7a267c0fa3eaafc9b1bc51838a41f4f4f 100644
---- a/dist/index.js
-+++ b/dist/index.js
-@@ -7932,7 +7932,10 @@ var require_src10 = __commonJS({
-                 logPacked(pkg2.packed);
-                 return pkg2;
-               }).catch((err) => {
--                if (err.code === "EPUBLISHCONFLICT") {
-+                if (
-+                  err.code === "EPUBLISHCONFLICT" ||
-+                  (err.code === "E403" &&
-+                    err.body?.error?.includes("You cannot publish over the previously published versions"))) {
-                   tracker.warn("publish", `Package is already published: ${pkg2.name}@${pkg2.version}`);
-                   tracker.completeWork(1);
-                   return pkg2;

scripts/nx-release.ts

@@ -1,29 +1,19 @@
 #!/usr/bin/env node
-import * as yargs from 'yargs';
-import { execSync } from 'child_process';
-import { existsSync, readFileSync, rmSync, writeFileSync } from 'fs';
+import { createProjectGraphAsync, workspaceRoot } from '@nx/devkit';
+import { execSync } from 'node:child_process';
+import { rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { URL } from 'node:url';
-import { join } from 'path';
+import { isRelativeVersionKeyword } from 'nx/src/command-line/release/utils/semver';
+import { ReleaseType, parse } from 'semver';
+import * as yargs from 'yargs';
 
-import { parse } from 'semver';
-
-const version = require('lerna/commands/version');
-const publish = require('lerna/commands/publish');
-
-const lernaJsonPath = join(__dirname, '../lerna.json');
-const originalLernaJson = readFileSync(lernaJsonPath);
-
-function hideFromGitIndex(uncommittedFiles: string[]) {
-  execSync(`git update-index --assume-unchanged ${uncommittedFiles.join(' ')}`);
-
-  return () =>
-    execSync(
-      `git update-index --no-assume-unchanged ${uncommittedFiles.join(' ')}`
-    );
-}
+const LARGE_BUFFER = 1024 * 1000000;
 
 (async () => {
   const options = parseArgs();
+  // Perform minimal logging by default
+  let isVerboseLogging = process.env.NX_VERBOSE_LOGGING === 'true';
 
   if (options.clearLocalRegistry) {
     rmSync(join(__dirname, '../build/local-registry/storage'), {
@@ -32,123 +22,135 @@ function hideFromGitIndex(uncommittedFiles: string[]) {
     });
   }
 
-  const currentLatestVersion = execSync('npm view nx version')
-    .toString()
-    .trim();
-
-  const parsedVersion = parse(options.version);
-  const parsedCurrentLatestVersion = parse(currentLatestVersion);
-
-  const distTag =
-    parsedVersion?.prerelease.length > 0
-      ? 'next'
-      : parsedVersion?.major < parsedCurrentLatestVersion.major
-      ? 'previous'
-      : 'latest';
-
   const buildCommand = 'pnpm build';
   console.log(`> ${buildCommand}`);
   execSync(buildCommand, {
     stdio: [0, 1, 2],
+    maxBuffer: LARGE_BUFFER,
   });
 
-  if (options.local) {
-    updateLernaJsonVersion(currentLatestVersion);
-  }
-
-  if (options.local) {
-    // Force all projects to be not private
-    const projects = JSON.parse(
-      execSync('npx lerna list --json --all').toString()
-    );
-    for (const proj of projects) {
-      if (proj.private) {
-        console.log('Publishing private package locally:', proj.name);
-        const packageJsonPath = join(proj.location, 'package.json');
-        const original = JSON.parse(readFileSync(packageJsonPath, 'utf-8'));
-        writeFileSync(
-          packageJsonPath,
-          JSON.stringify({ ...original, private: false })
-        );
-      }
-    }
-  }
+  // Ensure all the native-packages directories are available at the top level of the build directory, enabling consistent packageRoot structure
+  execSync(`pnpm nx copy-native-package-directories nx`, {
+    stdio: isVerboseLogging ? [0, 1, 2] : 'ignore',
+    maxBuffer: LARGE_BUFFER,
+  });
 
+  // Expected to run as part of the Github `publish` workflow
   if (!options.local && process.env.NPM_TOKEN) {
     // Delete all .node files that were built during the previous steps
     // Always run before the artifacts step because we still need the .node files for native-packages
     execSync('find ./build -name "*.node" -delete', {
       stdio: [0, 1, 2],
+      maxBuffer: LARGE_BUFFER,
     });
 
-    execSync('npx nx run-many --target=artifacts', {
+    execSync('pnpm nx run-many --target=artifacts', {
       stdio: [0, 1, 2],
+      maxBuffer: LARGE_BUFFER,
     });
   }
 
-  const versionOptions = {
-    bump: options.version ? options.version : undefined,
-    conventionalCommits: true,
-    conventionalPrerelease: options.tag === 'next',
-    preid: options.preid,
-    forcePublish: true,
-    createRelease: 'github',
-    tagVersionPrefix: '',
-    exact: true,
-    gitRemote: options.gitRemote,
-    gitTagVersion: !process.env.NPM_TOKEN,
-    message: 'chore(misc): publish %v',
-    loglevel: options.loglevel ?? 'info',
-    yes: !!process.env.NPM_TOKEN,
-  };
-
-  if (options.local) {
-    versionOptions.conventionalCommits = false;
-    delete versionOptions.createRelease;
-    versionOptions.gitTagVersion = false;
-    versionOptions.loglevel = options.loglevel ?? 'error';
-    versionOptions.yes = true;
-    versionOptions.bump = options.version ? options.version : 'minor';
-  }
-
-  if (options.local) {
-    /**
-     * Hide changes from Lerna
-     */
-    const uncommittedFiles = execSync('git diff --name-only --relative HEAD .')
-      .toString()
-      .split('\n')
-      .filter((i) => i.length > 0)
-      .filter((f) => existsSync(f));
-    const unhideFromGitIndex = hideFromGitIndex(uncommittedFiles);
-
-    process.on('exit', unhideFromGitIndex);
-    process.on('SIGTERM', unhideFromGitIndex);
-    process.on('SIGINT', unhideFromGitIndex);
-  }
-
-  const publishOptions: Record<string, boolean | string | undefined> = {
-    gitReset: false,
-    distTag: distTag,
+  const runNxReleaseVersion = () => {
+    let versionCommand = `pnpm nx release version${
+      options.version ? ` --specifier ${options.version}` : ''
+    }`;
+    if (options.dryRun) {
+      versionCommand += ' --dry-run';
+    }
+    console.log(`> ${versionCommand}`);
+    execSync(versionCommand, {
+      stdio: isVerboseLogging ? [0, 1, 2] : 'ignore',
+      maxBuffer: LARGE_BUFFER,
+    });
   };
 
+  // Intended for creating a github release which triggers the publishing workflow
   if (!options.local && !process.env.NPM_TOKEN) {
+    // For this important use-case it makes sense to always have full logs
+    isVerboseLogging = true;
+
     execSync('git status --ahead-behind');
 
-    await version(versionOptions);
+    if (isRelativeVersionKeyword(options.version)) {
+      throw new Error(
+        'When creating actual releases, you must use an exact semver version'
+      );
+    }
+
+    runNxReleaseVersion();
+
+    let changelogCommand = `pnpm nx release changelog ${options.version} --tagVersionPrefix="" --interactive`;
+    if (options.from) {
+      changelogCommand += ` --from ${options.from}`;
+    }
+    if (options.gitRemote) {
+      changelogCommand += ` --git-remote ${options.gitRemote}`;
+    }
+    if (options.dryRun) {
+      changelogCommand += ' --dry-run';
+    }
+    console.log(`> ${changelogCommand}`);
+    execSync(changelogCommand, {
+      stdio: isVerboseLogging ? [0, 1, 2] : 'ignore',
+      maxBuffer: LARGE_BUFFER,
+    });
+
     console.log(
       'Check github: https://github.com/nrwl/nx/actions/workflows/publish.yml'
     );
-  } else if (!options.skipPublish) {
-    await publish({ ...versionOptions, ...publishOptions });
-  } else {
-    await version(versionOptions);
-    console.warn('Not Publishing because --dryRun was passed');
+    process.exit(0);
   }
 
-  if (options.local) {
-    restoreOriginalLernaJson();
+  runNxReleaseVersion();
+
+  if (options.dryRun) {
+    console.warn('Not Publishing because --dryRun was passed');
+  } else {
+    // If publishing locally, force all projects to not be private first
+    if (options.local) {
+      console.log(
+        '\nPublishing locally, so setting all resolved packages to not be private'
+      );
+      const projectGraph = await createProjectGraphAsync();
+      for (const proj of Object.values(projectGraph.nodes)) {
+        if (proj.data.targets?.['nx-release-publish']) {
+          const packageJsonPath = join(
+            workspaceRoot,
+            proj.data.targets?.['nx-release-publish']?.options.packageRoot,
+            'package.json'
+          );
+          try {
+            const packageJson = require(packageJsonPath);
+            if (packageJson.private) {
+              console.log(
+                '- Publishing private package locally:',
+                packageJson.name
+              );
+              writeFileSync(
+                packageJsonPath,
+                JSON.stringify({ ...packageJson, private: false })
+              );
+            }
+          } catch {}
+        }
+      }
+    }
+
+    const distTag = determineDistTag(options.version);
+
+    // Run with dynamic output-style so that we have more minimal logs by default but still always see errors
+    let publishCommand = `pnpm nx release publish --registry=${getRegistry()} --tag=${distTag} --output-style=dynamic --parallel=8`;
+    if (options.dryRun) {
+      publishCommand += ' --dry-run';
+    }
+    console.log(`\n> ${publishCommand}`);
+    execSync(publishCommand, {
+      stdio: [0, 1, 2],
+      maxBuffer: LARGE_BUFFER,
+    });
   }
+
+  process.exit(0);
 })();
 
 function parseArgs() {
@@ -161,9 +163,9 @@ function parseArgs() {
       '$0 [version]',
       'This script is for publishing Nx both locally and publically'
     )
-    .option('skipPublish', {
+    .option('dryRun', {
       type: 'boolean',
-      description: 'Skips the actual publishing for testing out versioning',
+      description: 'Dry-run flag to be passed to all `nx release` commands',
     })
     .option('clearLocalRegistry', {
       type: 'boolean',
@@ -182,6 +184,11 @@ function parseArgs() {
       description: "Don't use this unless you really know what it does",
       hidden: true,
     })
+    .option('from', {
+      type: 'string',
+      description:
+        'Git ref to pass to `nx release changelog`. Not applicable for local publishing or e2e tests.',
+    })
     .positional('version', {
       type: 'string',
       description:
@@ -193,11 +200,6 @@ function parseArgs() {
         'Alternate git remote name to publish tags to (useful for testing changelog)',
       default: 'origin',
     })
-    .option('loglevel', {
-      type: 'string',
-      description: 'Log Level',
-      choices: ['error', 'info', 'debug'],
-    })
     .example(
       '$0',
       `By default, this will locally publish a minor version bump as latest. Great for local development. Most developers should only need this.`
@@ -222,6 +224,7 @@ function parseArgs() {
       ['gitRemote', 'force'],
       'Real Publishing Options for actually publishing to NPM'
     )
+    .demandOption('version')
     .check((args) => {
       const registry = getRegistry();
       const registryIsLocalhost = registry.hostname === 'localhost';
@@ -244,23 +247,48 @@ function parseArgs() {
     })
     .parseSync();
 
-  parsedArgs.tag ??= parsedArgs.local ? 'latest' : 'next';
-
   return parsedArgs;
 }
 
-function updateLernaJsonVersion(version: string) {
-  const json = JSON.parse(readFileSync(lernaJsonPath).toString());
-
-  json.version = version;
-
-  writeFileSync(lernaJsonPath, JSON.stringify(json));
-}
-
-function restoreOriginalLernaJson() {
-  writeFileSync(lernaJsonPath, originalLernaJson);
-}
-
 function getRegistry() {
   return new URL(execSync('npm config get registry').toString().trim());
 }
+
+function determineDistTag(newVersion: string): 'latest' | 'next' | 'previous' {
+  // For a relative version keyword, it cannot be previous
+  if (isRelativeVersionKeyword(newVersion)) {
+    const prereleaseKeywords: ReleaseType[] = [
+      'premajor',
+      'preminor',
+      'prepatch',
+      'prerelease',
+    ];
+    return prereleaseKeywords.includes(newVersion) ? 'next' : 'latest';
+  }
+
+  const parsedGivenVersion = parse(newVersion);
+  if (!parsedGivenVersion) {
+    throw new Error(
+      `Unable to parse the given version: "${newVersion}". Is it valid semver?`
+    );
+  }
+
+  const currentLatestVersion = execSync('npm view nx version')
+    .toString()
+    .trim();
+  const parsedCurrentLatestVersion = parse(currentLatestVersion);
+  if (!parsedCurrentLatestVersion) {
+    throw new Error(
+      `The current version resolved from the npm registry could not be parsed (resolved "${currentLatestVersion}")`
+    );
+  }
+
+  const distTag =
+    parsedGivenVersion.prerelease.length > 0
+      ? 'next'
+      : parsedGivenVersion.major < parsedCurrentLatestVersion.major
+      ? 'previous'
+      : 'latest';
+
+  return distTag;
+}