chore(repo): refactor publish.yml for PR releases (#26509)

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,6 +4,8 @@
 <!-- Please make sure that your commit message follows our format -->
 <!-- Example: `fix(nx): must begin with lowercase` -->
 
+<!-- If this is a particularly complex change or feature addition, you can request a dedicated Nx release for this pull request branch. Mention someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they will confirm if the PR warrants its own release for testing purposes, and generate it for you if appropriate. -->
+
 ## Current Behavior
 <!-- This is the behavior we have today -->
 

.github/workflows/publish.yml

@@ -1,19 +1,122 @@
 name: publish
 
 on:
+  # Automated schedule - canary releases from master
   schedule:
     - cron: "0 3 * * 2-6" # Tuesdays - Saturdays, at 3am UTC
+  # Manual trigger - PR releases or dry-runs (based on workflow inputs)
   workflow_dispatch:
+    inputs:
+      pr:
+        description: "If set, a real release will be created for the branch associated with the given PR number. If blank, a dry-run of the currently selected branch will be performed."
+        required: false
+        type: number
   release:
     types: [ published ]
 
+# Dynamically generate the display name for the GitHub UI based on the event type and inputs
+run-name: ${{ github.event.inputs.pr && format('PR Release for {0}', github.event.inputs.pr) || github.event_name == 'schedule' && 'Canary Release' || github.event_name == 'workflow_dispatch' && !github.event.inputs.pr && 'Release Dry-Run' || github.ref_name }}
+
 env:
   DEBUG: napi:*
   NX_RUN_GROUP: ${{ github.run_id }}-${{ github.run_attempt }}
   CYPRESS_INSTALL_BINARY: 0
+  NODE_VERSION: 18
+  PNPM_VERSION: 8.15.7 # Aligned with root package.json (pnpm/action-setup will helpfully error if out of sync)
 
 jobs:
+  # We first need to determine the version we are releasing, and if we need a custom repo or ref to use for the git checkout in subsequent steps.
+  # These values depend upon the event type that triggered the workflow:
+  #
+  # - schedule:
+  #   - We are running a canary release which always comes from the master branch, we can use default ref resolution
+  #     in actions/checkout. The exact version will be generated within scripts/nx-release.ts.
+  #
+  # - release:
+  #   - We are running a full release which is based on the tag that triggered the release event, we can use default
+  #     ref resolution in actions/checkout. The exact version will be generated within scripts/nx-release.ts.
+  #
+  # - workflow_dispatch:
+  #   - We are either running a dry-run on the current branch, in which case the version will be statica and we can use
+  #     default ref resolution in actions/checkout, or we are creating a PR release for the given PR number, in which case
+  #     we should generate an applicable version number within publish-resolve-data.js and use a custom ref of the PR branch name.
+  resolve-required-data:
+    name: Resolve Required Data
+    if: ${{ github.repository_owner == 'nrwl' }}
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.script.outputs.version }}
+      dry_run_flag: ${{ steps.script.outputs.dry_run_flag }}
+      success_comment: ${{ steps.script.outputs.success_comment }}
+      publish_branch: ${{ steps.script.outputs.publish_branch }}
+      ref: ${{ steps.script.outputs.ref }}
+      repo: ${{ steps.script.outputs.repo }}
+    env:
+      NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+    steps:
+      # Default checkout on the triggering branch so that the latest publish-resolve-data.js script is available
+      - uses: actions/checkout@v4
+
+      # Set up pnpm and node so that we can verify our setup and that the NPM_TOKEN secret will work later
+      - uses: pnpm/action-setup@v4
+        with:
+          version: ${{ env.PNPM_VERSION }}
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          registry-url: 'https://registry.npmjs.org'
+          check-latest: true
+          cache: 'pnpm'
+
+      # Ensure that the NPM_TOKEN secret is still valid before wasting any time deriving data or building projects
+      - name: Check NPM Credentials
+        run: npm whoami && echo "NPM credentials are valid" || (echo "NPM credentials are invalid or have expired." && exit 1)
+
+      - name: Resolve and set checkout and version data to use for release
+        id: script
+        uses: actions/github-script@v7
+        env:
+          PR_NUMBER: ${{ github.event.inputs.pr }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const script = require('${{ github.workspace }}/scripts/publish-resolve-data.js');
+            await script({ github, context, core });
+
+      - name: (PR Release Only) Check out latest master
+        if: ${{ steps.script.outputs.ref != '' }}
+        uses: actions/checkout@v4
+        with:
+          # Check out the latest master branch to get its copy of nx-release.ts
+          repository: nrwl/nx
+          ref: master
+          path: latest-master-checkout
+
+      - name: (PR Release Only) Check out PR branch
+        if: ${{ steps.script.outputs.ref != '' }}
+        uses: actions/checkout@v4
+        with:
+          # Check out the PR branch to get its copy of nx-release.ts
+          repository: ${{ steps.script.outputs.repo }}
+          ref: ${{ steps.script.outputs.ref }}
+          path: pr-branch-checkout
+
+      - name: (PR Release Only) Ensure that nx-release.ts has not changed in the PR being released
+        if: ${{ steps.script.outputs.ref != '' }}
+        env:
+          FILE_TO_COMPARE: "scripts/nx-release.ts"
+        run: |
+          if ! cmp -s "latest-master-checkout/${{ env.FILE_TO_COMPARE }}" "pr-branch-checkout/${{ env.FILE_TO_COMPARE }}"; then
+            echo "🛑 Error: The file ${{ env.FILE_TO_COMPARE }} is different on the ${{ steps.script.outputs.ref }} branch on ${{ steps.script.outputs.repo }} vs latest master on nrwl/nx, cancelling workflow."
+            exit 1
+          else
+            echo "✅ The file ${{ env.FILE_TO_COMPARE }} is identical between the ${{ steps.script.outputs.ref }} branch on ${{ steps.script.outputs.repo }} and latest master on nrwl/nx."
+          fi
+
   build:
+    needs: [resolve-required-data]
     if: ${{ github.repository_owner == 'nrwl' }}
     strategy:
       fail-fast: false
@@ -100,16 +203,19 @@ jobs:
     runs-on: ${{ matrix.settings.host }}
     steps:
       - uses: actions/checkout@v4
-
-      - uses: pnpm/action-setup@v2
         with:
-          version: 8
+          repository: ${{ needs.resolve-required-data.outputs.repo }} 
+          ref: ${{ needs.resolve-required-data.outputs.ref }}
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: ${{ env.PNPM_VERSION }}
 
       - name: Setup node
         uses: actions/setup-node@v4
         if: ${{ !matrix.settings.docker }}
         with:
-          node-version: 18
+          node-version: ${{ env.NODE_VERSION }}
           check-latest: true
           cache: 'pnpm'
 
@@ -120,7 +226,7 @@ jobs:
           targets: ${{ matrix.settings.target }}
 
       - name: Cache cargo
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             ~/.cargo/registry/index/
@@ -129,30 +235,36 @@ jobs:
             .cargo-cache
             target/
           key: ${{ matrix.settings.target }}-cargo-registry
+
       - uses: goto-bus-stop/setup-zig@v2
         if: ${{ matrix.settings.target == 'armv7-unknown-linux-gnueabihf' }}
         with:
           version: 0.10.0
+
       - name: Setup toolchain
         run: ${{ matrix.settings.setup }}
         if: ${{ matrix.settings.setup }}
         shell: bash
+
       - name: Setup node x86
         if: matrix.settings.target == 'i686-pc-windows-msvc'
         run: yarn config set supportedArchitectures.cpu "ia32"
         shell: bash
+
       - name: Install dependencies
         if: ${{ !matrix.settings.docker }}
         run: pnpm install --frozen-lockfile
         timeout-minutes: 30
+
       - name: Setup node x86
         uses: actions/setup-node@v4
         if: matrix.settings.target == 'i686-pc-windows-msvc'
         with:
-          node-version: 18
+          node-version: ${{ env.NODE_VERSION }}
           check-latest: true
           cache: pnpm
           architecture: x86
+
       - name: Build in docker
         uses: addnab/docker-run-action@v3
         if: ${{ matrix.settings.docker }}
@@ -160,28 +272,35 @@ jobs:
           image: ${{ matrix.settings.docker }}
           options: --user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index -v ${{ github.workspace }}:/build -w /build
           run: ${{ matrix.settings.build }}
+
       - name: Build
         run: ${{ matrix.settings.build }}
         if: ${{ !matrix.settings.docker }}
         shell: bash
+
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: bindings-${{ matrix.settings.target }}
           path: packages/**/*.node
           if-no-files-found: error
 
   build-freebsd:
+    needs: [resolve-required-data]
     if: ${{ github.repository_owner == 'nrwl' }}
     runs-on: macos-13-large
     name: Build FreeBSD
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
-        if: ${{ github.event_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' && !github.event.inputs.pr }}
+        with:
+          repository: ${{ needs.resolve-required-data.outputs.repo }} 
+          ref: ${{ needs.resolve-required-data.outputs.ref }}
+
       - name: Build
         id: build
-        if: ${{ github.event_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' && !github.event.inputs.pr }}
         uses: cross-platform-actions/action@v0.22.0
         env:
           DEBUG: napi:*
@@ -223,9 +342,10 @@ jobs:
             echo "KILL ALL NODE PROCESSES"
             killall node || true
             echo "COMPLETE"
+
       - name: Upload artifact
-        if: ${{ github.event_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' && !github.event.inputs.pr }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: bindings-freebsd
           path: packages/**/*.node
@@ -238,7 +358,9 @@ jobs:
     permissions:
       id-token: write
       contents: write
+      pull-requests: write
     needs:
+      - resolve-required-data
       - build-freebsd
       - build
     env:
@@ -247,45 +369,91 @@ jobs:
       NPM_CONFIG_PROVENANCE: true
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
         with:
-          version: 8
+          repository: ${{ needs.resolve-required-data.outputs.repo }}
+          ref: ${{ needs.resolve-required-data.outputs.ref }}
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: ${{ env.PNPM_VERSION }}
+
       - name: Setup node
         uses: actions/setup-node@v4
         with:
-          node-version: 18
+          node-version: ${{ env.NODE_VERSION }}
           registry-url: 'https://registry.npmjs.org'
           check-latest: true
           cache: 'pnpm'
-      - name: Check NPM Credentials
+
-        run: npm whoami && echo "NPM credentials are valid" || (echo "NPM credentials are invalid or have expired." && exit 1)
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
+
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
+
+      # This command will appropriately fail if no artifacts are available
       - name: List artifacts
         run: ls -R artifacts
         shell: bash
+
       - name: Publish
+        env:
+          VERSION: ${{ needs.resolve-required-data.outputs.version }}
+          DRY_RUN: ${{ needs.resolve-required-data.outputs.dry_run_flag }}
+          PUBLISH_BRANCH: ${{ needs.resolve-required-data.outputs.publish_branch }}
         run: |
-          git checkout -b publish/$GITHUB_REF_NAME
+          echo ""
-          # If triggered by the cron, create a canary release
+          # Create and check out the publish branch
-          if [ "${{ github.event_name }}" = "schedule" ]; then
+          git checkout -b $PUBLISH_BRANCH
-            VERSION="canary"
+          echo ""
-          else
+          echo "Version set to: $VERSION"
-            # Otherwise, use the tag name (if triggered via release), or explicit version (if triggered via workflow_dispatch)
+          echo "DRY_RUN set to: $DRY_RUN"
-            VERSION="${GITHUB_REF_NAME}"
+          echo ""
-          fi
-          # If triggered via workflow_dispatch, perform a dry-run
-          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
-            DRY_RUN="--dry-run"
-          else
-            DRY_RUN=""
-          fi
           pnpm nx-release --local=false $VERSION $DRY_RUN
-      - name: Trigger Docs Release
+
-        # Publish docs only on a full release
+      - name: (Stable Release Only) Trigger Docs Release
-        if: ${{ !github.event.release.prerelease }}
+        if: ${{ !github.event.release.prerelease && github.event_name == 'release' }}
         run: npx ts-node ./scripts/release-docs.ts
+
+      - name: (PR Release Only) Create comment for successful PR release
+        if: success() && github.event.inputs.pr
+        uses: actions/github-script@v7
+        env:
+          SUCCESS_COMMENT: ${{ needs.resolve-required-data.outputs.success_comment }}
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const successComment = JSON.parse(process.env.SUCCESS_COMMENT);
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ github.event.inputs.pr }},
+              body: successComment
+            });
+
+  pr_failure_comment:
+    # Run this job if it is a PR release, running on the nrwl origin, and any of the required jobs failed
+    if: ${{ github.repository_owner == 'nrwl' && github.event.inputs.pr && always() && contains(needs.*.result, 'failure') }}
+    needs: [resolve-required-data, build, build-freebsd, publish]
+    name: (PR Release Failure Only) Create comment for failed PR release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create comment for failed PR release
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          # This script is intentionally kept inline (and e.g. not generated in publish-resolve-data.js)
+          # to ensure that an error within the data generation itself is not missed.
+          script: |
+            const message = `
+              Failed to publish a PR release of this pull request, triggered by @${{ github.triggering_actor }}. 
+              See the failed workflow run at: https://github.com/nrwl/nx/actions/runs/${{ github.run_id }}
+            `;
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: ${{ github.event.inputs.pr }},
+              body: message
+            });

CONTRIBUTING.md

@@ -355,3 +355,7 @@ Closes #157
 To simplify and automate the process of committing with this format,
 **Nx is a [Commitizen](https://github.com/commitizen/cz-cli) friendly repository**, just do `git add` and
 execute `pnpm commit`.
+
+#### PR releases
+
+If you are working on a particularly complex change or feature addition, you can request a dedicated Nx release for the associated pull request branch. Mention someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they will confirm if the PR warrants its own release for testing purposes, and generate it for you if appropriate.

scripts/nx-release.ts

@@ -130,9 +130,6 @@ const VALID_AUTHORS_FOR_LATEST = [
 
   const distTag = determineDistTag(options.version);
 
-  if (options.dryRun) {
-    console.warn('Not Publishing because --dryRun was passed');
-  } else {
   // If publishing locally, force all projects to not be private first
   if (options.local) {
     console.log(
@@ -186,6 +183,7 @@ const VALID_AUTHORS_FOR_LATEST = [
     maxBuffer: LARGE_BUFFER,
   });
 
+  if (!options.dryRun) {
     let version;
     if (['minor', 'major', 'patch'].includes(options.version)) {
       version = execSync(`npm view nx@${distTag} version`).toString().trim();
@@ -358,12 +356,17 @@ function getRegistry() {
 
 function determineDistTag(
   newVersion: string
-): 'latest' | 'next' | 'previous' | 'canary' {
+): 'latest' | 'next' | 'previous' | 'canary' | 'pull-request' {
   // Special case of canary
   if (newVersion.includes('canary')) {
     return 'canary';
   }
 
+  // Special case of PR release
+  if (newVersion.startsWith('0.0.0-pr-')) {
+    return 'pull-request';
+  }
+
   // For a relative version keyword, it cannot be previous
   if (isRelativeVersionKeyword(newVersion)) {
     const prereleaseKeywords: ReleaseType[] = [

scripts/publish-resolve-data.js

@@ -0,0 +1,199 @@
+// @ts-check
+
+/**
+ * This function is invoked by the publish.yml GitHub Action workflow and contains all of the dynamic logic needed
+ * for the various workflow trigger types. This avoids the need for the logic to be stored in fragile inline
+ * shell commands.
+ *
+ * @typedef {'--dry-run' | ''} DryRunFlag
+ *
+ * @typedef {{
+ *   version: string;
+ *   dry_run_flag: DryRunFlag;
+ *   success_comment: string;
+ *   publish_branch: string;
+ *   repo: string;
+ *   ref: string;
+ * }} PublishResolveData
+ *
+ * Partial from https://github.com/actions/toolkit/blob/c6b487124a61d7dc6c7bd6ea0208368af3513a6e/packages/github/src/context.ts
+ * @typedef {{
+ *   actor: string;
+ *   runId: number;
+ *   repo: { owner: string; repo: string };
+ * }} GitHubContext
+ *
+ * @param {{
+ *  github: import('octokit/dist-types').Octokit & { ref_name: string };
+ *  context: GitHubContext;
+ *  core: import('@actions/core');
+ * }} param
+ */
+module.exports = async ({ github, context, core }) => {
+  const data = await getPublishResolveData({ github, context });
+
+  // Set the outputs to be consumed in later steps
+  core.setOutput('version', data.version);
+  core.setOutput('dry_run_flag', data.dry_run_flag);
+  core.setOutput('success_comment', JSON.stringify(data.success_comment)); // Escape the multi-line string
+  core.setOutput('publish_branch', data.publish_branch);
+  core.setOutput('ref', data.ref);
+  core.setOutput('repo', data.repo);
+};
+
+/**
+ * @param {{
+ *  github: import('octokit/dist-types').Octokit & { ref_name: string };
+ *  context: GitHubContext;
+ * }} param
+ *
+ * @returns {Promise<PublishResolveData>}
+ */
+async function getPublishResolveData({ github, context }) {
+  // We use empty strings as default values so that we can let the `actions/checkout` action apply its default resolution
+  const DEFAULT_REF = '';
+  const DEFAULT_REPO = '';
+
+  const DEFAULT_PUBLISH_BRANCH = `publish/${github.ref_name}`;
+
+  /** @type {DryRunFlag} */
+  const DRY_RUN_DISABLED = '';
+  /** @type {DryRunFlag} */
+  const DRY_RUN_ENABLED = '--dry-run';
+
+  switch (process.env.GITHUB_EVENT_NAME) {
+    case 'schedule': {
+      const data = {
+        version: 'canary',
+        dry_run_flag: DRY_RUN_DISABLED,
+        success_comment: '',
+        publish_branch: DEFAULT_PUBLISH_BRANCH,
+        // In this case the default checkout logic should use the default (master) branch
+        repo: DEFAULT_REPO,
+        ref: DEFAULT_REF,
+      };
+      console.log('"schedule" trigger detected', { data });
+      return data;
+    }
+
+    case 'release': {
+      const data = {
+        version: github.ref_name,
+        dry_run_flag: DRY_RUN_DISABLED,
+        success_comment: '',
+        publish_branch: DEFAULT_PUBLISH_BRANCH,
+        // In this case the default checkout logic should use the tag that triggered the release event
+        ref: DEFAULT_REF,
+        repo: DEFAULT_REPO,
+      };
+      console.log('"release" trigger detected', { data });
+      return data;
+    }
+
+    case 'workflow_dispatch': {
+      const prNumber = process.env.PR_NUMBER;
+
+      if (!prNumber) {
+        const data = {
+          version: '0.0.0-dry-run.0',
+          dry_run_flag: DRY_RUN_ENABLED,
+          success_comment: '',
+          publish_branch: DEFAULT_PUBLISH_BRANCH,
+          // In this case the default checkout logic should use the branch/tag selected when triggering the workflow
+          repo: DEFAULT_REPO,
+          ref: DEFAULT_REF,
+        };
+        console.log(
+          '"workflow_dispatch" trigger detected, no PR number provided',
+          { data }
+        );
+        return data;
+      }
+
+      const pr = await github.rest.pulls.get({
+        owner: context.repo.owner,
+        repo: context.repo.repo,
+        pull_number: Number(prNumber),
+      });
+      if (!pr?.data?.head?.repo) {
+        throw new Error(
+          `The PR data for PR number ${prNumber} is missing the head branch information`
+        );
+      }
+
+      const fullSHA = pr.data.head.sha;
+      const shortSHA = fullSHA.slice(0, 7);
+      const version = `0.0.0-pr-${prNumber}-${shortSHA}`;
+      const repo = pr.data.head.repo.full_name;
+      const ref = pr.data.head.ref;
+
+      const data = {
+        version,
+        dry_run_flag: DRY_RUN_DISABLED,
+        success_comment: getSuccessCommentForPR({
+          context,
+          version,
+          repo,
+          ref,
+          pr_short_sha: shortSHA,
+          pr_full_sha: fullSHA,
+        }),
+        // Custom publish branch name for PRs
+        publish_branch: `publish/pr-${prNumber}`,
+        // In this case we instruct the checkout action what repo and ref to use
+        repo,
+        ref,
+      };
+      console.log(
+        `"workflow_dispatch" trigger detected, PR number ${prNumber} provided`,
+        { data }
+      );
+
+      console.log(`Owner: ${context.repo.owner}`);
+      console.log(`Repo: ${context.repo.repo}`);
+      console.log(`Fork repo:`, pr.data.head.repo.full_name);
+      console.log(`Fetched PR details: ${pr.data.head.ref}`);
+      console.log(`Full PR SHA: ${pr.data.head.sha}`);
+
+      return data;
+    }
+
+    default:
+      throw new Error(
+        `The publish.yml workflow was triggered by an unexpected event: "${process.env.GITHUB_EVENT_NAME}"`
+      );
+  }
+}
+
+function getSuccessCommentForPR({
+  context,
+  version,
+  repo,
+  ref,
+  pr_short_sha,
+  pr_full_sha,
+}) {
+  return `## 🐳 We have a release for that!
+
+  This PR has a release associated with it. You can try it out using this command:
+  
+  \`\`\`bash
+  npx create-nx-workspace@${version} my-workspace
+  \`\`\`
+
+  Or just copy this version and use it in your own command:
+  \`\`\`bash
+  ${version}
+  \`\`\`
+
+  | Release details | 📑 |
+  | ------------- | ------------- |
+  | **Published version** | [${version}](https://www.npmjs.com/package/nx/v/${version}) |
+  | **Triggered by** | @${context.actor} |
+  | **Branch** | [${ref}](https://github.com/${repo}/tree/${ref}) |
+  | **Commit** | [${pr_short_sha}](https://github.com/${repo}/commit/${pr_full_sha}) |
+  | **Workflow run** | [${context.runId}](https://github.com/nrwl/nx/actions/runs/${context.runId}) |
+
+  To request a new release for this pull request, mention someone from the Nx team or the \`@nrwl/nx-pipelines-reviewers\`.
+`;
+}