feat(repo): add golden list of projects to our Nightly CI (#31414)

This pull request introduces several updates to the CI/CD workflows and
matrix configuration files.

The aim is to highlight critical Nx failures contained in each project
for maintainers to address.

### Changes
- Improvements to workflow caching.
- Improvements to macOS simulator handling.
- Updates to Slack notifications.
- Update matrix data processing for golden projects. 
- Support for Windows has been temporarily disabled due to build issues.

.github/workflows/e2e-matrix.yml

@@ -25,7 +25,8 @@ jobs:
         os:
           - ubuntu-latest
           - macos-latest
-          - windows-latest
+          # - windows-latest Windows fails to build gradle wrapper which always runs when we build nx.
+          ## https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment
         node_version:
           - 20
           - 22
@@ -36,9 +37,9 @@ jobs:
             node_version: 22
           # - os: macos-latest
           #   node_version: 23
-          - os: windows-latest
-            node_version: 22
-          # - os: windows-latest
+          # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
+            # node_version: 22
+          # - os: windows-latest TODO (emily): Windows fails to build gradle wrapper which always runs when we build nx. Re-enable when we fix this.
           #   node_version: 23
 
     name: Cache install (${{ matrix.os }}, node v${{ matrix.node_version }})
@@ -61,13 +62,17 @@ jobs:
           node-version: ${{ matrix.node_version }}
           cache: 'pnpm'
 
-      - name: Cache node_modules
-        id: cache-modules
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: echo "path=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache pnpm store
         uses: actions/cache@v4
         with:
-          lookup-only: true
-          path: '**/node_modules'
-          key: ${{ runner.os }}-modules-${{ matrix.node_version }}-${{ hashFiles('pnpm-lock.yaml') }}
+          path: ${{ steps.pnpm-cache.outputs.path }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
 
       - name: Ensure Python setuptools Installed on Macos
         if: ${{ matrix.os == 'macos-latest' }}
@@ -75,7 +80,6 @@ jobs:
         run: brew install python-setuptools
 
       - name: Install packages
-        if: steps.cache-modules.outputs.cache-hit != 'true'
         run: pnpm install --frozen-lockfile
 
       - name: Homebrew cache directory path
@@ -132,7 +136,7 @@ jobs:
     permissions:
       contents: read
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 70     # <- cap each job to 70 minutes
+    timeout-minutes: 150     # <- cap each job to 150 minutes
     strategy:
       matrix: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} # Load matrix from previous job
       fail-fast: false
@@ -220,33 +224,115 @@ jobs:
       - name: Configure Detox Environment, Install applesimutils
         if: ${{ matrix.os == 'macos-latest' }}
         run: |
-          # Check if applesimutils is already installed
+          # Ensure Xcode command line tools are installed and configured
+          xcode-select --print-path || sudo xcode-select --reset
+          sudo xcode-select -s /Applications/Xcode.app
+          
+          # Install or update applesimutils with error handling
           if ! brew list applesimutils &>/dev/null; then
+            echo "Installing applesimutils..."
             HOMEBREW_NO_AUTO_UPDATE=1 brew tap wix/brew >/dev/null
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null
+            HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils >/dev/null || {
+              echo "Failed to install applesimutils, retrying with update..."
+              brew update
+              HOMEBREW_NO_AUTO_UPDATE=1 brew install applesimutils
+            }
           else
-            echo "applesimutils is already installed, skipping installation"
+            echo "Updating applesimutils..."
+            HOMEBREW_NO_AUTO_UPDATE=1 brew upgrade applesimutils || true
           fi
+          
+          # Verify applesimutils installation
+          applesimutils --version || (echo "applesimutils installation failed" && exit 1)
+          
+          # Configure environment for M-series Mac
+          echo "DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer" >> $GITHUB_ENV
+          echo "PLATFORM_NAME=iOS Simulator" >> $GITHUB_ENV
+          
+          # Set additional environment variables for better debugging
+          echo "DETOX_DISABLE_TELEMETRY=1" >> $GITHUB_ENV
+          echo "DETOX_LOG_LEVEL=trace" >> $GITHUB_ENV
+          
+          # Verify Xcode installation
+          xcodebuild -version
+          
+          # List available simulators
+          xcrun simctl list devices available
         timeout-minutes: 10
+        continue-on-error: false
 
       - name: Reset iOS Simulators
         if: ${{ matrix.os == 'macos-latest' }}
+        id: reset-simulators
         run: |
-          xcrun simctl shutdown all && xcrun simctl erase all
+          echo "Resetting iOS Simulators..."
+          
+          # Kill simulator processes
+          sudo killall -9 com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
+          killall "Simulator" 2>/dev/null || true
+          killall "iOS Simulator" 2>/dev/null || true
+          
+          # Wait for processes to terminate
+          sleep 3
+          
+          # Shutdown and erase all simulators (ignore failures)
+          xcrun simctl shutdown all 2>/dev/null || true
+          sleep 5
+          xcrun simctl erase all 2>/dev/null || true
+          
+          # If erase failed, try the nuclear option
+          if xcrun simctl list devices | grep -q "Booted" 2>/dev/null; then
+            echo "Standard reset failed, using nuclear option..."
+            rm -rf ~/Library/Developer/CoreSimulator/Devices/* 2>/dev/null || true
+            launchctl remove com.apple.CoreSimulator.CoreSimulatorService 2>/dev/null || true
+            sleep 3
+          fi
+          
+          # Clean up additional directories
+          rm -rf ~/Library/Developer/CoreSimulator/Caches/* 2>/dev/null || true
+          rm -rf ~/Library/Logs/CoreSimulator/* 2>/dev/null || true
+          rm -rf ~/Library/Developer/Xcode/DerivedData/* 2>/dev/null || true
+          
+          echo "Simulator reset completed"
+        timeout-minutes: 5
+        continue-on-error: true
+
+      - name: Verify Simulator Reset
+        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
+        run: |
+          # Verify CoreSimulator service restarted
+          pgrep -fl "CoreSimulator" || (echo "CoreSimulator service not running" && exit 1)
+          
+          # Check simulator list is clean
+          xcrun simctl list devices
+          
+          # Verify simulator runtime paths exist and are writable
+          test -d ~/Library/Developer/CoreSimulator/Devices || (echo "Simulator devices directory missing" && exit 1)
+          touch ~/Library/Developer/CoreSimulator/Devices/test || (echo "Simulator devices directory not writable" && exit 1)
+          rm ~/Library/Developer/CoreSimulator/Devices/test
         timeout-minutes: 5
 
+      - name: Diagnose Simulator Reset Failure
+        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'failure' }}
+        run: |
+          echo "Simulator reset failed. Collecting diagnostic information..."
+          xcrun simctl list
+          echo "Checking simulator logs..."
+          ls -la ~/Library/Logs/CoreSimulator/ || echo "No simulator logs found"
+
       - name: Configure git metadata (needed for lerna smoke tests)
+        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
         run: |
           git config --global user.email test@test.com
           git config --global user.name "Test Test"
 
       - name: Set starting timestamp
+        if: ${{ (matrix.os != 'macos-latest') || (matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success') }}
         id: before-e2e
         shell: bash
         run: |
           echo "timestamp=$(date +%s)" >> $GITHUB_OUTPUT
 
-
       - name: Run e2e tests with pnpm (Linux/Windows)
         id: e2e-run-pnpm
         if: ${{ matrix.os != 'macos-latest' }}
@@ -270,7 +356,7 @@ jobs:
   
       - name: Run e2e tests with npm (macOS)
         id: e2e-run-npm
-        if: ${{ matrix.os == 'macos-latest' }}
+        if: ${{ matrix.os == 'macos-latest' && steps.reset-simulators.outcome == 'success' }}
         run: |
           # Run the tests
           if [[ "${{ matrix.project }}" == "e2e-detox" ]] || [[ "${{ matrix.project }}" == "e2e-react-native" ]] || [[ "${{ matrix.project }}" == "e2e-expo" ]]; then
@@ -328,16 +414,23 @@ jobs:
           sudo: ${{ matrix.os != 'windows-latest' }} # disable sudo for windows debugging
 
   process-result:
-    if: ${{ always() && github.repository_owner == 'nrwl' }}
+    if: ${{ always() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     needs: e2e
     timeout-minutes: 10
     outputs:
-      message: ${{ steps.process-json.outputs.SLACK_MESSAGE }}
-      proj-duration: ${{ steps.process-json.outputs.SLACK_PROJ_DURATION }}
-      pm-duration: ${{ steps.process-json.outputs.SLACK_PM_DURATION }}
-      codeowners: ${{ steps.process-json.outputs.CODEOWNERS }}
+      message: ${{ steps.process-json.outputs.slack_message }}
+      proj_duration: ${{ steps.process-json.outputs.slack_proj_duration }}
+      pm_duration: ${{ steps.process-json.outputs.slack_pm_duration }}
+      codeowners: ${{ steps.process-json.outputs.codeowners }}
+      has_golden_failures: ${{ steps.process-json.outputs.has_golden_failures }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          filter: tree:0
+
       - name: Prepare dir for output
         run: mkdir -p outputs
 
@@ -349,145 +442,16 @@ jobs:
       - name: Join and stringify matrix configs
         id: combine-json
         run: |
-          combined=$((jq -s . outputs/*/matrix.json) | jq tostring)
+          combined=$(jq -sc . outputs/*/matrix.json)
           echo "combined=$combined" >> $GITHUB_OUTPUT
 
-      - name: Make slack outputs
+      - name: Process results with TypeScript script
         id: process-json
-        uses: actions/github-script@v7
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          script: |
-            const combined = JSON.parse(${{ steps.combine-json.outputs.combined }});
-            const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project));
-
-            // codeowners
-            const codeowners = new Set();
-            failedProjects.forEach(c => {
-              codeowners.add(c.codeowners);
-            });
-            core.setOutput('CODEOWNERS', Array.from(codeowners).join(','));
-
-            function trimSpace(res) {
-              return res.split('\n').map((l) => l.trim()).join('\n');
-            }
-
-            // failed message
-            let lastProject;
-            let result = `
-              \`\`\`
-              | Failed project                 | PM   | OS    | Node |
-              |--------------------------------|------|-------|------|`;
-            failedProjects.forEach(matrix => {
-              const project = matrix.project !== lastProject ? matrix.project : '...';
-              result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name} | v${matrix.node_version}  |`
-              lastProject = matrix.project;
-            });
-            result += `\`\`\``;
-            core.setOutput('SLACK_MESSAGE', trimSpace(result));
-            console.log(trimSpace(result));
-
-            function humanizeDuration(num) {
-              let res = '';
-              const hours = Math.floor(num / 3600);
-              if (hours) {
-                res += `${hours}h `;
-              }
-              const mins = Math.floor((num % 3600) / 60);
-              if (mins) {
-                res += `${mins}m `;
-              }
-              const sec = num % 60;
-              if (sec) {
-                res += `${sec}s`
-              }
-              return res;
-            }
-
-            // duration message
-            const timeReport = {};
-            const pmReport = {
-              npm: 0,
-              yarn: 0,
-              pnpm: 0
-            };
-            const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native'];
-            combined.forEach((matrix) => {
-              if (matrix.os_name === 'Linux' && matrix.node_version === 20) {
-                pmReport[matrix.package_manager] += matrix.duration;
-              }
-              if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) {
-                if (timeReport[matrix.project]) {
-                  if (matrix.duration > timeReport[matrix.project].max) {
-                    timeReport[matrix.project].max = matrix.duration;
-                    timeReport[
-                      matrix.project
-                    ].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`;
-                  }
-                  if (matrix.duration < timeReport[matrix.project].min) {
-                    timeReport[matrix.project].min = matrix.duration;
-                    timeReport[
-                      matrix.project
-                    ].minEnv = `${matrix.os_name}, ${matrix.package_manager}`;
-                  }
-                } else {
-                  timeReport[matrix.project] = {
-                    min: matrix.duration,
-                    max: matrix.duration,
-                    minEnv: `${matrix.os_name}, ${matrix.package_manager}`,
-                    maxEnv: `${matrix.os_name}, ${matrix.package_manager}`,
-                  };
-                }
-              }
-            });
-
-            // project time report
-            let resultPkg = `
-                \`\`\`
-                | Project                        | Time                      |
-                |--------------------------------|---------------------------|`;
-            function mapProjectTime(proj, section) {
-              let res = '';
-              res += `${humanizeDuration(timeReport[proj][section])}`;
-              res += ` (${timeReport[proj][section + 'Env']})`
-              return res;
-            }
-            function durationIcon(proj, section) {
-              if (timeReport[proj][section] < 12 * 60) {
-                return `${section} ✅`;
-              }
-              if (timeReport[proj][section] < 15 * 60) {
-                return `${section} ❗`;
-              }
-              return `${section} ❌`;
-            }
-            Object.keys(timeReport).forEach(proj => {
-              resultPkg += `\n| ${proj.padEnd(30)} |                           |`;
-              resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`;
-              resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`;
-            });
-            resultPkg += `\`\`\``;
-            core.setOutput('SLACK_PROJ_DURATION', trimSpace(resultPkg));
-
-            // Print project duration report inline to allow reviewing on manual runs (when no slack message will be sent)
-            console.log(trimSpace(resultPkg));
-
-            let resultPm = `
-                \`\`\`
-                | PM   | Total time  |
-                |------|-------------|`;
-            Object.keys(pmReport).forEach(pm => {
-              resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm]).padEnd(11)} |`
-            });
-            resultPm += `\`\`\``;
-            core.setOutput('SLACK_PM_DURATION', trimSpace(resultPm));
-
-            // Print package manager duration report inline to allow reviewing on manual runs (when no slack message will be sent)
-            console.log(trimSpace(resultPm));
+        run: |
+          echo '${{ steps.combine-json.outputs.combined }}' | npx tsx .github/workflows/nightly/process-result.ts
 
   report-failure:
-    if: ${{ failure() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
+    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'true' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
     needs: process-result
     runs-on: ubuntu-latest
     name: Report failure
@@ -497,16 +461,16 @@ jobs:
         uses: ravsamhq/notify-slack-action@v2
         with:
           status: 'failure'
-          message_format: '{emoji} Workflow has {status_message} ${{ needs.process-result.outputs.message }}'
-          notification_title: '{workflow}'
+          message_format: '${{ needs.process-result.outputs.message }}'
+          notification_title: 'Golden Test Failure'
           footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
           mention_groups: ${{ needs.process-result.outputs.codeowners }}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
   report-success:
-    if: ${{ success() && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
-    needs: e2e
+    if: ${{ always() && needs.process-result.outputs.has_golden_failures == 'false' && github.repository_owner == 'nrwl' && github.event_name != 'workflow_dispatch' }}
+    needs: process-result
     runs-on: ubuntu-latest
     name: Report status
     timeout-minutes: 10
@@ -514,9 +478,9 @@ jobs:
       - name: Send notification
         uses: ravsamhq/notify-slack-action@v2
         with:
-          status: ${{ needs.e2e.result }}
-          message_format: '{emoji} Workflow has {status_message}'
-          notification_title: '{workflow}'
+          status: 'success'
+          message_format: '${{ needs.process-result.outputs.message }}'
+          notification_title: '✅ Golden Tests: All Passed!'
           footer: '<{run_url}|View Run> / Last commit <{commit_url}|{commit_sha}>'
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
@@ -532,8 +496,8 @@ jobs:
         uses: ravsamhq/notify-slack-action@v2
         with:
           status: 'skipped'
-          message_format: '${{ needs.process-result.outputs.pm-duration }}'
-          notification_title: 'Total duration per package manager (ubuntu only)'
+          message_format: '${{ needs.process-result.outputs.pm_duration }}'
+          notification_title: '⌛ Total duration per package manager (ubuntu only)'
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
 
@@ -542,13 +506,13 @@ jobs:
     needs: process-result
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    name: Report duration per package manager
+    name: Report duration per project
     steps:
       - name: Send notification
         uses: ravsamhq/notify-slack-action@v2
         with:
           status: 'skipped'
-          message_format: '${{ needs.process-result.outputs.proj-duration }}'
-          notification_title: 'E2E Project duration stats'
+          message_format: '${{ needs.process-result.outputs.proj_duration }}'
+          notification_title: '⌛ E2E Project duration stats'
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}

.github/workflows/nightly/process-matrix.ts

@@ -1,6 +1,7 @@
 type MatrixDataProject = {
   name: string,
   codeowners: string,
+  is_golden?: boolean, // true if this is a golden project, false otherwise
 };
 
 type MatrixDataOS = {
@@ -19,15 +20,26 @@ type MatrixData = {
   setup: MatrixDataOS[],
 }
 
+export type MatrixItem = {
+  project: string,
+  codeowners: string,
+  node_version: number | string,
+  package_manager: string,
+  os: string,
+  os_name: string,
+  os_timeout: number,
+  is_golden?: boolean,
+};
+
 // TODO: Extract Slack groups into named groups for easier maintenance
 const matrixData: MatrixData = {
   coreProjects: [
-    { name: 'e2e-lerna-smoke-tests', codeowners: 'S04TNCVEETS' },
-    { name: 'e2e-js', codeowners: 'S04SJ6HHP0X' },
-    { name: 'e2e-nx-init', codeowners: 'S04SYHYKGNP' },
-    { name: 'e2e-nx', codeowners: 'S04SYHYKGNP' },
-    { name: 'e2e-release', codeowners: 'S04SYHYKGNP' },
-    { name: 'e2e-workspace-create', codeowners: 'S04SYHYKGNP' }
+    { name: 'e2e-lerna-smoke-tests', codeowners: 'S04TNCVEETS', is_golden: true },
+    { name: 'e2e-js', codeowners: 'S04SJ6HHP0X', is_golden: true },
+    { name: 'e2e-nx-init', codeowners: 'S04SYHYKGNP', is_golden: true },
+    { name: 'e2e-nx', codeowners: 'S04SYHYKGNP', is_golden: true },
+    { name: 'e2e-release', codeowners: 'S04SYHYKGNP', is_golden: true },
+    { name: 'e2e-workspace-create', codeowners: 'S04SYHYKGNP', is_golden: true },
   ],
   projects: [
     { name: 'e2e-angular', codeowners: 'S04SS457V38' },
@@ -58,19 +70,12 @@ const matrixData: MatrixData = {
   setup: [
     { os: 'ubuntu-latest', os_name: 'Linux', os_timeout: 60, package_managers: ['npm', 'pnpm', 'yarn'], node_versions: ['20.19.0', "22.12.0"], excluded: ['e2e-detox', 'e2e-react-native', 'e2e-expo'] },
     { os: 'macos-latest', os_name: 'MacOS', os_timeout: 90, package_managers: ['npm'], node_versions: ['20.19.0'] },
-    { os: 'windows-latest', os_name: 'WinOS', os_timeout: 180, package_managers: ['npm'], node_versions: ['20.19.0'], excluded: ['e2e-detox', 'e2e-react-native', 'e2e-expo'] }
+    // TODO (emily): Fix Windows support as gradle fails when running nx build https://staging.nx.app/runs/LgD4vxGn8w?utm_source=pull-request&utm_medium=comment
+    // { os: 'windows-latest', os_name: 'WinOS', os_timeout: 180, package_managers: ['npm'], node_versions: ['20.19.0'], excluded: ['e2e-detox', 'e2e-react-native', 'e2e-expo'] }
   ]
 };
 
-const matrix: Array<{
-  project: string,
-  codeowners: string,
-  node_version: number | string,
-  package_manager: string,
-  os: string,
-  os_name: string,
-  os_timeout: number
-}> = [];
+const matrix: Array<MatrixItem> = [];
 
 function addMatrixCombo(project: MatrixDataProject, nodeVersion: number | string, pm: number, os: number) {
   matrix.push({
@@ -81,6 +86,7 @@ function addMatrixCombo(project: MatrixDataProject, nodeVersion: number | string
     os: matrixData.setup[os].os,
     os_name: matrixData.setup[os].os_name,
     os_timeout: matrixData.setup[os].os_timeout,
+    is_golden: !!project.is_golden, // Mark golden projects as true, others as false
   });
 }
 

.github/workflows/nightly/process-result.ts

@@ -0,0 +1,180 @@
+import * as fs from 'fs';
+import { MatrixItem } from './process-matrix';
+
+interface MatrixResult extends MatrixItem {
+  status: 'success' | 'failure' | 'cancelled';
+  duration: number;
+}
+
+interface ProcessedResults {
+  codeowners: string;
+  slack_message: string;
+  slack_proj_duration: string;
+  slack_pm_duration: string;
+  has_golden_failures: string;
+}
+
+function trimSpace(res: string): string {
+  return res.split('\n').map((l) => l.trim()).join('\n');
+}
+
+function humanizeDuration(num: number): string {
+  let res = '';
+  const hours = Math.floor(num / 3600);
+  if (hours) res += `${hours}h `;
+  const mins = Math.floor((num % 3600) / 60);
+  if (mins) res += `${mins}m `;
+  const sec = num % 60;
+  if (sec) res += `${sec}s`;
+  return res;
+}
+
+function processResults(combined: MatrixResult[]): ProcessedResults {
+  const failedProjects = combined.filter(c => c.status === 'failure').sort((a, b) => a.project.localeCompare(b.project));
+  const failedGoldenProjects = failedProjects.filter(c => c.is_golden);
+  const hasGoldenFailures = failedGoldenProjects.length > 0;
+  const codeowners = new Set<string>();
+  failedGoldenProjects.forEach(c => codeowners.add(c.codeowners));
+
+  let result = '';
+  let lastProject: string | undefined;
+
+  if (failedGoldenProjects.length > 0) {
+    result += `
+🔥 **Golden Test Failures (${failedGoldenProjects.length})**
+\`\`\`
+| Failed project                 | PM   | OS    | Node     |
+|--------------------------------|------|-------|----------|`;
+    lastProject = undefined;
+    failedGoldenProjects.forEach(matrix => {
+      const project = matrix.project !== lastProject ? matrix.project : '...';
+      result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name.padEnd(5)} | v${matrix.node_version.toString().padEnd(7)} |`;
+      lastProject = matrix.project;
+    });
+    result += `\`\`\``;
+  } else {
+    result += '\n✅ **Golden Tests: All Passed!**';
+  }
+
+  const failedRegularProjects = failedProjects.filter(c => !c.is_golden);
+  if (failedRegularProjects.length > 0) {
+    if (failedGoldenProjects.length > 0 || result.length > 0) result += '\n\n';
+    result += `
+📋 **Other Project Failures (${failedRegularProjects.length})**
+\`\`\`
+| Failed project                 | PM   | OS    | Node     |
+|--------------------------------|------|-------|----------|`;
+    lastProject = undefined;
+    failedRegularProjects.forEach(matrix => {
+      const project = matrix.project !== lastProject ? matrix.project : '...';
+      result += `\n| ${project.padEnd(30)} | ${matrix.package_manager.padEnd(4)} | ${matrix.os_name.padEnd(5)} | v${matrix.node_version.toString().padEnd(7)} |`;
+      lastProject = matrix.project;
+    });
+    result += `\`\`\``;
+  }
+
+  if (failedProjects.length === 0) {
+    result = '✅ **No test failures detected!**';
+  }
+
+  const timeReport: Record<string, { min: number; max: number; minEnv: string; maxEnv: string }> = {};
+  const pmReport = { npm: 0, yarn: 0, pnpm: 0 };
+  const macosProjects = ['e2e-detox', 'e2e-expo', 'e2e-react-native'];
+
+  combined.forEach(matrix => {
+    const nodeVersion = parseInt(matrix.node_version.toString());
+    if (matrix.os_name === 'Linux' && nodeVersion === 20 && matrix.package_manager in pmReport) {
+      pmReport[matrix.package_manager as keyof typeof pmReport] += matrix.duration;
+    }
+    if (matrix.os_name === 'Linux' || macosProjects.includes(matrix.project)) {
+      if (timeReport[matrix.project]) {
+        if (matrix.duration > timeReport[matrix.project].max) {
+          timeReport[matrix.project].max = matrix.duration;
+          timeReport[matrix.project].maxEnv = `${matrix.os_name}, ${matrix.package_manager}`;
+        }
+        if (matrix.duration < timeReport[matrix.project].min) {
+          timeReport[matrix.project].min = matrix.duration;
+          timeReport[matrix.project].minEnv = `${matrix.os_name}, ${matrix.package_manager}`;
+        }
+      } else {
+        timeReport[matrix.project] = {
+          min: matrix.duration,
+          max: matrix.duration,
+          minEnv: `${matrix.os_name}, ${matrix.package_manager}`,
+          maxEnv: `${matrix.os_name}, ${matrix.package_manager}`,
+        };
+      }
+    }
+  });
+
+  let resultPkg = `
+    \`\`\`
+    | Project                        | Time                      |
+    |--------------------------------|---------------------------|`;
+
+  function mapProjectTime(proj: string, section: 'min' | 'max'): string {
+    return `${humanizeDuration(timeReport[proj][section])} (${timeReport[proj][`${section}Env`]})`;
+  }
+
+  function durationIcon(proj: string, section: 'min' | 'max'): string {
+    const duration = timeReport[proj][section];
+    if (duration < 12 * 60) return `${section} ✅`;
+    if (duration < 15 * 60) return `${section} ❗`;
+    return `${section} ❌`;
+  }
+
+  Object.keys(timeReport).forEach(proj => {
+    resultPkg += `\n| ${proj.padEnd(30)} |                           |`;
+    resultPkg += `\n| ${durationIcon(proj, 'min').padStart(29)} | ${mapProjectTime(proj, 'min').padEnd(25)} |`;
+    resultPkg += `\n| ${durationIcon(proj, 'max').padStart(29)} | ${mapProjectTime(proj, 'max').padEnd(25)} |`;
+  });
+  resultPkg += `\`\`\``;
+
+  let resultPm = `
+    \`\`\`
+    | PM   | Total time  |
+    |------|-------------|`;
+  Object.keys(pmReport).forEach(pm => {
+    resultPm += `\n| ${pm.padEnd(4)} | ${humanizeDuration(pmReport[pm as keyof typeof pmReport]).padEnd(11)} |`;
+  });
+  resultPm += `\`\`\``;
+
+  return {
+    codeowners: Array.from(codeowners).join(','),
+    slack_message: trimSpace(result),
+    slack_proj_duration: trimSpace(resultPkg),
+    slack_pm_duration: trimSpace(resultPm),
+    has_golden_failures: hasGoldenFailures.toString(),
+  };
+}
+
+function setOutput(key: string, value: string) {
+  const outputPath = process.env.GITHUB_OUTPUT;
+  if (!outputPath) {
+    console.warn(`GITHUB_OUTPUT not set. Skipping output for "${key}".`);
+    return;
+  }
+
+  if (value.includes('\n')) {
+    const delimiter = `EOF_${key}_${Date.now()}`;
+    fs.appendFileSync(outputPath, `${key}<<${delimiter}\n${value}\n${delimiter}\n`);
+  } else {
+    fs.appendFileSync(outputPath, `${key}=${value}\n`);
+  }
+}
+
+try {
+  const combinedInput = process.argv[2]
+    ? process.argv[2]
+    : fs.readFileSync(0, 'utf-8').trim();
+
+  const combined: MatrixResult[] = JSON.parse(combinedInput);
+  const results = processResults(combined);
+
+  Object.entries(results).forEach(([key, value]) => {
+    setOutput(key, value);
+  });
+} catch (error) {
+  console.error('Error processing results:', error);
+  process.exit(1);
+}