chore(repo): improve e2e test stability (#6658)

* chore(repo): rest mac failing tests

* chore(repo): update mac to bigsur

* chore(repo): revert mac to catalina

* chore(react): fix flaky react report test

* chore(web): check if content exists serving file server

* chore(core): fix flaky arguments test

* chore(repo): increase duration of web file serve test

* chore(repo): fix tests

* chore(repo): fix cleanup console logs

* chore(repo): test latest changes

* chore(repo): fix typo

* chore(repo): remove content check from file-server

* chore(repo): check workspace

* chore(repo): check workspace

* chore(repo): turn on all the tests

.github/workflows/e2e-matrix.yml

@@ -28,7 +28,7 @@ jobs:
           #  os-name: windows
         exclude:
           - os: macos-latest
-            package_manager: npm
+            package_manager: yarn
           - os: macos-latest
             package_manager: pnpm
         #   - os: windows-latest
@@ -110,7 +110,9 @@ jobs:
         NX_E2E_RUN_CYPRESS: ${{ 'true' }}
         NODE_OPTIONS: --max_old_space_size=8192
         SELECTED_PM: ${{ matrix.package_manager }}
+        npm_config_registry: http://localhost:4872
         YARN_REGISTRY: http://localhost:4872
+        VERBOSE_OUTPUT: ${{ 'true' }}
 
     - name: Setup tmate session
       if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled && failure() }}

e2e/next/src/next.test.ts

@@ -82,8 +82,8 @@ describe('Next.js Applications', () => {
     try {
       await promisifiedTreeKill(p.pid, 'SIGKILL');
       expect(await killPorts(port)).toBeTruthy();
-    } catch {
+    } catch (err) {
-      expect('process running').toBeFalsy();
+      expect(err).toBeFalsy();
     }
   }, 300000);
 
@@ -609,8 +609,8 @@ describe('Next.js Applications', () => {
       await promisifiedTreeKill(p.pid, 'SIGKILL');
       // expect(await killPorts(port)).toBeTruthy();
       await killPorts(port);
-    } catch {
+    } catch (err) {
-      expect('process running').toBeFalsy();
+      expect(err).toBeFalsy();
     }
   }, 300000);
 });

e2e/nx-plugin/src/nx-plugin.test.ts

@@ -7,7 +7,6 @@ import {
   readJson,
   runCLI,
   runCLIAsync,
-  runCypressTests,
   uniq,
   workspaceConfigName,
 } from '@nrwl/e2e/utils';

e2e/react/src/react-package.test.ts

@@ -173,7 +173,9 @@ describe('Build React libraries and apps', () => {
     });
 
     it('should build the library when it does not have any deps', () => {
-      const output = runCLI(`build ${childLib}`);
+      const output = runCLI(`build ${childLib}`)
+        // FIX for windows and OSX where output names might get broken to multipleline
+        .replace(/\s\s\s││\s\s\s/gm, '');
       expect(output).toContain(`${childLib}.esm.js`);
       expect(output).toContain(`Bundle complete: ${childLib}`);
       checkFilesExist(`dist/libs/${childLib}/assets/hello.txt`);
@@ -186,9 +188,15 @@ describe('Build React libraries and apps', () => {
     });
 
     it('should properly add references to any dependency into the parent package.json', () => {
-      const childLibOutput = runCLI(`build ${childLib}`);
+      const childLibOutput = runCLI(`build ${childLib}`)
-      const childLib2Output = runCLI(`build ${childLib2}`);
+        // FIX for windows and OSX where output names might get broken to multipleline
-      const parentLibOutput = runCLI(`build ${parentLib}`);
+        .replace(/\s\s\s││\s\s\s/gm, '');
+      const childLib2Output = runCLI(`build ${childLib2}`)
+        // FIX for windows and OSX where output names might get broken to multipleline
+        .replace(/\s\s\s││\s\s\s/gm, '');
+      const parentLibOutput = runCLI(`build ${parentLib}`)
+        // FIX for windows and OSX where output names might get broken to multipleline
+        .replace(/\s\s\s││\s\s\s/gm, '');
 
       expect(childLibOutput).toContain(`${childLib}.esm.js`);
       expect(childLibOutput).toContain(`${childLib}.umd.js`);

e2e/utils/index.ts

@@ -275,7 +275,11 @@ export function runCommandAsync(
         if (!opts.silenceError && err) {
           reject(err);
         }
-        resolve({ stdout, stderr, combinedOutput: `${stdout}${stderr}` });
+        resolve({
+          stdout: stripConsoleColors(stdout),
+          stderr: stripConsoleColors(stderr),
+          combinedOutput: stripConsoleColors(`${stdout}${stderr}`),
+        });
       }
     );
   });
@@ -300,7 +304,7 @@ export function runCommandUntil(
     let complete = false;
 
     function checkCriteria(c) {
-      output += c.toString();
+      output += stripConsoleColors(c.toString());
       if (criteria(output) && !complete) {
         complete = true;
         res(p);
@@ -376,15 +380,13 @@ export function runCLI(
 ): string {
   try {
     const pm = getPackageManagerCommand();
-    let r = execSync(`${pm.runNx} ${command}`, {
+    let r = stripConsoleColors(
-      cwd: opts.cwd || tmpProjPath(),
+      execSync(`${pm.runNx} ${command}`, {
-      env: { ...(opts.env || process.env), NX_INVOKED_BY_RUNNER: undefined },
+        cwd: opts.cwd || tmpProjPath(),
-      encoding: 'utf-8',
+        env: { ...(opts.env || process.env), NX_INVOKED_BY_RUNNER: undefined },
-      maxBuffer: 50 * 1024 * 1024,
+        encoding: 'utf-8',
-    }).toString();
+        maxBuffer: 50 * 1024 * 1024,
-    r = r.replace(
+      })
-      /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g,
-      ''
     );
     if (process.env.VERBOSE_OUTPUT) {
       logInfo(`result of running: ${command}`, r);
@@ -409,6 +411,18 @@ export function runCLI(
   }
 }
 
+/**
+ * Remove log colors for fail proof string search
+ * @param log
+ * @returns
+ */
+function stripConsoleColors(log: string): string {
+  return log.replace(
+    /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g,
+    ''
+  );
+}
+
 export function expectTestsPass(v: { stdout: string; stderr: string }) {
   expect(v.stderr).toContain('Ran all test suites');
   expect(v.stderr).not.toContain('fail');

e2e/web/src/file-server.test.ts

@@ -27,16 +27,17 @@ describe('file-server', () => {
       `serve ${appName} --port=${port}`,
       (output) => {
         return (
-          output.indexOf('Built at') > -1 && output.indexOf('Available on') > -1
+          output.indexOf('Built at') > -1 &&
+          output.indexOf(`localhost:${port}`) > -1
         );
       }
     );
+
     try {
       await promisifiedTreeKill(p.pid, 'SIGKILL');
-      await killPorts(port);
+      expect(await killPorts(port)).toBeTruthy();
-      // expect(await killPorts(port)).toBeTruthy();
+    } catch (err) {
-    } catch {
+      expect(err).toBeFalsy();
-      expect('process running').toBeFalsy();
     }
-  }, 300000);
+  }, 1000000);
 });

e2e/workspace/src/run-commands.test.ts

@@ -80,10 +80,10 @@ describe('Run Commands', () => {
       executor: '@nrwl/workspace:run-commands',
       options: {
         commands: [
-          `echo 'var1: {args.var1}'`,
+          `echo "var1: {args.var1}"`,
-          `echo 'var2: {args.var2}'`,
+          `echo "var2: {args.var2}"`,
-          `echo 'hyphen: {args.var-hyphen}'`,
+          `echo "hyphen: {args.var-hyphen}"`,
-          `echo 'camel: {args.varCamelCase}'`,
+          `echo "camel: {args.varCamelCase}"`,
         ],
       },
     };