feat(testing): split atomized outputs for Playwright and Cypress CI targets (#28682)

This PR updates `@nx/playwright/plugin` to create non-conflicting
outputs for atomized tasks.



<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
Outputs from `outputDir` and reporters are the same for each atomized
task, which results in missing outputs.

## Expected Behavior
All outputs should be captured and cached correctly.

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

packages/cypress/src/plugins/plugin.spec.ts

@@ -375,7 +375,7 @@ describe('@nx/cypress/plugin', () => {
                   },
                   "e2e-ci--src/test.cy.ts": {
                     "cache": true,
-                    "command": "cypress run --env webServerCommand="my-app:serve-static" --spec src/test.cy.ts",
+                    "command": "cypress run --env webServerCommand="my-app:serve-static" --spec src/test.cy.ts --config="{\\"e2e\\":{\\"videosFolder\\":\\"dist/videos/src-test-cy-ts\\",\\"screenshotsFolder\\":\\"dist/screenshots/src-test-cy-ts\\"}}"",
                     "inputs": [
                       "default",
                       "^production",
@@ -404,8 +404,8 @@ describe('@nx/cypress/plugin', () => {
                       "cwd": ".",
                     },
                     "outputs": [
-                      "{projectRoot}/dist/videos",
+                      "{projectRoot}/dist/videos/src-test-cy-ts",
-                      "{projectRoot}/dist/screenshots",
+                      "{projectRoot}/dist/screenshots/src-test-cy-ts",
                     ],
                     "parallelism": false,
                   },

packages/cypress/src/plugins/plugin.ts

@@ -130,6 +130,65 @@ async function createNodesInternal(
   };
 }
 
+function getTargetOutputs(outputs: string[], subfolder?: string): string[] {
+  return outputs.map((output) =>
+    subfolder ? join(output, subfolder) : output
+  );
+}
+
+function getTargetConfig(
+  cypressConfig: any,
+  outputSubfolder: string,
+  ciBaseUrl?: string
+): string {
+  const config = {};
+  if (ciBaseUrl) {
+    config['baseUrl'] = ciBaseUrl;
+  }
+
+  const { screenshotsFolder, videosFolder, e2e, component } = cypressConfig;
+
+  if (videosFolder) {
+    config['videosFolder'] = join(videosFolder, outputSubfolder);
+  }
+
+  if (screenshotsFolder) {
+    config['screenshotsFolder'] = join(screenshotsFolder, outputSubfolder);
+  }
+
+  if (e2e) {
+    config['e2e'] = {};
+    if (e2e.videosFolder) {
+      config['e2e']['videosFolder'] = join(e2e.videosFolder, outputSubfolder);
+    }
+    if (e2e.screenshotsFolder) {
+      config['e2e']['screenshotsFolder'] = join(
+        e2e.screenshotsFolder,
+        outputSubfolder
+      );
+    }
+  }
+
+  if (component) {
+    config['component'] = {};
+    if (component.videosFolder) {
+      config['component']['videosFolder'] = join(
+        component.videosFolder,
+        outputSubfolder
+      );
+    }
+    if (component.screenshotsFolder) {
+      config['component']['screenshotsFolder'] = join(
+        component.screenshotsFolder,
+        outputSubfolder
+      );
+    }
+  }
+
+  // Stringify twice to escape the quotes.
+  return JSON.stringify(JSON.stringify(config));
+}
+
 function getOutputs(
   projectRoot: string,
   cypressConfig: any,
@@ -266,18 +325,24 @@ async function buildCypressTargets(
       const groupName = 'E2E (CI)';
       metadata = { targetGroups: { [groupName]: [] } };
       const ciTargetGroup = metadata.targetGroups[groupName];
+
       for (const file of specFiles) {
         const relativeSpecFilePath = normalizePath(relative(projectRoot, file));
         const targetName = options.ciTargetName + '--' + relativeSpecFilePath;
+        const outputSubfolder = relativeSpecFilePath
+          .replace(/[\/\\]/g, '-')
+          .replace(/\./g, '-');
 
         ciTargetGroup.push(targetName);
         targets[targetName] = {
-          outputs,
+          outputs: getTargetOutputs(outputs, outputSubfolder),
           inputs,
           cache: true,
-          command: `cypress run --env webServerCommand="${ciWebServerCommand}" --spec ${relativeSpecFilePath}${
+          command: `cypress run --env webServerCommand="${ciWebServerCommand}" --spec ${relativeSpecFilePath} --config=${getTargetConfig(
-            ciBaseUrl ? ` --config='{"baseUrl": "${ciBaseUrl}"}'` : ''
+            cypressConfig,
-          }`,
+            outputSubfolder,
+            ciBaseUrl
+          )}`,
           options: {
             cwd: projectRoot,
           },

packages/playwright/src/plugins/plugin.spec.ts

@@ -197,10 +197,10 @@ describe('@nx/playwright/plugin', () => {
                       "cwd": "{projectRoot}",
                     },
                     "outputs": [
+                      "{projectRoot}/test-results",
                       "{projectRoot}/playwright-report",
                       "{projectRoot}/test-results/report.json",
                       "{projectRoot}/test-results/html",
-                      "{projectRoot}/test-results",
                     ],
                     "parallelism": false,
                   },
@@ -233,10 +233,10 @@ describe('@nx/playwright/plugin', () => {
                       ],
                     },
                     "outputs": [
+                      "{projectRoot}/test-results",
                       "{projectRoot}/playwright-report",
                       "{projectRoot}/test-results/report.json",
                       "{projectRoot}/test-results/html",
-                      "{projectRoot}/test-results",
                     ],
                     "parallelism": false,
                   },
@@ -255,6 +255,9 @@ describe('@nx/playwright/plugin', () => {
       `module.exports = {
       testDir: 'tests',
       testIgnore: [/.*skip.*/, '**/ignored/**'],
+      reporter: [
+        ['html', { outputFolder: 'test-results/html' }],
+      ],
     }`
     );
     await tempFs.createFiles({
@@ -326,6 +329,7 @@ describe('@nx/playwright/plugin', () => {
         },
         "outputs": [
           "{projectRoot}/test-results",
+          "{projectRoot}/test-results/html",
         ],
         "parallelism": false,
       }
@@ -333,7 +337,7 @@ describe('@nx/playwright/plugin', () => {
     expect(targets['e2e-ci--tests/run-me.spec.ts']).toMatchInlineSnapshot(`
       {
         "cache": true,
-        "command": "playwright test tests/run-me.spec.ts",
+        "command": "playwright test tests/run-me.spec.ts --output=test-results/tests-run-me-spec-ts",
         "inputs": [
           "default",
           "^production",
@@ -359,9 +363,14 @@ describe('@nx/playwright/plugin', () => {
         },
         "options": {
           "cwd": "{projectRoot}",
+          "env": {
+            "PLAYWRIGHT_HTML_OUTPUT_DIR": "test-results/html/tests-run-me-spec-ts",
+            "PLAYWRIGHT_HTML_REPORT": "test-results/html/tests-run-me-spec-ts",
+          },
         },
         "outputs": [
-          "{projectRoot}/test-results",
+          "{projectRoot}/test-results/tests-run-me-spec-ts",
+          "{projectRoot}/test-results/html/tests-run-me-spec-ts",
         ],
         "parallelism": false,
       }
@@ -369,7 +378,7 @@ describe('@nx/playwright/plugin', () => {
     expect(targets['e2e-ci--tests/run-me-2.spec.ts']).toMatchInlineSnapshot(`
       {
         "cache": true,
-        "command": "playwright test tests/run-me-2.spec.ts",
+        "command": "playwright test tests/run-me-2.spec.ts --output=test-results/tests-run-me-2-spec-ts",
         "inputs": [
           "default",
           "^production",
@@ -395,9 +404,14 @@ describe('@nx/playwright/plugin', () => {
         },
         "options": {
           "cwd": "{projectRoot}",
+          "env": {
+            "PLAYWRIGHT_HTML_OUTPUT_DIR": "test-results/html/tests-run-me-2-spec-ts",
+            "PLAYWRIGHT_HTML_REPORT": "test-results/html/tests-run-me-2-spec-ts",
+          },
         },
         "outputs": [
-          "{projectRoot}/test-results",
+          "{projectRoot}/test-results/tests-run-me-2-spec-ts",
+          "{projectRoot}/test-results/html/tests-run-me-2-spec-ts",
         ],
         "parallelism": false,
       }

packages/playwright/src/plugins/plugin.ts

@@ -1,5 +1,5 @@
-import { existsSync, readdirSync } from 'fs';
+import { existsSync, readdirSync } from 'node:fs';
-import { dirname, join, relative } from 'path';
+import { dirname, join, parse, relative } from 'node:path';
 
 import {
   CreateNodes,
@@ -157,6 +157,8 @@ async function buildPlaywrightTargets(
   const targets: ProjectConfiguration['targets'] = {};
   let metadata: ProjectConfiguration['metadata'];
 
+  const testOutput = getTestOutput(playwrightConfig);
+  const reporterOutputs = getReporterOutputs(playwrightConfig);
   const baseTargetConfig: TargetConfiguration = {
     command: 'playwright test',
     options: {
@@ -186,7 +188,7 @@ async function buildPlaywrightTargets(
         : ['default', '^default']),
       { externalDependencies: ['@playwright/test'] },
     ],
-    outputs: getOutputs(projectRoot, playwrightConfig),
+    outputs: getTargetOutputs(testOutput, reporterOutputs, projectRoot),
   };
 
   if (options.ciTargetName) {
@@ -199,7 +201,7 @@ async function buildPlaywrightTargets(
           : ['default', '^default']),
         { externalDependencies: ['@playwright/test'] },
       ],
-      outputs: getOutputs(projectRoot, playwrightConfig),
+      outputs: getTargetOutputs(testOutput, reporterOutputs, projectRoot),
     };
 
     const groupName = 'E2E (CI)';
@@ -214,8 +216,12 @@ async function buildPlaywrightTargets(
     playwrightConfig.testMatch ??= '**/*.@(spec|test).?(c|m)[jt]s?(x)';
 
     const dependsOn: TargetConfiguration['dependsOn'] = [];
+
     await forEachTestFile(
       (testFile) => {
+        const outputSubfolder = relative(projectRoot, testFile)
+          .replace(/[\/\\]/g, '-')
+          .replace(/\./g, '-');
         const relativeSpecFilePath = normalizePath(
           relative(projectRoot, testFile)
         );
@@ -223,7 +229,22 @@ async function buildPlaywrightTargets(
         ciTargetGroup.push(targetName);
         targets[targetName] = {
           ...ciBaseTargetConfig,
-          command: `${baseTargetConfig.command} ${relativeSpecFilePath}`,
+          options: {
+            ...ciBaseTargetConfig.options,
+            env: getOutputEnvVars(reporterOutputs, outputSubfolder),
+          },
+          outputs: getTargetOutputs(
+            testOutput,
+            reporterOutputs,
+            projectRoot,
+            outputSubfolder
+          ),
+          command: `${
+            baseTargetConfig.command
+          } ${relativeSpecFilePath} --output=${join(
+            testOutput,
+            outputSubfolder
+          )}`,
           metadata: {
             technologies: ['playwright'],
             description: `Runs Playwright Tests in ${relativeSpecFilePath} in CI`,
@@ -319,56 +340,6 @@ function createMatcher(pattern: string | RegExp | Array<string | RegExp>) {
   }
 }
 
-function getOutputs(
-  projectRoot: string,
-  playwrightConfig: PlaywrightTestConfig
-): string[] {
-  function getOutput(path: string): string {
-    if (path.startsWith('..')) {
-      return join('{workspaceRoot}', join(projectRoot, path));
-    } else {
-      return join('{projectRoot}', path);
-    }
-  }
-
-  const outputs = [];
-
-  const { reporter, outputDir } = playwrightConfig;
-
-  if (reporter) {
-    const DEFAULT_REPORTER_OUTPUT = getOutput('playwright-report');
-    if (reporter === 'html' || reporter === 'json') {
-      // Reporter is a string, so it uses the default output directory.
-      outputs.push(DEFAULT_REPORTER_OUTPUT);
-    } else if (Array.isArray(reporter)) {
-      for (const r of reporter) {
-        const [, opts] = r;
-        // There are a few different ways to specify an output file or directory
-        // depending on the reporter. This is a best effort to find the output.
-        if (!opts) {
-          outputs.push(DEFAULT_REPORTER_OUTPUT);
-        } else if (opts.outputFile) {
-          outputs.push(getOutput(opts.outputFile));
-        } else if (opts.outputDir) {
-          outputs.push(getOutput(opts.outputDir));
-        } else if (opts.outputFolder) {
-          outputs.push(getOutput(opts.outputFolder));
-        } else {
-          outputs.push(DEFAULT_REPORTER_OUTPUT);
-        }
-      }
-    }
-  }
-
-  if (outputDir) {
-    outputs.push(getOutput(outputDir));
-  } else {
-    outputs.push(getOutput('./test-results'));
-  }
-
-  return outputs;
-}
-
 function normalizeOptions(options: PlaywrightPluginOptions): NormalizedOptions {
   return {
     ...options,
@@ -376,3 +347,94 @@ function normalizeOptions(options: PlaywrightPluginOptions): NormalizedOptions {
     ciTargetName: options.ciTargetName ?? 'e2e-ci',
   };
 }
+
+function getTestOutput(playwrightConfig: PlaywrightTestConfig): string {
+  const { outputDir } = playwrightConfig;
+  if (outputDir) {
+    return outputDir;
+  } else {
+    return './test-results';
+  }
+}
+
+function getReporterOutputs(
+  playwrightConfig: PlaywrightTestConfig
+): Array<[string, string]> {
+  const outputs: Array<[string, string]> = [];
+
+  const { reporter } = playwrightConfig;
+
+  if (reporter) {
+    const DEFAULT_REPORTER_OUTPUT = 'playwright-report';
+    if (reporter === 'html') {
+      outputs.push([reporter, DEFAULT_REPORTER_OUTPUT]);
+    } else if (reporter === 'json') {
+      outputs.push([reporter, DEFAULT_REPORTER_OUTPUT]);
+    } else if (Array.isArray(reporter)) {
+      for (const r of reporter) {
+        const [reporter, opts] = r;
+        // There are a few different ways to specify an output file or directory
+        // depending on the reporter. This is a best effort to find the output.
+        if (opts?.outputFile) {
+          outputs.push([reporter, opts.outputFile]);
+        } else if (opts?.outputDir) {
+          outputs.push([reporter, opts.outputDir]);
+        } else if (opts?.outputFolder) {
+          outputs.push([reporter, opts.outputFolder]);
+        } else {
+          outputs.push([reporter, DEFAULT_REPORTER_OUTPUT]);
+        }
+      }
+    }
+  }
+
+  return outputs;
+}
+
+function getTargetOutputs(
+  testOutput: string,
+  reporterOutputs: Array<[string, string]>,
+  projectRoot: string,
+  scope?: string
+): string[] {
+  const outputs = new Set<string>();
+  outputs.add(
+    normalizeOutput(projectRoot, scope ? join(testOutput, scope) : testOutput)
+  );
+  for (const [, output] of reporterOutputs) {
+    outputs.add(
+      normalizeOutput(projectRoot, scope ? join(output, scope) : output)
+    );
+  }
+  return Array.from(outputs);
+}
+
+function normalizeOutput(projectRoot: string, path: string): string {
+  if (path.startsWith('..')) {
+    return join('{workspaceRoot}', join(projectRoot, path));
+  } else {
+    return join('{projectRoot}', path);
+  }
+}
+
+function getOutputEnvVars(
+  reporterOutputs: Array<[string, string]>,
+  outputSubfolder: string
+): Record<string, string> {
+  const env: Record<string, string> = {};
+  for (let [reporter, output] of reporterOutputs) {
+    if (outputSubfolder) {
+      const isFile = parse(output).ext !== '';
+      const envVarName = `PLAYWRIGHT_${reporter.toUpperCase()}_OUTPUT_${
+        isFile ? 'FILE' : 'DIR'
+      }`;
+      env[envVarName] = join(output, outputSubfolder);
+      // Also set PLAYWRIGHT_HTML_REPORT for Playwright prior to 1.45.0.
+      // HTML prior to this version did not follow the pattern of "PLAYWRIGHT_<REPORTER>_OUTPUT_<FILE|DIR>".
+      if (reporter === 'html') {
+        env['PLAYWRIGHT_HTML_REPORT'] = env[envVarName];
+      }
+    }
+  }
+  return env;
+}