feat(misc): updated documentation generation scripts and removed shelljs (#5381)

cleanup(repo): updated documentation generation scripts and removed shelljs

package.json

@@ -22,7 +22,7 @@
     "lint": "nx run-many --target=lint --all --parallel",
     "depcheck": "ts-node -P ./scripts/tsconfig.scripts.json ./scripts/depcheck",
     "local-registry": "./scripts/local-registry.sh",
-    "documentation": "./scripts/documentation/documentation.sh && ./scripts/documentation/check-documentation.sh && yarn check-documentation-map && yarn check-internal-links",
+    "documentation": "ts-node -P scripts/tsconfig.scripts.json ./scripts/documentation/documentation.ts && yarn check-documentation-map && yarn check-internal-links",
     "submit-plugin": "node ./scripts/submit-plugin.js"
   },
   "devDependencies": {
@@ -209,7 +209,6 @@
     "sass": "1.26.3",
     "sass-loader": "8.0.2",
     "semver": "7.3.4",
-    "shelljs": "^0.8.3",
     "source-map": "0.7.3",
     "source-map-loader": "0.2.4",
     "source-map-support": "0.5.16",

scripts/check-versions.ts

@@ -16,8 +16,8 @@ import { join } from 'path';
 import { gt } from 'semver';
 import * as chalk from 'chalk';
 import { dasherize } from '../packages/workspace/src/utils/strings';
-import * as shell from 'shelljs';
 import * as glob from 'glob';
+import { execSync } from 'child_process';
 
 const excluded = ['nxVersion'];
 const scoped = [
@@ -123,7 +123,9 @@ function getVersionData(
 } {
   try {
     const latest = JSON.parse(
-      shell.exec(`npm view ${p} version --json --silent`, { silent: true })
+      execSync(`npm view ${p} version --json --silent`, {
+        stdio: ['ignore'],
+      }).toString('utf-8')
     );
     if (gt(latest, v)) {
       return { package: p, outdated: true, invalid: false, latest, prev: v };

scripts/chmod.js

@@ -1,3 +1,3 @@
-const shell = require('shelljs');
+const fs = require('fs');
 
-shell.chmod('+x', process.argv[2]);
+fs.chmodSync(process.argv[2], 0o777);

scripts/documentation/check-documentation.sh

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-if [ -z "$(git status --porcelain ./docs)" ]; then
-  echo "📄 Documentation not modified";
-  exit 0;
-else
-  echo "📄 Documentation has been modified, you need to commit the changes.";
-  git status --porcelain ./docs
-  exit 1;
-fi

scripts/documentation/documentation.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-echo "Generating API documentation"
-ts-node -r tsconfig-paths/register --project=scripts/tsconfig.scripts.json ./scripts/documentation/generate-executors-data.ts
-ts-node -r tsconfig-paths/register --project=scripts/tsconfig.scripts.json ./scripts/documentation/generate-generators-data.ts
-ts-node -r tsconfig-paths/register --project=scripts/tsconfig.scripts.json ./scripts/documentation/generate-cli-data.ts
-echo 'Done generating all Documentation'

scripts/documentation/documentation.ts

@@ -0,0 +1,51 @@
+import * as chalk from 'chalk';
+import { execSync } from 'child_process';
+
+import { generateCLIDocumentation } from './generate-cli-data';
+import { generateExecutorsDocumentation } from './generate-executors-data';
+import { generateGeneratorsDocumentation } from './generate-generators-data';
+
+async function generate() {
+  console.log(`${chalk.blue('i')} Generating Documentation`);
+
+  await generateGeneratorsDocumentation();
+  await generateExecutorsDocumentation();
+  await generateCLIDocumentation();
+
+  console.log(`\n${chalk.green('🗸')} Generated Documentation\n`);
+}
+
+function checkDocumentation() {
+  const output = execSync('git status --porcelain ./docs').toString('utf-8');
+
+  if (output) {
+    console.log(
+      `${chalk.red(
+        '!'
+      )} 📄 Documentation has been modified, you need to commit the changes. ${chalk.red(
+        '!'
+      )} `
+    );
+
+    console.log('\nChanged Docs:');
+    execSync('git status --porcelain ./docs', { stdio: 'inherit' });
+
+    process.exit(1);
+  } else {
+    console.log('📄 Documentation not modified');
+  }
+}
+
+generate().then(() => {
+  checkDocumentation();
+});
+
+function printInfo(
+  str: string,
+  newLine: boolean = true,
+  newLineAfter: boolean = true
+) {
+  console.log(
+    `${newLine ? '\n' : ''}${chalk.blue('i')} ${str}${newLineAfter ? '\n' : ''}`
+  );
+}

scripts/documentation/frameworks.ts

@@ -0,0 +1,2 @@
+export const Frameworks = ['angular', 'react', 'node'] as const;
+export type Framework = typeof Frameworks[number];

scripts/documentation/generate-cli-data.ts

@@ -1,9 +1,10 @@
+import * as chalk from 'chalk';
 import * as fs from 'fs-extra';
 import * as path from 'path';
 import { dedent } from 'tslint/lib/utils';
 import { commandsObject } from '../../packages/workspace';
+import { Framework, Frameworks } from './frameworks';
 import { generateMarkdownFile, sortAlphabeticallyFunction } from './utils';
-
 const importFresh = require('import-fresh');
 
 const examples = {
@@ -395,9 +396,11 @@ const sharedCommands = [
   'test',
 ];
 
-console.log('Generating Nx Commands Documentation');
+export async function generateCLIDocumentation() {
-Promise.all(
+  console.log(`\n${chalk.blue('i')} Generating Documentation for Nx Commands`);
-  ['angular', 'react', 'node'].map(async (framework) => {
+
+  await Promise.all(
+    Frameworks.map(async (framework: Framework) => {
       const commandsOutputDirectory = path.join(
         __dirname,
         '../../docs/',
@@ -427,7 +430,9 @@ Promise.all(
         const builder = await command.builder(
           importFresh('yargs')().resetOptions()
         );
-      const builderDescriptions = builder.getUsageInstance().getDescriptions();
+        const builderDescriptions = builder
+          .getUsageInstance()
+          .getDescriptions();
         const builderDefaultOptions = builder.getOptions().default;
         return {
           command: name,
@@ -537,6 +542,7 @@ Promise.all(
         })
       );
     })
-).then(() => {
+  );
-  console.log('Finished generating Nx Commands Documentation');
+
-});
+  console.log(`${chalk.green('🗸')} Generated Documentation for Nx Commands`);
+}

scripts/documentation/generate-executors-data.ts

@@ -18,6 +18,8 @@ import {
   Configuration,
   getPackageConfigurations,
 } from './get-package-configurations';
+import { Framework } from './frameworks';
+import * as chalk from 'chalk';
 
 /**
  * @WhatItDoes: Generates default documentation from the builders' schema.
@@ -69,7 +71,7 @@ function generateSchematicList(
 }
 
 function generateTemplate(
-  framework,
+  framework: Framework,
   builder
 ): { name: string; template: string } {
   const filename = framework === 'angular' ? 'angular.json' : 'workspace.json';
@@ -160,41 +162,62 @@ function generateTemplate(
   return { name: builder.name, template };
 }
 
-Promise.all(
+export async function generateExecutorsDocumentation() {
+  console.log(`\n${chalk.blue('i')} Generating Documentation for Executors\n`);
+
+  await Promise.all(
     getPackageConfigurations().map(({ framework, configs }) => {
       return Promise.all(
         configs
           .filter((item) => item.hasBuilders)
-        .map((config) => {
+          .map(async (config) => {
-          Promise.all(generateSchematicList(config, registry))
+            const buildersList = await Promise.all(
-            .then((builderList) =>
+              generateSchematicList(config, registry)
-              builderList.map((b) => generateTemplate(framework, b))
+            );
-            )
+
-            .then((markdownList) =>
+            const markdownList = buildersList.map((b) =>
-              Promise.all(
+              generateTemplate(framework, b)
+            );
+
+            await Promise.all(
               markdownList.map((template) =>
                 generateMarkdownFile(config.builderOutput, template)
               )
-              )
-            )
-            .then(() =>
-              console.log(
-                `Generated documentation for ${config.root} to ${config.output}`
-              )
             );
-        })
-    );
-  })
-).then(() => {
-  console.log('Done generating documentation for executors');
-});
 
-getPackageConfigurations().forEach(async ({ framework, configs }) => {
+            console.log(
+              ` - ${chalk.blue(
+                config.framework
+              )} Documentation for ${chalk.magenta(
+                path.relative(process.cwd(), config.root)
+              )} generated at ${chalk.grey(
+                path.relative(process.cwd(), config.builderOutput)
+              )}`
+            );
+          })
+      );
+    })
+  );
+
+  console.log();
+  await Promise.all(
+    getPackageConfigurations().map(async ({ framework, configs }) => {
       const builders = configs
         .filter((item) => item.hasBuilders)
         .map((item) => item.name);
+
       await generateJsonFile(
         path.join(__dirname, '../../docs', framework, 'executors.json'),
         builders
       );
-});
+
+      console.log(
+        `${chalk.green('🗸')} Generated ${chalk.blue(
+          framework
+        )} executors.json at ${chalk.grey(`docs/${framework}/executors.json`)}`
+      );
+    })
+  );
+
+  console.log(`\n${chalk.green('🗸')} Generated Documentation for Executors`);
+}

scripts/documentation/generate-generators-data.ts

@@ -1,4 +1,5 @@
 import * as fs from 'fs-extra';
+import * as chalk from 'chalk';
 import * as path from 'path';
 import { dedent } from 'tslint/lib/utils';
 import { FileSystemSchematicJsonDescription } from '@angular-devkit/schematics/tools';
@@ -16,6 +17,7 @@ import {
   Configuration,
   getPackageConfigurations,
 } from './get-package-configurations';
+import { Framework } from './frameworks';
 import { parseJsonSchemaToOptions } from './json-parser';
 
 /**
@@ -61,7 +63,7 @@ function generateSchematicList(
 }
 
 function generateTemplate(
-  framework: string,
+  framework: Framework,
   schematic
 ): { name: string; template: string } {
   const cliCommand = 'nx';
@@ -163,43 +165,63 @@ function generateTemplate(
   return { name: schematic.name, template };
 }
 
-Promise.all(
+export async function generateGeneratorsDocumentation() {
+  console.log(`\n${chalk.blue('i')} Generating Documentation for Generators\n`);
+
+  await Promise.all(
     getPackageConfigurations().map(({ framework, configs }) => {
       return Promise.all(
         configs
           .filter((item) => item.hasSchematics)
-        .map((config) => {
+          .map(async (config) => {
-          return Promise.all(generateSchematicList(config, registry))
+            const schematicList = await Promise.all(
-            .then((schematicList) => {
+              generateSchematicList(config, registry)
-              return schematicList
+            );
+
+            const markdownList = schematicList
               .filter((s) => !s['hidden'])
-                .map((s) => generateTemplate(framework, s));
+              .map((s_1) => generateTemplate(framework, s_1));
-            })
+
-            .then((markdownList) =>
+            await Promise.all(
-              Promise.all(
               markdownList.map((template) =>
                 generateMarkdownFile(config.schematicOutput, template)
               )
-              )
+            );
-            )
+
-            .then(() => {
             console.log(
-                `Documentation from ${config.root} generated to ${config.schematicOutput}`
+              ` - ${chalk.blue(
+                config.framework
+              )} Documentation for ${chalk.magenta(
+                path.relative(process.cwd(), config.root)
+              )} generated at ${chalk.grey(
+                path.relative(process.cwd(), config.schematicOutput)
+              )}`
+            );
+          })
+      );
+    })
+  );
+
+  console.log();
+  await Promise.all(
+    getPackageConfigurations().map(({ framework, configs }) => {
+      const schematics = configs
+        .filter((item) => item.hasSchematics)
+        .map((item) => item.name);
+      return generateJsonFile(
+        path.join(__dirname, '../../docs', framework, 'generators.json'),
+        schematics
+      ).then(() => {
+        console.log(
+          `${chalk.green('🗸')} Generated ${chalk.blue(
+            framework
+          )} generators.json at ${chalk.grey(
+            `docs/${framework}/generators.json`
+          )}`
         );
       });
     })
   );
-  })
-).then(() => {
-  console.log('Finished Generating Documentation for Generators');
-});
 
-getPackageConfigurations().forEach(async ({ framework, configs }) => {
+  console.log(`\n${chalk.green('🗸')} Generated Documentation for Generators`);
-  const schematics = configs
+}
-    .filter((item) => item.hasSchematics)
-    .map((item) => item.name);
-  await generateJsonFile(
-    path.join(__dirname, '../../docs', framework, 'generators.json'),
-    schematics
-  );
-});

scripts/documentation/get-package-configurations.ts

@@ -1,15 +1,17 @@
+import * as glob from 'glob';
 import * as path from 'path';
-import * as shelljs from 'shelljs';
+import { Framework, Frameworks } from './frameworks';
 
 export interface Configuration {
   name: string;
   root: string;
+  framework: Framework;
   source: string;
   output: string;
   builderOutput: string;
   schematicOutput: string;
-  hasBuilders: string;
+  hasBuilders: boolean;
-  hasSchematics: string;
+  hasSchematics: boolean;
 }
 
 /**
@@ -21,16 +23,21 @@ export interface Configuration {
 export function getPackageConfigurations(
   packagesDirectory: string = 'packages',
   documentationsDirectory: string = 'docs'
-): { framework: 'angular' | 'react' | 'node'; configs: Configuration[] }[] {
+): { framework: Framework; configs: Configuration[] }[] {
-  return ['angular', 'react', 'node'].map((framework) => {
+  return Frameworks.map((framework: Framework) => {
     const packagesDir = path.resolve(
       path.join(__dirname, '../../', packagesDirectory)
     );
     const documentationDir = path.resolve(
       path.join(__dirname, '../../', documentationsDirectory)
     );
-    const configs = shelljs.ls(packagesDir).map((folderName) => {
+
-      const itemList = shelljs.ls(path.join(packagesDir, folderName));
+    const configs = glob.sync(`${packagesDir}/*`).map(
+      (folderPath): Configuration => {
+        const folderName = folderPath.substring(packagesDir.length + 1);
+        const itemList = glob
+          .sync(`${folderPath}/*`)
+          .map((item) => item.split(folderPath + '/')[1]);
         const output = path.join(
           documentationDir,
           framework,
@@ -38,8 +45,8 @@ export function getPackageConfigurations(
         );
         return {
           name: folderName,
-        root: path.join(packagesDir, folderName),
+          root: folderPath,
-        source: path.join(packagesDir, `${folderName}/src`),
+          source: path.join(folderPath, '/src'),
           output,
           framework,
           builderOutput: path.join(output, 'executors'),
@@ -51,7 +58,8 @@ export function getPackageConfigurations(
             itemList.includes('collection.json') ||
             itemList.includes('generators.json'),
         };
-    });
+      }
-    return { framework: framework as any, configs };
+    );
+    return { framework, configs };
   });
 }

scripts/documentation/internal-link-checker.ts

@@ -1,8 +1,10 @@
-import { green, red } from 'chalk';
+import * as chalk from 'chalk';
-import * as shell from 'shelljs';
 import * as fs from 'fs';
 import * as parseLinks from 'parse-markdown-links';
 import * as path from 'path';
+import * as glob from 'glob';
+
+console.log(`${chalk.blue('i')} Internal Link Check`);
 
 const LOGGING_KEYS = [
   'LOG_DOC_TREE',
@@ -76,7 +78,7 @@ function expandFrameworks(linkPaths: string[]): string[] {
 }
 
 function extractAllInternalLinks(): Record<string, string[]> {
-  return shell.ls(`${BASE_PATH}/**/*.md`).reduce((acc, path) => {
+  return glob.sync(`${BASE_PATH}/**/*.md`).reduce((acc, path) => {
     const fileContents = readFileContents(path);
     const directLinks = fileContents
       .split(/[ ,]+/)
@@ -103,7 +105,7 @@ function extractAllInternalLinks(): Record<string, string[]> {
 }
 
 function extractAllInternalLinksWithAnchors(): Record<string, string[]> {
-  return shell.ls(`${BASE_PATH}/**/*.md`).reduce((acc, path) => {
+  return glob.sync(`${BASE_PATH}/**/*.md`).reduce((acc, path) => {
     const links = parseLinks(readFileContents(path))
       .filter(isLinkInternal)
       .filter(isNotAsset)
@@ -138,7 +140,7 @@ function isCategoryNode(
 
 function getDocumentMap(): DocumentTree[] {
   return JSON.parse(
-    fs.readFileSync(path.join(BASE_PATH, 'map.json'), { encoding: 'utf-8' })
+    fs.readFileSync(path.join(BASE_PATH, 'map.json'), 'utf-8')
   ) as DocumentTree[];
 }
 
@@ -253,29 +255,27 @@ function checkInternalAnchoredLinks(
 }
 
 if (!erroneousInternalLinks) {
-  console.log(green('All internal links appear to be valid!!'));
+  console.log(`${chalk.green('🗸')} All internal links appear to be valid!`);
-  console.log('Moving on to check internal anchors...');
   const erroneousAnchoredInternalLinks = checkInternalAnchoredLinks(
     validInternalLinksMap
   );
   if (!erroneousAnchoredInternalLinks) {
-    console.log(green('All internal anchored links appear to be valid!!'));
+    console.log(
+      `${chalk.green('🗸')} All internal anchored links appear to be valid!`
+    );
     process.exit(0);
   } else {
-    console.log(
+    console.log(`${chalk.red(
-      red(
+      'ERROR'
-        'The following files appear to contain the following invalid anchored internal links:'
+    )} The following files appear to contain the following invalid anchored internal links:
-      )
+    ${JSON.stringify(erroneousAnchoredInternalLinks, null, 2)}`);
-    );
-    console.log(red(JSON.stringify(erroneousAnchoredInternalLinks, null, 2)));
     process.exit(1);
   }
 } else {
-  console.log(
+  console.log(`${chalk.red(
-    red(
+    'ERROR'
-      'The following files appear to contain the following invalid internal links:'
+  )} The following files appear to contain the following invalid internal links:
-    )
+  ${JSON.stringify(erroneousInternalLinks, null, 2)}`);
-  );
+
-  console.log(red(JSON.stringify(erroneousInternalLinks, null, 2)));
   process.exit(1);
 }

scripts/documentation/map-link-checker.ts

@@ -1,12 +1,14 @@
-import { green, red } from 'chalk';
 import * as fs from 'fs';
-import * as shell from 'shelljs';
+import * as glob from 'glob';
+import * as chalk from 'chalk';
+
+console.log(`${chalk.blue('i')} Documentation Map Check`);
 
 const basePath = 'docs';
 const sharedFilesPattern = 'shared/cli';
 
-const readmePathList: string[] = shell
+const readmePathList: string[] = glob
-  .ls(`${basePath}/**/*.md`)
+  .sync(`${basePath}/**/*.md`)
   .map((path: string) => path.split(basePath)[1])
   .map((path: string) => path.slice(1, -3)) // Removing first `/` and `.md`
   .filter((path: string) => !path.startsWith(sharedFilesPattern));
@@ -49,38 +51,44 @@ let scriptError = false;
 
 if (!!readmeMissList.length) {
   console.error(
-    red("\n⚠️  Documentation files and 'map.json' file are out of sync!\n")
+    chalk.red(
+      "\n⚠️  Documentation files and 'map.json' file are out of sync!\n"
+    )
   );
   console.log(readmeMissList.map((x) => x.concat('.md')).join('\n'));
   console.error(
-    red(
+    chalk.red(
       `\nSome documentation files exist without any reference in \'map.json\', make sure to add an entry.`
     )
   );
   scriptError = true;
 } else {
   console.log(
-    green("Markdown files are in sync with 'map.json', everything is good 👍")
+    `${chalk.green('🗸')} Markdown files are in sync with ${chalk.grey(
+      'docs/maps.json'
+    )}.`
   );
 }
 
 if (!!mapMissList.length) {
-  console.error(
+  console.log(
-    red(
+    `\n${chalk.red(
-      "\n⚠️  The 'map.json' file and the documentation files are out of sync!\n"
+      'ERROR'
-    )
+    )} The 'map.json' file and the documentation files are out of sync!\n`
   );
   console.log(mapMissList.map((x) => x.concat('.md')).join('\n'));
-  console.error(
+  console.log(
-    red(
+    `\n${chalk.red(
-      `\nThe \'map.json\' file is linking documenation files that do not exist.`
+      'ERROR'
-    )
+    )} The \'map.json\' file is linking documenation files that do not exist.`
   );
   scriptError = true;
 } else {
   console.log(
-    green(
+    console.log(
-      "The 'map.json' file and the documentation files are in sync, everything is good 👍"
+      `${chalk.green(
+        '🗸'
+      )} The 'map.json' file and the documentation files are in sync.`
     )
   );
 }

yarn.lock

@@ -3190,6 +3190,7 @@
     url-loader "^4.0.0"
     util-deprecate "^1.0.2"
     webpack "^4.44.2"
+    
     webpack-dev-middleware "^3.7.0"
     webpack-filter-warnings-plugin "^1.2.1"
     webpack-hot-middleware "^2.25.0"