diff --git a/docs/shared/monorepo-ci-azure.md b/docs/shared/monorepo-ci-azure.md index 890001e139..78a0596e71 100644 --- a/docs/shared/monorepo-ci-azure.md +++ b/docs/shared/monorepo-ci-azure.md @@ -29,7 +29,7 @@ pr: variables: - name: NX_BRANCH ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - value: $(System.PullRequest.PullRequestNumber) # Pull request Number + value: $(System.PullRequest.PullRequestNumber) ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: value: $(Build.SourceBranchName) - name: TARGET_BRANCH @@ -46,9 +46,14 @@ jobs: pool: vmImage: 'ubuntu-latest' steps: - - script: npm i - - script: npx nx affected --target=build --parallel=3 - - script: npx nx affected --target=test --parallel=2 + - script: npm ci + + - script: npx nx workspace-lint + - script: npx nx format:check + + - script: npx nx affected --base=$(BASE_SHA) --target=lint --parallel=3 + - script: npx nx affected --base=$(BASE_SHA) --target=test --parallel=3 --ci --code-coverage + - script: npx nx affected --base=$(BASE_SHA) --target=build --parallel=3 ``` The `main` job implements the CI workflow. @@ -59,4 +64,58 @@ A computation cache is created on your local machine to make the developer exper Nx Cloud allows this cache to be shared across your entire organization, meaning that any cacheable operation completed on your workspace only needs to be run once. Nx Cloud also allows you to distribute your CI across multiple machines to make sure the CI is fast even for very large repos. +In order to use distributed task execution, we need to start agents and set the `NX_CLOUD_DISTRIBUTED_EXECUTION` flag to `true`. Once all tasks are finished, we must not forget to cleanup used agents. + +```yaml +trigger: + - main +pr: + - main + +variables: + - name: NX_CLOUD_DISTRIBUTED_EXECUTION + value: 'true' + - name: NX_BRANCH + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + value: $(System.PullRequest.PullRequestNumber) + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + value: $(Build.SourceBranchName) + - name: TARGET_BRANCH + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + value: $[replace(variables['System.PullRequest.TargetBranch'],'refs/heads/','origin/')] + - name: BASE_SHA + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + value: $(git merge-base $(TARGET_BRANCH) HEAD) + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + value: $(git rev-parse HEAD~1) + +jobs: + - job: agents + strategy: + matrix: + agent: [1, 2, 3] + displayName: 'Agent $(imageName)' + pool: + vmImage: 'ubuntu-latest' + steps: + - script: npm ci + - script: npx nx-cloud start-agent + + - job: main + pool: + vmImage: 'ubuntu-latest' + steps: + - script: npm ci + - script: npx nx-cloud start-ci-run + + - script: npx nx workspace-lint + - script: npx nx format:check + - script: npx nx affected --base=$(BASE_SHA) --target=lint --parallel=3 + - script: npx nx affected --base=$(BASE_SHA) --target=test --parallel=3 --ci --code-coverage + - script: npx nx affected --base=$(BASE_SHA) --target=build --parallel=3 + + - script: npx nx-cloud stop-all-agents + condition: always() +``` + Learn more about [configuring your CI](https://nx.app/docs/configuring-ci) environment using Nx Cloud with [Distributed Caching](https://nx.app/docs/distributed-caching) and [Distributed Task Execution](https://nx.app/docs/distributed-execution) in the Nx Cloud docs. diff --git a/docs/shared/monorepo-ci-bitbucket-pipelines.md b/docs/shared/monorepo-ci-bitbucket-pipelines.md index 3fa19afa2d..ad04f3e2c4 100644 --- a/docs/shared/monorepo-ci-bitbucket-pipelines.md +++ b/docs/shared/monorepo-ci-bitbucket-pipelines.md @@ -17,8 +17,7 @@ But they come with their own technical challenges. The more code you add into yo Below is an example of a Bitbucket Pipeline setup for an Nx workspace only building and testing what is affected. ```yaml -... - +image: node:16 pipelines: pull-requests: '**': @@ -27,9 +26,12 @@ pipelines: caches: # optional - node script: - - npm i + - npm ci + - npx nx workspace-lint + - npx nx format:check + - npx nx affected --target=lint --base=origin/master --parallel --max-parallel=3 + - npx nx affected --target=test --base=origin/master --parallel --max-parallel=3 --ci --code-coverage - npx nx affected --target=build --base=origin/master --head=HEAD --parallel --max-parallel=3 - - npx nx affected --target=test --base=origin/master --head=HEAD --parallel --max-parallel=2 branches: main: @@ -38,9 +40,12 @@ pipelines: caches: # optional - node script: - - npm i + - npm ci + - npx nx workspace-lint + - npx nx format:check + - npx nx affected --target=lint --base=origin/master --parallel --max-parallel=3 + - npx nx affected --target=test --base=HEAD~1 --parallel --max-parallel=3 --ci --code-coverage - npx nx affected --target=build --base=HEAD~1 --parallel --max-parallel=3 - - npx nx affected --target=test --base=HEAD~1 --parallel --max-parallel=2 ``` The `pull-requests` and `main` jobs implement the CI workflow. diff --git a/docs/shared/monorepo-ci-circle-ci.md b/docs/shared/monorepo-ci-circle-ci.md index 10a3b335e8..89cae1f0b7 100644 --- a/docs/shared/monorepo-ci-circle-ci.md +++ b/docs/shared/monorepo-ci-circle-ci.md @@ -25,30 +25,22 @@ orbs: nx: nrwl/nx@1.4.0 jobs: main: + docker: + - image: cimg/node:lts-browsers steps: - checkout - - run: npm install + - run: npm ci - nx/set-shas - - run: npx nx affected --base=$NX_BASE --target=build --parallel=3 - - run: npx nx affected --base=$NX_BASE --target=test --parallel=2 - pr: - steps: - - checkout - - run: npm install - - nx/set-shas - - run: npx nx affected --base=$NX_BASE --target=build --parallel=3 - - run: npx nx affected --base=$NX_BASE --target=test --parallel=2 + + - run: npx nx workspace-lint + - run: npx nx format:check + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=lint --parallel=3 + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=test --parallel=3 --ci --code-coverage + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=build --parallel=3 workflows: build: jobs: - - main: - filters: - branches: - only: main - - pr: - filters: - branches: - ignore: main + - main ``` The `pr` and `main` jobs implement the CI workflow. @@ -65,4 +57,53 @@ A computation cache is created on your local machine to make the developer exper Nx Cloud allows this cache to be shared across your entire organization, meaning that any cacheable operation completed on your workspace only needs to be run once. Nx Cloud also allows you to distribute your CI across multiple machines to make sure the CI is fast even for very large repos. +In order to use distributed task execution, we need to start agents and set the `NX_CLOUD_DISTRIBUTED_EXECUTION` flag to `true`. Once all tasks are finished, we must not forget to cleanup used agents. + +```yaml +version: 2.1 +orbs: + nx: nrwl/nx@1.4.0 +jobs: + agent: + docker: + - image: cimg/node:lts-browsers + parameters: + ordinal: + type: integer + steps: + - run: echo "export NX_RUN_GROUP=\"run-group-$CIRCLE_WORKFLOW_ID\";" >> $BASH_ENV + - checkout + - run: npm ci + - run: + command: npx nx-cloud start-agent + no_output_timeout: 60m + main: + docker: + - image: cimg/node:lts-browsers + environment: + NX_CLOUD_DISTRIBUTED_EXECUTION: 'true' + steps: + - run: echo "export NX_RUN_GROUP=\"run-group-$CIRCLE_WORKFLOW_ID\";" >> $BASH_ENV + - checkout + - run: npm ci + - nx/set-shas + - run: npx nx-cloud start-ci-run + + - run: npx nx workspace-lint + - run: npx nx format:check + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=lint --parallel=3 + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=test --parallel=3 --ci --code-coverage + - run: npx nx affected --base=$NX_BASE --head=$NX_HEAD --target=build --parallel=3 + + - run: npx nx-cloud stop-all-agents +workflows: + build: + jobs: + - agent: + matrix: + parameters: + ordinal: [1, 2, 3] + - main +``` + Learn more about [configuring your CI](https://nx.app/docs/configuring-ci) environment using Nx Cloud with [Distributed Caching](https://nx.app/docs/distributed-caching) and [Distributed Task Execution](https://nx.app/docs/distributed-execution) in the Nx Cloud docs. diff --git a/docs/shared/monorepo-ci-github-actions.md b/docs/shared/monorepo-ci-github-actions.md index b3bbba74bd..a88c384747 100644 --- a/docs/shared/monorepo-ci-github-actions.md +++ b/docs/shared/monorepo-ci-github-actions.md @@ -30,36 +30,18 @@ on: jobs: main: runs-on: ubuntu-latest - if: ${{ github.event_name != 'pull_request' }} - steps: - - uses: actions/checkout@v2 - name: Checkout [main] - with: - fetch-depth: 0 - - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - - uses: actions/setup-node@v1 - with: - node-version: '16' - - run: npm install - - run: npx nx affected --target=build --parallel=3 - - run: npx nx affected --target=test --parallel=2 - pr: - runs-on: ubuntu-latest - if: ${{ github.event_name == 'pull_request' }} steps: - uses: actions/checkout@v2 with: - ref: ${{ github.event.pull_request.head.ref }} fetch-depth: 0 - - name: Derive appropriate SHAs for base and head for `nx affected` commands - uses: nrwl/nx-set-shas@v2 - - uses: actions/setup-node@v1 - with: - node-version: '16' - - run: npm install + - uses: nrwl/nx-set-shas@v2 + - run: npm ci + + - run: npx nx workspace-lint + - run: npx nx format:check + - run: npx nx affected --target=lint --parallel=3 + - run: npx nx affected --target=test --parallel=3 --ci --code-coverage - run: npx nx affected --target=build --parallel=3 - - run: npx nx affected --target=test --parallel=2 ``` The `pr` and `main` jobs implement the CI workflow. Setting `timeout-minutes` is needed only if you have very slow tasks. @@ -70,4 +52,34 @@ A computation cache is created on your local machine to make the developer exper Nx Cloud allows this cache to be shared across your entire organization, meaning that any cacheable operation completed on your workspace only needs to be run once. Nx Cloud also allows you to distribute your CI across multiple machines to make sure the CI is fast even for very large repos. +In order to use distributed task execution, we need to start agents and set the `NX_CLOUD_DISTRIBUTED_EXECUTION` flag to `true`. Once all tasks are finished, we must not forget to cleanup used agents. + +```yaml +name: CI +on: + push: + branches: + - main + pull_request: + +jobs: + main: + name: Nx Cloud - Main Job + uses: nrwl/ci/.github/workflows/nx-cloud-main.yml@v0.2 + with: + parallel-commands: | + npx nx workspace-lint + npx nx format:check + parallel-commands-on-agents: | + npx nx affected --target=lint --parallel=3 + npx nx affected --target=test --parallel=3 --ci --code-coverage + npx nx affected --target=build --parallel=3 + + agents: + name: Nx Cloud - Agents + uses: nrwl/ci/.github/workflows/nx-cloud-agents.yml@v0.2 + with: + number-of-agents: 3 +``` + Learn more about [configuring your CI](https://nx.app/docs/configuring-ci) environment using Nx Cloud with [Distributed Caching](https://nx.app/docs/distributed-caching) and [Distributed Task Execution](https://nx.app/docs/distributed-execution) in the Nx Cloud docs. diff --git a/docs/shared/monorepo-ci-gitlab.md b/docs/shared/monorepo-ci-gitlab.md index ad0444a0d2..ef6d7a4ba5 100644 --- a/docs/shared/monorepo-ci-gitlab.md +++ b/docs/shared/monorepo-ci-gitlab.md @@ -31,15 +31,11 @@ install-dependencies: cache: key: files: - - yarn.lock + - package-lock.json paths: - - node_modules - - .yarn - script: - - yarn install --pure-lockfile --cache-folder .yarn - artifacts: - paths: - - node_modules + - .npm/ + before_script: + - npm ci --cache .npm --prefer-offline .distributed: interruptible: true @@ -52,17 +48,35 @@ install-dependencies: paths: - node_modules/.cache/nx -build: +workspace-lint: stage: test extends: .distributed script: - - yarn nx affected --base=HEAD~1 --target=build --parallel=3 + - npx nx workspace-lint + +format-check: + stage: test + extends: .distributed + script: + - npx nx format:check + +lint: + stage: test + extends: .distributed + script: + - npx nx affected --base=HEAD~1 --target=lint --parallel=3 test: stage: test extends: .distributed script: - - yarn nx affected --base=HEAD~1 --target=test --parallel=2 + - npx nx affected --base=HEAD~1 --target=test --parallel=3 --ci --code-coverage + +build: + stage: test + extends: .distributed + script: + - npx nx affected --base=HEAD~1 --target=build --parallel=3 ``` The `build` and `test` jobs implement the CI workflow using `.distributed` as template to keep diff --git a/docs/shared/monorepo-ci-jenkins.md b/docs/shared/monorepo-ci-jenkins.md index 820c608e90..e704f5b3d6 100644 --- a/docs/shared/monorepo-ci-jenkins.md +++ b/docs/shared/monorepo-ci-jenkins.md @@ -16,7 +16,7 @@ But they come with their own technical challenges. The more code you add into yo Below is an example of a Jenkins setup for an Nx workspace only building and testing what is affected. -Unlike `GitHub Actions` and `CircleCI`, you don't have the metadata to help you track the last successful run on `main`. In the example below, the base is set to `HEAD~1`, but a more robust solution would be to tag a SHA in the main job once it succeeds, and then use this tag as a base. +Unlike `GitHub Actions` and `CircleCI`, you don't have the metadata to help you track the last successful run on `main`. In the example below, the base is set to `HEAD~1` (for push) or branching point (for pull requests), but a more robust solution would be to tag a SHA in the main job once it succeeds, and then use this tag as a base. See the [nx-tag-successful-ci-run](https://github.com/nrwl/nx-tag-successful-ci-run) and [nx-set-shas](https://github.com/nrwl/nx-set-shas) (version 1 implements tagging mechanism) repos for more information. We also have to set `NX_BRANCH` explicitly. @@ -35,11 +35,12 @@ pipeline { } agent any steps { - sh "npm install" - sh "npx nx-cloud start-ci-run" + sh "npm ci" + sh "npx nx workspace-lint" + sh "npx nx format:check" + sh "npx nx affected --base=HEAD~1 --target=lint --parallel=3" + sh "npx nx affected --base=HEAD~1 --target=test --parallel=3" sh "npx nx affected --base=HEAD~1 --target=build --parallel=3" - sh "npx nx affected --base=HEAD~1 --target=test --parallel=2" - sh "npx nx-cloud stop-all-agents" } } stage('PR') { @@ -48,11 +49,12 @@ pipeline { } agent any steps { - sh "npm install" - sh "npx nx-cloud start-ci-run" + sh "npm ci" + sh "npx nx workspace-lint" + sh "npx nx format:check" + sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=lint --parallel=3" + sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=test --parallel=3 --ci --code-coverage" sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=build --parallel=3" - sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=test --parallel=2" - sh "npx nx-cloud stop-all-agents" } } } @@ -69,4 +71,54 @@ A computation cache is created on your local machine to make the developer exper Nx Cloud allows this cache to be shared across your entire organization, meaning that any cacheable operation completed on your workspace only needs to be run once. Nx Cloud also allows you to distribute your CI across multiple machines to make sure the CI is fast even for very large repos. +In order to use distributed task execution, we need to start agents and set the `NX_CLOUD_DISTRIBUTED_EXECUTION` flag to `true`. Once all tasks are finished, we must not forget to cleanup used agents. + +```groovy +pipeline { + agent none + environment { + NX_BRANCH = env.BRANCH_NAME.replace('PR-', '') + NX_CLOUD_DISTRIBUTED_EXECUTION = true + } + stages { + stage('Pipeline') { + parallel { + stage('Main') { + when { + branch 'main' + } + agent any + steps { + sh "npm ci" + sh "npx nx-cloud start-ci-run" + sh "npx nx workspace-lint" + sh "npx nx format:check" + sh "npx nx affected --base=HEAD~1 --target=lint --parallel=3" + sh "npx nx affected --base=HEAD~1 --target=test --parallel=3 --ci --code-coverage" + sh "npx nx affected --base=HEAD~1 --target=build --parallel=3" + sh "npx nx-cloud stop-all-agents" + } + } + stage('PR') { + when { + not { branch 'main' } + } + agent any + steps { + sh "npm ci" + sh "npx nx-cloud start-ci-run" + sh "npx nx workspace-lint" + sh "npx nx format:check" + sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=lint --parallel=3" + sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=test --parallel=3 --ci --code-coverage" + sh "npx nx affected --base origin/${env.CHANGE_TARGET} --target=build --parallel=3" + sh "npx nx-cloud stop-all-agents" + } + } + } + } + } +} +``` + Learn more about [configuring your CI](https://nx.app/docs/configuring-ci) environment using Nx Cloud with [Distributed Caching](https://nx.app/docs/distributed-caching) and [Distributed Task Execution](https://nx.app/docs/distributed-execution) in the Nx Cloud docs.