diff --git a/docs/blog/2024-09-25-introducing-nx-powerpack.md b/docs/blog/2024-09-25-introducing-nx-powerpack.md index a0b8a4e63a..c4902ddaa3 100644 --- a/docs/blog/2024-09-25-introducing-nx-powerpack.md +++ b/docs/blog/2024-09-25-introducing-nx-powerpack.md @@ -156,9 +156,6 @@ permissions: id-token: write ... -env: - NX_DB_CACHE: true - jobs: main: runs-on: ubuntu-latest diff --git a/docs/external-generated/packages-metadata.json b/docs/external-generated/packages-metadata.json index 552b25ad04..ff64b85ac8 100644 --- a/docs/external-generated/packages-metadata.json +++ b/docs/external-generated/packages-metadata.json @@ -1,4 +1,27 @@ [ + { + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Azure Blob Storage.", + "documents": [ + { + "id": "overview", + "name": "Overview", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Azure Blob Storage.", + "file": "external-generated/packages/powerpack-azure-cache/documents/overview", + "itemList": [], + "isExternal": false, + "path": "powerpack-azure-cache/documents/overview", + "tags": [], + "originalFilePath": "shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin" + } + ], + "executors": [], + "generators": [], + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-azure-cache", + "packageName": "@nx/powerpack-azure-cache", + "root": "/libs/nx-packages/powerpack-azure-cache", + "source": "/libs/nx-packages/powerpack-azure-cache/src" + }, { "description": "A Nx Powerpack plugin which allows users to write and apply rules for your entire workspace that help with consistency, maintainability, reliability and security.", "documents": [ @@ -32,6 +55,50 @@ "root": "/libs/nx-packages/powerpack-conformance", "source": "/libs/nx-packages/powerpack-conformance/src" }, + { + "description": "A Nx Powerpack plugin which is specific to Nx Enterprise Cloud workspaces.", + "documents": [], + "executors": [], + "generators": [ + { + "description": "Initialize Nx Powerpack Enterprise Cloud config", + "file": "external-generated/packages/powerpack-enterprise-cloud/generators/init.json", + "hidden": false, + "name": "init", + "originalFilePath": "/libs/nx-packages/powerpack-enterprise-cloud/src/generators/init/schema.json", + "path": "powerpack-enterprise-cloud/generators/init", + "type": "generator" + } + ], + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-enterprise-cloud", + "packageName": "@nx/powerpack-enterprise-cloud", + "root": "/libs/nx-packages/powerpack-enterprise-cloud", + "source": "/libs/nx-packages/powerpack-enterprise-cloud/src" + }, + { + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Google Cloud Storage.", + "documents": [ + { + "id": "overview", + "name": "Overview", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Google Cloud Storage.", + "file": "external-generated/packages/powerpack-gcs-cache/documents/overview", + "itemList": [], + "isExternal": false, + "path": "powerpack-gcs-cache/documents/overview", + "tags": [], + "originalFilePath": "shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin" + } + ], + "executors": [], + "generators": [], + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-gcs-cache", + "packageName": "@nx/powerpack-gcs-cache", + "root": "/libs/nx-packages/powerpack-gcs-cache", + "source": "/libs/nx-packages/powerpack-gcs-cache/src" + }, { "description": "Package to provide the ability to activate and read licenses for Nx Powerpack.", "documents": [], diff --git a/docs/external-generated/packages/powerpack-azure-cache/documents/overview.md b/docs/external-generated/packages/powerpack-azure-cache/documents/overview.md new file mode 100644 index 0000000000..1663e7ae2b --- /dev/null +++ b/docs/external-generated/packages/powerpack-azure-cache/documents/overview.md @@ -0,0 +1,83 @@ +--- +title: Overview of the Nx powerpack-azure-cache Plugin +description: The powerpack-azure-cache Nx plugin enables you to use Azure Storage to host your remote cache instead of Nx Cloud +--- + +The `@nx/powerpack-azure-cache` plugin enables you to use [Azure Storage](https://azure.microsoft.com/en-us/products/storage/blobs) instead of Nx Cloud to host your remote cache. + +This plugin will enable the remote cache for your Nx workspace, but does not provide any of the other features of Nx Cloud. If you want to leverage [distributed task execution](/ci/features/distribute-task-execution), [re-running flaky tasks](/ci/features/flaky-tasks) or [automatically splitting tasks](/ci/features/split-e2e-tasks), you'll need to [connect to Nx Cloud](/ci/intro/connect-to-nx-cloud) and use [Nx Replay](/ci/features/remote-cache) instead. + +{% callout type="warning" title="Potential Cache Poisoning" %} +Using your own Azure Storage account to host the remote cache opens you up to the possibility of [cache poisoning](/troubleshooting/unknown-local-cache). To avoid this, use [Nx Replay](/ci/features/remote-cache). +{% /callout %} + +{% callout title="This plugin requires an active Nx Powerpack license" %} +In order to use `@nx/powerpack-azure-cache`, you need to have an active Powerpack license. If you don't have a license or it has expired, your cache will no longer be shared and each machine will use its local cache. +{% /callout %} + +## Set Up @nx/powerpack-azure-cache + +### 1. Install the Package + +1. [Activate Powerpack](/nx-enterprise/activate-powerpack) if you haven't already +2. Install the package + +```shell +nx add @nx/powerpack-azure-cache +``` + +### 2. Authenticate with Azure + +There are several ways to [authenticate with Azure Storage](https://github.com/Azure/login#login-with-openid-connect-oidc-recommended), but the method recommended by Azure is to use OpenID Connect, like this: + +```yaml {% fileName=".github/workflows/ci.yml" %} +name: CI +... +permissions: + id-token: write + ... + +jobs: + main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} + runs-on: ubuntu-latest + steps: + ... + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ... + + - run: pnpm exec nx affected -t lint test build +``` + +You need to set the `AZURE_CLIENT_ID`, `AZURE_TENANT_ID` and `AZURE_SUBSCRIPTION_ID` secrets as defined in the [Azure documentation](https://github.com/Azure/login#login-with-openid-connect-oidc-recommended). + +Note: Any authentication method that [sets up the `DefaultAzureCredentials`](https://learn.microsoft.com/en-us/azure/developer/javascript/sdk/credential-chains#use-defaultazurecredential-for-flexibility) will enable the plugin to work. + +{% callout type="note" title="Custom Azure Endpoint" %} +If you are using a custom Azure endpoint, you will need to authenticate by [setting the `AZURE_STORAGE_CONNECTION_STRING` environment variable](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). The `@nx/powerpack-azure-cache` plugin will detect the environment variable and automatically use it to connect to Azure. +{% /callout %} + +### 3. Configure the Nx Cache to Use Azure Storage + +Finally, you need to configure your Nx cache in the `nx.json` file. The `container` that you specify needs to already exist - Nx doesn't create it for you. + +```jsonc {% fileName="nx.json" %} +{ + "azure": { + "container": "mycontainer", + "accountName": "myaccount" + } +} +``` + +| **Property** | **Description** | +| --------------- | -------------------------------- | +| **container** | The name of the container to use | +| **accountName** | The name of blob storage account | diff --git a/docs/external-generated/packages/powerpack-enterprise-cloud/generators/init.json b/docs/external-generated/packages/powerpack-enterprise-cloud/generators/init.json new file mode 100644 index 0000000000..f88d65e804 --- /dev/null +++ b/docs/external-generated/packages/powerpack-enterprise-cloud/generators/init.json @@ -0,0 +1,21 @@ +{ + "name": "init", + "factory": "./src/generators/init/init", + "schema": { + "$schema": "http://json-schema.org/schema", + "id": "NxPowerpackEnterpriseCloudInit", + "title": "Add Powerpack Enterprise Cloud plugin to the workspace", + "type": "object", + "cli": "nx", + "properties": {}, + "additionalProperties": false, + "required": [], + "presets": [] + }, + "description": "Initialize Nx Powerpack Enterprise Cloud config", + "implementation": "/libs/nx-packages/powerpack-enterprise-cloud/src/generators/init/init.ts", + "aliases": [], + "hidden": false, + "path": "/libs/nx-packages/powerpack-enterprise-cloud/src/generators/init/schema.json", + "type": "generator" +} diff --git a/docs/external-generated/packages/powerpack-gcs-cache/documents/overview.md b/docs/external-generated/packages/powerpack-gcs-cache/documents/overview.md new file mode 100644 index 0000000000..2212c9ac99 --- /dev/null +++ b/docs/external-generated/packages/powerpack-gcs-cache/documents/overview.md @@ -0,0 +1,82 @@ +--- +title: Overview of the Nx powerpack-gcs-cache Plugin +description: The powerpack-gcs-cache Nx plugin enables you to use Google Cloud Storage to host your remote cache instead of Nx Cloud +--- + +The `@nx/powerpack-gcs-cache` plugin enables you to use [Google Cloud Storage](https://cloud.google.com/storage) instead of Nx Cloud to host your remote cache. + +This plugin will enable the remote cache for your Nx workspace, but does not provide any of the other features of Nx Cloud. If you want to leverage [distributed task execution](/ci/features/distribute-task-execution), [re-running flaky tasks](/ci/features/flaky-tasks) or [automatically splitting tasks](/ci/features/split-e2e-tasks), you'll need to [connect to Nx Cloud](/ci/intro/connect-to-nx-cloud) and use [Nx Replay](/ci/features/remote-cache) instead. + +{% callout type="warning" title="Potential Cache Poisoning" %} +Using your own Google Cloud Storage account to host the remote cache opens you up to the possibility of [cache poisoning](/troubleshooting/unknown-local-cache). To avoid this, use [Nx Replay](/ci/features/remote-cache). +{% /callout %} + +{% callout title="This plugin requires an active Nx Powerpack license" %} +In order to use `@nx/powerpack-gcs-cache`, you need to have an active Powerpack license. If you don't have a license or it has expired, your cache will no longer be shared and each machine will use its local cache. +{% /callout %} + +## Set Up @nx/powerpack-gcs-cache + +### 1. Install the Package + +1. [Activate Powerpack](/nx-enterprise/activate-powerpack) if you haven't already +2. Install the package + +```shell +nx add @nx/powerpack-gcs-cache +``` + +### 2. Authenticate with Google Cloud + +There are several ways to [authenticate with Google Cloud Storage](https://github.com/google-github-actions/setup-gcloud#authorization), but the method recommended by Google is to use Workload Identity Federation, like this: + +```yaml {% fileName=".github/workflows/ci.yml" %} +name: CI +... +permissions: + id-token: write + ... + +jobs: + main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} + runs-on: ubuntu-latest + steps: + ... + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v2' + with: + token_format: 'access_token' + workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' + service_account: 'my-service-account@my-project.iam.gserviceaccount.com' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + ... + + - run: pnpm exec nx affected -t lint test build +``` + +Note: Any authentication method that [sets up the Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) will enable the plugin to work. + +### 3. Configure the Nx Cache to Use Google Cloud Storage + +Finally, you need to configure your Nx cache in the `nx.json` file. The `bucket` that you specify needs to already exist - Nx doesn't create it for you. + +```jsonc {% fileName="nx.json" %} +{ + "gcs": { + "bucket": "my-bucket" + } +} +``` + +| **Property** | **Description** | +| ------------ | ----------------------------- | +| **bucket** | The name of the bucket to use | diff --git a/docs/external-generated/packages/powerpack-s3-cache/documents/overview.md b/docs/external-generated/packages/powerpack-s3-cache/documents/overview.md index dc1ed6a229..f5faddceab 100644 --- a/docs/external-generated/packages/powerpack-s3-cache/documents/overview.md +++ b/docs/external-generated/packages/powerpack-s3-cache/documents/overview.md @@ -57,11 +57,10 @@ permissions: id-token: write ... -env: - NX_DB_CACHE: true - jobs: main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} runs-on: ubuntu-latest steps: ... diff --git a/docs/generated/manifests/menus.json b/docs/generated/manifests/menus.json index 265016dde5..558cdcf0ba 100644 --- a/docs/generated/manifests/menus.json +++ b/docs/generated/manifests/menus.json @@ -10501,6 +10501,32 @@ "isExternal": false, "disableCollapsible": false }, + { + "id": "powerpack-azure-cache", + "path": "/nx-api/powerpack-azure-cache", + "name": "powerpack-azure-cache", + "children": [ + { + "id": "documents", + "path": "/nx-api/powerpack-azure-cache/documents", + "name": "documents", + "children": [ + { + "name": "Overview", + "path": "/nx-api/powerpack-azure-cache/documents/overview", + "id": "overview", + "isExternal": false, + "children": [], + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + }, { "id": "powerpack-conformance", "path": "/nx-api/powerpack-conformance", @@ -10544,6 +10570,58 @@ "isExternal": false, "disableCollapsible": false }, + { + "id": "powerpack-enterprise-cloud", + "path": "/nx-api/powerpack-enterprise-cloud", + "name": "powerpack-enterprise-cloud", + "children": [ + { + "id": "generators", + "path": "/nx-api/powerpack-enterprise-cloud/generators", + "name": "generators", + "children": [ + { + "id": "init", + "path": "/nx-api/powerpack-enterprise-cloud/generators/init", + "name": "init", + "children": [], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + }, + { + "id": "powerpack-gcs-cache", + "path": "/nx-api/powerpack-gcs-cache", + "name": "powerpack-gcs-cache", + "children": [ + { + "id": "documents", + "path": "/nx-api/powerpack-gcs-cache/documents", + "name": "documents", + "children": [ + { + "name": "Overview", + "path": "/nx-api/powerpack-gcs-cache/documents/overview", + "id": "overview", + "isExternal": false, + "children": [], + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + } + ], + "isExternal": false, + "disableCollapsible": false + }, { "id": "powerpack-license", "path": "/nx-api/powerpack-license", diff --git a/docs/generated/manifests/nx-api.json b/docs/generated/manifests/nx-api.json index 78f4f3bf96..bf0d87f8db 100644 --- a/docs/generated/manifests/nx-api.json +++ b/docs/generated/manifests/nx-api.json @@ -3606,6 +3606,30 @@ }, "path": "/nx-api/workspace" }, + "powerpack-azure-cache": { + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-azure-cache", + "packageName": "@nx/powerpack-azure-cache", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Azure Blob Storage.", + "documents": { + "/nx-api/powerpack-azure-cache/documents/overview": { + "id": "overview", + "name": "Overview", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Azure Blob Storage.", + "file": "external-generated/packages/powerpack-azure-cache/documents/overview", + "itemList": [], + "isExternal": false, + "path": "/nx-api/powerpack-azure-cache/documents/overview", + "tags": [], + "originalFilePath": "shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin" + } + }, + "root": "/libs/nx-packages/powerpack-azure-cache", + "source": "/libs/nx-packages/powerpack-azure-cache/src", + "executors": {}, + "generators": {}, + "path": "/nx-api/powerpack-azure-cache" + }, "powerpack-conformance": { "githubRoot": "https://github.com/nrwl/nx/blob/master", "name": "powerpack-conformance", @@ -3640,6 +3664,52 @@ "generators": {}, "path": "/nx-api/powerpack-conformance" }, + "powerpack-enterprise-cloud": { + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-enterprise-cloud", + "packageName": "@nx/powerpack-enterprise-cloud", + "description": "A Nx Powerpack plugin which is specific to Nx Enterprise Cloud workspaces.", + "documents": {}, + "root": "/libs/nx-packages/powerpack-enterprise-cloud", + "source": "/libs/nx-packages/powerpack-enterprise-cloud/src", + "executors": {}, + "generators": { + "/nx-api/powerpack-enterprise-cloud/generators/init": { + "description": "Initialize Nx Powerpack Enterprise Cloud config", + "file": "external-generated/packages/powerpack-enterprise-cloud/generators/init.json", + "hidden": false, + "name": "init", + "originalFilePath": "/libs/nx-packages/powerpack-enterprise-cloud/src/generators/init/schema.json", + "path": "/nx-api/powerpack-enterprise-cloud/generators/init", + "type": "generator" + } + }, + "path": "/nx-api/powerpack-enterprise-cloud" + }, + "powerpack-gcs-cache": { + "githubRoot": "https://github.com/nrwl/nx/blob/master", + "name": "powerpack-gcs-cache", + "packageName": "@nx/powerpack-gcs-cache", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Google Cloud Storage.", + "documents": { + "/nx-api/powerpack-gcs-cache/documents/overview": { + "id": "overview", + "name": "Overview", + "description": "A Nx Powerpack plugin which provides a Nx cache which can be self hosted on Google Cloud Storage.", + "file": "external-generated/packages/powerpack-gcs-cache/documents/overview", + "itemList": [], + "isExternal": false, + "path": "/nx-api/powerpack-gcs-cache/documents/overview", + "tags": [], + "originalFilePath": "shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin" + } + }, + "root": "/libs/nx-packages/powerpack-gcs-cache", + "source": "/libs/nx-packages/powerpack-gcs-cache/src", + "executors": {}, + "generators": {}, + "path": "/nx-api/powerpack-gcs-cache" + }, "powerpack-license": { "githubRoot": "https://github.com/nrwl/nx/blob/master", "name": "powerpack-license", diff --git a/docs/map.json b/docs/map.json index b52c4e750d..d0cb38e5e4 100644 --- a/docs/map.json +++ b/docs/map.json @@ -2617,6 +2617,32 @@ } ] }, + { + "name": "powerpack-azure-cache", + "id": "powerpack-azure-cache", + "description": "powerpack-azure-cache package.", + "itemList": [ + { + "name": "Overview", + "id": "overview", + "path": "/nx-api/powerpack-azure-cache", + "file": "shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin" + } + ] + }, + { + "name": "powerpack-gcs-cache", + "id": "powerpack-gcs-cache", + "description": "powerpack-gcs-cache package.", + "itemList": [ + { + "name": "Overview", + "id": "overview", + "path": "/nx-api/powerpack-gcs-cache", + "file": "shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin" + } + ] + }, { "name": "powerpack-s3-cache", "id": "powerpack-s3-cache", diff --git a/docs/shared/deprecated/custom-task-runners.md b/docs/shared/deprecated/custom-task-runners.md index fd4565cab3..0a2f689e14 100644 --- a/docs/shared/deprecated/custom-task-runners.md +++ b/docs/shared/deprecated/custom-task-runners.md @@ -8,5 +8,5 @@ As of Nx 20, the `tasksRunnerOptions` property in `nx.json` is deprecated. This For most organizations, this feature is a net positive. If you are currently using a custom task runner, you are most likely using it to define your own custom [remote cache](/ci/features/remote-cache) storage location. You have several options moving forward: 1. Use [Nx Cloud](/nx-cloud) for your remote cache -2. Use an [Nx Powerpack](/powerpack) plugin to store your remote cache on an [Amazon S3 bucket](/nx-api/powerpack-s3-cache) or a [network drive](/nx-api/powerpack-shared-fs-cache) +2. Use an [Nx Powerpack](/powerpack) plugin to store your remote cache on [Amazon S3](/nx-api/powerpack-s3-cache), [Google Cloud](/nx-api/powerpack-gcs-cache), [Google Cloud](/nx-api/powerpack-gcs-cache) or a [network drive](/nx-api/powerpack-shared-fs-cache) 3. Use the deprecated custom task runner feature until Nx 21 diff --git a/docs/shared/features/powerpack/custom-caching.md b/docs/shared/features/powerpack/custom-caching.md index b94f964045..8b194a90bf 100644 --- a/docs/shared/features/powerpack/custom-caching.md +++ b/docs/shared/features/powerpack/custom-caching.md @@ -5,14 +5,16 @@ The recommended way to enable the [remote cache](/ci/features/remote-cache) is to use Nx Replay and have Nx Cloud share the task cache across your organization. For those organizations that are unable to use Nx Cloud, Nx offers official plugins that are enabled by [Nx Powerpack](/powerpack) to self-host the remote cache in a fast and secure manner. Powerpack is available for Nx version 19.8 and higher. The recommended ways to host the remote cache are, in order of preference: 1. [Nx Replay](/ci/features/remote-cache): Cache is hosted on Nx Cloud servers or on-premise with an [Nx Enterprise](/enterprise) contract -2. [@nx/powerpack-s3-cache](/nx-api/powerpack-s3-cache): Cache is on a self-hosted Amazon S3 bucket -3. [@nx/powerpack-shared-fs-cache](/nx-api/powerpack-shared-fs-cache): Cache is self-hosted on a shared file system location +2. [@nx/powerpack-s3-cache](/nx-api/powerpack-s3-cache): Cache is self-hosted on an Amazon S3 bucket +3. [@nx/powerpack-gcs-cache](/nx-api/powerpack-gcs-cache): Cache is self-hosted on Google Cloud storage +4. [@nx/powerpack-azure-cache](/nx-api/powerpack-azure-cache): Cache is self-hosted on Azure +5. [@nx/powerpack-shared-fs-cache](/nx-api/powerpack-shared-fs-cache): Cache is self-hosted on a shared file system location The options range from fully opting in to Nx's management of the remote cache to fully managing the configuration and security of your own remote cache. ## Setup -The `@nx/powerpack-s3-cache` and `@nx/powerpack-shared-fs-cache` plugins require an Nx Powerpack license to function. [Activating Powerpack](/nx-enterprise/activate-powerpack) is a simple process. +All the self-hosted cache storage plugins require an Nx Powerpack license to function. [Activating Powerpack](/nx-enterprise/activate-powerpack) is a simple process. {% call-to-action title="Buy a Powerpack License" icon="nx" description="Unlock all the features of the Nx CLI" url="https://cloud.nx.app/powerpack/purchase" /%} @@ -20,8 +22,9 @@ Then, choose the appropriate cache plugin for your situation. {% cards cols="2" lgCols="2" mdCols="2" smCols="2" %} -{% link-card title="Amazon S3 Bucket Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-s3-cache" icon="AwsIcon" /%} - +{% link-card title="Amazon S3 Bucket Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-s3-cache" icon="AmazonS3Icon" /%} +{% link-card title="Google Cloud Storage Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-gcs-cache" icon="GoogleCloudIcon" /%} +{% link-card title="Azure Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-azure-cache" icon="AzureDevOpsIcon" /%} {% link-card title="Shared Network Drive Remote Cache" type="Nx Plugin" url="/nx-api/powerpack-shared-fs-cache" icon="ServerIcon" /%} {% /cards %} diff --git a/docs/shared/features/powerpack/index.md b/docs/shared/features/powerpack/index.md index d55b12b7a4..756236f708 100644 --- a/docs/shared/features/powerpack/index.md +++ b/docs/shared/features/powerpack/index.md @@ -8,6 +8,6 @@ The following features are available after you [activate a Powerpack license](/n {% link-card title="Conformance" type="Powerpack Feature" url="/nx-enterprise/powerpack/conformance" icon="CheckBadgeIcon" /%} {% link-card title="Owners" type="Powerpack Feature" url="/nx-enterprise/powerpack/owners" icon="UserGroupIcon" /%} - {% link-card title="Self-Hosted Remote Cache" type="Powerpack Feature" url="/nx-enterprise/powerpack/custom-caching" icon="AwsIcon" /%} + {% link-card title="Self-Hosted Remote Cache" type="Powerpack Feature" url="/nx-enterprise/powerpack/custom-caching" icon="ServerIcon" /%} {% /cards %} diff --git a/docs/shared/guides/unknown-local-cache.md b/docs/shared/guides/unknown-local-cache.md index 7175b47283..9068ed19f8 100644 --- a/docs/shared/guides/unknown-local-cache.md +++ b/docs/shared/guides/unknown-local-cache.md @@ -80,6 +80,8 @@ Nx Replay is not the only remote cache you can use. You can also use one of the If you can't use Nx Replay, Nx provides plugins that enable you to [self-host the remote cache](/nx-enterprise/powerpack/custom-caching). These plugins are available as part of the [Nx Powerpack](/powerpack), which you can unlock by [activating your license](/nx-enterprise/activate-powerpack). There are plugins to self-host your remote cache in the following locations: - [Amazon S3 Bucket](/nx-api/powerpack-s3-cache) +- [Google Cloud Storage](/nx-api/powerpack-gcs-cache) +- [Azure](/nx-api/powerpack-azure-cache) - [Shared File System](/nx-api/powerpack-shared-fs-cache) These plugins will ensure that the task metadata and project graph information are correctly associated with the file artifacts in your cache. diff --git a/docs/shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin.md b/docs/shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin.md new file mode 100644 index 0000000000..1663e7ae2b --- /dev/null +++ b/docs/shared/packages/powerpack-azure-cache/powerpack-azure-cache-plugin.md @@ -0,0 +1,83 @@ +--- +title: Overview of the Nx powerpack-azure-cache Plugin +description: The powerpack-azure-cache Nx plugin enables you to use Azure Storage to host your remote cache instead of Nx Cloud +--- + +The `@nx/powerpack-azure-cache` plugin enables you to use [Azure Storage](https://azure.microsoft.com/en-us/products/storage/blobs) instead of Nx Cloud to host your remote cache. + +This plugin will enable the remote cache for your Nx workspace, but does not provide any of the other features of Nx Cloud. If you want to leverage [distributed task execution](/ci/features/distribute-task-execution), [re-running flaky tasks](/ci/features/flaky-tasks) or [automatically splitting tasks](/ci/features/split-e2e-tasks), you'll need to [connect to Nx Cloud](/ci/intro/connect-to-nx-cloud) and use [Nx Replay](/ci/features/remote-cache) instead. + +{% callout type="warning" title="Potential Cache Poisoning" %} +Using your own Azure Storage account to host the remote cache opens you up to the possibility of [cache poisoning](/troubleshooting/unknown-local-cache). To avoid this, use [Nx Replay](/ci/features/remote-cache). +{% /callout %} + +{% callout title="This plugin requires an active Nx Powerpack license" %} +In order to use `@nx/powerpack-azure-cache`, you need to have an active Powerpack license. If you don't have a license or it has expired, your cache will no longer be shared and each machine will use its local cache. +{% /callout %} + +## Set Up @nx/powerpack-azure-cache + +### 1. Install the Package + +1. [Activate Powerpack](/nx-enterprise/activate-powerpack) if you haven't already +2. Install the package + +```shell +nx add @nx/powerpack-azure-cache +``` + +### 2. Authenticate with Azure + +There are several ways to [authenticate with Azure Storage](https://github.com/Azure/login#login-with-openid-connect-oidc-recommended), but the method recommended by Azure is to use OpenID Connect, like this: + +```yaml {% fileName=".github/workflows/ci.yml" %} +name: CI +... +permissions: + id-token: write + ... + +jobs: + main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} + runs-on: ubuntu-latest + steps: + ... + + - name: Azure login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + ... + + - run: pnpm exec nx affected -t lint test build +``` + +You need to set the `AZURE_CLIENT_ID`, `AZURE_TENANT_ID` and `AZURE_SUBSCRIPTION_ID` secrets as defined in the [Azure documentation](https://github.com/Azure/login#login-with-openid-connect-oidc-recommended). + +Note: Any authentication method that [sets up the `DefaultAzureCredentials`](https://learn.microsoft.com/en-us/azure/developer/javascript/sdk/credential-chains#use-defaultazurecredential-for-flexibility) will enable the plugin to work. + +{% callout type="note" title="Custom Azure Endpoint" %} +If you are using a custom Azure endpoint, you will need to authenticate by [setting the `AZURE_STORAGE_CONNECTION_STRING` environment variable](https://learn.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string). The `@nx/powerpack-azure-cache` plugin will detect the environment variable and automatically use it to connect to Azure. +{% /callout %} + +### 3. Configure the Nx Cache to Use Azure Storage + +Finally, you need to configure your Nx cache in the `nx.json` file. The `container` that you specify needs to already exist - Nx doesn't create it for you. + +```jsonc {% fileName="nx.json" %} +{ + "azure": { + "container": "mycontainer", + "accountName": "myaccount" + } +} +``` + +| **Property** | **Description** | +| --------------- | -------------------------------- | +| **container** | The name of the container to use | +| **accountName** | The name of blob storage account | diff --git a/docs/shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin.md b/docs/shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin.md new file mode 100644 index 0000000000..2212c9ac99 --- /dev/null +++ b/docs/shared/packages/powerpack-gcs-cache/powerpack-gcs-cache-plugin.md @@ -0,0 +1,82 @@ +--- +title: Overview of the Nx powerpack-gcs-cache Plugin +description: The powerpack-gcs-cache Nx plugin enables you to use Google Cloud Storage to host your remote cache instead of Nx Cloud +--- + +The `@nx/powerpack-gcs-cache` plugin enables you to use [Google Cloud Storage](https://cloud.google.com/storage) instead of Nx Cloud to host your remote cache. + +This plugin will enable the remote cache for your Nx workspace, but does not provide any of the other features of Nx Cloud. If you want to leverage [distributed task execution](/ci/features/distribute-task-execution), [re-running flaky tasks](/ci/features/flaky-tasks) or [automatically splitting tasks](/ci/features/split-e2e-tasks), you'll need to [connect to Nx Cloud](/ci/intro/connect-to-nx-cloud) and use [Nx Replay](/ci/features/remote-cache) instead. + +{% callout type="warning" title="Potential Cache Poisoning" %} +Using your own Google Cloud Storage account to host the remote cache opens you up to the possibility of [cache poisoning](/troubleshooting/unknown-local-cache). To avoid this, use [Nx Replay](/ci/features/remote-cache). +{% /callout %} + +{% callout title="This plugin requires an active Nx Powerpack license" %} +In order to use `@nx/powerpack-gcs-cache`, you need to have an active Powerpack license. If you don't have a license or it has expired, your cache will no longer be shared and each machine will use its local cache. +{% /callout %} + +## Set Up @nx/powerpack-gcs-cache + +### 1. Install the Package + +1. [Activate Powerpack](/nx-enterprise/activate-powerpack) if you haven't already +2. Install the package + +```shell +nx add @nx/powerpack-gcs-cache +``` + +### 2. Authenticate with Google Cloud + +There are several ways to [authenticate with Google Cloud Storage](https://github.com/google-github-actions/setup-gcloud#authorization), but the method recommended by Google is to use Workload Identity Federation, like this: + +```yaml {% fileName=".github/workflows/ci.yml" %} +name: CI +... +permissions: + id-token: write + ... + +jobs: + main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} + runs-on: ubuntu-latest + steps: + ... + + - id: 'auth' + name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v2' + with: + token_format: 'access_token' + workload_identity_provider: 'projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider' + service_account: 'my-service-account@my-project.iam.gserviceaccount.com' + + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v2' + with: + version: '>= 363.0.0' + + ... + + - run: pnpm exec nx affected -t lint test build +``` + +Note: Any authentication method that [sets up the Application Default Credentials](https://cloud.google.com/docs/authentication/application-default-credentials) will enable the plugin to work. + +### 3. Configure the Nx Cache to Use Google Cloud Storage + +Finally, you need to configure your Nx cache in the `nx.json` file. The `bucket` that you specify needs to already exist - Nx doesn't create it for you. + +```jsonc {% fileName="nx.json" %} +{ + "gcs": { + "bucket": "my-bucket" + } +} +``` + +| **Property** | **Description** | +| ------------ | ----------------------------- | +| **bucket** | The name of the bucket to use | diff --git a/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md b/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md index dc1ed6a229..f5faddceab 100644 --- a/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md +++ b/docs/shared/packages/powerpack-s3-cache/powerpack-s3-cache-plugin.md @@ -57,11 +57,10 @@ permissions: id-token: write ... -env: - NX_DB_CACHE: true - jobs: main: + env: + NX_POWERPACK_LICENSE: ${{ secrets.NX_POWERPACK_LICENSE }} runs-on: ubuntu-latest steps: ... diff --git a/docs/shared/reference/sitemap.md b/docs/shared/reference/sitemap.md index 54496eddbd..8f99f60aef 100644 --- a/docs/shared/reference/sitemap.md +++ b/docs/shared/reference/sitemap.md @@ -772,11 +772,20 @@ - [npm-package](/nx-api/workspace/generators/npm-package) - [ci-workflow](/nx-api/workspace/generators/ci-workflow) - [infer-targets](/nx-api/workspace/generators/infer-targets) + - [powerpack-azure-cache](/nx-api/powerpack-azure-cache) + - [documents](/nx-api/powerpack-azure-cache/documents) + - [Overview](/nx-api/powerpack-azure-cache/documents/overview) - [powerpack-conformance](/nx-api/powerpack-conformance) - [documents](/nx-api/powerpack-conformance/documents) - [Overview](/nx-api/powerpack-conformance/documents/overview) - [executors](/nx-api/powerpack-conformance/executors) - [bundle-rule](/nx-api/powerpack-conformance/executors/bundle-rule) + - [powerpack-enterprise-cloud](/nx-api/powerpack-enterprise-cloud) + - [generators](/nx-api/powerpack-enterprise-cloud/generators) + - [init](/nx-api/powerpack-enterprise-cloud/generators/init) + - [powerpack-gcs-cache](/nx-api/powerpack-gcs-cache) + - [documents](/nx-api/powerpack-gcs-cache/documents) + - [Overview](/nx-api/powerpack-gcs-cache/documents/overview) - [powerpack-license](/nx-api/powerpack-license) - [powerpack-owners](/nx-api/powerpack-owners) - [documents](/nx-api/powerpack-owners/documents) diff --git a/nx-dev/nx-dev/pages/nx-api/index.tsx b/nx-dev/nx-dev/pages/nx-api/index.tsx index 93eb3cb42c..9619e0ec0e 100644 --- a/nx-dev/nx-dev/pages/nx-api/index.tsx +++ b/nx-dev/nx-dev/pages/nx-api/index.tsx @@ -40,23 +40,8 @@ export default function Packages({ ), }, packages: useMemo(() => { - const storybookIdx = packages.findIndex((p) => p.name === 'storybook'); - const packagesWithRspack = [ - ...packages.slice(0, storybookIdx), - { - description: - 'The Nx Plugin for Rspack contains executors and generators that support building applications using Rspack.', - githubRoot: 'https://github.com/nrwl/nx/blob/master', - name: 'rspack', - packageName: '@nrwl/rspack', - path: '/nx-api/rspack', - root: '/nx-api/rspack', - source: '/nx-api/rspack/src', - }, - ...packages.slice(storybookIdx), - ]; return sortCorePackagesFirst( - filterMigrationPackages(packagesWithRspack), + filterMigrationPackages(packages), 'name' ); }, [packages]), diff --git a/nx-dev/nx-dev/public/images/icons/powerpack-azure-cache.svg b/nx-dev/nx-dev/public/images/icons/powerpack-azure-cache.svg new file mode 100644 index 0000000000..31b29cffc0 --- /dev/null +++ b/nx-dev/nx-dev/public/images/icons/powerpack-azure-cache.svg @@ -0,0 +1,10 @@ + + Azure DevOps + + \ No newline at end of file diff --git a/nx-dev/nx-dev/public/images/icons/powerpack-gcs-cache.svg b/nx-dev/nx-dev/public/images/icons/powerpack-gcs-cache.svg new file mode 100644 index 0000000000..cadf402c76 --- /dev/null +++ b/nx-dev/nx-dev/public/images/icons/powerpack-gcs-cache.svg @@ -0,0 +1,10 @@ + + Google Cloud + + \ No newline at end of file diff --git a/nx-dev/ui-powerpack/src/lib/powerpack-features.tsx b/nx-dev/ui-powerpack/src/lib/powerpack-features.tsx index 409c43fd32..3b02bbdd73 100644 --- a/nx-dev/ui-powerpack/src/lib/powerpack-features.tsx +++ b/nx-dev/ui-powerpack/src/lib/powerpack-features.tsx @@ -17,7 +17,12 @@ import { import { cx } from '@nx/nx-dev/ui-primitives'; import { AnimatedCurvedBeam } from '@nx/nx-dev/ui-animations'; import { CircleStackIcon, ServerIcon } from '@heroicons/react/24/outline'; -import { AzureDevOpsIcon, GoogleCloudIcon, NxIcon } from '@nx/nx-dev/ui-icons'; +import { + AmazonS3Icon, + AzureDevOpsIcon, + GoogleCloudIcon, + NxIcon, +} from '@nx/nx-dev/ui-icons'; import Link from 'next/link'; import { AnimatePresence, motion } from 'framer-motion'; @@ -398,9 +403,9 @@ export function CustomRemoteCacheAnimation(): ReactElement { )} >
- AWS + Amazon S3
-
S3
+