docs(core): nx cloud intro (#13132)
This commit is contained in:
parent
c4ebef2803
commit
20fe12b1b9
@ -230,6 +230,12 @@
|
|||||||
"tags": ["cache-task-results"],
|
"tags": ["cache-task-results"],
|
||||||
"file": "shared/core-features/cache-task-results"
|
"file": "shared/core-features/cache-task-results"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "Share Your Cache",
|
||||||
|
"id": "share-your-cache",
|
||||||
|
"tags": ["share-your-cache"],
|
||||||
|
"file": "shared/core-features/share-your-cache"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "Explore the Graph",
|
"name": "Explore the Graph",
|
||||||
"id": "explore-graph",
|
"id": "explore-graph",
|
||||||
@ -1026,21 +1032,6 @@
|
|||||||
"id": "set-up",
|
"id": "set-up",
|
||||||
"description": "Learn how to set up Nx Cloud for your workspace.",
|
"description": "Learn how to set up Nx Cloud for your workspace.",
|
||||||
"itemList": [
|
"itemList": [
|
||||||
{
|
|
||||||
"name": "Adding Nx Cloud to an Nx Workspace",
|
|
||||||
"id": "add-nx-cloud",
|
|
||||||
"file": "nx-cloud/set-up/add-nx-cloud"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Set Up Distributed Caching",
|
|
||||||
"id": "set-up-caching",
|
|
||||||
"file": "nx-cloud/set-up/set-up-caching"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "Set Up Distributed Task Execution",
|
|
||||||
"id": "set-up-dte",
|
|
||||||
"file": "nx-cloud/set-up/set-up-dte"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "Record Non-Nx Commands",
|
"name": "Record Non-Nx Commands",
|
||||||
"id": "record-commands",
|
"id": "record-commands",
|
||||||
@ -1152,6 +1143,11 @@
|
|||||||
"id": "config",
|
"id": "config",
|
||||||
"file": "nx-cloud/reference/config"
|
"file": "nx-cloud/reference/config"
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"name": "nx-cloud CLI",
|
||||||
|
"id": "nx-cloud-cli",
|
||||||
|
"file": "nx-cloud/reference/nx-cloud-cli"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"name": "Environment Variables",
|
"name": "Environment Variables",
|
||||||
"id": "env-vars",
|
"id": "env-vars",
|
||||||
|
|||||||
@ -1,27 +1,30 @@
|
|||||||
# What is Nx Cloud?
|
# What is Nx Cloud?
|
||||||
|
|
||||||
## Save Time With Distributed Task Execution & Distributed Cache
|
Nx partitions a command into smaller tasks and runs them in parallel, in the correct order. Nx Cloud takes it one step further and [runs any command across multiple machines](/core-features/distribute-task-execution), while giving you a consolidated view of the command as if it ran locally.
|
||||||
|
|
||||||
Nx partitions a command into smaller tasks and runs them in parallel, in the correct order. Nx Cloud takes it one step further and [runs any command across multiple machines](/nx-cloud/set-up/set-up-dte), while giving you a consolidated view of the command as if it ran locally.
|
Nx caches the output of any previously run command such as testing and building, so it can replay the cached results instead of rerunning it. Nx Cloud allows you to [share the computation cache](/core-features/share-your-cache) across everyone in your team and CI.
|
||||||
|
|
||||||
Nx caches the output of any previously run command such as testing and building, so it can replay the cached results instead of rerunning it. Nx Cloud allows you to [share the computation cache](/nx-cloud/set-up/set-up-caching) across everyone in your team and CI.
|
|
||||||
|
|
||||||
And it takes five minutes to set up.
|
And it takes five minutes to set up.
|
||||||
|
|
||||||
|
{% personas %}
|
||||||
|
{% persona type="cache" title="Share Your Cache" url="/core-features/share-your-cache" %}
|
||||||
|
Share the computation cache across CI and developer machines
|
||||||
|
|
||||||
|
- [Set up distributed caching](/core-features/share-your-cache)
|
||||||
|
|
||||||
|
{% /persona %}
|
||||||
|
|
||||||
|
{% persona type="distribute" title="Distribute Tasks in CI" url="/core-features/distribute-task-execution" %}
|
||||||
|
|
||||||
|
Improve the worst case CI build by efficiently parallelizing tasks
|
||||||
|
|
||||||
|
- [Set up distributed task execution](/core-features/distribute-task-execution)
|
||||||
|
|
||||||
|
{% /persona %}
|
||||||
|
{% /personas %}
|
||||||
|
|
||||||
## Clean User Interface
|
## Clean User Interface
|
||||||
|
|
||||||
Most developers will benefit from Nx Cloud without ever-changing any of their workflow. Commands will just execute faster. For those developers that are tasked with making the most of Nx Cloud or those that want to get more insight into the different commands executed in the repository, Nx Cloud has a clean, informative user interface.
|
Most developers will benefit from Nx Cloud without ever-changing any of their workflow. Commands will just execute faster. For those developers that are tasked with making the most of Nx Cloud or those that want to get more insight into the different commands executed in the repository, Nx Cloud has a clean, informative user interface.
|
||||||
|
|
||||||
The [top level organization page](https://nx.app/orgs/5e38af6de037b5000598b2d6/workspaces/5edaf12087863a0005781f17) displays recent runs and a helpful dashboard of information about commands run in the repository.
|
Take a look at the [Nx repo's Nx Cloud dashboard](https://nx.app/orgs/5e38af6de037b5000598b2d6/workspaces/5edaf12087863a0005781f17) for a real world example of how Nx Cloud is saving us CI time.
|
||||||
|
|
||||||
{% iframe
|
|
||||||
src="https://staging.nx.app/orgs/62d013d4d26f260059f7765e/workspaces/62d013ea0852fe0a2df74438?hideHeader=true"
|
|
||||||
title="Nx Cloud dashboard"
|
|
||||||
width="100%" /%}
|
|
||||||
|
|
||||||
Each branch in the repository has its own [dedicated page](https://nx.app/branch?workspaceId=5edaf12087863a0005781f17&branchName=master) where you can view agent utilization and a waterfall task execution graph for the most recent runs against that branch.
|
|
||||||
|
|
||||||
{% iframe
|
|
||||||
src="https://staging.nx.app/branch?workspaceId=62d013ea0852fe0a2df74438&branchName=master&hideHeader=true"
|
|
||||||
title="Nx Cloud branch view"
|
|
||||||
width="100%" /%}
|
|
||||||
|
|||||||
@ -32,6 +32,12 @@ Setting `NX_CLOUD_ENV_NAME` will prefix all your commands so you can easily dist
|
|||||||
|
|
||||||
By default, Nx Cloud requests will time out after 10 seconds. `NX_CLOUD_NO_TIMEOUTS` disables the timeout.
|
By default, Nx Cloud requests will time out after 10 seconds. `NX_CLOUD_NO_TIMEOUTS` disables the timeout.
|
||||||
|
|
||||||
|
### NX_RUN_GROUP
|
||||||
|
|
||||||
|
- The `@nrwl/nx-cloud` requires the `NX_RUN_GROUP` environment variables to be set. For many CI providers (e.g., GitHub
|
||||||
|
Actions), the runner is able to set it automatically. For others, the variable will have to be set manually. If you set
|
||||||
|
it manually, note that `NX_RUN_GROUP` has to be a unique value associated with a CI run.
|
||||||
|
|
||||||
### NX_VERBOSE_LOGGING
|
### NX_VERBOSE_LOGGING
|
||||||
|
|
||||||
Setting `NX_VERBOSE_LOGGING` to true will output the debug information about agents communicating with the main job. This can be useful for debugging unexpected cache misses and issues with on-prem setups.
|
Setting `NX_VERBOSE_LOGGING` to true will output the debug information about agents communicating with the main job. This can be useful for debugging unexpected cache misses and issues with on-prem setups.
|
||||||
|
|||||||
45
docs/nx-cloud/reference/nx-cloud-cli.md
Normal file
45
docs/nx-cloud/reference/nx-cloud-cli.md
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
# `nx-cloud` CLI
|
||||||
|
|
||||||
|
## npx nx-cloud start-ci-run
|
||||||
|
|
||||||
|
At the beginning of your main job, invoke `npx nx-cloud start-ci-run`. This tells Nx Cloud that the following series of
|
||||||
|
command correspond to the same CI run.
|
||||||
|
|
||||||
|
You can configure your CI run by passing the following flags:
|
||||||
|
|
||||||
|
### --use-dte-by-default
|
||||||
|
|
||||||
|
By default, invoking `npx nx-cloud start-ci-run` will configure Nx to distribute all commands by default. You can
|
||||||
|
disable this as follows: `npx nx-cloud start-ci-run --use-dte-by-default=false`.
|
||||||
|
|
||||||
|
### --stop-agents-on-failure
|
||||||
|
|
||||||
|
By default, a failure in one of the commands is going to terminate the whole CI run and will stop all the
|
||||||
|
agents. You can disable this as follows: `npx nx-cloud start-ci-run --stop-agents-on-failure=false`.
|
||||||
|
|
||||||
|
### --stop-agents-after
|
||||||
|
|
||||||
|
By default, Nx Cloud won't terminate any agents until you invoke `npx nx-cloud stop-all-agents` because Nx Cloud
|
||||||
|
doesn't know if you will need agents to run another command. This can result in agents being idle at the end of a CI
|
||||||
|
run.
|
||||||
|
|
||||||
|
You can fix it by telling Nx Cloud that it can terminate agents after it sees a certain
|
||||||
|
target: `npx nx-cloud start-ci-run --stop-agents-after=e2e`.
|
||||||
|
|
||||||
|
> Earlier versions of `@nrwl/nx-cloud` required you to set the `NX_CLOUD_DISTRIBUTED_EXECUTION` env variable to `true`
|
||||||
|
> to
|
||||||
|
> enable DTE, but in the latest version `npx nx-cloud start-ci-run` does it automatically.
|
||||||
|
|
||||||
|
## Enabling/Disabling DTE
|
||||||
|
|
||||||
|
Invoking `npx nx-cloud start-ci-run` will tell Nx to distribute by default. You can enable/disable distribution for
|
||||||
|
individual commands as follows:
|
||||||
|
|
||||||
|
- `nx affected --target=build --dte` (explicitly enable distribution, Nx >= 14.7)
|
||||||
|
- `nx affected --target=build --no-dte` (explicitly disable distribution, Nx >= 14.7)
|
||||||
|
- `NX_CLOUD_DISTRIBUTED_EXECUTION=true nx affected --target=build` (explicitly enable distribution)
|
||||||
|
- `NX_CLOUD_DISTRIBUTED_EXECUTION=false nx affected --target=build` (explicitly disable distribution)
|
||||||
|
|
||||||
|
## npx nx-cloud stop-all-agents
|
||||||
|
|
||||||
|
This command tells Nx Cloud to terminate all agents associated with this CI run.
|
||||||
@ -1,57 +0,0 @@
|
|||||||
# Adding Nx Cloud to an Nx Workspace
|
|
||||||
|
|
||||||
## Adding Nx Cloud When Creating a Workspace
|
|
||||||
|
|
||||||
You can enable Nx Cloud when creating a new workspace with `npx create-nx-workspace`. Just reply with `Yes` to the prompt that asks about adding Nx Cloud to your workspace.
|
|
||||||
|
|
||||||
```shell
|
|
||||||
? Enable distributed caching to make your CI faster
|
|
||||||
(Use arrow keys)
|
|
||||||
❯ Yes I want faster builds
|
|
||||||
No
|
|
||||||
```
|
|
||||||
|
|
||||||
This will connect your workspace to Nx Cloud, so you can start testing it out right away.
|
|
||||||
|
|
||||||
## Adding Nx Cloud to an Existing Workspace
|
|
||||||
|
|
||||||
If you are on the latest version of Nx, you can **connect your existing Nx workspace** to Nx Cloud by
|
|
||||||
running:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
npx nx connect-to-nx-cloud
|
|
||||||
```
|
|
||||||
|
|
||||||
If you are using an older version of Nx, you can connect your workspace by adding the `@nrwl/nx-cloud`
|
|
||||||
package to your workspace and running `nx g @nrwl/nx-cloud:init`.
|
|
||||||
|
|
||||||
## Connecting Your Workspace to Your Nx Cloud Account
|
|
||||||
|
|
||||||
After you have enabled Nx Cloud in your workspace, you will see the following:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
> NX NOTE Nx Cloud has been enabled
|
|
||||||
|
|
||||||
Your workspace is currently public. Anybody with code access
|
|
||||||
can view the workspace on nx.app.
|
|
||||||
|
|
||||||
You can connect the workspace to your Nx Cloud account at
|
|
||||||
https://nx.app/orgs/workspace-setup?accessToken=N2Y3NzcyO...
|
|
||||||
(You can do this later.)
|
|
||||||
```
|
|
||||||
|
|
||||||
Click on this link to associate the workspace with your Nx Cloud account. If you don't have an Nx Cloud account, you can
|
|
||||||
create one on the spot.
|
|
||||||
|
|
||||||
After you claim your workspace, you will be able to manage permissions, create access tokens, set up billing, and so
|
|
||||||
forth.
|
|
||||||
|
|
||||||
**You will also see an interactive tutorial helping you explore distributed caching and the Nx Cloud user interface.**
|
|
||||||
|
|
||||||
If you lose this link, you can still connect your workspace to Nx Cloud. Go to [nx.app](https://nx.app), create an
|
|
||||||
account, and connect your workspace using the access token from `nx.json`.
|
|
||||||
|
|
||||||
## Skipping Cloud
|
|
||||||
|
|
||||||
Similar to how `--skip-nx-cache` will instruct Nx not to use the cache, passing `--no-cloud` will tell Nx not to use Nx
|
|
||||||
Cloud.
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
# Enabling Distributed Caching
|
|
||||||
|
|
||||||
After connecting to Nx Cloud, you will immediately start taking advantage of the distributed caching. To see this in
|
|
||||||
action
|
|
||||||
run:
|
|
||||||
|
|
||||||
- `nx build YOUR_APP_NAME`
|
|
||||||
- `nx reset` to remove local cache
|
|
||||||
- `nx build YOUR_APP_NAME` again to fetch the file artifacts and the terminal output from the cloud
|
|
||||||
|
|
||||||
## Skipping Nx Cloud Cache
|
|
||||||
|
|
||||||
Similar to how `--skip-nx-cache` will instruct Nx not to use the cache, passing `--no-cloud` will tell Nx not to use Nx
|
|
||||||
Cloud, only local cache will be used.
|
|
||||||
|
|
||||||
### Relevant Documentation
|
|
||||||
|
|
||||||
- [Cache Task Results](/core-features/cache-task-results)
|
|
||||||
- [How Caching Works](/concepts/how-caching-works)
|
|
||||||
@ -1,162 +0,0 @@
|
|||||||
# Set Up Distributed Task Execution
|
|
||||||
|
|
||||||
> Make sure to use the latest version of `@nrwl/nx-cloud`. The latest version works with any version of Nx >= 13.0.
|
|
||||||
|
|
||||||
## How It Works
|
|
||||||
|
|
||||||
When using Nx Cloud Distributed Task Execution (DTE), you split your CI setup into two parts:
|
|
||||||
|
|
||||||
- The main job that controls what is going to be executed
|
|
||||||
- The agent jobs that actually execute the tasks
|
|
||||||
|
|
||||||
The main job execution flow looks like this:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# Coordinate the agents to run the tasks
|
|
||||||
- npx nx-cloud start-ci-run
|
|
||||||
# Run any commands you want here
|
|
||||||
- nx affected --target=lint & nx affected --target=test & nx affected --target=build
|
|
||||||
# Stop any run away agents
|
|
||||||
- npx nx-cloud stop-all-agents
|
|
||||||
```
|
|
||||||
|
|
||||||
The agent job execution flow is very simple:
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
# Wait for tasks to execute
|
|
||||||
- npx nx-cloud start-agent
|
|
||||||
```
|
|
||||||
|
|
||||||
The main job looks more or less the same way as if you haven't used any distribution. The only thing you need to do is
|
|
||||||
to invoke `npx nx-cloud start-ci-run` at the beginning and optionally invoke `npx nx-cloud stop-all-agents` at the end.
|
|
||||||
|
|
||||||
The agent jobs run long-running `start-agent` processes that execute all the tasks associated with a given CI run. The
|
|
||||||
only thing you need to do to set them up is to invoke `npx nx-cloud start-agent`. This process will keep running until
|
|
||||||
Nx Cloud tells it to terminate.
|
|
||||||
|
|
||||||
> Note it's important that the main job and the agent jobs have the same environment and the same source code. They start
|
|
||||||
> around the same time. And, once the main job completes, all the agents
|
|
||||||
> will be stopped.
|
|
||||||
|
|
||||||
It's also important to note that an Nx Cloud agent isn't a machine but rather a long-running process that runs on a
|
|
||||||
machine. I.e., Nx Cloud doesn't manage your agents--you need to do it in your CI config (check out CI examples below).
|
|
||||||
|
|
||||||
Nx Cloud is an orchestrator. The main job tells Nx Cloud what you want to run, and Nx Cloud will distribute those tasks
|
|
||||||
across the agents. Nx Cloud will automatically move files from one agent to another, from the agents to the main job.
|
|
||||||
|
|
||||||
The end result is that when say `nx affected --target=build` completes on the main job, all the file artifacts created
|
|
||||||
on agents are copied over to the main job, as if the main job had built everything locally.
|
|
||||||
|
|
||||||
### Relevant Documentation
|
|
||||||
|
|
||||||
- [Distribute Task Execution](/core-features/distribute-task-execution)
|
|
||||||
- [Distributed Task Execution Illustrated Guide](/concepts/dte).
|
|
||||||
|
|
||||||
## npx nx-cloud start-ci-run
|
|
||||||
|
|
||||||
At the beginning of your main job, invoke `npx nx-cloud start-ci-run`. This tells Nx Cloud that the following series of
|
|
||||||
command correspond to the same CI run.
|
|
||||||
|
|
||||||
You can configure your CI run by passing the following flags:
|
|
||||||
|
|
||||||
### --use-dte-by-default
|
|
||||||
|
|
||||||
By default, invoking `npx nx-cloud start-ci-run` will configure Nx to distribute all commands by default. You can
|
|
||||||
disable this as follows: `npx nx-cloud start-ci-run --use-dte-by-default=false`.
|
|
||||||
|
|
||||||
### --stop-agents-on-failure
|
|
||||||
|
|
||||||
By default, a failure in one of the commands is going to terminate the whole CI run and will stop all the
|
|
||||||
agents. You can disable this as follows: `npx nx-cloud start-ci-run --stop-agents-on-failure=false`.
|
|
||||||
|
|
||||||
### --stop-agents-after
|
|
||||||
|
|
||||||
By default, Nx Cloud won't terminate any agents until you invoke `npx nx-cloud stop-all-agents` because Nx Cloud
|
|
||||||
doesn't know if you will need agents to run another command. This can result in agents being idle at the end of a CI
|
|
||||||
run.
|
|
||||||
|
|
||||||
You can fix it by telling Nx Cloud that it can terminate agents after it sees a certain
|
|
||||||
target: `npx nx-cloud start-ci-run --stop-agents-after=e2e`.
|
|
||||||
|
|
||||||
> Earlier versions of `@nrwl/nx-cloud` required you to set the `NX_CLOUD_DISTRIBUTED_EXECUTION` env variable to `true`
|
|
||||||
> to
|
|
||||||
> enable DTE, but in the latest version `npx nx-cloud start-ci-run` does it automatically.
|
|
||||||
|
|
||||||
## Enabling/Disabling DTE
|
|
||||||
|
|
||||||
Invoking `npx nx-cloud start-ci-run` will tell Nx to distribute by default. You can enable/disable distribution for
|
|
||||||
individual commands as follows:
|
|
||||||
|
|
||||||
- `nx affected --target=build --dte` (explicitly enable distribution, Nx >= 14.7)
|
|
||||||
- `nx affected --target=build --no-dte` (explicitly disable distribution, Nx >= 14.7)
|
|
||||||
- `NX_CLOUD_DISTRIBUTED_EXECUTION=true nx affected --target=build` (explicitly enable distribution)
|
|
||||||
- `NX_CLOUD_DISTRIBUTED_EXECUTION=false nx affected --target=build` (explicitly disable distribution)
|
|
||||||
|
|
||||||
## npx nx-cloud stop-all-agents
|
|
||||||
|
|
||||||
This command tells Nx Cloud to terminate all agents associated with this CI run.
|
|
||||||
|
|
||||||
## Running Things in Parallel
|
|
||||||
|
|
||||||
`--parallel` is propagated to the agents. E.g., `npx nx affected --target=build --parallel=3 --dte` tells Nx Cloud to run
|
|
||||||
up to 3 build targets in parallel on each agent. So if you have say 10 agents, you will run up to 30 builds in parallel
|
|
||||||
across all of them.
|
|
||||||
|
|
||||||
You also want to run as many commands in parallel as you can. For instance,
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- nx affected --target=build
|
|
||||||
- nx affected --target=test
|
|
||||||
- nx affected --target=lint
|
|
||||||
```
|
|
||||||
|
|
||||||
is worse than
|
|
||||||
|
|
||||||
```yaml
|
|
||||||
- nx affected --target=build & nx affected --target=test & nx affected --target=lint
|
|
||||||
```
|
|
||||||
|
|
||||||
The latter is going to schedule all the three commands at the same time, so if an agent cannot find anything to build, it will start running tests and lints. The result is better agent utilization and shorter CI time.
|
|
||||||
|
|
||||||
## CI/CD Examples
|
|
||||||
|
|
||||||
The examples below show how to set up CI using Nx and Nx Cloud using distributed task execution and distributed caching.
|
|
||||||
|
|
||||||
Every organization manages their CI/CD pipelines differently, so the examples don't cover org-specific aspects of
|
|
||||||
CI/CD (e.g., deployment). They mainly focus on configuring Nx correctly.
|
|
||||||
|
|
||||||
Read the guides for more information on how to configure them in CI.
|
|
||||||
|
|
||||||
- [Overview](/recipes/ci/ci-setup#distributed-ci-with-nx-cloud)
|
|
||||||
- [Azure Pipelines](/recipes/ci/monorepo-ci-azure#distributed-ci-with-nx-cloud)
|
|
||||||
- [Circle CI](/recipes/ci/monorepo-ci-circle-ci#distributed-ci-with-nx-cloud)
|
|
||||||
- [GitHub Actions](/recipes/ci/monorepo-ci-github-actions#distributed-ci-with-nx-cloud)
|
|
||||||
- [Jenkins](/recipes/ci/monorepo-ci-jenkins#distributed-ci-with-nx-cloud)
|
|
||||||
|
|
||||||
Note that only cacheable operations can be distributed because they have to be replayed on the main job.
|
|
||||||
|
|
||||||
## Additional Notes
|
|
||||||
|
|
||||||
### Env Variables
|
|
||||||
|
|
||||||
- The `@nrwl/nx-cloud` requires the `NX_BRANCH` environment variables to be set. For many CI providers (e.g., GitHub
|
|
||||||
Actions), the runner is able to set it automatically. For others, the variable will have to be set manually. If you set
|
|
||||||
it manually, note that `NX_BRANCH` has to be set to a PR number for the GitHub integration to work.
|
|
||||||
|
|
||||||
- The `@nrwl/nx-cloud` requires the `NX_RUN_GROUP` environment variables to be set. For many CI providers (e.g., GitHub
|
|
||||||
Actions), the runner is able to set it automatically. For others, the variable will have to be set manually. If you set
|
|
||||||
it manually, note that `NX_RUN_GROUP` has to be a unique value associated with a CI run.
|
|
||||||
|
|
||||||
- Setting `NX_CLOUD_DISTRIBUTED_EXECUTION` to true enables distributed task execution.
|
|
||||||
|
|
||||||
- Setting `NX_VERBOSE_LOGGING` to true will output the debug information about agents communicating with the main job.
|
|
||||||
It's a good way to troubleshoot issues.
|
|
||||||
|
|
||||||
- Setting `NX_CLOUD_ENV_NAME` will prefix all your commands so you can easily distinguish them in the UI and in GitHub
|
|
||||||
comments. For instance, if you run the same set of commands on Windows and Linux machines, you can
|
|
||||||
set `NX_CLOUD_ENV_NAME` to win on the Windows agent, and linux on Linux agents.
|
|
||||||
|
|
||||||
## Relevant Repositories and Examples
|
|
||||||
|
|
||||||
- [Nx: On how to make your CI 16 times faster with a small config change](https://github.com/vsavkin/interstellar)
|
|
||||||
- ["Lerna & Distributed Task Execution" Example](https://github.com/vsavkin/lerna-dte)
|
|
||||||
@ -43,7 +43,7 @@ Once all tasks are finished, we must not forget to cleanup used agents:
|
|||||||
npx nx-cloud stop-all-agents
|
npx nx-cloud stop-all-agents
|
||||||
```
|
```
|
||||||
|
|
||||||
Learn more about configuring your CI environment using Nx Cloud with [Distributed Caching](/nx-cloud/set-up/set-up-caching) and [Distributed Task Execution](/nx-cloud/set-up/set-up-dte) in the Nx Cloud docs.
|
Learn more about configuring your CI environment using Nx Cloud with [Distributed Caching](/core-features/share-your-cache) and [Distributed Task Execution](/core-features/distribute-task-execution).
|
||||||
|
|
||||||
{% /nx-cloud-section %}
|
{% /nx-cloud-section %}
|
||||||
|
|
||||||
|
|||||||
@ -75,89 +75,6 @@ For a more in-depth understanding of the caching implementation and to fine-tune
|
|||||||
By default, Nx uses a local computation cache. Nx stores the cached values only for a week, after which they
|
By default, Nx uses a local computation cache. Nx stores the cached values only for a week, after which they
|
||||||
are deleted. To clear the cache run [`nx reset`](/nx/reset), and Nx will create a new one the next time it tries to access it.
|
are deleted. To clear the cache run [`nx reset`](/nx/reset), and Nx will create a new one the next time it tries to access it.
|
||||||
|
|
||||||
## Distributed Computation Caching
|
|
||||||
|
|
||||||
The computation cache provided by Nx can be distributed across multiple machines. You can either build an implementation
|
|
||||||
of the cache or use Nx Cloud. Nx Cloud is an app that provides a fast and zero-config implementation of distributed
|
|
||||||
caching. It's completely free for OSS projects and for most closed-sourced
|
|
||||||
projects ([read more here](https://dev.to/nrwl/more-time-saved-for-free-with-nx-cloud-4a2j)).
|
|
||||||
|
|
||||||
You can connect your workspace to Nx Cloud by running:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
npx nx connect-to-nx-cloud
|
|
||||||
```
|
|
||||||
|
|
||||||
```{% command="npx nx connect-to-nx-cloud"%}
|
|
||||||
✔ Enable distributed caching to make your CI faster · Yes
|
|
||||||
|
|
||||||
> NX Generating @nrwl/nx-cloud:init
|
|
||||||
|
|
||||||
UPDATE nx.json
|
|
||||||
|
|
||||||
> NX Distributed caching via Nx Cloud has been enabled
|
|
||||||
|
|
||||||
In addition to the caching, Nx Cloud provides config-free distributed execution,
|
|
||||||
UI for viewing complex runs and GitHub integration. Learn more at https://nx.app
|
|
||||||
|
|
||||||
Your workspace is currently unclaimed. Run details from unclaimed workspaces can be viewed on cloud.nx.app by anyone
|
|
||||||
with the link. Claim your workspace at the following link to restrict access.
|
|
||||||
|
|
||||||
https://cloud.nx.app/orgs/workspace-setup?accessToken=YOURACCESSTOKEN
|
|
||||||
```
|
|
||||||
|
|
||||||
To see the remote cache in action, run:
|
|
||||||
|
|
||||||
```shell
|
|
||||||
nx build header && nx reset && nx build header
|
|
||||||
```
|
|
||||||
|
|
||||||
```{% command="nx build header && nx reset && nx build header"%}
|
|
||||||
> nx run header:build
|
|
||||||
|
|
||||||
> header@0.0.0 build
|
|
||||||
> rimraf dist && rollup --config
|
|
||||||
|
|
||||||
src/index.tsx → dist...
|
|
||||||
created dist in 786ms
|
|
||||||
|
|
||||||
—————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————
|
|
||||||
|
|
||||||
> NX Successfully ran target build for project header (2s)
|
|
||||||
|
|
||||||
See logs and investigate cache misses at https://cloud.nx.app/runs/k0HDHACpL8
|
|
||||||
|
|
||||||
|
|
||||||
> NX Resetting the Nx workspace cache and stopping the Nx Daemon.
|
|
||||||
|
|
||||||
This might take a few minutes.
|
|
||||||
|
|
||||||
|
|
||||||
> NX Daemon Server - Stopped
|
|
||||||
|
|
||||||
|
|
||||||
> NX Successfully reset the Nx workspace.
|
|
||||||
|
|
||||||
|
|
||||||
> nx run header:build [remote cache]
|
|
||||||
|
|
||||||
|
|
||||||
> header@0.0.0 build
|
|
||||||
> rimraf dist && rollup --config
|
|
||||||
|
|
||||||
|
|
||||||
src/index.tsx → dist...
|
|
||||||
created dist in 786ms
|
|
||||||
|
|
||||||
—————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————
|
|
||||||
|
|
||||||
> NX Successfully ran target build for project header (664ms)
|
|
||||||
|
|
||||||
Nx read the output from the cache instead of running the command for 1 out of 1 tasks.
|
|
||||||
|
|
||||||
Nx Cloud made it possible to reuse header: https://nx.app/runs/P0X6ZGTkqZ
|
|
||||||
```
|
|
||||||
|
|
||||||
## See Also:
|
## See Also:
|
||||||
|
|
||||||
- [Nx Cloud Documentation](/nx-cloud/intro/what-is-nx-cloud)
|
- [Nx Cloud Documentation](/nx-cloud/intro/what-is-nx-cloud)
|
||||||
|
|||||||
@ -22,6 +22,10 @@ To distribute your task execution, you need to (1) connect to Nx Cloud and (2) e
|
|||||||
nx connect-to-nx-cloud
|
nx connect-to-nx-cloud
|
||||||
```
|
```
|
||||||
|
|
||||||
|
{% callout type="note" title="Use the latest version of Nx Cloud" %}
|
||||||
|
This command installs the latest version of `@nrwl/nx-cloud`. The latest version works with any version of Nx >= 13.0.
|
||||||
|
{% /callout %}
|
||||||
|
|
||||||
```shell title="2. Enable DTE in CI"
|
```shell title="2. Enable DTE in CI"
|
||||||
nx generate @nrwl/workspace:ci-workflow --ci=github
|
nx generate @nrwl/workspace:ci-workflow --ci=github
|
||||||
```
|
```
|
||||||
@ -37,7 +41,7 @@ Distributed task execution can work on any CI provider. You are responsible for
|
|||||||
|
|
||||||
The main job execution flow looks like this:
|
The main job execution flow looks like this:
|
||||||
|
|
||||||
```yml
|
```yaml
|
||||||
# Coordinate the agents to run the tasks
|
# Coordinate the agents to run the tasks
|
||||||
- npx nx-cloud start-ci-run
|
- npx nx-cloud start-ci-run
|
||||||
# Run any commands you want here
|
# Run any commands you want here
|
||||||
@ -48,17 +52,81 @@ The main job execution flow looks like this:
|
|||||||
|
|
||||||
The agent job execution flow is very simple:
|
The agent job execution flow is very simple:
|
||||||
|
|
||||||
```yml
|
```yaml
|
||||||
# Wait for tasks to execute
|
# Wait for tasks to execute
|
||||||
- npx nx-cloud start-agent
|
- npx nx-cloud start-agent
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The main job looks more or less the same way as if you haven't used any distribution. The only thing you need to do is
|
||||||
|
to invoke `npx nx-cloud start-ci-run` at the beginning and optionally invoke `npx nx-cloud stop-all-agents` at the end.
|
||||||
|
|
||||||
|
The agent jobs run long-running `start-agent` processes that execute all the tasks associated with a given CI run. The
|
||||||
|
only thing you need to do to set them up is to invoke `npx nx-cloud start-agent`. This process will keep running until
|
||||||
|
Nx Cloud tells it to terminate.
|
||||||
|
|
||||||
|
> Note it's important that the main job and the agent jobs have the same environment and the same source code. They start
|
||||||
|
> around the same time. And, once the main job completes, all the agents
|
||||||
|
> will be stopped.
|
||||||
|
|
||||||
|
It's also important to note that an Nx Cloud agent isn't a machine but rather a long-running process that runs on a
|
||||||
|
machine. I.e., Nx Cloud doesn't manage your agents--you need to do it in your CI config (check out CI examples below).
|
||||||
|
|
||||||
|
Nx Cloud is an orchestrator. The main job tells Nx Cloud what you want to run, and Nx Cloud will distribute those tasks
|
||||||
|
across the agents. Nx Cloud will automatically move files from one agent to another, from the agents to the main job.
|
||||||
|
|
||||||
|
The end result is that when say `nx affected --target=build` completes on the main job, all the file artifacts created
|
||||||
|
on agents are copied over to the main job, as if the main job had built everything locally.
|
||||||
|
|
||||||
|
## Running Things in Parallel
|
||||||
|
|
||||||
|
`--parallel` is propagated to the agents. E.g., `npx nx affected --target=build --parallel=3 --dte` tells Nx Cloud to run
|
||||||
|
up to 3 build targets in parallel on each agent. So if you have say 10 agents, you will run up to 30 builds in parallel
|
||||||
|
across all of them.
|
||||||
|
|
||||||
|
You also want to run as many commands in parallel as you can. For instance,
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- nx affected --target=build
|
||||||
|
- nx affected --target=test
|
||||||
|
- nx affected --target=lint
|
||||||
|
```
|
||||||
|
|
||||||
|
is worse than
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
- nx affected --target=build & nx affected --target=test & nx affected --target=lint
|
||||||
|
```
|
||||||
|
|
||||||
|
The latter is going to schedule all the three commands at the same time, so if an agent cannot find anything to build, it will start running tests and lints. The result is better agent utilization and shorter CI time.
|
||||||
|
|
||||||
|
## CI/CD Examples
|
||||||
|
|
||||||
|
The examples below show how to set up CI using Nx and Nx Cloud using distributed task execution and distributed caching.
|
||||||
|
|
||||||
|
Every organization manages their CI/CD pipelines differently, so the examples don't cover org-specific aspects of
|
||||||
|
CI/CD (e.g., deployment). They mainly focus on configuring Nx correctly.
|
||||||
|
|
||||||
|
Read the guides for more information on how to configure them in CI.
|
||||||
|
|
||||||
|
- [Overview](/recipes/ci/ci-setup#distributed-ci-with-nx-cloud)
|
||||||
|
- [Azure Pipelines](/recipes/ci/monorepo-ci-azure#distributed-ci-with-nx-cloud)
|
||||||
|
- [Circle CI](/recipes/ci/monorepo-ci-circle-ci#distributed-ci-with-nx-cloud)
|
||||||
|
- [GitHub Actions](/recipes/ci/monorepo-ci-github-actions#distributed-ci-with-nx-cloud)
|
||||||
|
- [Jenkins](/recipes/ci/monorepo-ci-jenkins#distributed-ci-with-nx-cloud)
|
||||||
|
|
||||||
|
Note that only cacheable operations can be distributed because they have to be replayed on the main job.
|
||||||
|
|
||||||
## Illustrated Guide
|
## Illustrated Guide
|
||||||
|
|
||||||
For more details about how distributed task execution works, check out the [illustrated guide](/concepts/dte) by Nrwlian [Nicole Oliver](https://twitter.com/nixcodes).
|
For more details about how distributed task execution works, check out the [illustrated guide](/concepts/dte) by Nrwlian [Nicole Oliver](https://twitter.com/nixcodes).
|
||||||
|
|
||||||
[](/concepts/dte)
|
[](/concepts/dte)
|
||||||
|
|
||||||
|
## Relevant Repositories and Examples
|
||||||
|
|
||||||
|
- [Nx: On how to make your CI 16 times faster with a small config change](https://github.com/vsavkin/interstellar)
|
||||||
|
- ["Lerna & Distributed Task Execution" Example](https://github.com/vsavkin/lerna-dte)
|
||||||
|
|
||||||
## See Also
|
## See Also
|
||||||
|
|
||||||
- [Nx Cloud Documentation](/nx-cloud/intro/what-is-nx-cloud)
|
- [Nx Cloud Documentation](/nx-cloud/intro/what-is-nx-cloud)
|
||||||
|
|||||||
113
docs/shared/core-features/share-your-cache.md
Normal file
113
docs/shared/core-features/share-your-cache.md
Normal file
@ -0,0 +1,113 @@
|
|||||||
|
# Share Your Cache
|
||||||
|
|
||||||
|
The computation cache provided by Nx can be distributed across multiple machines. You can either build an implementation
|
||||||
|
of the cache or use Nx Cloud. Nx Cloud is an app that provides a fast and zero-config implementation of distributed
|
||||||
|
caching. It's completely free for OSS projects and for most closed-sourced
|
||||||
|
projects ([read more here](https://dev.to/nrwl/more-time-saved-for-free-with-nx-cloud-4a2j)).
|
||||||
|
|
||||||
|
You can connect your workspace to Nx Cloud by running:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
npx nx connect-to-nx-cloud
|
||||||
|
```
|
||||||
|
|
||||||
|
```{% command="npx nx connect-to-nx-cloud"%}
|
||||||
|
✔ Enable distributed caching to make your CI faster · Yes
|
||||||
|
|
||||||
|
> NX Generating @nrwl/nx-cloud:init
|
||||||
|
|
||||||
|
UPDATE nx.json
|
||||||
|
|
||||||
|
> NX Distributed caching via Nx Cloud has been enabled
|
||||||
|
|
||||||
|
In addition to the caching, Nx Cloud provides config-free distributed execution,
|
||||||
|
UI for viewing complex runs and GitHub integration. Learn more at https://nx.app
|
||||||
|
|
||||||
|
Your workspace is currently unclaimed. Run details from unclaimed workspaces can be viewed on cloud.nx.app by anyone
|
||||||
|
with the link. Claim your workspace at the following link to restrict access.
|
||||||
|
|
||||||
|
https://cloud.nx.app/orgs/workspace-setup?accessToken=YOURACCESSTOKEN
|
||||||
|
```
|
||||||
|
|
||||||
|
To see the remote cache in action, run:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
nx build header && nx reset && nx build header
|
||||||
|
```
|
||||||
|
|
||||||
|
```{% command="nx build header && nx reset && nx build header"%}
|
||||||
|
> nx run header:build
|
||||||
|
|
||||||
|
> header@0.0.0 build
|
||||||
|
> rimraf dist && rollup --config
|
||||||
|
|
||||||
|
src/index.tsx → dist...
|
||||||
|
created dist in 786ms
|
||||||
|
|
||||||
|
—————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————
|
||||||
|
|
||||||
|
> NX Successfully ran target build for project header (2s)
|
||||||
|
|
||||||
|
See logs and investigate cache misses at https://cloud.nx.app/runs/k0HDHACpL8
|
||||||
|
|
||||||
|
|
||||||
|
> NX Resetting the Nx workspace cache and stopping the Nx Daemon.
|
||||||
|
|
||||||
|
This might take a few minutes.
|
||||||
|
|
||||||
|
|
||||||
|
> NX Daemon Server - Stopped
|
||||||
|
|
||||||
|
|
||||||
|
> NX Successfully reset the Nx workspace.
|
||||||
|
|
||||||
|
|
||||||
|
> nx run header:build [remote cache]
|
||||||
|
|
||||||
|
|
||||||
|
> header@0.0.0 build
|
||||||
|
> rimraf dist && rollup --config
|
||||||
|
|
||||||
|
|
||||||
|
src/index.tsx → dist...
|
||||||
|
created dist in 786ms
|
||||||
|
|
||||||
|
—————————————————————————————————————————————————————————————————————————————————————————————————————————————————————————
|
||||||
|
|
||||||
|
> NX Successfully ran target build for project header (664ms)
|
||||||
|
|
||||||
|
Nx read the output from the cache instead of running the command for 1 out of 1 tasks.
|
||||||
|
|
||||||
|
Nx Cloud made it possible to reuse header: https://nx.app/runs/P0X6ZGTkqZ
|
||||||
|
```
|
||||||
|
|
||||||
|
## Connecting Your Workspace to Your Nx Cloud Account
|
||||||
|
|
||||||
|
After you have enabled Nx Cloud in your workspace, you will see the following:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
> NX NOTE Nx Cloud has been enabled
|
||||||
|
|
||||||
|
Your workspace is currently public. Anybody with code access
|
||||||
|
can view the workspace on nx.app.
|
||||||
|
|
||||||
|
You can connect the workspace to your Nx Cloud account at
|
||||||
|
https://nx.app/orgs/workspace-setup?accessToken=N2Y3NzcyO...
|
||||||
|
(You can do this later.)
|
||||||
|
```
|
||||||
|
|
||||||
|
Click on this link to associate the workspace with your Nx Cloud account. If you don't have an Nx Cloud account, you can
|
||||||
|
create one on the spot.
|
||||||
|
|
||||||
|
After you claim your workspace, you will be able to manage permissions, create access tokens, set up billing, and so
|
||||||
|
forth.
|
||||||
|
|
||||||
|
**You will also see an interactive tutorial helping you explore distributed caching and the Nx Cloud user interface.**
|
||||||
|
|
||||||
|
If you lose this link, you can still connect your workspace to Nx Cloud. Go to [nx.app](https://nx.app), create an
|
||||||
|
account, and connect your workspace using the access token from `nx.json`.
|
||||||
|
|
||||||
|
## Skipping Cloud
|
||||||
|
|
||||||
|
Similar to how `--skip-nx-cache` will instruct Nx not to use the cache, passing `--no-cloud` will tell Nx not to use Nx
|
||||||
|
Cloud.
|
||||||
@ -38,7 +38,7 @@ For example:
|
|||||||
from `apps/my-app/.env`, it will notice that `NX_API_URL` already exists, so it will ignore it.
|
from `apps/my-app/.env`, it will notice that `NX_API_URL` already exists, so it will ignore it.
|
||||||
|
|
||||||
We recommend nesting your **app** specific `env` files in `apps/your-app`, and creating workspace/root level `env` files
|
We recommend nesting your **app** specific `env` files in `apps/your-app`, and creating workspace/root level `env` files
|
||||||
for workspace-specific settings (like the [Nx Cloud token](/concepts/how-caching-works#distributed-computation-caching)).
|
for workspace-specific settings (like the [Nx Cloud token](/core-features/share-your-cache)).
|
||||||
{% /callout %}
|
{% /callout %}
|
||||||
|
|
||||||
### Pointing to custom env files
|
### Pointing to custom env files
|
||||||
|
|||||||
@ -7,14 +7,6 @@ import { assertTextOnPage } from './helpers';
|
|||||||
describe('nx-dev: Nx Cloud section', () => {
|
describe('nx-dev: Nx Cloud section', () => {
|
||||||
(<{ title: string; path: string }[]>[
|
(<{ title: string; path: string }[]>[
|
||||||
{ title: 'What is Nx Cloud?', path: '/nx-cloud/intro/what-is-nx-cloud' },
|
{ title: 'What is Nx Cloud?', path: '/nx-cloud/intro/what-is-nx-cloud' },
|
||||||
{
|
|
||||||
title: 'Enabling Distributed Caching',
|
|
||||||
path: '/nx-cloud/set-up/set-up-caching',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
title: 'Set Up Distributed Task Execution',
|
|
||||||
path: '/nx-cloud/set-up/set-up-dte',
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
title: 'Recording Non-Nx Commands',
|
title: 'Recording Non-Nx Commands',
|
||||||
path: '/nx-cloud/set-up/record-commands',
|
path: '/nx-cloud/set-up/record-commands',
|
||||||
|
|||||||
@ -188,6 +188,14 @@ module.exports = withNx({
|
|||||||
permanent: true,
|
permanent: true,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
// Nx Cloud restructure
|
||||||
|
for (let s of Object.keys(redirectRules.nxCloudUrls)) {
|
||||||
|
rules.push({
|
||||||
|
source: s,
|
||||||
|
destination: redirectRules.nxCloudUrls[s],
|
||||||
|
permanent: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// Landing pages
|
// Landing pages
|
||||||
rules.push({
|
rules.push({
|
||||||
|
|||||||
@ -321,6 +321,15 @@ const recipesUrls = {
|
|||||||
'/recipe/adding-to-monorepo': '/recipes/adopting-nx/adding-to-monorepo',
|
'/recipe/adding-to-monorepo': '/recipes/adopting-nx/adding-to-monorepo',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Nx Cloud
|
||||||
|
*/
|
||||||
|
const nxCloudUrls = {
|
||||||
|
'/nx-cloud/set-up/add-nx-cloud': '/core-features/share-your-cache',
|
||||||
|
'/nx-cloud/set-up/set-up-caching': '/core-features/share-your-cache',
|
||||||
|
'/nx-cloud/set-up/set-up-dte': '/core-features/distribute-task-execution',
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Tutorial Updates
|
* Tutorial Updates
|
||||||
*/
|
*/
|
||||||
@ -372,6 +381,7 @@ module.exports = {
|
|||||||
guideUrls,
|
guideUrls,
|
||||||
overviewUrls,
|
overviewUrls,
|
||||||
recipesUrls,
|
recipesUrls,
|
||||||
|
nxCloudUrls,
|
||||||
schemaUrls,
|
schemaUrls,
|
||||||
tutorialRedirects,
|
tutorialRedirects,
|
||||||
};
|
};
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user