feat(core): allow using Nx Cloud without nx-cloud installed (#19553)

This commit is contained in:
Jason Jean 2023-10-13 10:47:43 -04:00 committed by GitHub
parent 6857155822
commit d62acecec6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
39 changed files with 1040 additions and 112 deletions

View File

@ -23,6 +23,14 @@ Type: `boolean`
Show help
### interactive
Type: `boolean`
Default: `true`
Prompt for confirmation
### version
Type: `boolean`

View File

@ -7666,6 +7666,23 @@
],
"isExternal": false,
"disableCollapsible": false
},
{
"id": "generators",
"path": "/nx-api/nx/generators",
"name": "generators",
"children": [
{
"id": "connect-to-nx-cloud",
"path": "/nx-api/nx/generators/connect-to-nx-cloud",
"name": "connect-to-nx-cloud",
"children": [],
"isExternal": false,
"disableCollapsible": false
}
],
"isExternal": false,
"disableCollapsible": false
}
],
"isExternal": false,

View File

@ -1815,7 +1815,17 @@
"type": "executor"
}
},
"generators": {},
"generators": {
"/nx-api/nx/generators/connect-to-nx-cloud": {
"description": "Connect a workspace to Nx Cloud",
"file": "generated/packages/nx/generators/connect-to-nx-cloud.json",
"hidden": false,
"name": "connect-to-nx-cloud",
"originalFilePath": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json",
"path": "/nx-api/nx/generators/connect-to-nx-cloud",
"type": "generator"
}
},
"path": "/nx-api/nx"
},
"playwright": {

View File

@ -1794,7 +1794,17 @@
"type": "executor"
}
],
"generators": [],
"generators": [
{
"description": "Connect a workspace to Nx Cloud",
"file": "generated/packages/nx/generators/connect-to-nx-cloud.json",
"hidden": false,
"name": "connect-to-nx-cloud",
"originalFilePath": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json",
"path": "nx/generators/connect-to-nx-cloud",
"type": "generator"
}
],
"githubRoot": "https://github.com/nrwl/nx/blob/master",
"name": "nx",
"packageName": "nx",

View File

@ -23,6 +23,14 @@ Type: `boolean`
Show help
### interactive
Type: `boolean`
Default: `true`
Prompt for confirmation
### version
Type: `boolean`

View File

@ -0,0 +1,34 @@
{
"name": "connect-to-nx-cloud",
"factory": "./src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud",
"schema": {
"$schema": "http://json-schema.org/schema",
"id": "NxCloudInit",
"title": "Add Nx Cloud Configuration to the workspace",
"description": "Connect a workspace to Nx Cloud.",
"type": "object",
"cli": "nx",
"properties": {
"analytics": {
"type": "boolean",
"description": "Anonymously store hashed machine ID for task runs",
"default": false
},
"installationSource": {
"type": "string",
"description": "Name of Nx Cloud installation invoker (ex. user, add-nx-to-monorepo, create-nx-workspace, nx-upgrade",
"default": "user"
}
},
"additionalProperties": false,
"required": [],
"presets": []
},
"description": "Connect a workspace to Nx Cloud",
"x-hidden": true,
"implementation": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud.ts",
"aliases": [],
"hidden": false,
"path": "/packages/nx/src/nx-cloud/generators/connect-to-nx-cloud/schema.json",
"type": "generator"
}

View File

@ -44,11 +44,6 @@
"type": "string"
},
"appName": { "type": "string", "description": "Application name." },
"nxCloud": {
"description": "Connect the workspace to the free tier of the distributed cache provided by Nx Cloud.",
"type": "boolean",
"default": false
},
"linter": {
"description": "The tool to use for running lint checks.",
"type": "string",

View File

@ -511,6 +511,8 @@
- [noop](/nx-api/nx/executors/noop)
- [run-commands](/nx-api/nx/executors/run-commands)
- [run-script](/nx-api/nx/executors/run-script)
- [generators](/nx-api/nx/generators)
- [connect-to-nx-cloud](/nx-api/nx/generators/connect-to-nx-cloud)
- [playwright](/nx-api/playwright)
- [documents](/nx-api/playwright/documents)
- [Overview](/nx-api/playwright/documents/overview)

View File

@ -0,0 +1,37 @@
import { cleanupProject, newProject, runCLI } from '@nx/e2e/utils';
describe('Nx Cloud', () => {
beforeAll(() =>
newProject({
unsetProjectNameAndRootFormat: false,
})
);
const libName = 'test-lib';
beforeAll(() => {
runCLI('connect --no-interactive', {
env: {
...process.env,
NX_CLOUD_API: 'https://staging.nx.app',
},
});
runCLI(`generate @nx/js:lib ${libName} --no-interactive`);
});
afterAll(() => cleanupProject());
it('should cache tests', async () => {
// Should be able to view logs with Nx Cloud
expect(runCLI(`test ${libName}`)).toContain(
`View logs and investigate cache misses at https://staging.nx.app`
);
// Reset Local cache
runCLI(`reset`);
// Should be pull cache from Nx Cloud
expect(runCLI(`test ${libName}`)).toContain(
`Nx Cloud made it possible to reuse test-lib: https://staging.nx.app`
);
});
});

View File

@ -13,7 +13,7 @@ export async function setupNxCloud(
try {
const pmc = getPackageManagerCommand(packageManager);
const res = await execAndWait(
`${pmc.exec} nx g nx-cloud:init --no-analytics --installationSource=create-nx-workspace`,
`${pmc.exec} nx g nx:connect-to-nx-cloud --no-interactive --quiet`,
directory
);
nxCloudSpinner.succeed('NxCloud has been set up successfully');

View File

@ -0,0 +1,65 @@
#!/usr/bin/env node
import { findAncestorNodeModules } from '../src/nx-cloud/resolution-helpers';
import { getCloudOptions } from '../src/nx-cloud/utilities/get-cloud-options';
import {
NxCloudClientUnavailableError,
NxCloudEnterpriseOutdatedError,
verifyOrUpdateNxCloudClient,
} from '../src/nx-cloud/update-manager';
import type { CloudTaskRunnerOptions } from '../src/nx-cloud/nx-cloud-tasks-runner-shell';
import { output } from '../src/utils/output';
const command = process.argv[2];
const options = getCloudOptions();
Promise.resolve().then(async () => invokeCommandWithNxCloudClient(options));
async function invokeCommandWithNxCloudClient(options: CloudTaskRunnerOptions) {
try {
const { nxCloudClient } = await verifyOrUpdateNxCloudClient(options);
const paths = findAncestorNodeModules(__dirname, []);
nxCloudClient.configureLightClientRequire()(paths);
if (command in nxCloudClient.commands) {
nxCloudClient.commands[command]()
.then(() => process.exit(0))
.catch((e) => {
console.error(e);
process.exit(1);
});
} else {
output.error({
title: `Unknown Command "${command}"`,
});
output.log({
title: 'Available Commands:',
bodyLines: Object.keys(nxCloudClient.commands).map((c) => `- ${c}`),
});
process.exit(1);
}
} catch (e: any) {
const body = ['Cannot run commands from the `nx-cloud` CLI.'];
if (e instanceof NxCloudEnterpriseOutdatedError) {
body.push(
'If you are an Nx Enterprise customer, please reach out to your assigned Developer Productivity Engineer.',
'If you are NOT an Nx Enterprise customer but are seeing this message, please reach out to cloud-support@nrwl.io.'
);
}
if (e instanceof NxCloudClientUnavailableError) {
body.unshift(
'You may be offline. Please try again when you are back online.'
);
}
output.error({
title: e.message,
bodyLines: body,
});
process.exit(1);
}
}

View File

@ -0,0 +1,10 @@
{
"generators": {
"connect-to-nx-cloud": {
"factory": "./src/nx-cloud/generators/connect-to-nx-cloud/connect-to-nx-cloud",
"schema": "./src/nx-cloud/generators/connect-to-nx-cloud/schema.json",
"description": "Connect a workspace to Nx Cloud",
"x-hidden": true
}
}
}

View File

@ -68,7 +68,7 @@
},
"17.0.0-use-minimal-config-for-tasks-runner-options": {
"cli": "nx",
"version": "17.0.0-beta.2",
"version": "17.0.0-beta.3",
"description": "Use minimal config for tasksRunnerOptions",
"implementation": "./src/migrations/update-17-0-0/use-minimal-config-for-tasks-runner-options"
}

View File

@ -23,7 +23,8 @@
"CLI"
],
"bin": {
"nx": "./bin/nx.js"
"nx": "./bin/nx.js",
"nx-cloud": "./bin/nx-cloud.js"
},
"author": "Victor Savkin",
"license": "MIT",
@ -153,6 +154,7 @@
}
]
},
"generators": "./generators.json",
"executors": "./executors.json",
"builders": "./executors.json",
"publishConfig": {

View File

@ -1,13 +1,24 @@
import { CommandModule } from 'yargs';
import { linkToNxDevAndExamples } from '../yargs-utils/documentation';
import type { ConnectToNxCloudOptions } from './connect-to-nx-cloud';
export const yargsConnectCommand: CommandModule = {
export const yargsConnectCommand: CommandModule<{}, ConnectToNxCloudOptions> = {
command: 'connect',
aliases: ['connect-to-nx-cloud'],
describe: `Connect workspace to Nx Cloud`,
builder: (yargs) => linkToNxDevAndExamples(yargs, 'connect-to-nx-cloud'),
handler: async () => {
await (await import('./connect-to-nx-cloud')).connectToNxCloudCommand();
builder: (yargs) =>
linkToNxDevAndExamples(
yargs.option('interactive', {
type: 'boolean',
description: 'Prompt for confirmation',
default: true,
}),
'connect-to-nx-cloud'
),
handler: async (options) => {
await (
await import('./connect-to-nx-cloud')
).connectToNxCloudCommand(options);
process.exit(0);
},
};

View File

@ -1,6 +1,4 @@
import { output } from '../../utils/output';
import { getPackageManagerCommand } from '../../utils/package-manager';
import { execSync } from 'child_process';
import { readNxJson } from '../../config/configuration';
import {
getNxCloudToken,
@ -48,9 +46,15 @@ export async function connectToNxCloudIfExplicitlyAsked(
}
}
export async function connectToNxCloudCommand(
promptOverride?: string
): Promise<boolean> {
export interface ConnectToNxCloudOptions {
interactive: boolean;
promptOverride?: string;
}
export async function connectToNxCloudCommand({
promptOverride,
interactive,
}: ConnectToNxCloudOptions): Promise<boolean> {
const nxJson = readNxJson();
if (isNxCloudUsed(nxJson)) {
output.log({
@ -68,21 +72,9 @@ export async function connectToNxCloudCommand(
return false;
}
const res = await connectToNxCloudPrompt(promptOverride);
const res = interactive ? await connectToNxCloudPrompt(promptOverride) : true;
if (!res) return false;
const pmc = getPackageManagerCommand();
if (pmc) {
execSync(`${pmc.addDev} nx-cloud@latest`);
} else {
const nxJson = readNxJson();
if (nxJson.installation) {
nxJson.installation.plugins ??= {};
nxJson.installation.plugins['nx-cloud'] = execSync(
`npm view nx-cloud@latest version`
).toString();
}
}
runNxSync(`g nx-cloud:init`, {
runNxSync(`g nx:connect-to-nx-cloud --quiet --no-interactive`, {
stdio: [0, 1, 2],
});
return true;

View File

@ -42,26 +42,17 @@ export async function viewLogs(): Promise<number> {
if (!installCloud) return;
const pmc = getPackageManagerCommand();
try {
output.log({
title: 'Installing nx-cloud',
});
execSync(`${pmc.addDev} nx-cloud@latest`, { stdio: 'ignore' });
} catch (e) {
output.log({
title: 'Installation failed',
});
console.log(e);
return 1;
}
try {
output.log({
title: 'Connecting to Nx Cloud',
});
runNxSync(`g nx-cloud:init --installation-source=view-logs`, {
runNxSync(
`g nx:connect-to-nx-cloud --installation-source=view-logs --quiet --no-interactive`,
{
stdio: 'ignore',
});
}
);
} catch (e) {
output.log({
title: 'Failed to connect to Nx Cloud',

View File

@ -88,7 +88,7 @@ export async function addNxToMonorepo(options: Options) {
scriptOutputs
);
addDepsToPackageJson(repoRoot, useNxCloud);
addDepsToPackageJson(repoRoot);
output.log({ title: '📦 Installing dependencies' });
runInstall(repoRoot);

View File

@ -117,7 +117,7 @@ export async function addNxToNest(options: Options, packageJson: PackageJson) {
const pmc = getPackageManagerCommand();
addDepsToPackageJson(repoRoot, useNxCloud);
addDepsToPackageJson(repoRoot);
addNestPluginToPackageJson(repoRoot);
markRootPackageJsonAsNxProject(
repoRoot,

View File

@ -72,7 +72,7 @@ export async function addNxToNpmRepo(options: Options) {
const pmc = getPackageManagerCommand();
addDepsToPackageJson(repoRoot, useNxCloud);
addDepsToPackageJson(repoRoot);
markRootPackageJsonAsNxProject(
repoRoot,
cacheableOperations,

View File

@ -54,7 +54,7 @@ export async function addNxToAngularCliRepo(options: Options) {
options.nxCloud ?? (options.interactive ? await askAboutNxCloud() : false);
output.log({ title: '📦 Installing dependencies' });
installDependencies(useNxCloud);
installDependencies();
output.log({ title: '📝 Setting up workspace' });
await setupWorkspace(cacheableOperations, options.integrated);
@ -107,8 +107,8 @@ async function collectCacheableOperations(options: Options): Promise<string[]> {
return cacheableOperations;
}
function installDependencies(useNxCloud: boolean): void {
addDepsToPackageJson(repoRoot, useNxCloud);
function installDependencies(): void {
addDepsToPackageJson(repoRoot);
addPluginDependencies();
runInstall(repoRoot);
}

View File

@ -140,14 +140,6 @@ async function installDependencies(
json.devDependencies[`${pkgInfo.pkgScope}/tao`] = pkgInfo.pkgVersion;
}
if (useNxCloud) {
// get the latest nx-cloud version compatible with the Nx major
// version being installed
json.devDependencies['nx-cloud'] = await resolvePackageVersion(
'nx-cloud',
`^${major(pkgInfo.pkgVersion)}.0.0`
);
}
json.devDependencies = sortObjectByKeys(json.devDependencies);
if (pkgInfo.unscopedPkgName === 'angular') {

View File

@ -113,14 +113,11 @@ function deduceDefaultBase() {
}
}
export function addDepsToPackageJson(repoRoot: string, useCloud: boolean) {
export function addDepsToPackageJson(repoRoot: string) {
const path = joinPathFragments(repoRoot, `package.json`);
const json = readJsonFile(path);
if (!json.devDependencies) json.devDependencies = {};
json.devDependencies['nx'] = nxVersion;
if (useCloud) {
json.devDependencies['nx-cloud'] = 'latest';
}
writeJsonFile(path, json);
}
@ -140,10 +137,13 @@ export function initCloud(
| 'nx-init-nest'
| 'nx-init-npm-repo'
) {
runNxSync(`g nx-cloud:init --installationSource=${installationSource}`, {
runNxSync(
`g nx:connect-to-nx-cloud --installationSource=${installationSource} --quiet --no-interactive`,
{
stdio: [0, 1, 2],
cwd: repoRoot,
});
}
);
}
export function addVsCodeRecommendedExtensions(

View File

@ -1218,9 +1218,10 @@ async function generateMigrationsJsonAndUpdatePackageJson(
!isCI() &&
!isNxCloudUsed(originalNxJson)
) {
const useCloud = await connectToNxCloudCommand(
messages.getPromptMessage('nxCloudMigration')
);
const useCloud = await connectToNxCloudCommand({
promptOverride: messages.getPromptMessage('nxCloudMigration'),
interactive: true,
});
await recordStat({
command: 'migrate',
nxVersion,

View File

@ -2,7 +2,10 @@ import chalk = require('chalk');
import yargs = require('yargs');
import { examples } from '../examples';
export function linkToNxDevAndExamples(yargs: yargs.Argv, command: string) {
export function linkToNxDevAndExamples<T>(
yargs: yargs.Argv<T>,
command: string
) {
(examples[command] || []).forEach((t) => {
yargs = yargs.example(t.command, t.description);
});

View File

@ -81,6 +81,12 @@ describe('use-minimal-config-for-tasks-runner-options migration', () => {
},
},
});
writeJson(tree, 'package.json', {
devDependencies: {
'nx-cloud': 'latest',
nx: 'latest',
},
});
await migrate(tree);
@ -89,6 +95,10 @@ describe('use-minimal-config-for-tasks-runner-options migration', () => {
expect(nxJson.nxCloudUrl).toEqual('https://nx.app');
expect(nxJson.nxCloudEncryptionKey).toEqual('secret');
expect(nxJson.tasksRunnerOptions).not.toBeDefined();
expect(readJson(tree, 'package.json').devDependencies).toEqual({
nx: 'latest',
});
});
it('should move nxCloudAccessToken and nxCloudUrl for @nrwl/nx-cloud', async () => {

View File

@ -1,6 +1,8 @@
import { updateJson } from '../../generators/utils/json';
import { Tree } from '../../generators/tree';
import { NxJsonConfiguration } from '../../config/nx-json';
import { PackageJson } from '../../utils/package-json';
import { formatChangedFilesWithPrettierIfAvailable } from '../../generators/internal-utils/format-changed-files-with-prettier-if-available';
export default async function migrate(tree: Tree) {
if (!tree.exists('nx.json')) {
@ -33,6 +35,19 @@ export default async function migrate(tree: Tree) {
if (options.url) {
nxJson.nxCloudUrl = options.url;
delete options.url;
if (
[
'https://nx.app',
'https://cloud.nx.app',
'https://staging.nx.app',
'https://snapshot.nx.app',
].includes(nxJson.nxCloudUrl)
) {
removeNxCloudDependency(tree);
}
} else {
removeNxCloudDependency(tree);
}
if (options.encryptionKey) {
nxJson.nxCloudEncryptionKey = options.encryptionKey;
@ -69,4 +84,18 @@ export default async function migrate(tree: Tree) {
}
return nxJson;
});
await formatChangedFilesWithPrettierIfAvailable(tree);
}
function removeNxCloudDependency(tree: Tree) {
if (tree.exists('package.json')) {
updateJson<PackageJson>(tree, 'package.json', (packageJson) => {
delete packageJson.dependencies?.['nx-cloud'];
delete packageJson.devDependencies?.['nx-cloud'];
delete packageJson.dependencies?.['@nrwl/nx-cloud'];
delete packageJson.devDependencies?.['@nrwl/nx-cloud'];
return packageJson;
});
}
}

View File

@ -0,0 +1,5 @@
export function debugLog(...args: any[]) {
if (process.env['NX_VERBOSE_LOGGING'] === 'true') {
console.log('[NX CLOUD]', ...args);
}
}

View File

@ -0,0 +1,139 @@
import { execSync } from 'child_process';
import { URL } from 'node:url';
import { output } from '../../../utils/output';
import { Tree } from '../../../generators/tree';
import { readJson } from '../../../generators/utils/json';
import { NxJsonConfiguration } from '../../../config/nx-json';
import { readNxJson, updateNxJson } from '../../../generators/utils/nx-json';
import { formatChangedFilesWithPrettierIfAvailable } from '../../../generators/internal-utils/format-changed-files-with-prettier-if-available';
function printCloudConnectionDisabledMessage() {
output.error({
title: `Connections to Nx Cloud are disabled for this workspace`,
bodyLines: [
`This was an intentional decision by someone on your team.`,
`Nx Cloud cannot and will not be enabled.`,
``,
`To allow connections to Nx Cloud again, remove the 'neverConnectToCloud'`,
`property in nx.json.`,
],
});
}
function getRootPackageName(tree: Tree): string {
let packageJson;
try {
packageJson = readJson(tree, 'package.json');
} catch (e) {}
return packageJson?.name ?? 'my-workspace';
}
function removeTrailingSlash(apiUrl: string) {
return apiUrl[apiUrl.length - 1] === '/'
? apiUrl.substr(0, apiUrl.length - 1)
: apiUrl;
}
function getNxInitDate(): string | null {
try {
const nxInitIso = execSync(
'git log --diff-filter=A --follow --format=%aI -- nx.json | tail -1',
{ stdio: 'pipe' }
)
.toString()
.trim();
const nxInitDate = new Date(nxInitIso);
return nxInitDate.toISOString();
} catch (e) {
return null;
}
}
async function createNxCloudWorkspace(
workspaceName: string,
installationSource: string,
nxInitDate: string | null
): Promise<{ token: string; url: string }> {
const apiUrl = removeTrailingSlash(
process.env.NX_CLOUD_API || process.env.NRWL_API || `https://cloud.nx.app`
);
const response = await require('axios').post(
`${apiUrl}/nx-cloud/create-org-and-workspace`,
{
workspaceName,
installationSource,
nxInitDate,
}
);
if (response.data.message) {
throw new Error(response.data.message);
}
return response.data;
}
function printSuccessMessage(url: string) {
let host = 'nx.app';
try {
host = new URL(url).host;
} catch (e) {}
output.note({
title: `Distributed caching via Nx Cloud has been enabled`,
bodyLines: [
`In addition to the caching, Nx Cloud provides config-free distributed execution,`,
`UI for viewing complex runs and GitHub integration. Learn more at https://nx.app`,
``,
`Your workspace is currently unclaimed. Run details from unclaimed workspaces can be viewed on ${host} by anyone`,
`with the link. Claim your workspace at the following link to restrict access.`,
``,
`${url}`,
],
});
}
interface ConnectToNxCloudOptions {
analytics: boolean;
installationSource: string;
}
function addNxCloudOptionsToNxJson(
tree: Tree,
nxJson: NxJsonConfiguration,
token: string
) {
nxJson.nxCloudAccessToken = token;
const overrideUrl = process.env.NX_CLOUD_API || process.env.NRWL_API;
if (overrideUrl) {
(nxJson as any).nxCloudUrl = overrideUrl;
}
updateNxJson(tree, nxJson);
}
export async function connectToNxCloud(
tree: Tree,
schema: ConnectToNxCloudOptions
) {
const nxJson = readNxJson(tree);
if ((nxJson as any).neverConnectToCloud) {
return () => {
printCloudConnectionDisabledMessage();
};
} else {
// TODO: Change to using loading light client when that is enabled by default
const r = await createNxCloudWorkspace(
getRootPackageName(tree),
schema.installationSource,
getNxInitDate()
);
addNxCloudOptionsToNxJson(tree, nxJson, r.token);
await formatChangedFilesWithPrettierIfAvailable(tree);
return () => printSuccessMessage(r.url);
}
}
export default connectToNxCloud;

View File

@ -0,0 +1,22 @@
{
"$schema": "http://json-schema.org/schema",
"id": "NxCloudInit",
"title": "Add Nx Cloud Configuration to the workspace",
"description": "Connect a workspace to Nx Cloud.",
"type": "object",
"cli": "nx",
"properties": {
"analytics": {
"type": "boolean",
"description": "Anonymously store hashed machine ID for task runs",
"default": false
},
"installationSource": {
"type": "string",
"description": "Name of Nx Cloud installation invoker (ex. user, add-nx-to-monorepo, create-nx-workspace, nx-upgrade",
"default": "user"
}
},
"additionalProperties": false,
"required": []
}

View File

@ -0,0 +1,68 @@
import { findAncestorNodeModules } from './resolution-helpers';
import {
NxCloudClientUnavailableError,
NxCloudEnterpriseOutdatedError,
verifyOrUpdateNxCloudClient,
} from './update-manager';
import {
defaultTasksRunner,
DefaultTasksRunnerOptions,
} from '../tasks-runner/default-tasks-runner';
import { TasksRunner } from '../tasks-runner/tasks-runner';
import { output } from '../utils/output';
import { Task } from '../config/task-graph';
export interface CloudTaskRunnerOptions extends DefaultTasksRunnerOptions {
accessToken?: string;
canTrackAnalytics?: boolean;
encryptionKey?: string;
maskedProperties?: string[];
showUsageWarnings?: boolean;
customProxyConfigPath?: string;
useLatestApi?: boolean;
url?: string;
useLightClient?: boolean;
clientVersion?: string;
}
export const nxCloudTasksRunnerShell: TasksRunner<
CloudTaskRunnerOptions
> = async (tasks: Task[], options: CloudTaskRunnerOptions, context) => {
try {
const { nxCloudClient, version } = await verifyOrUpdateNxCloudClient(
options
);
options.clientVersion = version;
const paths = findAncestorNodeModules(__dirname, []);
nxCloudClient.configureLightClientRequire()(paths);
return nxCloudClient.nxCloudTasksRunner(tasks, options, context);
} catch (e: any) {
const body =
e instanceof NxCloudEnterpriseOutdatedError
? [
'If you are an Nx Enterprise customer, please reach out to your assigned Developer Productivity Engineer.',
'If you are NOT an Nx Enterprise customer but are seeing this message, please reach out to cloud-support@nrwl.io.',
]
: e instanceof NxCloudClientUnavailableError
? [
'You might be offline. Nx Cloud will be re-enabled when you are back online.',
]
: [];
if (e instanceof NxCloudEnterpriseOutdatedError) {
output.warn({
title: e.message,
bodyLines: ['Nx Cloud will not used for this command.', ...body],
});
}
const results = await defaultTasksRunner(tasks, options, context);
output.warn({
title: e.message,
bodyLines: ['Nx Cloud was not used for this command.', ...body],
});
return results;
}
};

View File

@ -0,0 +1,21 @@
import { existsSync } from 'fs';
import { dirname, isAbsolute, join, resolve } from 'path';
export function findAncestorNodeModules(startPath, collector) {
let currentPath = isAbsolute(startPath) ? startPath : resolve(startPath);
while (currentPath !== dirname(currentPath)) {
const potentialNodeModules = join(currentPath, 'node_modules');
if (existsSync(potentialNodeModules)) {
collector.push(potentialNodeModules);
}
if (existsSync(join(currentPath, 'nx.json'))) {
break;
}
currentPath = dirname(currentPath);
}
return collector;
}

View File

@ -0,0 +1,351 @@
import { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios';
import {
createWriteStream,
existsSync,
mkdirSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from 'fs';
import { createGunzip } from 'zlib';
import { join } from 'path';
import { createApiAxiosInstance } from './utilities/axios';
import { debugLog } from './debug-logger';
import type { CloudTaskRunnerOptions } from './nx-cloud-tasks-runner-shell';
import * as tar from 'tar-stream';
import { cacheDir } from '../utils/cache-directory';
import { createHash } from 'crypto';
import { TasksRunner } from '../tasks-runner/tasks-runner';
interface CloudBundleInstall {
version: string;
fullPath: string;
}
type ValidVerifyClientBundleResponse = {
valid: true;
url: null;
version: null;
};
type InvalidVerifyClientBundleResponse = {
valid: false;
url: string;
version: string;
};
type VerifyClientBundleResponse =
| ValidVerifyClientBundleResponse
| InvalidVerifyClientBundleResponse;
export class NxCloudEnterpriseOutdatedError extends Error {
constructor(url: string) {
super(`Nx Cloud instance hosted at ${url} is outdated`);
}
}
export class NxCloudClientUnavailableError extends Error {
constructor() {
super('No existing Nx Cloud client and failed to download new version');
}
}
export interface NxCloudClient {
configureLightClientRequire: () => (paths: string[]) => void;
commands: Record<string, () => Promise<void>>;
nxCloudTasksRunner: TasksRunner<CloudTaskRunnerOptions>;
}
export async function verifyOrUpdateNxCloudClient(
options: CloudTaskRunnerOptions
): Promise<{ nxCloudClient: NxCloudClient; version: string } | null> {
debugLog('Verifying current cloud bundle');
const currentBundle = getLatestInstalledRunnerBundle();
if (shouldVerifyInstalledRunnerBundle(currentBundle)) {
const axios = createApiAxiosInstance(options);
let verifyBundleResponse: AxiosResponse<VerifyClientBundleResponse>;
try {
verifyBundleResponse = await verifyCurrentBundle(axios, currentBundle);
} catch (e: any) {
// Enterprise image compatibility, to be removed
if (e.message === 'Request failed with status code 404' && options.url) {
throw new NxCloudEnterpriseOutdatedError(options.url);
}
debugLog(
'Could not verify bundle. Resetting validation timer and using previously installed or default runner. Error: ',
e
);
writeBundleVerificationLock();
if (currentBundle === null) {
throw new NxCloudClientUnavailableError();
}
if (currentBundle.version === 'NX_ENTERPRISE_OUTDATED_IMAGE') {
throw new NxCloudEnterpriseOutdatedError(options.url);
}
const nxCloudClient = require(currentBundle.fullPath);
if (nxCloudClient.commands === undefined) {
throw new NxCloudEnterpriseOutdatedError(options.url);
}
return {
version: currentBundle.version,
nxCloudClient,
};
}
if (verifyBundleResponse.data.valid) {
debugLog('Currently installed bundle is valid');
writeBundleVerificationLock();
return {
version: currentBundle.version,
nxCloudClient: require(currentBundle.fullPath),
};
}
const { version, url } = verifyBundleResponse.data;
debugLog(
'Currently installed bundle is invalid, downloading version',
version,
' from ',
url
);
if (version === 'NX_ENTERPRISE_OUTDATED_IMAGE') {
throw new NxCloudEnterpriseOutdatedError(options.url);
}
const fullPath = await downloadAndExtractClientBundle(
axios,
runnerBundleInstallDirectory,
version,
url
);
debugLog('Done: ', fullPath);
const nxCloudClient = require(fullPath);
if (nxCloudClient.commands === undefined) {
throw new NxCloudEnterpriseOutdatedError(options.url);
}
return { version, nxCloudClient };
}
if (currentBundle === null) {
throw new NxCloudClientUnavailableError();
}
debugLog('Done: ', currentBundle.fullPath);
return {
version: currentBundle.version,
nxCloudClient: require(currentBundle.fullPath),
};
}
const runnerBundleInstallDirectory = join(cacheDir, 'cloud');
function getLatestInstalledRunnerBundle(): CloudBundleInstall | null {
if (!existsSync(runnerBundleInstallDirectory)) {
mkdirSync(runnerBundleInstallDirectory, { recursive: true });
}
try {
const installedBundles: CloudBundleInstall[] = readdirSync(
runnerBundleInstallDirectory
)
.filter((potentialDirectory) => {
return statSync(
join(runnerBundleInstallDirectory, potentialDirectory)
).isDirectory();
})
.map((fileOrDirectory) => ({
version: fileOrDirectory,
fullPath: join(runnerBundleInstallDirectory, fileOrDirectory),
}));
if (installedBundles.length === 0) {
// No installed bundles
return null;
}
return installedBundles[0];
} catch (e: any) {
console.log('Could not read runner bundle path:', e.message);
return null;
}
}
function shouldVerifyInstalledRunnerBundle(
currentBundle: CloudBundleInstall | null
): boolean {
if (process.env.NX_CLOUD_FORCE_REVALIDATE === 'true') {
return true;
}
// No bundle, need to download anyway
if (currentBundle != null) {
debugLog('A local bundle currently exists: ', currentBundle);
const lastVerification = getLatestBundleVerificationTimestamp();
// Never been verified, need to verify
if (lastVerification != null) {
// If last verification was less than 30 minutes ago, return the current installed bundle
const THIRTY_MINUTES = 30 * 60 * 1000;
if (Date.now() - lastVerification < THIRTY_MINUTES) {
debugLog(
'Last verification was within the past 30 minutes, will not verify this time'
);
return false;
}
debugLog(
'Last verification was more than 30 minutes ago, verifying bundle is still valid'
);
}
}
return true;
}
async function verifyCurrentBundle(
axios: AxiosInstance,
currentBundle: CloudBundleInstall | null
): Promise<AxiosResponse<VerifyClientBundleResponse>> {
const contentHash = getBundleContentHash(currentBundle);
const queryParams =
currentBundle && contentHash
? `?${new URLSearchParams({
version: currentBundle.version,
contentHash: contentHash,
}).toString()}`
: '';
return axios.get('/nx-cloud/client/verify' + queryParams);
}
function getLatestBundleVerificationTimestamp(): number | null {
const lockfilePath = join(runnerBundleInstallDirectory, 'verify.lock');
if (existsSync(lockfilePath)) {
const timestampAsString = readFileSync(lockfilePath, 'utf-8');
let timestampAsNumber: number;
try {
timestampAsNumber = Number(timestampAsString);
return timestampAsNumber;
} catch (e) {
return null;
}
}
return null;
}
function writeBundleVerificationLock() {
const lockfilePath = join(runnerBundleInstallDirectory, 'verify.lock');
writeFileSync(lockfilePath, new Date().getTime().toString(), 'utf-8');
}
function getBundleContentHash(
bundle: CloudBundleInstall | null
): string | null {
if (bundle == null) {
return null;
}
return hashDirectory(bundle.fullPath);
}
function hashDirectory(dir: string): string {
const files = readdirSync(dir).sort();
const hashes = files.map((file) => {
const filePath = join(dir, file);
const stat = statSync(filePath);
// If the current path is a directory, recursively hash the contents
if (stat.isDirectory()) {
return hashDirectory(filePath);
}
// If it's a file, hash the file contents
const content = readFileSync(filePath);
return createHash('sha256').update(content).digest('hex');
});
// Hash the combined hashes of the directory's contents
const combinedHashes = hashes.sort().join('');
return createHash('sha256').update(combinedHashes).digest('hex');
}
async function downloadAndExtractClientBundle(
axios: AxiosInstance,
runnerBundleInstallDirectory: string,
version: string,
url: string
): Promise<string> {
let resp;
try {
resp = await axios.get(url, {
responseType: 'stream',
} as AxiosRequestConfig);
} catch (e: any) {
console.error('Error while updating Nx Cloud client bundle');
throw e;
}
const bundleExtractLocation = join(runnerBundleInstallDirectory, version);
if (!existsSync(bundleExtractLocation)) {
mkdirSync(bundleExtractLocation);
}
return new Promise((res, rej) => {
const extract = tar.extract();
extract.on('entry', function (headers, stream, next) {
if (headers.type === 'directory') {
const directoryPath = join(bundleExtractLocation, headers.name);
if (!existsSync(directoryPath)) {
mkdirSync(directoryPath, { recursive: true });
}
next();
stream.resume();
} else if (headers.type === 'file') {
const outputFilePath = join(bundleExtractLocation, headers.name);
const writeStream = createWriteStream(outputFilePath);
stream.pipe(writeStream);
stream.on('end', function () {
next();
});
stream.resume();
}
});
extract.on('error', (e) => {
rej(e);
});
extract.on('finish', function () {
removeOldClientBundles(version);
writeBundleVerificationLock();
res(bundleExtractLocation);
});
resp.data.pipe(createGunzip()).pipe(extract);
});
}
function removeOldClientBundles(currentInstallVersion: string) {
const filesAndFolders = readdirSync(runnerBundleInstallDirectory);
for (let fileOrFolder of filesAndFolders) {
const fileOrFolderPath = join(runnerBundleInstallDirectory, fileOrFolder);
if (fileOrFolder !== currentInstallVersion) {
rmSync(fileOrFolderPath, { recursive: true });
}
}
}

View File

@ -0,0 +1,39 @@
import { AxiosRequestConfig } from 'axios';
import { join } from 'path';
import {
ACCESS_TOKEN,
NX_CLOUD_NO_TIMEOUTS,
UNLIMITED_TIMEOUT,
} from './environment';
import { CloudTaskRunnerOptions } from '../nx-cloud-tasks-runner-shell';
const axios = require('axios');
export function createApiAxiosInstance(options: CloudTaskRunnerOptions) {
let axiosConfigBuilder = (axiosConfig: AxiosRequestConfig) => axiosConfig;
const baseUrl =
process.env.NX_CLOUD_API || options.url || 'https://cloud.nx.app';
const accessToken = ACCESS_TOKEN ? ACCESS_TOKEN : options.accessToken!;
if (!accessToken) {
throw new Error(
`Unable to authenticate. Either define accessToken in nx.json or set the NX_CLOUD_ACCESS_TOKEN env variable.`
);
}
if (options.customProxyConfigPath) {
const { nxCloudProxyConfig } = require(join(
process.cwd(),
options.customProxyConfigPath
));
axiosConfigBuilder = nxCloudProxyConfig ?? axiosConfigBuilder;
}
return axios.create(
axiosConfigBuilder({
baseURL: baseUrl,
timeout: NX_CLOUD_NO_TIMEOUTS ? UNLIMITED_TIMEOUT : 10000,
headers: { authorization: accessToken },
})
);
}

View File

@ -0,0 +1,48 @@
import * as dotenv from 'dotenv';
import { readFileSync } from 'fs';
import { join } from 'path';
import { isCI } from '../../utils/is-ci';
import { workspaceRoot } from '../../utils/workspace-root';
// Set once
export const UNLIMITED_TIMEOUT = 9999999;
process.env.NX_CLOUD_AGENT_TIMEOUT_MS
? Number(process.env.NX_CLOUD_AGENT_TIMEOUT_MS)
: 3600000;
// 60 minutes
process.env.NX_CLOUD_ORCHESTRATOR_TIMEOUT_MS
? Number(process.env.NX_CLOUD_ORCHESTRATOR_TIMEOUT_MS)
: 3600000;
// 60 minutes
process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_AGENT_COUNT
? Number(process.env.NX_CLOUD_DISTRIBUTED_EXECUTION_AGENT_COUNT)
: null;
process.env.NX_CLOUD_NUMBER_OF_RETRIES
? Number(process.env.NX_CLOUD_NUMBER_OF_RETRIES)
: isCI()
? 10
: 1;
export let ACCESS_TOKEN;
export let NX_CLOUD_NO_TIMEOUTS;
loadEnvVars();
function parseEnv() {
try {
const envContents = readFileSync(join(workspaceRoot, 'nx-cloud.env'));
return dotenv.parse(envContents);
} catch (e) {
return {};
}
}
function loadEnvVars() {
const parsed = parseEnv();
ACCESS_TOKEN =
process.env.NX_CLOUD_AUTH_TOKEN ||
process.env.NX_CLOUD_ACCESS_TOKEN ||
parsed.NX_CLOUD_AUTH_TOKEN ||
parsed.NX_CLOUD_ACCESS_TOKEN;
NX_CLOUD_NO_TIMEOUTS =
process.env.NX_CLOUD_NO_TIMEOUTS === 'true' ||
parsed.NX_CLOUD_NO_TIMEOUTS === 'true';
}

View File

@ -0,0 +1,10 @@
import { CloudTaskRunnerOptions } from '../nx-cloud-tasks-runner-shell';
import { readNxJson } from '../../config/nx-json';
import { getRunnerOptions } from '../../tasks-runner/run-command';
export function getCloudOptions(): CloudTaskRunnerOptions {
const nxJson = readNxJson();
// TODO: The default is not always cloud? But it's not handled at the moment
return getRunnerOptions('default', nxJson, {}, true);
}

View File

@ -419,6 +419,27 @@ function shouldUseDynamicLifeCycle(
return !tasks.find((t) => shouldStreamOutput(t, null));
}
function loadTasksRunner(modulePath: string) {
try {
const maybeTasksRunner = require(modulePath) as
| TasksRunner
| { default: TasksRunner };
// to support both babel and ts formats
return 'default' in maybeTasksRunner
? maybeTasksRunner.default
: maybeTasksRunner;
} catch (e) {
if (
e.code === 'MODULE_NOT_FOUND' &&
(modulePath === 'nx-cloud' || modulePath === '@nrwl/nx-cloud')
) {
return require('../nx-cloud/nx-cloud-tasks-runner-shell')
.nxCloudTasksRunnerShell;
}
throw e;
}
}
export function getRunner(
nxArgs: NxArgs,
nxJson: NxJsonConfiguration
@ -435,11 +456,8 @@ export function getRunner(
const modulePath: string = getTasksRunnerPath(runner, nxJson);
let tasksRunner = require(modulePath);
// to support both babel and ts formats
if (tasksRunner.default) {
tasksRunner = tasksRunner.default;
}
try {
const tasksRunner = loadTasksRunner(modulePath);
return {
tasksRunner,
@ -450,6 +468,9 @@ export function getRunner(
modulePath === 'nx-cloud'
),
};
} catch {
throw new Error(`Could not find runner configuration for ${runner}`);
}
}
function getTasksRunnerPath(
runner: string,
@ -473,7 +494,7 @@ function getTasksRunnerPath(
return isCloudRunner ? 'nx-cloud' : require.resolve('./default-tasks-runner');
}
function getRunnerOptions(
export function getRunnerOptions(
runner: string,
nxJson: NxJsonConfiguration<string[] | '*'>,
nxArgs: NxArgs,

View File

@ -1,5 +1,4 @@
import {
addDependenciesToPackageJson,
getPackageManagerCommand,
installPackagesTask,
joinPathFragments,
@ -21,7 +20,6 @@ interface Schema {
appName?: string;
skipInstall?: boolean;
style?: string;
nxCloud?: boolean;
preset: string;
defaultBase: string;
framework?: string;
@ -49,8 +47,6 @@ export async function newGenerator(tree: Tree, opts: Schema) {
addPresetDependencies(tree, options);
addCloudDependencies(tree, options);
return async () => {
const pmc = getPackageManagerCommand(options.packageManager);
if (pmc.preInstall) {
@ -78,9 +74,6 @@ function validateOptions(options: Schema, host: Tree) {
) {
throw new Error(`Cannot select a preset when skipInstall is set to true.`);
}
if (options.skipInstall && options.nxCloud) {
throw new Error(`Cannot select nxCloud when skipInstall is set to true.`);
}
if (
(options.preset === Preset.NodeStandalone ||
@ -144,14 +137,3 @@ function normalizeOptions(options: Schema): NormalizedSchema {
return normalized as NormalizedSchema;
}
function addCloudDependencies(host: Tree, options: Schema) {
if (options.nxCloud) {
return addDependenciesToPackageJson(
host,
{},
{ 'nx-cloud': 'latest' },
join(options.directory, 'package.json')
);
}
}

View File

@ -47,11 +47,6 @@
"type": "string",
"description": "Application name."
},
"nxCloud": {
"description": "Connect the workspace to the free tier of the distributed cache provided by Nx Cloud.",
"type": "boolean",
"default": false
},
"linter": {
"description": "The tool to use for running lint checks.",
"type": "string",