From a5682d1ca59cb71389e19a11867c1bade4ae6141 Mon Sep 17 00:00:00 2001 From: Craigory Coppola Date: Thu, 30 May 2024 15:28:59 -0400 Subject: [PATCH] feat(core): add create nodes v2 for batch processing config files (#26250) --- .../devkit/AggregateCreateNodesError.md | 197 +++++++++++++ docs/generated/devkit/CreateNodes.md | 12 + docs/generated/devkit/CreateNodesContext.md | 14 + docs/generated/devkit/CreateNodesContextV2.md | 26 ++ .../generated/devkit/CreateNodesFunctionV2.md | 25 ++ docs/generated/devkit/CreateNodesResultV2.md | 3 + docs/generated/devkit/CreateNodesV2.md | 12 + docs/generated/devkit/NxPluginV2.md | 13 +- docs/generated/devkit/README.md | 6 + docs/generated/devkit/createNodesFromFiles.md | 22 ++ .../packages/devkit/documents/nx_devkit.md | 6 + packages/gradle/src/plugin/dependencies.ts | 10 +- packages/gradle/src/plugin/nodes.spec.ts | 5 +- packages/gradle/src/plugin/nodes.ts | 78 ++++- .../gradle/src/utils/get-gradle-report.ts | 23 +- packages/nx/src/devkit-exports.ts | 8 + .../project-json/build-nodes/project-json.ts | 1 - packages/nx/src/project-graph/error-types.ts | 72 +++-- .../nx/src/project-graph/plugins/index.ts | 1 + .../src/project-graph/plugins/internal-api.ts | 62 +++- .../src/project-graph/plugins/public-api.ts | 48 +++- .../src/project-graph/plugins/utils.spec.ts | 215 +++++++++----- .../nx/src/project-graph/plugins/utils.ts | 75 ++--- .../utils/project-configuration-utils.ts | 268 ++++++++++-------- 24 files changed, 885 insertions(+), 317 deletions(-) create mode 100644 docs/generated/devkit/AggregateCreateNodesError.md create mode 100644 docs/generated/devkit/CreateNodesContextV2.md create mode 100644 docs/generated/devkit/CreateNodesFunctionV2.md create mode 100644 docs/generated/devkit/CreateNodesResultV2.md create mode 100644 docs/generated/devkit/CreateNodesV2.md create mode 100644 docs/generated/devkit/createNodesFromFiles.md diff --git a/docs/generated/devkit/AggregateCreateNodesError.md b/docs/generated/devkit/AggregateCreateNodesError.md new file mode 100644 index 0000000000..2fc55fbe79 --- /dev/null +++ b/docs/generated/devkit/AggregateCreateNodesError.md @@ -0,0 +1,197 @@ +# Class: AggregateCreateNodesError + +This error should be thrown when a `createNodesV2` function hits a recoverable error. +It allows Nx to recieve partial results and continue processing for better UX. + +## Hierarchy + +- `Error` + + ↳ **`AggregateCreateNodesError`** + +## Table of contents + +### Constructors + +- [constructor](../../devkit/documents/AggregateCreateNodesError#constructor) + +### Properties + +- [cause](../../devkit/documents/AggregateCreateNodesError#cause): unknown +- [errors](../../devkit/documents/AggregateCreateNodesError#errors): [file: string, error: Error][] +- [message](../../devkit/documents/AggregateCreateNodesError#message): string +- [name](../../devkit/documents/AggregateCreateNodesError#name): string +- [partialResults](../../devkit/documents/AggregateCreateNodesError#partialresults): CreateNodesResultV2 +- [stack](../../devkit/documents/AggregateCreateNodesError#stack): string +- [prepareStackTrace](../../devkit/documents/AggregateCreateNodesError#preparestacktrace): Function +- [stackTraceLimit](../../devkit/documents/AggregateCreateNodesError#stacktracelimit): number + +### Methods + +- [captureStackTrace](../../devkit/documents/AggregateCreateNodesError#capturestacktrace) + +## Constructors + +### constructor + +• **new AggregateCreateNodesError**(`errors`, `partialResults`): [`AggregateCreateNodesError`](../../devkit/documents/AggregateCreateNodesError) + +Throwing this error from a `createNodesV2` function will allow Nx to continue processing and recieve partial results from your plugin. + +#### Parameters + +| Name | Type | Description | +| :--------------- | :------------------------------------------------------------------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `errors` | [file: string, error: Error][] | An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]] | +| `partialResults` | [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) | The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue. | + +#### Returns + +[`AggregateCreateNodesError`](../../devkit/documents/AggregateCreateNodesError) + +**`Example`** + +```ts +export async function createNodesV2(files: string[]) { + const partialResults = []; + const errors = []; + await Promise.all( + files.map(async (file) => { + try { + const result = await createNodes(file); + partialResults.push(result); + } catch (e) { + errors.push([file, e]); + } + }) + ); + if (errors.length > 0) { + throw new AggregateCreateNodesError(errors, partialResults); + } + return partialResults; +} +``` + +#### Overrides + +Error.constructor + +## Properties + +### cause + +• `Optional` **cause**: `unknown` + +#### Inherited from + +Error.cause + +--- + +### errors + +• `Readonly` **errors**: [file: string, error: Error][] + +An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]] + +--- + +### message + +• **message**: `string` + +#### Inherited from + +Error.message + +--- + +### name + +• **name**: `string` + +#### Inherited from + +Error.name + +--- + +### partialResults + +• `Readonly` **partialResults**: [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) + +The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue. + +--- + +### stack + +• `Optional` **stack**: `string` + +#### Inherited from + +Error.stack + +--- + +### prepareStackTrace + +▪ `Static` `Optional` **prepareStackTrace**: (`err`: `Error`, `stackTraces`: `CallSite`[]) => `any` + +Optional override for formatting stack traces + +**`See`** + +https://v8.dev/docs/stack-trace-api#customizing-stack-traces + +#### Type declaration + +▸ (`err`, `stackTraces`): `any` + +##### Parameters + +| Name | Type | +| :------------ | :----------- | +| `err` | `Error` | +| `stackTraces` | `CallSite`[] | + +##### Returns + +`any` + +#### Inherited from + +Error.prepareStackTrace + +--- + +### stackTraceLimit + +▪ `Static` **stackTraceLimit**: `number` + +#### Inherited from + +Error.stackTraceLimit + +## Methods + +### captureStackTrace + +▸ **captureStackTrace**(`targetObject`, `constructorOpt?`): `void` + +Create .stack property on a target object + +#### Parameters + +| Name | Type | +| :---------------- | :--------- | +| `targetObject` | `object` | +| `constructorOpt?` | `Function` | + +#### Returns + +`void` + +#### Inherited from + +Error.captureStackTrace diff --git a/docs/generated/devkit/CreateNodes.md b/docs/generated/devkit/CreateNodes.md index ed46857f91..229a34d071 100644 --- a/docs/generated/devkit/CreateNodes.md +++ b/docs/generated/devkit/CreateNodes.md @@ -4,6 +4,18 @@ A pair of file patterns and [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) +Nx 19.2+: Both original `CreateNodes` and `CreateNodesV2` are supported. Nx will only invoke `CreateNodesV2` if it is present. +Nx 20.X : The `CreateNodesV2` will be the only supported API. This typing will still exist, but be identical to `CreateNodesV2`. +Nx **will not** invoke the original `plugin.createNodes` callback. This should give plugin authors a window to transition. +Plugin authors should update their plugin's `createNodes` function to align with `CreateNodesV2` / the updated `CreateNodes`. +The plugin should contain something like: `export createNodes = createNodesV2;` during this period. This will allow the plugin +to maintain compatibility with Nx 19.2 and up. +Nx 21.X : The `CreateNodesV2` typing will be removed, as it has replaced `CreateNodes`. + +**`Deprecated`** + +Use [CreateNodesV2](../../devkit/documents/CreateNodesV2) instead. CreateNodesV2 will replace this API. Read more about the transition above. + #### Type parameters | Name | Type | diff --git a/docs/generated/devkit/CreateNodesContext.md b/docs/generated/devkit/CreateNodesContext.md index 861b8d7907..98fcabdcc6 100644 --- a/docs/generated/devkit/CreateNodesContext.md +++ b/docs/generated/devkit/CreateNodesContext.md @@ -2,6 +2,12 @@ Context for [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) +## Hierarchy + +- [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) + + ↳ **`CreateNodesContext`** + ## Table of contents ### Properties @@ -24,8 +30,16 @@ The subset of configuration files which match the createNodes pattern • `Readonly` **nxJsonConfiguration**: [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\> +#### Inherited from + +[CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2).[nxJsonConfiguration](../../devkit/documents/CreateNodesContextV2#nxjsonconfiguration) + --- ### workspaceRoot • `Readonly` **workspaceRoot**: `string` + +#### Inherited from + +[CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2).[workspaceRoot](../../devkit/documents/CreateNodesContextV2#workspaceroot) diff --git a/docs/generated/devkit/CreateNodesContextV2.md b/docs/generated/devkit/CreateNodesContextV2.md new file mode 100644 index 0000000000..bc4bb8bd89 --- /dev/null +++ b/docs/generated/devkit/CreateNodesContextV2.md @@ -0,0 +1,26 @@ +# Interface: CreateNodesContextV2 + +## Hierarchy + +- **`CreateNodesContextV2`** + + ↳ [`CreateNodesContext`](../../devkit/documents/CreateNodesContext) + +## Table of contents + +### Properties + +- [nxJsonConfiguration](../../devkit/documents/CreateNodesContextV2#nxjsonconfiguration): NxJsonConfiguration +- [workspaceRoot](../../devkit/documents/CreateNodesContextV2#workspaceroot): string + +## Properties + +### nxJsonConfiguration + +• `Readonly` **nxJsonConfiguration**: [`NxJsonConfiguration`](../../devkit/documents/NxJsonConfiguration)\<`string`[] \| `"*"`\> + +--- + +### workspaceRoot + +• `Readonly` **workspaceRoot**: `string` diff --git a/docs/generated/devkit/CreateNodesFunctionV2.md b/docs/generated/devkit/CreateNodesFunctionV2.md new file mode 100644 index 0000000000..b32121c677 --- /dev/null +++ b/docs/generated/devkit/CreateNodesFunctionV2.md @@ -0,0 +1,25 @@ +# Type alias: CreateNodesFunctionV2\ + +Ƭ **CreateNodesFunctionV2**\<`T`\>: (`projectConfigurationFiles`: readonly `string`[], `options`: `T` \| `undefined`, `context`: [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2)) => [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\> + +#### Type parameters + +| Name | Type | +| :--- | :-------- | +| `T` | `unknown` | + +#### Type declaration + +▸ (`projectConfigurationFiles`, `options`, `context`): [`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\> + +##### Parameters + +| Name | Type | +| :-------------------------- | :-------------------------------------------------------------------- | +| `projectConfigurationFiles` | readonly `string`[] | +| `options` | `T` \| `undefined` | +| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) | + +##### Returns + +[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2) \| `Promise`\<[`CreateNodesResultV2`](../../devkit/documents/CreateNodesResultV2)\> diff --git a/docs/generated/devkit/CreateNodesResultV2.md b/docs/generated/devkit/CreateNodesResultV2.md new file mode 100644 index 0000000000..fb7f8965cb --- /dev/null +++ b/docs/generated/devkit/CreateNodesResultV2.md @@ -0,0 +1,3 @@ +# Type alias: CreateNodesResultV2 + +Ƭ **CreateNodesResultV2**: readonly [configFileSource: string, result: CreateNodesResult][] diff --git a/docs/generated/devkit/CreateNodesV2.md b/docs/generated/devkit/CreateNodesV2.md new file mode 100644 index 0000000000..0972531f01 --- /dev/null +++ b/docs/generated/devkit/CreateNodesV2.md @@ -0,0 +1,12 @@ +# Type alias: CreateNodesV2\ + +Ƭ **CreateNodesV2**\<`T`\>: readonly [projectFilePattern: string, createNodesFunction: CreateNodesFunctionV2\] + +A pair of file patterns and [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2) +In Nx 20 [CreateNodes](../../devkit/documents/CreateNodes) will be replaced with this type. In Nx 21, this type will be removed. + +#### Type parameters + +| Name | Type | +| :--- | :-------- | +| `T` | `unknown` | diff --git a/docs/generated/devkit/NxPluginV2.md b/docs/generated/devkit/NxPluginV2.md index b3b448f975..8f4e539619 100644 --- a/docs/generated/devkit/NxPluginV2.md +++ b/docs/generated/devkit/NxPluginV2.md @@ -12,9 +12,10 @@ A plugin for Nx which creates nodes and dependencies for the [ProjectGraph](../. #### Type declaration -| Name | Type | Description | -| :-------------------- | :------------------------------------------------------------------------------ | :-------------------------------------------------------------------------------------------------------------------------------------------- | -| `createDependencies?` | [`CreateDependencies`](../../devkit/documents/CreateDependencies)\<`TOptions`\> | Provides a function to analyze files to create dependencies for the [ProjectGraph](../../devkit/documents/ProjectGraph) | -| `createMetadata?` | [`CreateMetadata`](../../devkit/documents/CreateMetadata)\<`TOptions`\> | Provides a function to create metadata for the [ProjectGraph](../../devkit/documents/ProjectGraph) | -| `createNodes?` | [`CreateNodes`](../../devkit/documents/CreateNodes)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '\*_/_.csproj': buildProjectsFromCsProjFile } | -| `name` | `string` | - | +| Name | Type | Description | +| :-------------------- | :------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `createDependencies?` | [`CreateDependencies`](../../devkit/documents/CreateDependencies)\<`TOptions`\> | Provides a function to analyze files to create dependencies for the [ProjectGraph](../../devkit/documents/ProjectGraph) | +| `createMetadata?` | [`CreateMetadata`](../../devkit/documents/CreateMetadata)\<`TOptions`\> | Provides a function to create metadata for the [ProjectGraph](../../devkit/documents/ProjectGraph) | +| `createNodes?` | [`CreateNodes`](../../devkit/documents/CreateNodes)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '**/\*.csproj': buildProjectsFromCsProjFile } **`Deprecated`\*\* Use createNodesV2 instead. In Nx 20 support for calling createNodes with a single file for the first argument will be removed. | +| `createNodesV2?` | [`CreateNodesV2`](../../devkit/documents/CreateNodesV2)\<`TOptions`\> | Provides a file pattern and function that retrieves configuration info from those files. e.g. { '\*_/_.csproj': buildProjectsFromCsProjFiles } In Nx 20 createNodes will be replaced with this property. In Nx 21, this property will be removed. | +| `name` | `string` | - | diff --git a/docs/generated/devkit/README.md b/docs/generated/devkit/README.md index c2dc580fd2..e3049681e7 100644 --- a/docs/generated/devkit/README.md +++ b/docs/generated/devkit/README.md @@ -18,12 +18,14 @@ It only uses language primitives and immutable objects ### Classes +- [AggregateCreateNodesError](../../devkit/documents/AggregateCreateNodesError) - [ProjectGraphBuilder](../../devkit/documents/ProjectGraphBuilder) ### Interfaces - [CreateDependenciesContext](../../devkit/documents/CreateDependenciesContext) - [CreateNodesContext](../../devkit/documents/CreateNodesContext) +- [CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2) - [CreateNodesResult](../../devkit/documents/CreateNodesResult) - [DefaultTasksRunnerOptions](../../devkit/documents/DefaultTasksRunnerOptions) - [ExecutorContext](../../devkit/documents/ExecutorContext) @@ -67,6 +69,9 @@ It only uses language primitives and immutable objects - [CreateMetadataContext](../../devkit/documents/CreateMetadataContext) - [CreateNodes](../../devkit/documents/CreateNodes) - [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) +- [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2) +- [CreateNodesResultV2](../../devkit/documents/CreateNodesResultV2) +- [CreateNodesV2](../../devkit/documents/CreateNodesV2) - [CustomHasher](../../devkit/documents/CustomHasher) - [DynamicDependency](../../devkit/documents/DynamicDependency) - [Executor](../../devkit/documents/Executor) @@ -109,6 +114,7 @@ It only uses language primitives and immutable objects - [applyChangesToString](../../devkit/documents/applyChangesToString) - [convertNxExecutor](../../devkit/documents/convertNxExecutor) - [convertNxGenerator](../../devkit/documents/convertNxGenerator) +- [createNodesFromFiles](../../devkit/documents/createNodesFromFiles) - [createProjectFileMapUsingProjectGraph](../../devkit/documents/createProjectFileMapUsingProjectGraph) - [createProjectGraphAsync](../../devkit/documents/createProjectGraphAsync) - [defaultTasksRunner](../../devkit/documents/defaultTasksRunner) diff --git a/docs/generated/devkit/createNodesFromFiles.md b/docs/generated/devkit/createNodesFromFiles.md new file mode 100644 index 0000000000..cbabd22fa4 --- /dev/null +++ b/docs/generated/devkit/createNodesFromFiles.md @@ -0,0 +1,22 @@ +# Function: createNodesFromFiles + +▸ **createNodesFromFiles**\<`T`\>(`createNodes`, `configFiles`, `options`, `context`): `Promise`\<[file: string, value: CreateNodesResult][]\> + +#### Type parameters + +| Name | Type | +| :--- | :-------- | +| `T` | `unknown` | + +#### Parameters + +| Name | Type | +| :------------ | :-------------------------------------------------------------------- | +| `createNodes` | [`CreateNodesFunction`](../../devkit/documents/CreateNodesFunction) | +| `configFiles` | readonly `string`[] | +| `options` | `T` | +| `context` | [`CreateNodesContextV2`](../../devkit/documents/CreateNodesContextV2) | + +#### Returns + +`Promise`\<[file: string, value: CreateNodesResult][]\> diff --git a/docs/generated/packages/devkit/documents/nx_devkit.md b/docs/generated/packages/devkit/documents/nx_devkit.md index c2dc580fd2..e3049681e7 100644 --- a/docs/generated/packages/devkit/documents/nx_devkit.md +++ b/docs/generated/packages/devkit/documents/nx_devkit.md @@ -18,12 +18,14 @@ It only uses language primitives and immutable objects ### Classes +- [AggregateCreateNodesError](../../devkit/documents/AggregateCreateNodesError) - [ProjectGraphBuilder](../../devkit/documents/ProjectGraphBuilder) ### Interfaces - [CreateDependenciesContext](../../devkit/documents/CreateDependenciesContext) - [CreateNodesContext](../../devkit/documents/CreateNodesContext) +- [CreateNodesContextV2](../../devkit/documents/CreateNodesContextV2) - [CreateNodesResult](../../devkit/documents/CreateNodesResult) - [DefaultTasksRunnerOptions](../../devkit/documents/DefaultTasksRunnerOptions) - [ExecutorContext](../../devkit/documents/ExecutorContext) @@ -67,6 +69,9 @@ It only uses language primitives and immutable objects - [CreateMetadataContext](../../devkit/documents/CreateMetadataContext) - [CreateNodes](../../devkit/documents/CreateNodes) - [CreateNodesFunction](../../devkit/documents/CreateNodesFunction) +- [CreateNodesFunctionV2](../../devkit/documents/CreateNodesFunctionV2) +- [CreateNodesResultV2](../../devkit/documents/CreateNodesResultV2) +- [CreateNodesV2](../../devkit/documents/CreateNodesV2) - [CustomHasher](../../devkit/documents/CustomHasher) - [DynamicDependency](../../devkit/documents/DynamicDependency) - [Executor](../../devkit/documents/Executor) @@ -109,6 +114,7 @@ It only uses language primitives and immutable objects - [applyChangesToString](../../devkit/documents/applyChangesToString) - [convertNxExecutor](../../devkit/documents/convertNxExecutor) - [convertNxGenerator](../../devkit/documents/convertNxGenerator) +- [createNodesFromFiles](../../devkit/documents/createNodesFromFiles) - [createProjectFileMapUsingProjectGraph](../../devkit/documents/createProjectFileMapUsingProjectGraph) - [createProjectGraphAsync](../../devkit/documents/createProjectGraphAsync) - [defaultTasksRunner](../../devkit/documents/defaultTasksRunner) diff --git a/packages/gradle/src/plugin/dependencies.ts b/packages/gradle/src/plugin/dependencies.ts index cd69c3be72..66f0d6a3bf 100644 --- a/packages/gradle/src/plugin/dependencies.ts +++ b/packages/gradle/src/plugin/dependencies.ts @@ -10,11 +10,9 @@ import { readFileSync } from 'node:fs'; import { basename } from 'node:path'; import { - getGradleReport, - invalidateGradleReportCache, + getCurrentGradleReport, newLineSeparator, } from '../utils/get-gradle-report'; -import { writeTargetsToCache } from './nodes'; export const createDependencies: CreateDependencies = async ( _, @@ -31,7 +29,7 @@ export const createDependencies: CreateDependencies = async ( gradleFileToGradleProjectMap, gradleProjectToProjectName, buildFileToDepsMap, - } = getGradleReport(); + } = getCurrentGradleReport(); for (const gradleFile of gradleFiles) { const gradleProject = gradleFileToGradleProjectMap.get(gradleFile); @@ -59,10 +57,6 @@ export const createDependencies: CreateDependencies = async ( gradleDependenciesEnd.name ); - writeTargetsToCache(); - if (dependencies.length) { - invalidateGradleReportCache(); - } return dependencies; }; diff --git a/packages/gradle/src/plugin/nodes.spec.ts b/packages/gradle/src/plugin/nodes.spec.ts index 5ed64367e9..496163e685 100644 --- a/packages/gradle/src/plugin/nodes.spec.ts +++ b/packages/gradle/src/plugin/nodes.spec.ts @@ -1,12 +1,13 @@ import { CreateNodesContext } from '@nx/devkit'; import { TempFs } from 'nx/src/internal-testing-utils/temp-fs'; -import type { GradleReport } from '../utils/get-gradle-report'; +import { type GradleReport } from '../utils/get-gradle-report'; let gradleReport: GradleReport; jest.mock('../utils/get-gradle-report.ts', () => { return { - getGradleReport: jest.fn().mockImplementation(() => gradleReport), + populateGradleReport: jest.fn().mockImplementation(() => void 0), + getCurrentGradleReport: jest.fn().mockImplementation(() => gradleReport), }; }); diff --git a/packages/gradle/src/plugin/nodes.ts b/packages/gradle/src/plugin/nodes.ts index 7e19b38cf2..9a52c7f601 100644 --- a/packages/gradle/src/plugin/nodes.ts +++ b/packages/gradle/src/plugin/nodes.ts @@ -1,10 +1,16 @@ import { CreateNodes, + CreateNodesV2, CreateNodesContext, + CreateNodesContextV2, ProjectConfiguration, TargetConfiguration, + createNodesFromFiles, readJsonFile, writeJsonFile, + CreateNodesResultV2, + CreateNodesFunction, + logger, } from '@nx/devkit'; import { calculateHashForCreateNodes } from '@nx/devkit/src/utils/calculate-hash-for-create-nodes'; import { existsSync } from 'node:fs'; @@ -12,7 +18,13 @@ import { dirname, join } from 'node:path'; import { projectGraphCacheDirectory } from 'nx/src/utils/cache-directory'; import { getGradleExecFile } from '../utils/exec-gradle'; -import { getGradleReport } from '../utils/get-gradle-report'; +import { + populateGradleReport, + getCurrentGradleReport, + GradleReport, + gradleConfigGlob, +} from '../utils/get-gradle-report'; +import { hashObject } from 'nx/src/hasher/file-hasher'; const cacheableTaskType = new Set(['Build', 'Verification']); const dependsOnMap = { @@ -33,8 +45,6 @@ export interface GradlePluginOptions { [taskTargetName: string]: string | undefined; } -const cachePath = join(projectGraphCacheDirectory, 'gradle.hash'); -const targetsCache = readTargetsCache(); type GradleTargets = Record< string, { @@ -44,20 +54,45 @@ type GradleTargets = Record< } >; -function readTargetsCache(): GradleTargets { +function readTargetsCache(cachePath: string): GradleTargets { return existsSync(cachePath) ? readJsonFile(cachePath) : {}; } -export function writeTargetsToCache() { - const oldCache = readTargetsCache(); - writeJsonFile(cachePath, { - ...oldCache, - ...targetsCache, - }); +export function writeTargetsToCache(cachePath: string, results: GradleTargets) { + writeJsonFile(cachePath, results); } -export const createNodes: CreateNodes = [ - '**/build.{gradle.kts,gradle}', +export const createNodesV2: CreateNodesV2 = [ + gradleConfigGlob, + async (configFiles, options, context) => { + const optionsHash = hashObject(options); + const cachePath = join( + projectGraphCacheDirectory, + `gradle-${optionsHash}.hash` + ); + const targetsCache = readTargetsCache(cachePath); + + populateGradleReport(context.workspaceRoot); + const gradleReport = getCurrentGradleReport(); + + try { + return await createNodesFromFiles( + makeCreateNodes(gradleReport, targetsCache), + configFiles, + options, + context + ); + } finally { + writeTargetsToCache(cachePath, targetsCache); + } + }, +]; + +export const makeCreateNodes = + ( + gradleReport: GradleReport, + targetsCache: GradleTargets + ): CreateNodesFunction => ( gradleFilePath, options: GradlePluginOptions | undefined, @@ -71,6 +106,7 @@ export const createNodes: CreateNodes = [ context ); targetsCache[hash] ??= createGradleProject( + gradleReport, gradleFilePath, options, context @@ -84,10 +120,26 @@ export const createNodes: CreateNodes = [ [projectRoot]: project, }, }; + }; + +/** + * @deprecated `{@link createNodesV2} is replacing this. Update your plugin to export its own `createNodesV2` function that wraps this one instead.` + */ +export const createNodes: CreateNodes = [ + gradleConfigGlob, + (configFile, options, context) => { + logger.warn( + '`createNodes` is deprecated. Update your plugin to utilize createNodesV2 instead. In Nx 20, this will error.' + ); + populateGradleReport(context.workspaceRoot); + const gradleReport = getCurrentGradleReport(); + const internalCreateNodes = makeCreateNodes(gradleReport, {}); + return internalCreateNodes(configFile, options, context); }, ]; function createGradleProject( + gradleReport: GradleReport, gradleFilePath: string, options: GradlePluginOptions | undefined, context: CreateNodesContext @@ -98,7 +150,7 @@ function createGradleProject( gradleFileToOutputDirsMap, gradleFileToGradleProjectMap, gradleProjectToProjectName, - } = getGradleReport(); + } = gradleReport; const gradleProject = gradleFileToGradleProjectMap.get( gradleFilePath diff --git a/packages/gradle/src/utils/get-gradle-report.ts b/packages/gradle/src/utils/get-gradle-report.ts index 2dd1d423cd..38c00fb9d8 100644 --- a/packages/gradle/src/utils/get-gradle-report.ts +++ b/packages/gradle/src/utils/get-gradle-report.ts @@ -4,6 +4,7 @@ import { join, relative } from 'node:path'; import { normalizePath, workspaceRoot } from '@nx/devkit'; import { execGradle } from './exec-gradle'; +import { hashWithWorkspaceContext } from 'nx/src/utils/workspace-context'; export const fileSeparator = process.platform.startsWith('win') ? 'file:///' @@ -22,14 +23,25 @@ export interface GradleReport { } let gradleReportCache: GradleReport; +let gradleCurrentConfigHash: string; -export function invalidateGradleReportCache() { - gradleReportCache = undefined; +export const gradleConfigGlob = '**/build.{gradle.kts,gradle}'; + +export function getCurrentGradleReport() { + if (!gradleReportCache) { + throw new Error( + 'Expected cached gradle report. Please open an issue at https://github.com/nrwl/nx/issues/new/choose' + ); + } + return gradleReportCache; } -export function getGradleReport(): GradleReport { - if (gradleReportCache) { - return gradleReportCache; +export function populateGradleReport(workspaceRoot: string): void { + const gradleConfigHash = hashWithWorkspaceContext(workspaceRoot, [ + gradleConfigGlob, + ]); + if (gradleReportCache && gradleConfigHash === gradleCurrentConfigHash) { + return; } const gradleProjectReportStart = performance.mark( @@ -47,7 +59,6 @@ export function getGradleReport(): GradleReport { gradleProjectReportEnd.name ); gradleReportCache = processProjectReports(projectReportLines); - return gradleReportCache; } export function processProjectReports( diff --git a/packages/nx/src/devkit-exports.ts b/packages/nx/src/devkit-exports.ts index 8e5d7dcffe..046030bdb0 100644 --- a/packages/nx/src/devkit-exports.ts +++ b/packages/nx/src/devkit-exports.ts @@ -46,6 +46,10 @@ export type { CreateNodesFunction, CreateNodesResult, CreateNodesContext, + CreateNodesContextV2, + CreateNodesFunctionV2, + CreateNodesResultV2, + CreateNodesV2, CreateDependencies, CreateDependenciesContext, CreateMetadata, @@ -53,6 +57,10 @@ export type { ProjectsMetadata, } from './project-graph/plugins'; +export { AggregateCreateNodesError } from './project-graph/error-types'; + +export { createNodesFromFiles } from './project-graph/plugins'; + export type { NxPluginV1, ProjectTargetConfigurator, diff --git a/packages/nx/src/plugins/project-json/build-nodes/project-json.ts b/packages/nx/src/plugins/project-json/build-nodes/project-json.ts index 59031d6d5c..21198e136b 100644 --- a/packages/nx/src/plugins/project-json/build-nodes/project-json.ts +++ b/packages/nx/src/plugins/project-json/build-nodes/project-json.ts @@ -4,7 +4,6 @@ import { ProjectConfiguration } from '../../../config/workspace-json-project-jso import { toProjectName } from '../../../config/to-project-name'; import { readJsonFile } from '../../../utils/fileutils'; import { NxPluginV2 } from '../../../project-graph/plugins'; -import { CreateNodesError } from '../../../project-graph/error-types'; export const ProjectJsonProjectsPlugin: NxPluginV2 = { name: 'nx/core/project-json', diff --git a/packages/nx/src/project-graph/error-types.ts b/packages/nx/src/project-graph/error-types.ts index c2c6992db7..9cda2cf023 100644 --- a/packages/nx/src/project-graph/error-types.ts +++ b/packages/nx/src/project-graph/error-types.ts @@ -1,14 +1,14 @@ -import { CreateNodesResultWithContext } from './plugins/internal-api'; import { ConfigurationResult, ConfigurationSourceMaps, } from './utils/project-configuration-utils'; import { ProjectConfiguration } from '../config/workspace-json-project-json'; import { ProjectGraph } from '../config/project-graph'; +import { CreateNodesFunctionV2 } from './plugins'; export class ProjectGraphError extends Error { readonly #errors: Array< - | CreateNodesError + | AggregateCreateNodesError | MergeNodesError | CreateMetadataError | ProjectsWithNoNameError @@ -22,7 +22,7 @@ export class ProjectGraphError extends Error { constructor( errors: Array< - | CreateNodesError + | AggregateCreateNodesError | MergeNodesError | ProjectsWithNoNameError | MultipleProjectsWithSameNameError @@ -168,7 +168,7 @@ export class ProjectConfigurationsError extends Error { constructor( public readonly errors: Array< | MergeNodesError - | CreateNodesError + | AggregateCreateNodesError | ProjectsWithNoNameError | MultipleProjectsWithSameNameError >, @@ -190,34 +190,39 @@ export function isProjectConfigurationsError( ); } -export class CreateNodesError extends Error { - file: string; - pluginName: string; - - constructor({ - file, - pluginName, - error, - }: { - file: string; - pluginName: string; - error: Error; - }) { - const msg = `The "${pluginName}" plugin threw an error while creating nodes from ${file}:`; - - super(msg, { cause: error }); - this.name = this.constructor.name; - this.file = file; - this.pluginName = pluginName; - this.stack = `${this.message}\n ${error.stack.split('\n').join('\n ')}`; - } -} - +/** + * This error should be thrown when a `createNodesV2` function hits a recoverable error. + * It allows Nx to recieve partial results and continue processing for better UX. + */ export class AggregateCreateNodesError extends Error { + /** + * Throwing this error from a `createNodesV2` function will allow Nx to continue processing and recieve partial results from your plugin. + * @example + * export async function createNodesV2( + * files: string[], + * ) { + * const partialResults = []; + * const errors = []; + * await Promise.all(files.map(async (file) => { + * try { + * const result = await createNodes(file); + * partialResults.push(result); + * } catch (e) { + * errors.push([file, e]); + * } + * })); + * if (errors.length > 0) { + * throw new AggregateCreateNodesError(errors, partialResults); + * } + * return partialResults; + * } + * + * @param errors An array of tuples that represent errors encountered when processing a given file. An example entry might look like ['path/to/project.json', [Error: 'Invalid JSON. Unexpected token 'a' in JSON at position 0]] + * @param partialResults The partial results of the `createNodesV2` function. This should be the results for each file that didn't encounter an issue. + */ constructor( - public readonly pluginName: string, - public readonly errors: Array, - public readonly partialResults: Array + public readonly errors: Array<[file: string | null, error: Error]>, + public readonly partialResults: Awaited> ) { super('Failed to create nodes'); this.name = this.constructor.name; @@ -335,13 +340,6 @@ export function isCreateMetadataError(e: unknown): e is CreateMetadataError { ); } -export function isCreateNodesError(e: unknown): e is CreateNodesError { - return ( - e instanceof CreateNodesError || - (typeof e === 'object' && 'name' in e && e?.name === CreateNodesError.name) - ); -} - export function isAggregateCreateNodesError( e: unknown ): e is AggregateCreateNodesError { diff --git a/packages/nx/src/project-graph/plugins/index.ts b/packages/nx/src/project-graph/plugins/index.ts index f48519b8af..3739a2e887 100644 --- a/packages/nx/src/project-graph/plugins/index.ts +++ b/packages/nx/src/project-graph/plugins/index.ts @@ -1,3 +1,4 @@ export * from './public-api'; export { readPluginPackageJson, registerPluginTSTranspiler } from './loader'; +export { createNodesFromFiles } from './utils'; diff --git a/packages/nx/src/project-graph/plugins/internal-api.ts b/packages/nx/src/project-graph/plugins/internal-api.ts index 874f750d66..5d07df6ced 100644 --- a/packages/nx/src/project-graph/plugins/internal-api.ts +++ b/packages/nx/src/project-graph/plugins/internal-api.ts @@ -13,7 +13,7 @@ import { CreateDependenciesContext, CreateMetadata, CreateMetadataContext, - CreateNodesContext, + CreateNodesContextV2, CreateNodesResult, NxPluginV2, } from './public-api'; @@ -21,9 +21,13 @@ import { ProjectGraph, ProjectGraphProcessor, } from '../../config/project-graph'; -import { runCreateNodesInParallel } from './utils'; import { loadNxPluginInIsolation } from './isolation'; import { loadNxPlugin, unregisterPluginTSTranspiler } from './loader'; +import { createNodesFromFiles } from './utils'; +import { + AggregateCreateNodesError, + isAggregateCreateNodesError, +} from '../error-types'; export class LoadedNxPlugin { readonly name: string; @@ -33,8 +37,10 @@ export class LoadedNxPlugin { // the result's context. fn: ( matchedFiles: string[], - context: CreateNodesContext - ) => Promise + context: CreateNodesContextV2 + ) => Promise< + Array + > ]; readonly createDependencies?: ( context: CreateDependenciesContext @@ -57,14 +63,56 @@ export class LoadedNxPlugin { this.exclude = pluginDefinition.exclude; } - if (plugin.createNodes) { + if (plugin.createNodes && !plugin.createNodesV2) { this.createNodes = [ plugin.createNodes[0], - (files, context) => - runCreateNodesInParallel(files, plugin, this.options, context), + (configFiles, context) => + createNodesFromFiles( + plugin.createNodes[1], + configFiles, + this.options, + context + ).then((results) => results.map((r) => [this.name, r[0], r[1]])), ]; } + if (plugin.createNodesV2) { + this.createNodes = [ + plugin.createNodesV2[0], + async (configFiles, context) => { + const result = await plugin.createNodesV2[1]( + configFiles, + this.options, + context + ); + return result.map((r) => [this.name, r[0], r[1]]); + }, + ]; + } + + if (this.createNodes) { + const inner = this.createNodes[1]; + this.createNodes[1] = async (...args) => { + performance.mark(`${plugin.name}:createNodes - start`); + try { + return await inner(...args); + } catch (e) { + if (isAggregateCreateNodesError(e)) { + throw e; + } + // The underlying plugin errored out. We can't know any partial results. + throw new AggregateCreateNodesError([null, e], []); + } finally { + performance.mark(`${plugin.name}:createNodes - end`); + performance.measure( + `${plugin.name}:createNodes`, + `${plugin.name}:createNodes - start`, + `${plugin.name}:createNodes - end` + ); + } + }; + } + if (plugin.createDependencies) { this.createDependencies = (context) => plugin.createDependencies(this.options, context); diff --git a/packages/nx/src/project-graph/plugins/public-api.ts b/packages/nx/src/project-graph/plugins/public-api.ts index 171e27b764..89e684f00c 100644 --- a/packages/nx/src/project-graph/plugins/public-api.ts +++ b/packages/nx/src/project-graph/plugins/public-api.ts @@ -16,15 +16,18 @@ import { RawProjectGraphDependency } from '../project-graph-builder'; /** * Context for {@link CreateNodesFunction} */ -export interface CreateNodesContext { - readonly nxJsonConfiguration: NxJsonConfiguration; - readonly workspaceRoot: string; +export interface CreateNodesContext extends CreateNodesContextV2 { /** * The subset of configuration files which match the createNodes pattern */ readonly configFiles: readonly string[]; } +export interface CreateNodesContextV2 { + readonly nxJsonConfiguration: NxJsonConfiguration; + readonly workspaceRoot: string; +} + /** * A function which parses a configuration file into a set of nodes. * Used for creating nodes for the {@link ProjectGraph} @@ -35,6 +38,16 @@ export type CreateNodesFunction = ( context: CreateNodesContext ) => CreateNodesResult | Promise; +export type CreateNodesResultV2 = Array< + readonly [configFileSource: string, result: CreateNodesResult] +>; + +export type CreateNodesFunctionV2 = ( + projectConfigurationFiles: readonly string[], + options: T | undefined, + context: CreateNodesContextV2 +) => CreateNodesResultV2 | Promise; + export type Optional = Omit & Partial>; export interface CreateNodesResult { @@ -51,12 +64,31 @@ export interface CreateNodesResult { /** * A pair of file patterns and {@link CreateNodesFunction} + * + * Nx 19.2+: Both original `CreateNodes` and `CreateNodesV2` are supported. Nx will only invoke `CreateNodesV2` if it is present. + * Nx 20.X : The `CreateNodesV2` will be the only supported API. This typing will still exist, but be identical to `CreateNodesV2`. + Nx **will not** invoke the original `plugin.createNodes` callback. This should give plugin authors a window to transition. + Plugin authors should update their plugin's `createNodes` function to align with `CreateNodesV2` / the updated `CreateNodes`. + The plugin should contain something like: `export createNodes = createNodesV2;` during this period. This will allow the plugin + to maintain compatibility with Nx 19.2 and up. + * Nx 21.X : The `CreateNodesV2` typing will be removed, as it has replaced `CreateNodes`. + * + * @deprecated Use {@link CreateNodesV2} instead. CreateNodesV2 will replace this API. Read more about the transition above. */ export type CreateNodes = readonly [ projectFilePattern: string, createNodesFunction: CreateNodesFunction ]; +/** + * A pair of file patterns and {@link CreateNodesFunctionV2} + * In Nx 20 {@link CreateNodes} will be replaced with this type. In Nx 21, this type will be removed. + */ +export type CreateNodesV2 = readonly [ + projectFilePattern: string, + createNodesFunction: CreateNodesFunctionV2 +]; + /** * Context for {@link CreateDependencies} */ @@ -123,9 +155,19 @@ export type NxPluginV2 = { /** * Provides a file pattern and function that retrieves configuration info from * those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFile } + * + * @deprecated Use {@link createNodesV2} instead. In Nx 20 support for calling createNodes with a single file for the first argument will be removed. */ createNodes?: CreateNodes; + /** + * Provides a file pattern and function that retrieves configuration info from + * those files. e.g. { '**\/*.csproj': buildProjectsFromCsProjFiles } + * + * In Nx 20 {@link createNodes} will be replaced with this property. In Nx 21, this property will be removed. + */ + createNodesV2?: CreateNodesV2; + /** * Provides a function to analyze files to create dependencies for the {@link ProjectGraph} */ diff --git a/packages/nx/src/project-graph/plugins/utils.spec.ts b/packages/nx/src/project-graph/plugins/utils.spec.ts index 0e3916f9d9..e2d81ee797 100644 --- a/packages/nx/src/project-graph/plugins/utils.spec.ts +++ b/packages/nx/src/project-graph/plugins/utils.spec.ts @@ -1,123 +1,186 @@ -import { runCreateNodesInParallel } from './utils'; +import { isAggregateCreateNodesError } from '../error-types'; +import { createNodesFromFiles } from './utils'; const configFiles = ['file1', 'file2'] as const; const context = { - file: 'file1', nxJsonConfiguration: {}, workspaceRoot: '', - configFiles, } as const; -describe('createNodesInParallel', () => { +describe('createNodesFromFiles', () => { it('should return results with context', async () => { - const plugin = { - name: 'test', - createNodes: [ - '*/**/*', - async (file: string) => { - return { - projects: { - [file]: { - root: file, - }, + const createNodes = [ + '*/**/*', + async (file: string) => { + return { + projects: { + [file]: { + root: file, }, - }; - }, - ], - } as const; + }, + }; + }, + ] as const; const options = {}; - const results = await runCreateNodesInParallel( + const results = await createNodesFromFiles( + createNodes[1], configFiles, - plugin, options, context ); expect(results).toMatchInlineSnapshot(` [ - { - "file": "file1", - "pluginName": "test", - "projects": { - "file1": { - "root": "file1", + [ + "file1", + { + "projects": { + "file1": { + "root": "file1", + }, }, }, - }, - { - "file": "file2", - "pluginName": "test", - "projects": { - "file2": { - "root": "file2", + ], + [ + "file2", + { + "projects": { + "file2": { + "root": "file2", + }, }, }, - }, + ], ] `); }); it('should handle async errors', async () => { - const plugin = { - name: 'test', - createNodes: [ - '*/**/*', - async () => { - throw new Error('Async Error'); - }, - ], - } as const; + const createNodes = [ + '*/**/*', + async () => { + throw new Error('Async Error'); + }, + ] as const; const options = {}; - const error = await runCreateNodesInParallel( + let error; + await createNodesFromFiles( + createNodes[1], configFiles, - plugin, options, context - ).catch((e) => e); + ).catch((e) => (error = e)); - expect(error).toMatchInlineSnapshot( - `[AggregateCreateNodesError: Failed to create nodes]` - ); + const isAggregateError = isAggregateCreateNodesError(error); + expect(isAggregateError).toBe(true); - expect(error.errors).toMatchInlineSnapshot(` - [ - [CreateNodesError: The "test" plugin threw an error while creating nodes from file1:], - [CreateNodesError: The "test" plugin threw an error while creating nodes from file2:], - ] - `); + if (isAggregateCreateNodesError(error)) { + expect(error.errors).toMatchInlineSnapshot(` + [ + [ + "file1", + [Error: Async Error], + ], + [ + "file2", + [Error: Async Error], + ], + ] + `); + } }); it('should handle sync errors', async () => { - const plugin = { - name: 'test', - createNodes: [ - '*/**/*', - () => { - throw new Error('Sync Error'); - }, - ], - } as const; + const createNodes = [ + '*/**/*', + () => { + throw new Error('Sync Error'); + }, + ] as const; const options = {}; - const error = await runCreateNodesInParallel( + let error; + await createNodesFromFiles( + createNodes[1], configFiles, - plugin, options, context - ).catch((e) => e); + ).catch((e) => (error = e)); - expect(error).toMatchInlineSnapshot( - `[AggregateCreateNodesError: Failed to create nodes]` - ); + const isAggregateError = isAggregateCreateNodesError(error); + expect(isAggregateError).toBe(true); - expect(error.errors).toMatchInlineSnapshot(` - [ - [CreateNodesError: The "test" plugin threw an error while creating nodes from file1:], - [CreateNodesError: The "test" plugin threw an error while creating nodes from file2:], - ] - `); + if (isAggregateCreateNodesError(error)) { + expect(error.errors).toMatchInlineSnapshot(` + [ + [ + "file1", + [Error: Sync Error], + ], + [ + "file2", + [Error: Sync Error], + ], + ] + `); + } + }); + + it('should handle partial errors', async () => { + const createNodes = [ + '*/**/*', + async (file: string) => { + if (file === 'file1') { + throw new Error('Error'); + } + return { + projects: { + [file]: { + root: file, + }, + }, + }; + }, + ] as const; + const options = {}; + + let error; + await createNodesFromFiles( + createNodes[1], + configFiles, + options, + context + ).catch((e) => (error = e)); + + const isAggregateError = isAggregateCreateNodesError(error); + expect(isAggregateError).toBe(true); + + if (isAggregateCreateNodesError(error)) { + expect(error.errors).toMatchInlineSnapshot(` + [ + [ + "file1", + [Error: Error], + ], + ] + `); + expect(error.partialResults).toMatchInlineSnapshot(` + [ + [ + "file2", + { + "projects": { + "file2": { + "root": "file2", + }, + }, + }, + ], + ] + `); + } }); }); diff --git a/packages/nx/src/project-graph/plugins/utils.ts b/packages/nx/src/project-graph/plugins/utils.ts index e9009fe1f5..4ab9c8e7aa 100644 --- a/packages/nx/src/project-graph/plugins/utils.ts +++ b/packages/nx/src/project-graph/plugins/utils.ts @@ -4,19 +4,16 @@ import { toProjectName } from '../../config/to-project-name'; import { combineGlobPatterns } from '../../utils/globs'; import type { NxPluginV1 } from '../../utils/nx-plugin.deprecated'; -import type { - CreateNodesResultWithContext, - LoadedNxPlugin, - NormalizedPlugin, -} from './internal-api'; +import type { LoadedNxPlugin, NormalizedPlugin } from './internal-api'; import { + CreateNodesContextV2, + CreateNodesFunction, + CreateNodesFunctionV2, CreateNodesResult, - type CreateNodesContext, type NxPlugin, type NxPluginV2, } from './public-api'; -import { AggregateCreateNodesError, CreateNodesError } from '../error-types'; -import { performance } from 'perf_hooks'; +import { AggregateCreateNodesError } from '../error-types'; export function isNxPluginV2(plugin: NxPlugin): plugin is NxPluginV2 { return 'createNodes' in plugin || 'createDependencies' in plugin; @@ -54,49 +51,37 @@ export function normalizeNxPlugin(plugin: NxPlugin): NormalizedPlugin { return plugin; } -export async function runCreateNodesInParallel( +export type AsyncFn = T extends ( + ...args: infer A +) => infer R + ? (...args: A) => Promise> + : never; + +export async function createNodesFromFiles( + createNodes: CreateNodesFunction, configFiles: readonly string[], - plugin: NormalizedPlugin, - options: unknown, - context: CreateNodesContext -): Promise { - performance.mark(`${plugin.name}:createNodes - start`); + options: T, + context: CreateNodesContextV2 +) { + const results: Array<[file: string, value: CreateNodesResult]> = []; + const errors: Array<[file: string, error: Error]> = []; - const errors: CreateNodesError[] = []; - const results: CreateNodesResultWithContext[] = []; - - const promises: Array> = configFiles.map(async (file) => { - try { - const value = await plugin.createNodes[1](file, options, context); - if (value) { - results.push({ - ...value, - file, - pluginName: plugin.name, + await Promise.all( + configFiles.map(async (file) => { + try { + const value = await createNodes(file, options, { + ...context, + configFiles, }); + results.push([file, value] as const); + } catch (e) { + errors.push([file, e] as const); } - } catch (e) { - errors.push( - new CreateNodesError({ - error: e, - pluginName: plugin.name, - file, - }) - ); - } - }); - - await Promise.all(promises).then(() => { - performance.mark(`${plugin.name}:createNodes - end`); - performance.measure( - `${plugin.name}:createNodes`, - `${plugin.name}:createNodes - start`, - `${plugin.name}:createNodes - end` - ); - }); + }) + ); if (errors.length > 0) { - throw new AggregateCreateNodesError(plugin.name, errors, results); + throw new AggregateCreateNodesError(errors, results); } return results; } diff --git a/packages/nx/src/project-graph/utils/project-configuration-utils.ts b/packages/nx/src/project-graph/utils/project-configuration-utils.ts index 8ff6752c7d..84abcac850 100644 --- a/packages/nx/src/project-graph/utils/project-configuration-utils.ts +++ b/packages/nx/src/project-graph/utils/project-configuration-utils.ts @@ -17,15 +17,10 @@ import { import { minimatch } from 'minimatch'; import { join } from 'path'; import { performance } from 'perf_hooks'; +import { LoadedNxPlugin } from '../plugins/internal-api'; import { - CreateNodesResultWithContext, - LoadedNxPlugin, -} from '../plugins/internal-api'; -import { - CreateNodesError, MergeNodesError, ProjectConfigurationsError, - isAggregateCreateNodesError, ProjectsWithNoNameError, MultipleProjectsWithSameNameError, isMultipleProjectsWithSameNameError, @@ -34,7 +29,10 @@ import { ProjectWithExistingNameError, isProjectWithExistingNameError, isProjectWithNoNameError, + isAggregateCreateNodesError, + AggregateCreateNodesError, } from '../error-types'; +import { CreateNodesResult } from '../plugins'; export type SourceInformation = [file: string | null, plugin: string]; export type ConfigurationSourceMaps = Record< @@ -347,9 +345,9 @@ export async function createProjectConfigurations( ): Promise { performance.mark('build-project-configs:start'); - const results: Array>> = []; + const results: Array> = []; const errors: Array< - | CreateNodesError + | AggregateCreateNodesError | MergeNodesError | ProjectsWithNoNameError | MultipleProjectsWithSameNameError @@ -357,10 +355,10 @@ export async function createProjectConfigurations( // We iterate over plugins first - this ensures that plugins specified first take precedence. for (const { - name: pluginName, createNodes: createNodesTuple, include, exclude, + name: pluginName, } of plugins) { const [pattern, createNodes] = createNodesTuple ?? []; @@ -368,120 +366,44 @@ export async function createProjectConfigurations( continue; } - const matchingConfigFiles: string[] = []; + const matchingConfigFiles: string[] = findMatchingConfigFiles( + projectFiles, + pattern, + include, + exclude + ); - for (const file of projectFiles) { - if (minimatch(file, pattern, { dot: true })) { - if (include) { - const included = include.some((includedPattern) => - minimatch(file, includedPattern, { dot: true }) - ); - if (!included) { - continue; - } - } - - if (exclude) { - const excluded = exclude.some((excludedPattern) => - minimatch(file, excludedPattern, { dot: true }) - ); - if (excluded) { - continue; - } - } - - matchingConfigFiles.push(file); - } - } let r = createNodes(matchingConfigFiles, { nxJsonConfiguration: nxJson, workspaceRoot: root, - configFiles: matchingConfigFiles, - }).catch((e) => { - if (isAggregateCreateNodesError(e)) { - errors.push(...e.errors); - return e.partialResults; - } else { - throw e; - } + }).catch((e: Error) => { + const errorBodyLines = [ + `An error occurred while processing files for the ${pluginName} plugin.`, + ]; + const error: AggregateCreateNodesError = isAggregateCreateNodesError(e) + ? // This is an expected error if something goes wrong while processing files. + e + : // This represents a single plugin erroring out with a hard error. + new AggregateCreateNodesError([[null, e]], []); + + errorBodyLines.push( + ...error.errors.map(([file, e]) => ` - ${file}: ${e.message}`) + ); + + error.message = errorBodyLines.join('\n'); + + // This represents a single plugin erroring out with a hard error. + errors.push(error); + // The plugin didn't return partial results, so we return an empty array. + return error.partialResults.map((r) => [pluginName, r[0], r[1]] as const); }); results.push(r); } return Promise.all(results).then((results) => { - performance.mark('createNodes:merge - start'); - const projectRootMap: Record = {}; - const externalNodes: Record = {}; - const configurationSourceMaps: Record< - string, - Record - > = {}; - - for (const result of results.flat()) { - const { - projects: projectNodes, - externalNodes: pluginExternalNodes, - file, - pluginName, - } = result; - - const sourceInfo: SourceInformation = [file, pluginName]; - - if (result[OVERRIDE_SOURCE_FILE]) { - sourceInfo[0] = result[OVERRIDE_SOURCE_FILE]; - } - - for (const node in projectNodes) { - // Handles `{projects: {'libs/foo': undefined}}`. - if (!projectNodes[node]) { - continue; - } - const project = { - root: node, - ...projectNodes[node], - }; - try { - mergeProjectConfigurationIntoRootMap( - projectRootMap, - project, - configurationSourceMaps, - sourceInfo - ); - } catch (error) { - errors.push( - new MergeNodesError({ - file, - pluginName, - error, - }) - ); - } - } - Object.assign(externalNodes, pluginExternalNodes); - } - - try { - validateAndNormalizeProjectRootMap(projectRootMap); - } catch (e) { - if ( - isProjectsWithNoNameError(e) || - isMultipleProjectsWithSameNameError(e) - ) { - errors.push(e); - } else { - throw e; - } - } - - const rootMap = createRootMap(projectRootMap); - - performance.mark('createNodes:merge - end'); - performance.measure( - 'createNodes:merge', - 'createNodes:merge - start', - 'createNodes:merge - end' - ); + const { projectRootMap, externalNodes, rootMap, configurationSourceMaps } = + mergeCreateNodesResults(results, errors); performance.mark('build-project-configs:end'); performance.measure( @@ -510,6 +432,126 @@ export async function createProjectConfigurations( }); } +function mergeCreateNodesResults( + results: (readonly [ + plugin: string, + file: string, + result: CreateNodesResult + ])[][], + errors: ( + | AggregateCreateNodesError + | MergeNodesError + | ProjectsWithNoNameError + | MultipleProjectsWithSameNameError + )[] +) { + performance.mark('createNodes:merge - start'); + const projectRootMap: Record = {}; + const externalNodes: Record = {}; + const configurationSourceMaps: Record< + string, + Record + > = {}; + + for (const result of results.flat()) { + const [file, pluginName, nodes] = result; + + const { projects: projectNodes, externalNodes: pluginExternalNodes } = + nodes; + + const sourceInfo: SourceInformation = [file, pluginName]; + + if (result[OVERRIDE_SOURCE_FILE]) { + sourceInfo[0] = result[OVERRIDE_SOURCE_FILE]; + } + + for (const node in projectNodes) { + // Handles `{projects: {'libs/foo': undefined}}`. + if (!projectNodes[node]) { + continue; + } + const project = { + root: node, + ...projectNodes[node], + }; + try { + mergeProjectConfigurationIntoRootMap( + projectRootMap, + project, + configurationSourceMaps, + sourceInfo + ); + } catch (error) { + errors.push( + new MergeNodesError({ + file, + pluginName, + error, + }) + ); + } + } + Object.assign(externalNodes, pluginExternalNodes); + } + + try { + validateAndNormalizeProjectRootMap(projectRootMap); + } catch (e) { + if ( + isProjectsWithNoNameError(e) || + isMultipleProjectsWithSameNameError(e) + ) { + errors.push(e); + } else { + throw e; + } + } + + const rootMap = createRootMap(projectRootMap); + + performance.mark('createNodes:merge - end'); + performance.measure( + 'createNodes:merge', + 'createNodes:merge - start', + 'createNodes:merge - end' + ); + return { projectRootMap, externalNodes, rootMap, configurationSourceMaps }; +} + +function findMatchingConfigFiles( + projectFiles: string[], + pattern: string, + include: string[], + exclude: string[] +) { + const matchingConfigFiles: string[] = []; + + for (const file of projectFiles) { + if (minimatch(file, pattern, { dot: true })) { + if (include) { + const included = include.some((includedPattern) => + minimatch(file, includedPattern, { dot: true }) + ); + if (!included) { + continue; + } + } + + if (exclude) { + const excluded = exclude.some((excludedPattern) => + minimatch(file, excludedPattern, { dot: true }) + ); + if (excluded) { + continue; + } + } + + matchingConfigFiles.push(file); + } + } + return matchingConfigFiles; +} + export function readProjectConfigurationsFromRootMap( projectRootMap: Record ) {