From 6a65101ec239f1b3eae28734e4109b56a3700599 Mon Sep 17 00:00:00 2001 From: Victor Savkin Date: Tue, 21 Jun 2022 08:18:30 -0400 Subject: [PATCH] feat(core): extend the default hasher to support different filesets --- docs/generated/packages/cypress.json | 1 - docs/generated/packages/jest.json | 1 - e2e/nx-run/src/cache.test.ts | 89 +- packages/cypress/executors.json | 1 - .../cypress/src/executors/cypress/hasher.ts | 28 - packages/jest/executors.json | 1 - packages/jest/src/executors/jest/hasher.ts | 15 - .../linter/src/executors/eslint/hasher.ts | 2 +- packages/nx/schemas/nx-schema.json | 30 + packages/nx/schemas/project-schema.json | 30 + packages/nx/src/config/nx-json.ts | 10 +- packages/nx/src/config/project-graph.ts | 4 + .../src/config/workspace-json-project-json.ts | 20 + .../generators/utils/project-configuration.ts | 2 + packages/nx/src/hasher/hasher.spec.ts | 835 +++++++++++------- packages/nx/src/hasher/hasher.ts | 457 +++++----- 16 files changed, 904 insertions(+), 622 deletions(-) delete mode 100644 packages/cypress/src/executors/cypress/hasher.ts delete mode 100644 packages/jest/src/executors/jest/hasher.ts diff --git a/docs/generated/packages/cypress.json b/docs/generated/packages/cypress.json index d5ae36477d..a0d444b256 100644 --- a/docs/generated/packages/cypress.json +++ b/docs/generated/packages/cypress.json @@ -230,7 +230,6 @@ "required": ["cypressConfig"], "presets": [] }, - "hasher": "./src/executors/cypress/hasher", "description": "Run Cypress E2E tests.", "aliases": [], "hidden": false, diff --git a/docs/generated/packages/jest.json b/docs/generated/packages/jest.json index 87e0979bef..e7796002b3 100644 --- a/docs/generated/packages/jest.json +++ b/docs/generated/packages/jest.json @@ -316,7 +316,6 @@ "required": ["jestConfig"], "presets": [] }, - "hasher": "./src/executors/jest/hasher", "description": "Run Jest unit tests.", "aliases": [], "hidden": false, diff --git a/e2e/nx-run/src/cache.test.ts b/e2e/nx-run/src/cache.test.ts index 76f4dac6a1..eb8644aaeb 100644 --- a/e2e/nx-run/src/cache.test.ts +++ b/e2e/nx-run/src/cache.test.ts @@ -7,6 +7,7 @@ import { runCLI, uniq, updateFile, + updateJson, updateProjectConfig, } from '@nrwl/e2e/utils'; @@ -163,43 +164,69 @@ describe('cache', () => { updateFile('nx.json', (c) => originalNxJson); }, 120000); - it('should only cache specific files if build outputs is configured with specific files', async () => { - const mylib1 = uniq('mylib1'); - runCLI(`generate @nrwl/react:lib ${mylib1} --buildable`); - - // Update outputs in workspace.json to just be a particular file - updateProjectConfig(mylib1, (config) => { - config.targets['build-base'] = { - ...config.targets.build, - }; - config.targets.build = { - executor: '@nrwl/workspace:run-commands', - outputs: [`dist/libs/${mylib1}/index.js`], - options: { - commands: [ - { - command: `npx nx run ${mylib1}:build-base`, - }, - ], - parallel: false, + it('should use consider filesets when hashing', async () => { + const parent = uniq('parent'); + const child1 = uniq('child1'); + const child2 = uniq('child2'); + runCLI(`generate @nrwl/js:lib ${parent}`); + runCLI(`generate @nrwl/js:lib ${child1}`); + runCLI(`generate @nrwl/js:lib ${child2}`); + updateJson(`nx.json`, (c) => { + c.filesets = { prod: ['!**/*.spec.ts'] }; + c.targetDefaults = { + test: { + dependsOnFilesets: ['default', '^prod'], }, }; - return config; + return c; }); - // run build with caching - // -------------------------------------------- - const outputThatPutsDataIntoCache = runCLI(`run ${mylib1}:build`); - // now the data is in cache - expect(outputThatPutsDataIntoCache).not.toContain('cache'); + updateJson(`libs/${parent}/project.json`, (c) => { + c.implicitDependencies = [child1, child2]; + return c; + }); - rmDist(); + updateJson(`libs/${child1}/project.json`, (c) => { + c.filesets = { prod: ['**/*.ts'] }; + return c; + }); - const outputWithBuildTasksCached = runCLI(`run ${mylib1}:build`); - expect(outputWithBuildTasksCached).toContain('cache'); - expectCached(outputWithBuildTasksCached, [mylib1]); - // Ensure that only the specific file in outputs was copied to cache - expect(listFiles(`dist/libs/${mylib1}`)).toEqual([`index.js`]); + const firstRun = runCLI(`test ${parent}`); + expect(firstRun).not.toContain('read the output from the cache'); + + // ----------------------------------------- + // change child2 spec + updateFile(`libs/${child2}/src/lib/${child2}.spec.ts`, (c) => { + return c + '\n// some change'; + }); + const child2RunSpecChange = runCLI(`test ${child2}`); + expect(child2RunSpecChange).not.toContain('read the output from the cache'); + + const parentRunSpecChange = runCLI(`test ${parent}`); + expect(parentRunSpecChange).toContain('read the output from the cache'); + + // ----------------------------------------- + // change child2 prod + updateFile(`libs/${child2}/src/lib/${child2}.ts`, (c) => { + return c + '\n// some change'; + }); + const child2RunProdChange = runCLI(`test ${child2}`); + expect(child2RunProdChange).not.toContain('read the output from the cache'); + + const parentRunProdChange = runCLI(`test ${parent}`); + expect(parentRunProdChange).not.toContain('read the output from the cache'); + + // ----------------------------------------- + // change child1 spec + updateFile(`libs/${child1}/src/lib/${child1}.spec.ts`, (c) => { + return c + '\n// some change'; + }); + + // this is a miss cause child1 redefined "prod" to include all files + const parentRunSpecChangeChild1 = runCLI(`test ${parent}`); + expect(parentRunSpecChangeChild1).not.toContain( + 'read the output from the cache' + ); }, 120000); function expectCached( diff --git a/packages/cypress/executors.json b/packages/cypress/executors.json index 505eaf6892..f04e75ca73 100644 --- a/packages/cypress/executors.json +++ b/packages/cypress/executors.json @@ -10,7 +10,6 @@ "cypress": { "implementation": "./src/executors/cypress/cypress.impl", "schema": "./src/executors/cypress/schema.json", - "hasher": "./src/executors/cypress/hasher", "description": "Run Cypress E2E tests." } } diff --git a/packages/cypress/src/executors/cypress/hasher.ts b/packages/cypress/src/executors/cypress/hasher.ts deleted file mode 100644 index 3b581787b2..0000000000 --- a/packages/cypress/src/executors/cypress/hasher.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { - Hash, - Hasher, - NxJsonConfiguration, - ProjectGraph, - Task, - TaskGraph, - ProjectsConfigurations, -} from '@nrwl/devkit'; - -export default async function run( - task: Task, - context: { - hasher: Hasher; - projectGraph: ProjectGraph; - taskGraph: TaskGraph; - workspaceConfig: ProjectsConfigurations & NxJsonConfiguration; - } -): Promise { - const cypressPluginConfig = context.workspaceConfig.pluginsConfig - ? (context.workspaceConfig.pluginsConfig['@nrwl/cypress'] as any) - : undefined; - const filter = - cypressPluginConfig && cypressPluginConfig.hashingExcludesTestsOfDeps - ? 'exclude-tests-of-deps' - : 'all-files'; - return context.hasher.hashTaskWithDepsAndContext(task, filter); -} diff --git a/packages/jest/executors.json b/packages/jest/executors.json index a5c9da9bd7..6a9afc165b 100644 --- a/packages/jest/executors.json +++ b/packages/jest/executors.json @@ -11,7 +11,6 @@ "implementation": "./src/executors/jest/jest.impl", "batchImplementation": "./src/executors/jest/jest.impl#batchJest", "schema": "./src/executors/jest/schema.json", - "hasher": "./src/executors/jest/hasher", "description": "Run Jest unit tests." } } diff --git a/packages/jest/src/executors/jest/hasher.ts b/packages/jest/src/executors/jest/hasher.ts deleted file mode 100644 index 61728d17a0..0000000000 --- a/packages/jest/src/executors/jest/hasher.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Task, Hash, HasherContext } from '@nrwl/devkit'; - -export default async function run( - task: Task, - context: HasherContext -): Promise { - const jestPluginConfig = context.workspaceConfig.pluginsConfig - ? (context.workspaceConfig.pluginsConfig['@nrwl/jest'] as any) - : undefined; - const filter = - jestPluginConfig && jestPluginConfig.hashingExcludesTestsOfDeps - ? 'exclude-tests-of-deps' - : 'all-files'; - return context.hasher.hashTaskWithDepsAndContext(task, filter); -} diff --git a/packages/linter/src/executors/eslint/hasher.ts b/packages/linter/src/executors/eslint/hasher.ts index 0c05df9163..1283056a59 100644 --- a/packages/linter/src/executors/eslint/hasher.ts +++ b/packages/linter/src/executors/eslint/hasher.ts @@ -20,7 +20,7 @@ export default async function run( return context.hasher.hashTaskWithDepsAndContext(task); } - const command = context.hasher.hashCommand(task); + const command = await context.hasher.hashCommand(task); const source = await context.hasher.hashSource(task); const deps = allDeps(task.id, context.taskGraph, context.projectGraph); const tags = context.hasher.hashArray( diff --git a/packages/nx/schemas/nx-schema.json b/packages/nx/schemas/nx-schema.json index 8d0f909105..acf852130b 100644 --- a/packages/nx/schemas/nx-schema.json +++ b/packages/nx/schemas/nx-schema.json @@ -111,6 +111,11 @@ }, "additionalProperties": false }, + "filesets": { + "type": "object", + "description": "Default filesets", + "additionalProperties": true + }, "targetDependencyConfig": { "type": "array", "items": { @@ -140,6 +145,31 @@ "type": "object", "description": "Target defaults", "properties": { + "dependsOnFilesets": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "projects": { + "type": "string", + "description": "The projects that the targets belong to.", + "enum": ["self", "dependencies"] + }, + "fileset": { + "type": "string", + "description": "The name of the target." + } + }, + "additionalProperties": false + } + ] + } + }, "dependsOn": { "type": "array", "items": { diff --git a/packages/nx/schemas/project-schema.json b/packages/nx/schemas/project-schema.json index abd33c4f9e..89cc249ae3 100644 --- a/packages/nx/schemas/project-schema.json +++ b/packages/nx/schemas/project-schema.json @@ -4,6 +4,11 @@ "title": "JSON schema for Nx projects", "type": "object", "properties": { + "filesets": { + "type": "object", + "description": "Filesets used by Nx to hash relevant files to a given target", + "additionalProperties": true + }, "targets": { "type": "object", "description": "Configures all the targets which define what tasks you can run against the project", @@ -30,6 +35,31 @@ "type": "object" } }, + "dependsOnFilesets": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "projects": { + "type": "string", + "description": "The projects that the targets belong to.", + "enum": ["self", "dependencies"] + }, + "fileset": { + "type": "string", + "description": "The name of the target." + } + }, + "additionalProperties": false + } + ] + } + }, "dependsOn": { "type": "array", "items": { diff --git a/packages/nx/src/config/nx-json.ts b/packages/nx/src/config/nx-json.ts index 31c8041083..eada30323e 100644 --- a/packages/nx/src/config/nx-json.ts +++ b/packages/nx/src/config/nx-json.ts @@ -1,5 +1,8 @@ import { PackageManager } from '../utils/package-manager'; -import { TargetDependencyConfig } from './workspace-json-project-json'; +import { + FilesetDependencyConfig, + TargetDependencyConfig, +} from './workspace-json-project-json'; export type ImplicitDependencyEntry = { [key: string]: T | ImplicitJsonSubsetDependency; @@ -21,6 +24,7 @@ export type TargetDefaults = Record< { outputs?: string[]; dependsOn?: (TargetDependencyConfig | string)[]; + dependsOnFilesets?: (FilesetDependencyConfig | string)[]; } >; @@ -46,6 +50,10 @@ export interface NxJsonConfiguration { * Dependencies between different target names across all projects */ targetDependencies?: TargetDependencies; + /** + * Default filesets used when no project-specific fileset is defined; + */ + filesets?: { [filesetName: string]: string[] }; /** * Dependencies between different target names across all projects */ diff --git a/packages/nx/src/config/project-graph.ts b/packages/nx/src/config/project-graph.ts index af145e3d1c..6b20539138 100644 --- a/packages/nx/src/config/project-graph.ts +++ b/packages/nx/src/config/project-graph.ts @@ -81,6 +81,10 @@ export interface ProjectGraphProjectNode { */ root: string; sourceRoot?: string; + /** + * Filesets associated with the project + */ + filesets?: { [filesetName: string]: string[] }; /** * Targets associated to the project */ diff --git a/packages/nx/src/config/workspace-json-project-json.ts b/packages/nx/src/config/workspace-json-project-json.ts index 6b0e3e2af4..f03f957114 100644 --- a/packages/nx/src/config/workspace-json-project-json.ts +++ b/packages/nx/src/config/workspace-json-project-json.ts @@ -109,6 +109,21 @@ export interface TargetDependencyConfig { target: string; } +export interface FilesetDependencyConfig { + /** + * This the projects that the filesets belong to + * + * 'self': This target depends on a fileset of the same project + * 'deps': This target depends on the deps' filesets. + */ + projects: 'self' | 'dependencies'; + + /** + * The name of the fileset + */ + fileset: string; +} + /** * Target's configuration */ @@ -131,6 +146,11 @@ export interface TargetConfiguration { */ dependsOn?: (TargetDependencyConfig | string)[]; + /** + * This describes filesets that a target depends on. + */ + dependsOnFilesets?: (FilesetDependencyConfig | string)[]; + /** * Target's options. They are passed in to the executor. */ diff --git a/packages/nx/src/generators/utils/project-configuration.ts b/packages/nx/src/generators/utils/project-configuration.ts index 5e7ff23091..5dae62cb32 100644 --- a/packages/nx/src/generators/utils/project-configuration.ts +++ b/packages/nx/src/generators/utils/project-configuration.ts @@ -131,6 +131,7 @@ export function updateWorkspaceConfiguration( plugins, pluginsConfig, npmScope, + filesets, targetDefaults, targetDependencies, workspaceLayout, @@ -144,6 +145,7 @@ export function updateWorkspaceConfiguration( plugins, pluginsConfig, npmScope, + filesets, targetDefaults, targetDependencies, workspaceLayout, diff --git a/packages/nx/src/hasher/hasher.spec.ts b/packages/nx/src/hasher/hasher.spec.ts index a449adc85c..e524a08f78 100644 --- a/packages/nx/src/hasher/hasher.spec.ts +++ b/packages/nx/src/hasher/hasher.spec.ts @@ -3,32 +3,24 @@ import { DependencyType } from '../config/project-graph'; jest.doMock('../utils/workspace-root', () => { return { - workspaceRoot: '', + workspaceRoot: '/root', }; }); -import fs = require('fs'); -import tsUtils = require('../utils/typescript'); -import { Hasher } from './hasher'; - -jest.mock('fs'); +jest.mock('fs', () => require('memfs').fs); +require('fs').existsSync = () => true; jest.mock('../utils/typescript'); -fs.existsSync = () => true; +import { vol } from 'memfs'; +import tsUtils = require('../utils/typescript'); +import { Hasher } from './hasher'; describe('Hasher', () => { const nxJson = { npmScope: 'nrwl', }; - const workSpaceJson = { - projects: { - parent: { root: 'libs/parent' }, - child: { root: 'libs/child' }, - }, - }; - - const tsConfigBaseJsonHash = JSON.stringify({ + const tsConfigBaseJson = JSON.stringify({ compilerOptions: { paths: { '@nrwl/parent': ['libs/parent/src/index.ts'], @@ -37,15 +29,15 @@ describe('Hasher', () => { }, }); let hashes = { - 'yarn.lock': 'yarn.lock.hash', - 'nx.json': 'nx.json.hash', - 'package-lock.json': 'package-lock.json.hash', - 'package.json': 'package.json.hash', - 'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', - 'tsconfig.base.json': tsConfigBaseJsonHash, - 'workspace.json': 'workspace.json.hash', - global1: 'global1.hash', - global2: 'global2.hash', + '/root/yarn.lock': 'yarn.lock.hash', + '/root/nx.json': 'nx.json.hash', + '/root/package-lock.json': 'package-lock.json.hash', + '/root/package.json': 'package.json.hash', + '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash', + '/root/tsconfig.base.json': tsConfigBaseJson, + '/root/workspace.json': 'workspace.json.hash', + '/root/global1': 'global1.hash', + '/root/global2': 'global2.hash', }; function createHashing(): any { @@ -55,24 +47,32 @@ describe('Hasher', () => { }; } - beforeAll(() => { - fs.readFileSync = (file) => { - if (file === 'workspace.json') { - return JSON.stringify(workSpaceJson); - } - if (file === 'nx.json') { - return JSON.stringify(nxJson); - } - if (file === 'tsconfig.base.json') { - return tsConfigBaseJsonHash; - } - return file; - }; - - tsUtils.getRootTsConfigFileName = () => 'tsconfig.base.json'; + /** + * const workSpaceJson = { + * projects: { + * parent: { root: 'libs/parent' }, + * child: { root: 'libs/child' }, + * }, + * }; + */ + beforeEach(() => { + vol.fromJSON( + { + 'nx.json': JSON.stringify(nxJson), + 'tsconfig.base.json': tsConfigBaseJson, + 'yarn.lock': 'content', + }, + '/root' + ); + tsUtils.getRootTsConfigFileName = () => '/root/tsconfig.base.json'; }); - it('should create project hash', async () => { + afterEach(() => { + jest.resetAllMocks(); + vol.reset(); + }); + + it('should create task hash', async () => { const hasher = new Hasher( { nodes: { @@ -80,7 +80,10 @@ describe('Hasher', () => { name: 'parent', type: 'lib', data: { - root: '', + root: 'libs/parent', + targets: { + build: {}, + }, files: [{ file: '/file', ext: '.ts', hash: 'file.hash' }], }, }, @@ -112,14 +115,14 @@ describe('Hasher', () => { expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.nodes).toEqual({ - parent: - '/file|file.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:default': + '/file|file.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', }); - expect(hash.details.implicitDeps).toEqual({ - 'nx.json': '{"npmScope":"nrwl"}', - 'yarn.lock': 'yarn.lock.hash', - 'package-lock.json': 'package-lock.json.hash', - 'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', + expect(hash.details.implicitDeps).toMatchObject({ + '/root/yarn.lock': 'yarn.lock.hash', + '/root/package-lock.json': 'package-lock.json.hash', + '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash', + '/root/nx.json': 'nx.json.hash', }); expect(hash.details.runtime).toEqual({ 'echo runtime123': 'runtime123', @@ -127,7 +130,7 @@ describe('Hasher', () => { }); }); - it('should create project hash with tsconfig.base.json cache', async () => { + it('should hash task where the project has dependencies', async () => { const hasher = new Hasher( { nodes: { @@ -135,8 +138,257 @@ describe('Hasher', () => { name: 'parent', type: 'lib', data: { - root: '', - files: [{ file: '/file.ts', hash: 'file.hash' }], + root: 'libs/parent', + targets: { build: {} }, + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + targets: { build: {} }, + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }); + + // note that the parent hash is based on parent source files only! + expect(hash.details.nodes).toEqual({ + 'child:$fileset:default': + '/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:default': + '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + it('should hash non-default filesets', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + filesets: { + prod: ['!**/*.spec.ts'], + }, + targets: { + build: { + dependsOnFilesets: ['prod', '^prod'], + }, + }, + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + filesets: { + prod: ['!**/*.spec.ts'], + }, + targets: { build: {} }, + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hash.details.nodes).toEqual({ + 'child:$fileset:prod': + '/fileb.ts|b.hash|{"root":"libs/child","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:prod': + '/filea.ts|a.hash|{"root":"libs/parent","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{"dependsOnFilesets":["prod","^prod"]}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + // write another test depending on 2 self and 2 children + + it('should use the default file set when cannot find the configured one', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + filesets: { + prod: ['!**/*.spec.ts'], + }, + targets: { + build: { + dependsOnFilesets: ['prod', '^prod'], + }, + }, + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + targets: { build: {} }, + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hash.details.nodes).toEqual({ + 'child:$fileset:prod': + '/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:prod': + '/filea.ts|a.hash|{"root":"libs/parent","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{"dependsOnFilesets":["prod","^prod"]}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + it('should use defaultFilesets and targetDefaults from nx.json', async () => { + vol.fromJSON( + { + 'nx.json': JSON.stringify({ + filesets: { + prod: ['!**/*.spec.ts'], + }, + targetDefaults: { + build: { + dependsOnFilesets: ['prod', '^prod'], + }, + }, + }), + 'tsconfig.base.json': tsConfigBaseJson, + 'yarn.lock': 'content', + }, + '/root' + ); + + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + targets: { + build: {}, + }, + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + targets: { build: {} }, + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hash.details.nodes).toEqual({ + 'child:$fileset:prod': + '/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:prod': + '/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + it('should be able to include only a part of the base tsconfig', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + targets: { build: {} }, + files: [{ file: '/file', hash: 'file.hash' }], }, }, }, @@ -168,14 +420,14 @@ describe('Hasher', () => { expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.nodes).toEqual({ - parent: - '/file.ts|file.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"]}}}', + 'parent:$fileset:default': + '/file|file.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"]}}}', }); - expect(hash.details.implicitDeps).toEqual({ - 'nx.json': '{"npmScope":"nrwl"}', - 'yarn.lock': 'yarn.lock.hash', - 'package-lock.json': 'package-lock.json.hash', - 'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', + expect(hash.details.implicitDeps).toMatchObject({ + '/root/nx.json': 'nx.json.hash', + '/root/yarn.lock': 'yarn.lock.hash', + '/root/package-lock.json': 'package-lock.json.hash', + '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash', }); expect(hash.details.runtime).toEqual({ 'echo runtime123': 'runtime123', @@ -183,6 +435,78 @@ describe('Hasher', () => { }); }); + it('should hash tasks where the project graph has circular dependencies', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: 'libs/parent', + targets: { build: {} }, + files: [{ file: '/filea.ts', hash: 'a.hash' }], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: 'libs/child', + targets: { build: {} }, + files: [{ file: '/fileb.ts', hash: 'b.hash' }], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + child: [{ source: 'child', target: 'parent', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const tasksHash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }); + + expect(tasksHash.value).toContain('yarn.lock.hash'); //implicits + expect(tasksHash.value).toContain('a.hash'); //project files + expect(tasksHash.value).toContain('b.hash'); //project files + expect(tasksHash.value).toContain('prop-value'); //overrides + expect(tasksHash.value).toContain('parent|build'); //project and target + expect(tasksHash.value).toContain('build'); //target + expect(tasksHash.details.nodes).toEqual({ + 'child:$fileset:default': + '/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:default': + '/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + + const hashb = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'child', target: 'build' }, + id: 'child-build', + overrides: { prop: 'prop-value' }, + }); + + expect(hashb.value).toContain('yarn.lock.hash'); //implicits + expect(hashb.value).toContain('a.hash'); //project files + expect(hashb.value).toContain('b.hash'); //project files + expect(hashb.value).toContain('prop-value'); //overrides + expect(hashb.value).toContain('child|build'); //project and target + expect(hashb.value).toContain('build'); //target + expect(hashb.details.nodes).toEqual({ + 'child:$fileset:default': + '/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'parent:$fileset:default': + '/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + it('should throw an error when failed to execute runtimeCacheInputs', async () => { const hasher = new Hasher( { @@ -191,8 +515,9 @@ describe('Hasher', () => { name: 'parent', type: 'lib', data: { - root: '', - files: [{ file: '/file.ts', hash: 'some-hash' }], + root: 'libs/parent', + targets: { build: {} }, + files: [{ file: '/file', hash: 'some-hash' }], }, }, }, @@ -225,283 +550,6 @@ describe('Hasher', () => { } }); - it('should hash projects with dependencies', async () => { - const hasher = new Hasher( - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: '', - files: [ - { file: '/filea.ts', hash: 'a.hash' }, - { file: '/filea.spec.ts', hash: 'a.spec.hash' }, - ], - }, - }, - child: { - name: 'child', - type: 'lib', - data: { - root: '', - files: [ - { file: '/fileb.ts', hash: 'b.hash' }, - { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, - ], - }, - }, - }, - dependencies: { - parent: [{ source: 'parent', target: 'child', type: 'static' }], - }, - }, - {} as any, - {}, - createHashing() - ); - - const hash = await hasher.hashTaskWithDepsAndContext({ - target: { project: 'parent', target: 'build' }, - id: 'parent-build', - overrides: { prop: 'prop-value' }, - }); - - // note that the parent hash is based on parent source files only! - expect(hash.details.nodes).toEqual({ - child: - '/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - parent: - '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - }); - }); - - it('should hash projects with dependencies (exclude spec files of dependencies)', async () => { - const hasher = new Hasher( - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: '', - files: [ - { file: '/filea.ts', hash: 'a.hash' }, - { file: '/filea.spec.ts', hash: 'a.spec.hash' }, - ], - }, - }, - child: { - name: 'child', - type: 'lib', - data: { - root: '', - files: [ - { file: '/fileb.ts', hash: 'b.hash' }, - { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, - ], - }, - }, - }, - dependencies: { - parent: [{ source: 'parent', target: 'child', type: 'static' }], - }, - }, - {} as any, - {}, - createHashing() - ); - - const hash = await hasher.hashTaskWithDepsAndContext( - { - target: { project: 'parent', target: 'build' }, - id: 'parent-build', - overrides: { prop: 'prop-value' }, - }, - 'exclude-tests-of-deps' - ); - - // note that the parent hash is based on parent source files only! - expect(hash.details.nodes).toEqual({ - child: - '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - parent: - '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - }); - }); - - it('should hash projects with dependencies (exclude spec files of all projects)', async () => { - const hasher = new Hasher( - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: '', - files: [ - { file: '/filea.ts', hash: 'a.hash' }, - { file: '/filea.spec.ts', hash: 'a.spec.hash' }, - ], - }, - }, - child: { - name: 'child', - type: 'lib', - data: { - root: '', - files: [ - { file: '/fileb.ts', hash: 'b.hash' }, - { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, - ], - }, - }, - }, - dependencies: { - parent: [{ source: 'parent', target: 'child', type: 'static' }], - }, - }, - {} as any, - {}, - createHashing() - ); - - const hash = await hasher.hashTaskWithDepsAndContext( - { - target: { project: 'parent', target: 'build' }, - id: 'parent-build', - overrides: { prop: 'prop-value' }, - }, - 'exclude-tests-of-all' - ); - - // note that the parent hash is based on parent source files only! - expect(hash.details.nodes).toEqual({ - child: - '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - parent: - '/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - }); - }); - - it('should hash dependent npm project versions', async () => { - const hasher = new Hasher( - { - nodes: { - app: { - name: 'app', - type: 'app', - data: { - root: '', - files: [{ file: '/filea.ts', hash: 'a.hash' }], - }, - }, - }, - externalNodes: { - 'npm:react': { - name: 'npm:react', - type: 'npm', - data: { - version: '17.0.0', - packageName: 'react', - }, - }, - }, - dependencies: { - 'npm:react': [], - app: [ - { source: 'app', target: 'npm:react', type: DependencyType.static }, - ], - }, - }, - {} as any, - {}, - createHashing() - ); - - const hash = await hasher.hashTaskWithDepsAndContext({ - target: { project: 'app', target: 'build' }, - id: 'app-build', - overrides: { prop: 'prop-value' }, - }); - - // note that the parent hash is based on parent source files only! - expect(hash.details.nodes).toEqual({ - app: '/filea.ts|a.hash|""|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - 'npm:react': '17.0.0', - }); - }); - - it('should hash when circular dependencies', async () => { - const hasher = new Hasher( - { - nodes: { - parent: { - name: 'parent', - type: 'lib', - data: { - root: '', - files: [{ file: '/filea.ts', hash: 'a.hash' }], - }, - }, - child: { - name: 'child', - type: 'lib', - data: { - root: '', - files: [{ file: '/fileb.ts', hash: 'b.hash' }], - }, - }, - }, - dependencies: { - parent: [{ source: 'parent', target: 'child', type: 'static' }], - child: [{ source: 'child', target: 'parent', type: 'static' }], - }, - }, - {} as any, - {}, - createHashing() - ); - - const tasksHash = await hasher.hashTaskWithDepsAndContext({ - target: { project: 'parent', target: 'build' }, - id: 'parent-build', - overrides: { prop: 'prop-value' }, - }); - - expect(tasksHash.value).toContain('yarn.lock.hash'); //implicits - expect(tasksHash.value).toContain('a.hash'); //project files - expect(tasksHash.value).toContain('b.hash'); //project files - expect(tasksHash.value).toContain('prop-value'); //overrides - expect(tasksHash.value).toContain('parent|build'); //project and target - expect(tasksHash.value).toContain('build'); //target - expect(tasksHash.details.nodes).toEqual({ - child: - '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - parent: - '/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - }); - - const hashb = await hasher.hashTaskWithDepsAndContext({ - target: { project: 'child', target: 'build' }, - id: 'child-build', - overrides: { prop: 'prop-value' }, - }); - - expect(hashb.value).toContain('yarn.lock.hash'); //implicits - expect(hashb.value).toContain('a.hash'); //project files - expect(hashb.value).toContain('b.hash'); //project files - expect(hashb.value).toContain('prop-value'); //overrides - expect(hashb.value).toContain('child|build'); //project and target - expect(hashb.value).toContain('build'); //target - expect(hashb.details.nodes).toEqual({ - child: - '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - parent: - '/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', - }); - }); - it('should hash implicit deps', async () => { const hasher = new Hasher( { @@ -510,7 +558,8 @@ describe('Hasher', () => { name: 'parent', type: 'lib', data: { - root: '', + root: 'libs/parents', + targets: { build: {} }, files: [], }, }, @@ -545,6 +594,110 @@ describe('Hasher', () => { expect(tasksHash.value).toContain('global1.hash'); expect(tasksHash.value).toContain('global2.hash'); }); + // ADDRESS + // it('should hash projects with dependencies (exclude spec files of dependencies)', async () => { + // const hasher = new Hasher( + // { + // nodes: { + // parent: { + // name: 'parent', + // type: 'lib', + // data: { + // root: '', + // files: [ + // { file: '/filea.ts', hash: 'a.hash' }, + // { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + // ], + // }, + // }, + // child: { + // name: 'child', + // type: 'lib', + // data: { + // root: '', + // files: [ + // { file: '/fileb.ts', hash: 'b.hash' }, + // { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + // ], + // }, + // }, + // }, + // dependencies: { + // parent: [{ source: 'parent', target: 'child', type: 'static' }], + // }, + // }, + // {} as any, + // {}, + // createHashing() + // ); + // + // const hash = await hasher.hashTaskWithDepsAndContext( + // { + // target: { project: 'parent', target: 'build' }, + // id: 'parent-build', + // overrides: { prop: 'prop-value' }, + // } + // ); + // + // // note that the parent hash is based on parent source files only! + // expect(hash.details.nodes).toEqual({ + // child: + // '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + // parent: + // '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + // }); + + // }); + + it('should hash npm project versions', async () => { + const hasher = new Hasher( + { + nodes: { + app: { + name: 'app', + type: 'app', + data: { + root: 'apps/app', + targets: { build: {} }, + files: [{ file: '/filea.ts', hash: 'a.hash' }], + }, + }, + }, + externalNodes: { + 'npm:react': { + name: 'npm:react', + type: 'npm', + data: { + version: '17.0.0', + packageName: 'react', + }, + }, + }, + dependencies: { + 'npm:react': [], + app: [ + { source: 'app', target: 'npm:react', type: DependencyType.static }, + ], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext({ + target: { project: 'app', target: 'build' }, + id: 'app-build', + overrides: { prop: 'prop-value' }, + }); + + // note that the parent hash is based on parent source files only! + expect(hash.details.nodes).toEqual({ + 'app:$fileset:default': + '/filea.ts|a.hash|{"root":"apps/app","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'npm:react': '17.0.0', + }); + }); it('should hash missing dependent npm project versions', async () => { const hasher = new Hasher( @@ -554,7 +707,8 @@ describe('Hasher', () => { name: 'app', type: 'app', data: { - root: '', + root: 'apps/app', + targets: { build: {} }, files: [{ file: '/filea.ts', hash: 'a.hash' }], }, }, @@ -584,7 +738,8 @@ describe('Hasher', () => { // note that the parent hash is based on parent source files only! expect(hash.details.nodes).toEqual({ - app: '/filea.ts|a.hash|""|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + 'app:$fileset:default': + '/filea.ts|a.hash|{"root":"apps/app","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', 'npm:react': '__npm:react__', }); }); diff --git a/packages/nx/src/hasher/hasher.ts b/packages/nx/src/hasher/hasher.ts index 51698d74aa..1c87cdab88 100644 --- a/packages/nx/src/hasher/hasher.ts +++ b/packages/nx/src/hasher/hasher.ts @@ -1,4 +1,3 @@ -import { resolveNewFormatWithInlineProjects } from '../config/workspaces'; import { exec } from 'child_process'; import { existsSync } from 'fs'; import * as minimatch from 'minimatch'; @@ -6,13 +5,18 @@ import { join } from 'path'; import { performance } from 'perf_hooks'; import { getRootTsConfigFileName } from '../utils/typescript'; import { workspaceRoot } from '../utils/workspace-root'; -import { workspaceFileName } from '../project-graph/file-utils'; import { defaultHashing, HashingImpl } from './hashing-impl'; -import { ProjectGraph } from '../config/project-graph'; +import { + FileData, + ProjectGraph, + ProjectGraphDependency, + ProjectGraphProjectNode, +} from '../config/project-graph'; import { NxJsonConfiguration } from '../config/nx-json'; import { Task } from '../config/task-graph'; import { readJsonFile } from '../utils/fileutils'; -import { ProjectsConfigurations } from '../config/workspace-json-project-json'; +import { FilesetDependencyConfig } from '../config/workspace-json-project-json'; +import { readNxJson } from '../config/configuration'; /** * A data structure returned by the default hasher. @@ -27,8 +31,9 @@ export interface Hash { }; } -interface ProjectHashResult { +interface TaskGraphResult { value: string; + command: string; nodes: { [name: string]: string }; } @@ -54,10 +59,10 @@ interface TsconfigJsonConfiguration { * The default hasher used by executors. */ export class Hasher { - static version = '2.0'; + static version = '3.0'; private implicitDependencies: Promise; private runtimeInputs: Promise; - private projectHashes: ProjectHasher; + private taskHasher: TaskHasher; private hashing: HashingImpl; constructor( @@ -72,45 +77,27 @@ export class Hasher { // this is only used for testing this.hashing = hashing; } - this.projectHashes = new ProjectHasher(this.projectGraph, this.hashing, { + this.taskHasher = new TaskHasher(this.projectGraph, this.hashing, { selectivelyHashTsConfig: this.options.selectivelyHashTsConfig ?? false, }); } - async hashTaskWithDepsAndContext( - task: Task, - filter: - | 'all-files' - | 'exclude-tests-of-all' - | 'exclude-tests-of-deps' = 'all-files' - ): Promise { - const command = this.hashCommand(task); - + async hashTaskWithDepsAndContext(task: Task): Promise { const values = (await Promise.all([ - this.projectHashes.hashProject( - task.target.project, - [task.target.project], - filter - ), + this.taskHasher.hashTask(task, [task.target.project]), this.implicitDepsHash(), this.runtimeInputsHash(), - ])) as [ - ProjectHashResult, - ImplicitHashResult, - RuntimeHashResult - // NodeModulesResult - ]; + ])) as [TaskGraphResult, ImplicitHashResult, RuntimeHashResult]; const value = this.hashing.hashArray([ Hasher.version, - command, ...values.map((v) => v.value), ]); return { value, details: { - command, + command: values[0].command, nodes: values[0].nodes, implicitDeps: values[1].files, runtime: values[2].runtime, @@ -118,22 +105,6 @@ export class Hasher { }; } - hashCommand(task: Task) { - const overrides = { ...task.overrides }; - delete overrides['__overrides_unparsed__']; - const sortedOverrides = {}; - for (let k of Object.keys(overrides).sort()) { - sortedOverrides[k] = overrides[k]; - } - - return this.hashing.hashArray([ - task.target.project ?? '', - task.target.target ?? '', - task.target.configuration ?? '', - JSON.stringify(sortedOverrides), - ]); - } - async hashContext(): Promise<{ implicitDeps: ImplicitHashResult; runtime: RuntimeHashResult; @@ -149,11 +120,13 @@ export class Hasher { }; } + async hashCommand(task: Task): Promise { + return (await this.taskHasher.hashTask(task, [task.target.project])) + .command; + } + async hashSource(task: Task): Promise { - return this.projectHashes.hashProjectNodeSource( - task.target.project, - 'all-files' - ); + return (await this.taskHasher.hashTask(task, [task.target.project])).value; } hashArray(values: string[]): string { @@ -244,6 +217,8 @@ export class Hasher { ...filesWithoutPatterns, ...implicitDepsFromPatterns, + 'nx.json', + //TODO: vsavkin move the special cases into explicit ts support 'package-lock.json', 'yarn.lock', @@ -269,7 +244,6 @@ export class Hasher { const hash = this.hashing.hashFile(file); return { file, hash }; }), - ...this.hashNxJson(), ]; const combinedHash = this.hashing.hashArray( @@ -291,175 +265,258 @@ export class Hasher { return this.implicitDependencies; } - - private hashNxJson() { - const nxJsonPath = join(workspaceRoot, 'nx.json'); - if (!existsSync(nxJsonPath)) { - return []; - } - - let nxJsonContents = '{}'; - try { - const nxJson = readJsonFile(nxJsonPath); - delete nxJson.projects; - nxJsonContents = JSON.stringify(nxJson); - } catch {} - - return [ - { - hash: this.hashing.hashArray([nxJsonContents]), - file: 'nx.json', - }, - ]; - } } -class ProjectHasher { - private sourceHashes: { [projectName: string]: Promise } = {}; - private workspaceJson: ProjectsConfigurations; - private nxJson: NxJsonConfiguration; +class TaskHasher { + private DEFAULT_FILESET_CONFIG = [ + { + projects: 'self', + fileset: 'default', + }, + { + projects: 'dependencies', + fileset: 'default', + }, + ]; + private filesetHashes: { + [taskId: string]: Promise<{ taskId: string; value: string }>; + } = {}; private tsConfigJson: TsconfigJsonConfiguration; + private nxJson: NxJsonConfiguration; constructor( private readonly projectGraph: ProjectGraph, private readonly hashing: HashingImpl, private readonly options: { selectivelyHashTsConfig: boolean } ) { - this.workspaceJson = this.readWorkspaceConfigFile(workspaceFileName()); - this.nxJson = this.readNxJsonConfigFile('nx.json'); this.tsConfigJson = this.readTsConfig(); + this.nxJson = readNxJson(); } - async hashProject( - projectName: string, - visited: string[], - filter: 'all-files' | 'exclude-tests-of-all' | 'exclude-tests-of-deps' - ): Promise { + async hashTask(task: Task, visited: string[]): Promise { return Promise.resolve().then(async () => { - const deps = this.projectGraph.dependencies[projectName] ?? []; - const depHashes = ( - await Promise.all( - deps.map(async (d) => { - if (visited.indexOf(d.target) > -1) { - return null; - } else { - visited.push(d.target); - return await this.hashProject(d.target, visited, filter); - } - }) - ) - ).filter((r) => !!r); - const filterForProject = - filter === 'all-files' - ? 'all-files' - : filter === 'exclude-tests-of-deps' && visited[0] === projectName - ? 'all-files' - : 'exclude-tests'; - const projectHash = await this.hashProjectNodeSource( - projectName, - filterForProject - ); - const nodes = depHashes.reduce( - (m, c) => { - return { ...m, ...c.nodes }; - }, - { [projectName]: projectHash } + const projectNode = this.projectGraph.nodes[task.target.project]; + if (!projectNode) { + return this.hashExternalDependency(task); + } + const projectGraphDeps = + this.projectGraph.dependencies[task.target.project] ?? []; + + const filesetConfigs = this.filesetConfigs(task, projectNode); + const self = await this.hashSelfFilesets(filesetConfigs, projectNode); + const deps = await this.hashDepsTasks( + filesetConfigs, + projectGraphDeps, + visited ); + + const command = this.hashCommand(task); + + const nodes = deps.reduce((m, c) => { + return { ...m, ...c.nodes }; + }, {}); + self.forEach((r) => (nodes[r.taskId] = r.value)); + const value = this.hashing.hashArray([ - ...depHashes.map((d) => d.value), - projectHash, + command, + ...self.map((d) => d.value), + ...deps.map((d) => d.value), ]); - return { value, nodes }; + + return { value, command, nodes }; }); } - async hashProjectNodeSource( - projectName: string, - filter: 'all-files' | 'exclude-tests' + private async hashDepsTasks( + config: FilesetDependencyConfig[], + projectGraphDeps: ProjectGraphDependency[], + visited: string[] ) { - const mapKey = `${projectName}-${filter}`; - if (!this.sourceHashes[mapKey]) { - this.sourceHashes[mapKey] = new Promise(async (res) => { + return ( + await Promise.all( + config + .filter((fileset) => fileset.projects === 'dependencies') + .map(async (fileset) => { + return await Promise.all( + projectGraphDeps.map(async (d) => { + if (visited.indexOf(d.target) > -1) { + return null; + } else { + visited.push(d.target); + return await this.hashTask( + { + id: `${d.target}:$fileset:${fileset.fileset}`, + target: { + project: d.target, + target: '$fileset', + configuration: fileset.fileset, + }, + overrides: {}, + }, + visited + ); + } + }) + ); + }) + ) + ) + .flat() + .filter((r) => !!r); + } + + private async hashSelfFilesets( + config: FilesetDependencyConfig[], + projectNode: ProjectGraphProjectNode + ) { + return await Promise.all( + config + .filter((fileset) => fileset.projects === 'self') + .map((fileset) => + this.hashFilesetSource(projectNode.name, fileset.fileset) + ) + ); + } + + private filesetConfigs( + task: Task, + projectNode: ProjectGraphProjectNode + ): FilesetDependencyConfig[] { + if (task.target.target === '$fileset') { + return [ + { + fileset: task.target.configuration, + projects: 'self', + }, + { + fileset: task.target.configuration, + projects: 'dependencies', + }, + ]; + } else { + const targetData = projectNode.data.targets[task.target.target]; + const targetDefaults = this.nxJson.targetDefaults[task.target.target]; + // task from TaskGraph can be added here + return expandFilesetConfigSyntaxSugar( + targetData.dependsOnFilesets || + targetDefaults?.dependsOnFilesets || + this.DEFAULT_FILESET_CONFIG + ); + } + } + + private hashExternalDependency(task: Task) { + const n = this.projectGraph.externalNodes[task.target.project]; + const version = n?.data?.version; + let hash: string; + if (version) { + hash = this.hashing.hashArray([version]); + } else { + // unknown dependency + // this may occur if a file has a dependency to a npm package + // which is not directly registestered in package.json + // but only indirectly through dependencies of registered + // npm packages + // when it is at a later stage registered in package.json + // the cache project graph will not know this module but + // the new project graph will know it + // The actual checksum added here is of no importance as + // the version is unknown and may only change when some + // other change occurs in package.json and/or package-lock.json + hash = `__${task.target.project}__`; + } + return { + value: hash, + command: '', + nodes: { + [task.target.project]: version || hash, + }, + }; + } + + private hashCommand(task: Task) { + const overrides = { ...task.overrides }; + delete overrides['__overrides_unparsed__']; + const sortedOverrides = {}; + for (let k of Object.keys(overrides).sort()) { + sortedOverrides[k] = overrides[k]; + } + + return this.hashing.hashArray([ + task.target.project ?? '', + task.target.target ?? '', + task.target.configuration ?? '', + JSON.stringify(sortedOverrides), + ]); + } + + private async hashFilesetSource( + projectName: string, + filesetName: string + ): Promise<{ taskId: string; value: string }> { + const mapKey = `${projectName}:$fileset:${filesetName}`; + if (!this.filesetHashes[mapKey]) { + this.filesetHashes[mapKey] = new Promise(async (res) => { const p = this.projectGraph.nodes[projectName]; - if (!p) { - const n = this.projectGraph.externalNodes[projectName]; - const version = n?.data?.version; - let hash: string; - if (version) { - hash = this.hashing.hashArray([version]); - } else { - // unknown dependency - // this may occur if a file has a dependency to a npm package - // which is not directly registestered in package.json - // but only indirectly through dependencies of registered - // npm packages - // when it is at a later stage registered in package.json - // the cache project graph will not know this module but - // the new project graph will know it - // The actual checksum added here is of no importance as - // the version is unknown and may only change when some - // other change occurs in package.json and/or package-lock.json - hash = `__${projectName}__`; - } - res(hash); - return; - } - - const filteredFiles = - filter === 'all-files' - ? p.data.files - : p.data.files.filter((f) => !this.isSpec(f.file)); + const filesetPatterns = this.selectFilesetPatterns(p, filesetName); + const filteredFiles = this.filterFiles(p.data.files, filesetPatterns); const fileNames = filteredFiles.map((f) => f.file); const values = filteredFiles.map((f) => f.hash); - const workspaceJson = JSON.stringify( - this.workspaceJson.projects[projectName] ?? '' - ); - let tsConfig: string; - - if (this.options.selectivelyHashTsConfig) { - tsConfig = this.removeOtherProjectsPathRecords(projectName); - } else { - tsConfig = JSON.stringify(this.tsConfigJson); - } - - res( - this.hashing.hashArray([ + tsConfig = this.hashTsConfig(p); + res({ + taskId: mapKey, + value: this.hashing.hashArray([ ...fileNames, ...values, - workspaceJson, + JSON.stringify({ ...p.data, files: undefined }), tsConfig, - ]) - ); + ]), + }); }); } - return this.sourceHashes[mapKey]; + return this.filesetHashes[mapKey]; } - private isSpec(file: string) { - return ( - file.endsWith('.spec.tsx') || - file.endsWith('.test.tsx') || - file.endsWith('-test.tsx') || - file.endsWith('-spec.tsx') || - file.endsWith('.spec.ts') || - file.endsWith('.test.ts') || - file.endsWith('-test.ts') || - file.endsWith('-spec.ts') || - file.endsWith('.spec.js') || - file.endsWith('.test.js') || - file.endsWith('-test.js') || - file.endsWith('-spec.js') + private selectFilesetPatterns( + p: ProjectGraphProjectNode, + filesetName: string + ) { + if (filesetName == undefined) { + filesetName = 'default'; + } + const projectFilesets = p.data.filesets + ? p.data.filesets[filesetName] + : null; + const defaultFilesets = this.nxJson.filesets + ? this.nxJson.filesets[filesetName] + : null; + if (projectFilesets) return projectFilesets; + if (defaultFilesets) return defaultFilesets; + return null; + } + + private filterFiles(files: FileData[], patterns: string[] | null) { + if (patterns === null) return files; + return files.filter( + (f) => !!patterns.find((pattern) => minimatch(f.file, pattern)) ); } - private removeOtherProjectsPathRecords(projectName: string) { - const { paths, ...compilerOptions } = this.tsConfigJson.compilerOptions; + private hashTsConfig(p: ProjectGraphProjectNode) { + if (this.options.selectivelyHashTsConfig) { + return this.removeOtherProjectsPathRecords(p); + } else { + return JSON.stringify(this.tsConfigJson); + } + } - const rootPath = this.workspaceJson.projects[projectName].root.split('/'); + private removeOtherProjectsPathRecords(p: ProjectGraphProjectNode) { + const { paths, ...compilerOptions } = this.tsConfigJson.compilerOptions; + const rootPath = p.data.root.split('/'); rootPath.shift(); const pathAlias = `@${this.nxJson.npmScope}/${rootPath.join('/')}`; @@ -484,24 +541,20 @@ class ProjectHasher { }; } } - - private readWorkspaceConfigFile(path: string): ProjectsConfigurations { - try { - const res = readJsonFile(path); - res.projects ??= {}; - return resolveNewFormatWithInlineProjects(res); - } catch { - return { projects: {}, version: 2 }; - } - } - - private readNxJsonConfigFile(path: string): NxJsonConfiguration { - try { - const res = readJsonFile(path); - res.projects ??= {}; - return res; - } catch { - return {}; - } - } +} + +function expandFilesetConfigSyntaxSugar( + deps: (FilesetDependencyConfig | string)[] +): FilesetDependencyConfig[] { + return deps.map((d) => { + if (typeof d === 'string') { + if (d.startsWith('^')) { + return { projects: 'dependencies', fileset: d.substring(1) }; + } else { + return { projects: 'self', fileset: d }; + } + } else { + return d; + } + }); }