feat(core): extend the default hasher to support different filesets

This commit is contained in:
Victor Savkin 2022-06-21 08:18:30 -04:00
parent 1d8369b646
commit 6a65101ec2
16 changed files with 904 additions and 622 deletions

View File

@ -230,7 +230,6 @@
"required": ["cypressConfig"], "required": ["cypressConfig"],
"presets": [] "presets": []
}, },
"hasher": "./src/executors/cypress/hasher",
"description": "Run Cypress E2E tests.", "description": "Run Cypress E2E tests.",
"aliases": [], "aliases": [],
"hidden": false, "hidden": false,

View File

@ -316,7 +316,6 @@
"required": ["jestConfig"], "required": ["jestConfig"],
"presets": [] "presets": []
}, },
"hasher": "./src/executors/jest/hasher",
"description": "Run Jest unit tests.", "description": "Run Jest unit tests.",
"aliases": [], "aliases": [],
"hidden": false, "hidden": false,

View File

@ -7,6 +7,7 @@ import {
runCLI, runCLI,
uniq, uniq,
updateFile, updateFile,
updateJson,
updateProjectConfig, updateProjectConfig,
} from '@nrwl/e2e/utils'; } from '@nrwl/e2e/utils';
@ -163,43 +164,69 @@ describe('cache', () => {
updateFile('nx.json', (c) => originalNxJson); updateFile('nx.json', (c) => originalNxJson);
}, 120000); }, 120000);
it('should only cache specific files if build outputs is configured with specific files', async () => { it('should use consider filesets when hashing', async () => {
const mylib1 = uniq('mylib1'); const parent = uniq('parent');
runCLI(`generate @nrwl/react:lib ${mylib1} --buildable`); const child1 = uniq('child1');
const child2 = uniq('child2');
// Update outputs in workspace.json to just be a particular file runCLI(`generate @nrwl/js:lib ${parent}`);
updateProjectConfig(mylib1, (config) => { runCLI(`generate @nrwl/js:lib ${child1}`);
config.targets['build-base'] = { runCLI(`generate @nrwl/js:lib ${child2}`);
...config.targets.build, updateJson(`nx.json`, (c) => {
}; c.filesets = { prod: ['!**/*.spec.ts'] };
config.targets.build = { c.targetDefaults = {
executor: '@nrwl/workspace:run-commands', test: {
outputs: [`dist/libs/${mylib1}/index.js`], dependsOnFilesets: ['default', '^prod'],
options: {
commands: [
{
command: `npx nx run ${mylib1}:build-base`,
},
],
parallel: false,
}, },
}; };
return config; return c;
}); });
// run build with caching updateJson(`libs/${parent}/project.json`, (c) => {
// -------------------------------------------- c.implicitDependencies = [child1, child2];
const outputThatPutsDataIntoCache = runCLI(`run ${mylib1}:build`); return c;
// now the data is in cache });
expect(outputThatPutsDataIntoCache).not.toContain('cache');
rmDist(); updateJson(`libs/${child1}/project.json`, (c) => {
c.filesets = { prod: ['**/*.ts'] };
return c;
});
const outputWithBuildTasksCached = runCLI(`run ${mylib1}:build`); const firstRun = runCLI(`test ${parent}`);
expect(outputWithBuildTasksCached).toContain('cache'); expect(firstRun).not.toContain('read the output from the cache');
expectCached(outputWithBuildTasksCached, [mylib1]);
// Ensure that only the specific file in outputs was copied to cache // -----------------------------------------
expect(listFiles(`dist/libs/${mylib1}`)).toEqual([`index.js`]); // change child2 spec
updateFile(`libs/${child2}/src/lib/${child2}.spec.ts`, (c) => {
return c + '\n// some change';
});
const child2RunSpecChange = runCLI(`test ${child2}`);
expect(child2RunSpecChange).not.toContain('read the output from the cache');
const parentRunSpecChange = runCLI(`test ${parent}`);
expect(parentRunSpecChange).toContain('read the output from the cache');
// -----------------------------------------
// change child2 prod
updateFile(`libs/${child2}/src/lib/${child2}.ts`, (c) => {
return c + '\n// some change';
});
const child2RunProdChange = runCLI(`test ${child2}`);
expect(child2RunProdChange).not.toContain('read the output from the cache');
const parentRunProdChange = runCLI(`test ${parent}`);
expect(parentRunProdChange).not.toContain('read the output from the cache');
// -----------------------------------------
// change child1 spec
updateFile(`libs/${child1}/src/lib/${child1}.spec.ts`, (c) => {
return c + '\n// some change';
});
// this is a miss cause child1 redefined "prod" to include all files
const parentRunSpecChangeChild1 = runCLI(`test ${parent}`);
expect(parentRunSpecChangeChild1).not.toContain(
'read the output from the cache'
);
}, 120000); }, 120000);
function expectCached( function expectCached(

View File

@ -10,7 +10,6 @@
"cypress": { "cypress": {
"implementation": "./src/executors/cypress/cypress.impl", "implementation": "./src/executors/cypress/cypress.impl",
"schema": "./src/executors/cypress/schema.json", "schema": "./src/executors/cypress/schema.json",
"hasher": "./src/executors/cypress/hasher",
"description": "Run Cypress E2E tests." "description": "Run Cypress E2E tests."
} }
} }

View File

@ -1,28 +0,0 @@
import {
Hash,
Hasher,
NxJsonConfiguration,
ProjectGraph,
Task,
TaskGraph,
ProjectsConfigurations,
} from '@nrwl/devkit';
export default async function run(
task: Task,
context: {
hasher: Hasher;
projectGraph: ProjectGraph;
taskGraph: TaskGraph;
workspaceConfig: ProjectsConfigurations & NxJsonConfiguration;
}
): Promise<Hash> {
const cypressPluginConfig = context.workspaceConfig.pluginsConfig
? (context.workspaceConfig.pluginsConfig['@nrwl/cypress'] as any)
: undefined;
const filter =
cypressPluginConfig && cypressPluginConfig.hashingExcludesTestsOfDeps
? 'exclude-tests-of-deps'
: 'all-files';
return context.hasher.hashTaskWithDepsAndContext(task, filter);
}

View File

@ -11,7 +11,6 @@
"implementation": "./src/executors/jest/jest.impl", "implementation": "./src/executors/jest/jest.impl",
"batchImplementation": "./src/executors/jest/jest.impl#batchJest", "batchImplementation": "./src/executors/jest/jest.impl#batchJest",
"schema": "./src/executors/jest/schema.json", "schema": "./src/executors/jest/schema.json",
"hasher": "./src/executors/jest/hasher",
"description": "Run Jest unit tests." "description": "Run Jest unit tests."
} }
} }

View File

@ -1,15 +0,0 @@
import { Task, Hash, HasherContext } from '@nrwl/devkit';
export default async function run(
task: Task,
context: HasherContext
): Promise<Hash> {
const jestPluginConfig = context.workspaceConfig.pluginsConfig
? (context.workspaceConfig.pluginsConfig['@nrwl/jest'] as any)
: undefined;
const filter =
jestPluginConfig && jestPluginConfig.hashingExcludesTestsOfDeps
? 'exclude-tests-of-deps'
: 'all-files';
return context.hasher.hashTaskWithDepsAndContext(task, filter);
}

View File

@ -20,7 +20,7 @@ export default async function run(
return context.hasher.hashTaskWithDepsAndContext(task); return context.hasher.hashTaskWithDepsAndContext(task);
} }
const command = context.hasher.hashCommand(task); const command = await context.hasher.hashCommand(task);
const source = await context.hasher.hashSource(task); const source = await context.hasher.hashSource(task);
const deps = allDeps(task.id, context.taskGraph, context.projectGraph); const deps = allDeps(task.id, context.taskGraph, context.projectGraph);
const tags = context.hasher.hashArray( const tags = context.hasher.hashArray(

View File

@ -111,6 +111,11 @@
}, },
"additionalProperties": false "additionalProperties": false
}, },
"filesets": {
"type": "object",
"description": "Default filesets",
"additionalProperties": true
},
"targetDependencyConfig": { "targetDependencyConfig": {
"type": "array", "type": "array",
"items": { "items": {
@ -140,6 +145,31 @@
"type": "object", "type": "object",
"description": "Target defaults", "description": "Target defaults",
"properties": { "properties": {
"dependsOnFilesets": {
"type": "array",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"projects": {
"type": "string",
"description": "The projects that the targets belong to.",
"enum": ["self", "dependencies"]
},
"fileset": {
"type": "string",
"description": "The name of the target."
}
},
"additionalProperties": false
}
]
}
},
"dependsOn": { "dependsOn": {
"type": "array", "type": "array",
"items": { "items": {

View File

@ -4,6 +4,11 @@
"title": "JSON schema for Nx projects", "title": "JSON schema for Nx projects",
"type": "object", "type": "object",
"properties": { "properties": {
"filesets": {
"type": "object",
"description": "Filesets used by Nx to hash relevant files to a given target",
"additionalProperties": true
},
"targets": { "targets": {
"type": "object", "type": "object",
"description": "Configures all the targets which define what tasks you can run against the project", "description": "Configures all the targets which define what tasks you can run against the project",
@ -30,6 +35,31 @@
"type": "object" "type": "object"
} }
}, },
"dependsOnFilesets": {
"type": "array",
"items": {
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"projects": {
"type": "string",
"description": "The projects that the targets belong to.",
"enum": ["self", "dependencies"]
},
"fileset": {
"type": "string",
"description": "The name of the target."
}
},
"additionalProperties": false
}
]
}
},
"dependsOn": { "dependsOn": {
"type": "array", "type": "array",
"items": { "items": {

View File

@ -1,5 +1,8 @@
import { PackageManager } from '../utils/package-manager'; import { PackageManager } from '../utils/package-manager';
import { TargetDependencyConfig } from './workspace-json-project-json'; import {
FilesetDependencyConfig,
TargetDependencyConfig,
} from './workspace-json-project-json';
export type ImplicitDependencyEntry<T = '*' | string[]> = { export type ImplicitDependencyEntry<T = '*' | string[]> = {
[key: string]: T | ImplicitJsonSubsetDependency<T>; [key: string]: T | ImplicitJsonSubsetDependency<T>;
@ -21,6 +24,7 @@ export type TargetDefaults = Record<
{ {
outputs?: string[]; outputs?: string[];
dependsOn?: (TargetDependencyConfig | string)[]; dependsOn?: (TargetDependencyConfig | string)[];
dependsOnFilesets?: (FilesetDependencyConfig | string)[];
} }
>; >;
@ -46,6 +50,10 @@ export interface NxJsonConfiguration<T = '*' | string[]> {
* Dependencies between different target names across all projects * Dependencies between different target names across all projects
*/ */
targetDependencies?: TargetDependencies; targetDependencies?: TargetDependencies;
/**
* Default filesets used when no project-specific fileset is defined;
*/
filesets?: { [filesetName: string]: string[] };
/** /**
* Dependencies between different target names across all projects * Dependencies between different target names across all projects
*/ */

View File

@ -81,6 +81,10 @@ export interface ProjectGraphProjectNode<T = any> {
*/ */
root: string; root: string;
sourceRoot?: string; sourceRoot?: string;
/**
* Filesets associated with the project
*/
filesets?: { [filesetName: string]: string[] };
/** /**
* Targets associated to the project * Targets associated to the project
*/ */

View File

@ -109,6 +109,21 @@ export interface TargetDependencyConfig {
target: string; target: string;
} }
export interface FilesetDependencyConfig {
/**
* This the projects that the filesets belong to
*
* 'self': This target depends on a fileset of the same project
* 'deps': This target depends on the deps' filesets.
*/
projects: 'self' | 'dependencies';
/**
* The name of the fileset
*/
fileset: string;
}
/** /**
* Target's configuration * Target's configuration
*/ */
@ -131,6 +146,11 @@ export interface TargetConfiguration<T = any> {
*/ */
dependsOn?: (TargetDependencyConfig | string)[]; dependsOn?: (TargetDependencyConfig | string)[];
/**
* This describes filesets that a target depends on.
*/
dependsOnFilesets?: (FilesetDependencyConfig | string)[];
/** /**
* Target's options. They are passed in to the executor. * Target's options. They are passed in to the executor.
*/ */

View File

@ -131,6 +131,7 @@ export function updateWorkspaceConfiguration(
plugins, plugins,
pluginsConfig, pluginsConfig,
npmScope, npmScope,
filesets,
targetDefaults, targetDefaults,
targetDependencies, targetDependencies,
workspaceLayout, workspaceLayout,
@ -144,6 +145,7 @@ export function updateWorkspaceConfiguration(
plugins, plugins,
pluginsConfig, pluginsConfig,
npmScope, npmScope,
filesets,
targetDefaults, targetDefaults,
targetDependencies, targetDependencies,
workspaceLayout, workspaceLayout,

View File

@ -3,32 +3,24 @@ import { DependencyType } from '../config/project-graph';
jest.doMock('../utils/workspace-root', () => { jest.doMock('../utils/workspace-root', () => {
return { return {
workspaceRoot: '', workspaceRoot: '/root',
}; };
}); });
import fs = require('fs'); jest.mock('fs', () => require('memfs').fs);
import tsUtils = require('../utils/typescript'); require('fs').existsSync = () => true;
import { Hasher } from './hasher';
jest.mock('fs');
jest.mock('../utils/typescript'); jest.mock('../utils/typescript');
fs.existsSync = () => true; import { vol } from 'memfs';
import tsUtils = require('../utils/typescript');
import { Hasher } from './hasher';
describe('Hasher', () => { describe('Hasher', () => {
const nxJson = { const nxJson = {
npmScope: 'nrwl', npmScope: 'nrwl',
}; };
const workSpaceJson = { const tsConfigBaseJson = JSON.stringify({
projects: {
parent: { root: 'libs/parent' },
child: { root: 'libs/child' },
},
};
const tsConfigBaseJsonHash = JSON.stringify({
compilerOptions: { compilerOptions: {
paths: { paths: {
'@nrwl/parent': ['libs/parent/src/index.ts'], '@nrwl/parent': ['libs/parent/src/index.ts'],
@ -37,15 +29,15 @@ describe('Hasher', () => {
}, },
}); });
let hashes = { let hashes = {
'yarn.lock': 'yarn.lock.hash', '/root/yarn.lock': 'yarn.lock.hash',
'nx.json': 'nx.json.hash', '/root/nx.json': 'nx.json.hash',
'package-lock.json': 'package-lock.json.hash', '/root/package-lock.json': 'package-lock.json.hash',
'package.json': 'package.json.hash', '/root/package.json': 'package.json.hash',
'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash',
'tsconfig.base.json': tsConfigBaseJsonHash, '/root/tsconfig.base.json': tsConfigBaseJson,
'workspace.json': 'workspace.json.hash', '/root/workspace.json': 'workspace.json.hash',
global1: 'global1.hash', '/root/global1': 'global1.hash',
global2: 'global2.hash', '/root/global2': 'global2.hash',
}; };
function createHashing(): any { function createHashing(): any {
@ -55,24 +47,32 @@ describe('Hasher', () => {
}; };
} }
beforeAll(() => { /**
fs.readFileSync = (file) => { * const workSpaceJson = {
if (file === 'workspace.json') { * projects: {
return JSON.stringify(workSpaceJson); * parent: { root: 'libs/parent' },
} * child: { root: 'libs/child' },
if (file === 'nx.json') { * },
return JSON.stringify(nxJson); * };
} */
if (file === 'tsconfig.base.json') { beforeEach(() => {
return tsConfigBaseJsonHash; vol.fromJSON(
} {
return file; 'nx.json': JSON.stringify(nxJson),
}; 'tsconfig.base.json': tsConfigBaseJson,
'yarn.lock': 'content',
tsUtils.getRootTsConfigFileName = () => 'tsconfig.base.json'; },
'/root'
);
tsUtils.getRootTsConfigFileName = () => '/root/tsconfig.base.json';
}); });
it('should create project hash', async () => { afterEach(() => {
jest.resetAllMocks();
vol.reset();
});
it('should create task hash', async () => {
const hasher = new Hasher( const hasher = new Hasher(
{ {
nodes: { nodes: {
@ -80,7 +80,10 @@ describe('Hasher', () => {
name: 'parent', name: 'parent',
type: 'lib', type: 'lib',
data: { data: {
root: '', root: 'libs/parent',
targets: {
build: {},
},
files: [{ file: '/file', ext: '.ts', hash: 'file.hash' }], files: [{ file: '/file', ext: '.ts', hash: 'file.hash' }],
}, },
}, },
@ -112,14 +115,14 @@ describe('Hasher', () => {
expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}');
expect(hash.details.nodes).toEqual({ expect(hash.details.nodes).toEqual({
parent: 'parent:$fileset:default':
'/file|file.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', '/file|file.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
}); });
expect(hash.details.implicitDeps).toEqual({ expect(hash.details.implicitDeps).toMatchObject({
'nx.json': '{"npmScope":"nrwl"}', '/root/yarn.lock': 'yarn.lock.hash',
'yarn.lock': 'yarn.lock.hash', '/root/package-lock.json': 'package-lock.json.hash',
'package-lock.json': 'package-lock.json.hash', '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash',
'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', '/root/nx.json': 'nx.json.hash',
}); });
expect(hash.details.runtime).toEqual({ expect(hash.details.runtime).toEqual({
'echo runtime123': 'runtime123', 'echo runtime123': 'runtime123',
@ -127,7 +130,7 @@ describe('Hasher', () => {
}); });
}); });
it('should create project hash with tsconfig.base.json cache', async () => { it('should hash task where the project has dependencies', async () => {
const hasher = new Hasher( const hasher = new Hasher(
{ {
nodes: { nodes: {
@ -135,8 +138,257 @@ describe('Hasher', () => {
name: 'parent', name: 'parent',
type: 'lib', type: 'lib',
data: { data: {
root: '', root: 'libs/parent',
files: [{ file: '/file.ts', hash: 'file.hash' }], targets: { build: {} },
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: 'libs/child',
targets: { build: {} },
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
'child:$fileset:default':
'/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:default':
'/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash non-default filesets', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: 'libs/parent',
filesets: {
prod: ['!**/*.spec.ts'],
},
targets: {
build: {
dependsOnFilesets: ['prod', '^prod'],
},
},
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: 'libs/child',
filesets: {
prod: ['!**/*.spec.ts'],
},
targets: { build: {} },
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
expect(hash.details.nodes).toEqual({
'child:$fileset:prod':
'/fileb.ts|b.hash|{"root":"libs/child","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:prod':
'/filea.ts|a.hash|{"root":"libs/parent","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{"dependsOnFilesets":["prod","^prod"]}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
// write another test depending on 2 self and 2 children
it('should use the default file set when cannot find the configured one', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: 'libs/parent',
filesets: {
prod: ['!**/*.spec.ts'],
},
targets: {
build: {
dependsOnFilesets: ['prod', '^prod'],
},
},
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: 'libs/child',
targets: { build: {} },
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
expect(hash.details.nodes).toEqual({
'child:$fileset:prod':
'/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:prod':
'/filea.ts|a.hash|{"root":"libs/parent","filesets":{"prod":["!**/*.spec.ts"]},"targets":{"build":{"dependsOnFilesets":["prod","^prod"]}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should use defaultFilesets and targetDefaults from nx.json', async () => {
vol.fromJSON(
{
'nx.json': JSON.stringify({
filesets: {
prod: ['!**/*.spec.ts'],
},
targetDefaults: {
build: {
dependsOnFilesets: ['prod', '^prod'],
},
},
}),
'tsconfig.base.json': tsConfigBaseJson,
'yarn.lock': 'content',
},
'/root'
);
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: 'libs/parent',
targets: {
build: {},
},
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: 'libs/child',
targets: { build: {} },
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
expect(hash.details.nodes).toEqual({
'child:$fileset:prod':
'/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:prod':
'/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should be able to include only a part of the base tsconfig', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: 'libs/parent',
targets: { build: {} },
files: [{ file: '/file', hash: 'file.hash' }],
}, },
}, },
}, },
@ -168,14 +420,14 @@ describe('Hasher', () => {
expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}'); expect(hash.details.command).toEqual('parent|build||{"prop":"prop-value"}');
expect(hash.details.nodes).toEqual({ expect(hash.details.nodes).toEqual({
parent: 'parent:$fileset:default':
'/file.ts|file.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"]}}}', '/file|file.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"]}}}',
}); });
expect(hash.details.implicitDeps).toEqual({ expect(hash.details.implicitDeps).toMatchObject({
'nx.json': '{"npmScope":"nrwl"}', '/root/nx.json': 'nx.json.hash',
'yarn.lock': 'yarn.lock.hash', '/root/yarn.lock': 'yarn.lock.hash',
'package-lock.json': 'package-lock.json.hash', '/root/package-lock.json': 'package-lock.json.hash',
'pnpm-lock.yaml': 'pnpm-lock.yaml.hash', '/root/pnpm-lock.yaml': 'pnpm-lock.yaml.hash',
}); });
expect(hash.details.runtime).toEqual({ expect(hash.details.runtime).toEqual({
'echo runtime123': 'runtime123', 'echo runtime123': 'runtime123',
@ -183,6 +435,78 @@ describe('Hasher', () => {
}); });
}); });
it('should hash tasks where the project graph has circular dependencies', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: 'libs/parent',
targets: { build: {} },
files: [{ file: '/filea.ts', hash: 'a.hash' }],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: 'libs/child',
targets: { build: {} },
files: [{ file: '/fileb.ts', hash: 'b.hash' }],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
child: [{ source: 'child', target: 'parent', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const tasksHash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
expect(tasksHash.value).toContain('yarn.lock.hash'); //implicits
expect(tasksHash.value).toContain('a.hash'); //project files
expect(tasksHash.value).toContain('b.hash'); //project files
expect(tasksHash.value).toContain('prop-value'); //overrides
expect(tasksHash.value).toContain('parent|build'); //project and target
expect(tasksHash.value).toContain('build'); //target
expect(tasksHash.details.nodes).toEqual({
'child:$fileset:default':
'/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:default':
'/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
const hashb = await hasher.hashTaskWithDepsAndContext({
target: { project: 'child', target: 'build' },
id: 'child-build',
overrides: { prop: 'prop-value' },
});
expect(hashb.value).toContain('yarn.lock.hash'); //implicits
expect(hashb.value).toContain('a.hash'); //project files
expect(hashb.value).toContain('b.hash'); //project files
expect(hashb.value).toContain('prop-value'); //overrides
expect(hashb.value).toContain('child|build'); //project and target
expect(hashb.value).toContain('build'); //target
expect(hashb.details.nodes).toEqual({
'child:$fileset:default':
'/fileb.ts|b.hash|{"root":"libs/child","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'parent:$fileset:default':
'/filea.ts|a.hash|{"root":"libs/parent","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should throw an error when failed to execute runtimeCacheInputs', async () => { it('should throw an error when failed to execute runtimeCacheInputs', async () => {
const hasher = new Hasher( const hasher = new Hasher(
{ {
@ -191,8 +515,9 @@ describe('Hasher', () => {
name: 'parent', name: 'parent',
type: 'lib', type: 'lib',
data: { data: {
root: '', root: 'libs/parent',
files: [{ file: '/file.ts', hash: 'some-hash' }], targets: { build: {} },
files: [{ file: '/file', hash: 'some-hash' }],
}, },
}, },
}, },
@ -225,283 +550,6 @@ describe('Hasher', () => {
} }
}); });
it('should hash projects with dependencies', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
'/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash projects with dependencies (exclude spec files of dependencies)', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext(
{
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
},
'exclude-tests-of-deps'
);
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
'/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash projects with dependencies (exclude spec files of all projects)', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [
{ file: '/filea.ts', hash: 'a.hash' },
{ file: '/filea.spec.ts', hash: 'a.spec.hash' },
],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [
{ file: '/fileb.ts', hash: 'b.hash' },
{ file: '/fileb.spec.ts', hash: 'b.spec.hash' },
],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext(
{
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
},
'exclude-tests-of-all'
);
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
child:
'/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash dependent npm project versions', async () => {
const hasher = new Hasher(
{
nodes: {
app: {
name: 'app',
type: 'app',
data: {
root: '',
files: [{ file: '/filea.ts', hash: 'a.hash' }],
},
},
},
externalNodes: {
'npm:react': {
name: 'npm:react',
type: 'npm',
data: {
version: '17.0.0',
packageName: 'react',
},
},
},
dependencies: {
'npm:react': [],
app: [
{ source: 'app', target: 'npm:react', type: DependencyType.static },
],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'app', target: 'build' },
id: 'app-build',
overrides: { prop: 'prop-value' },
});
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
app: '/filea.ts|a.hash|""|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'npm:react': '17.0.0',
});
});
it('should hash when circular dependencies', async () => {
const hasher = new Hasher(
{
nodes: {
parent: {
name: 'parent',
type: 'lib',
data: {
root: '',
files: [{ file: '/filea.ts', hash: 'a.hash' }],
},
},
child: {
name: 'child',
type: 'lib',
data: {
root: '',
files: [{ file: '/fileb.ts', hash: 'b.hash' }],
},
},
},
dependencies: {
parent: [{ source: 'parent', target: 'child', type: 'static' }],
child: [{ source: 'child', target: 'parent', type: 'static' }],
},
},
{} as any,
{},
createHashing()
);
const tasksHash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'parent', target: 'build' },
id: 'parent-build',
overrides: { prop: 'prop-value' },
});
expect(tasksHash.value).toContain('yarn.lock.hash'); //implicits
expect(tasksHash.value).toContain('a.hash'); //project files
expect(tasksHash.value).toContain('b.hash'); //project files
expect(tasksHash.value).toContain('prop-value'); //overrides
expect(tasksHash.value).toContain('parent|build'); //project and target
expect(tasksHash.value).toContain('build'); //target
expect(tasksHash.details.nodes).toEqual({
child:
'/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
const hashb = await hasher.hashTaskWithDepsAndContext({
target: { project: 'child', target: 'build' },
id: 'child-build',
overrides: { prop: 'prop-value' },
});
expect(hashb.value).toContain('yarn.lock.hash'); //implicits
expect(hashb.value).toContain('a.hash'); //project files
expect(hashb.value).toContain('b.hash'); //project files
expect(hashb.value).toContain('prop-value'); //overrides
expect(hashb.value).toContain('child|build'); //project and target
expect(hashb.value).toContain('build'); //target
expect(hashb.details.nodes).toEqual({
child:
'/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
parent:
'/filea.ts|a.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
});
});
it('should hash implicit deps', async () => { it('should hash implicit deps', async () => {
const hasher = new Hasher( const hasher = new Hasher(
{ {
@ -510,7 +558,8 @@ describe('Hasher', () => {
name: 'parent', name: 'parent',
type: 'lib', type: 'lib',
data: { data: {
root: '', root: 'libs/parents',
targets: { build: {} },
files: [], files: [],
}, },
}, },
@ -545,6 +594,110 @@ describe('Hasher', () => {
expect(tasksHash.value).toContain('global1.hash'); expect(tasksHash.value).toContain('global1.hash');
expect(tasksHash.value).toContain('global2.hash'); expect(tasksHash.value).toContain('global2.hash');
}); });
// ADDRESS
// it('should hash projects with dependencies (exclude spec files of dependencies)', async () => {
// const hasher = new Hasher(
// {
// nodes: {
// parent: {
// name: 'parent',
// type: 'lib',
// data: {
// root: '',
// files: [
// { file: '/filea.ts', hash: 'a.hash' },
// { file: '/filea.spec.ts', hash: 'a.spec.hash' },
// ],
// },
// },
// child: {
// name: 'child',
// type: 'lib',
// data: {
// root: '',
// files: [
// { file: '/fileb.ts', hash: 'b.hash' },
// { file: '/fileb.spec.ts', hash: 'b.spec.hash' },
// ],
// },
// },
// },
// dependencies: {
// parent: [{ source: 'parent', target: 'child', type: 'static' }],
// },
// },
// {} as any,
// {},
// createHashing()
// );
//
// const hash = await hasher.hashTaskWithDepsAndContext(
// {
// target: { project: 'parent', target: 'build' },
// id: 'parent-build',
// overrides: { prop: 'prop-value' },
// }
// );
//
// // note that the parent hash is based on parent source files only!
// expect(hash.details.nodes).toEqual({
// child:
// '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
// parent:
// '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
// });
// });
it('should hash npm project versions', async () => {
const hasher = new Hasher(
{
nodes: {
app: {
name: 'app',
type: 'app',
data: {
root: 'apps/app',
targets: { build: {} },
files: [{ file: '/filea.ts', hash: 'a.hash' }],
},
},
},
externalNodes: {
'npm:react': {
name: 'npm:react',
type: 'npm',
data: {
version: '17.0.0',
packageName: 'react',
},
},
},
dependencies: {
'npm:react': [],
app: [
{ source: 'app', target: 'npm:react', type: DependencyType.static },
],
},
},
{} as any,
{},
createHashing()
);
const hash = await hasher.hashTaskWithDepsAndContext({
target: { project: 'app', target: 'build' },
id: 'app-build',
overrides: { prop: 'prop-value' },
});
// note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({
'app:$fileset:default':
'/filea.ts|a.hash|{"root":"apps/app","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'npm:react': '17.0.0',
});
});
it('should hash missing dependent npm project versions', async () => { it('should hash missing dependent npm project versions', async () => {
const hasher = new Hasher( const hasher = new Hasher(
@ -554,7 +707,8 @@ describe('Hasher', () => {
name: 'app', name: 'app',
type: 'app', type: 'app',
data: { data: {
root: '', root: 'apps/app',
targets: { build: {} },
files: [{ file: '/filea.ts', hash: 'a.hash' }], files: [{ file: '/filea.ts', hash: 'a.hash' }],
}, },
}, },
@ -584,7 +738,8 @@ describe('Hasher', () => {
// note that the parent hash is based on parent source files only! // note that the parent hash is based on parent source files only!
expect(hash.details.nodes).toEqual({ expect(hash.details.nodes).toEqual({
app: '/filea.ts|a.hash|""|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', 'app:$fileset:default':
'/filea.ts|a.hash|{"root":"apps/app","targets":{"build":{}}}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}',
'npm:react': '__npm:react__', 'npm:react': '__npm:react__',
}); });
}); });

View File

@ -1,4 +1,3 @@
import { resolveNewFormatWithInlineProjects } from '../config/workspaces';
import { exec } from 'child_process'; import { exec } from 'child_process';
import { existsSync } from 'fs'; import { existsSync } from 'fs';
import * as minimatch from 'minimatch'; import * as minimatch from 'minimatch';
@ -6,13 +5,18 @@ import { join } from 'path';
import { performance } from 'perf_hooks'; import { performance } from 'perf_hooks';
import { getRootTsConfigFileName } from '../utils/typescript'; import { getRootTsConfigFileName } from '../utils/typescript';
import { workspaceRoot } from '../utils/workspace-root'; import { workspaceRoot } from '../utils/workspace-root';
import { workspaceFileName } from '../project-graph/file-utils';
import { defaultHashing, HashingImpl } from './hashing-impl'; import { defaultHashing, HashingImpl } from './hashing-impl';
import { ProjectGraph } from '../config/project-graph'; import {
FileData,
ProjectGraph,
ProjectGraphDependency,
ProjectGraphProjectNode,
} from '../config/project-graph';
import { NxJsonConfiguration } from '../config/nx-json'; import { NxJsonConfiguration } from '../config/nx-json';
import { Task } from '../config/task-graph'; import { Task } from '../config/task-graph';
import { readJsonFile } from '../utils/fileutils'; import { readJsonFile } from '../utils/fileutils';
import { ProjectsConfigurations } from '../config/workspace-json-project-json'; import { FilesetDependencyConfig } from '../config/workspace-json-project-json';
import { readNxJson } from '../config/configuration';
/** /**
* A data structure returned by the default hasher. * A data structure returned by the default hasher.
@ -27,8 +31,9 @@ export interface Hash {
}; };
} }
interface ProjectHashResult { interface TaskGraphResult {
value: string; value: string;
command: string;
nodes: { [name: string]: string }; nodes: { [name: string]: string };
} }
@ -54,10 +59,10 @@ interface TsconfigJsonConfiguration {
* The default hasher used by executors. * The default hasher used by executors.
*/ */
export class Hasher { export class Hasher {
static version = '2.0'; static version = '3.0';
private implicitDependencies: Promise<ImplicitHashResult>; private implicitDependencies: Promise<ImplicitHashResult>;
private runtimeInputs: Promise<RuntimeHashResult>; private runtimeInputs: Promise<RuntimeHashResult>;
private projectHashes: ProjectHasher; private taskHasher: TaskHasher;
private hashing: HashingImpl; private hashing: HashingImpl;
constructor( constructor(
@ -72,45 +77,27 @@ export class Hasher {
// this is only used for testing // this is only used for testing
this.hashing = hashing; this.hashing = hashing;
} }
this.projectHashes = new ProjectHasher(this.projectGraph, this.hashing, { this.taskHasher = new TaskHasher(this.projectGraph, this.hashing, {
selectivelyHashTsConfig: this.options.selectivelyHashTsConfig ?? false, selectivelyHashTsConfig: this.options.selectivelyHashTsConfig ?? false,
}); });
} }
async hashTaskWithDepsAndContext( async hashTaskWithDepsAndContext(task: Task): Promise<Hash> {
task: Task,
filter:
| 'all-files'
| 'exclude-tests-of-all'
| 'exclude-tests-of-deps' = 'all-files'
): Promise<Hash> {
const command = this.hashCommand(task);
const values = (await Promise.all([ const values = (await Promise.all([
this.projectHashes.hashProject( this.taskHasher.hashTask(task, [task.target.project]),
task.target.project,
[task.target.project],
filter
),
this.implicitDepsHash(), this.implicitDepsHash(),
this.runtimeInputsHash(), this.runtimeInputsHash(),
])) as [ ])) as [TaskGraphResult, ImplicitHashResult, RuntimeHashResult];
ProjectHashResult,
ImplicitHashResult,
RuntimeHashResult
// NodeModulesResult
];
const value = this.hashing.hashArray([ const value = this.hashing.hashArray([
Hasher.version, Hasher.version,
command,
...values.map((v) => v.value), ...values.map((v) => v.value),
]); ]);
return { return {
value, value,
details: { details: {
command, command: values[0].command,
nodes: values[0].nodes, nodes: values[0].nodes,
implicitDeps: values[1].files, implicitDeps: values[1].files,
runtime: values[2].runtime, runtime: values[2].runtime,
@ -118,22 +105,6 @@ export class Hasher {
}; };
} }
hashCommand(task: Task) {
const overrides = { ...task.overrides };
delete overrides['__overrides_unparsed__'];
const sortedOverrides = {};
for (let k of Object.keys(overrides).sort()) {
sortedOverrides[k] = overrides[k];
}
return this.hashing.hashArray([
task.target.project ?? '',
task.target.target ?? '',
task.target.configuration ?? '',
JSON.stringify(sortedOverrides),
]);
}
async hashContext(): Promise<{ async hashContext(): Promise<{
implicitDeps: ImplicitHashResult; implicitDeps: ImplicitHashResult;
runtime: RuntimeHashResult; runtime: RuntimeHashResult;
@ -149,11 +120,13 @@ export class Hasher {
}; };
} }
async hashCommand(task: Task): Promise<string> {
return (await this.taskHasher.hashTask(task, [task.target.project]))
.command;
}
async hashSource(task: Task): Promise<string> { async hashSource(task: Task): Promise<string> {
return this.projectHashes.hashProjectNodeSource( return (await this.taskHasher.hashTask(task, [task.target.project])).value;
task.target.project,
'all-files'
);
} }
hashArray(values: string[]): string { hashArray(values: string[]): string {
@ -244,6 +217,8 @@ export class Hasher {
...filesWithoutPatterns, ...filesWithoutPatterns,
...implicitDepsFromPatterns, ...implicitDepsFromPatterns,
'nx.json',
//TODO: vsavkin move the special cases into explicit ts support //TODO: vsavkin move the special cases into explicit ts support
'package-lock.json', 'package-lock.json',
'yarn.lock', 'yarn.lock',
@ -269,7 +244,6 @@ export class Hasher {
const hash = this.hashing.hashFile(file); const hash = this.hashing.hashFile(file);
return { file, hash }; return { file, hash };
}), }),
...this.hashNxJson(),
]; ];
const combinedHash = this.hashing.hashArray( const combinedHash = this.hashing.hashArray(
@ -291,175 +265,258 @@ export class Hasher {
return this.implicitDependencies; return this.implicitDependencies;
} }
private hashNxJson() {
const nxJsonPath = join(workspaceRoot, 'nx.json');
if (!existsSync(nxJsonPath)) {
return [];
}
let nxJsonContents = '{}';
try {
const nxJson = readJsonFile(nxJsonPath);
delete nxJson.projects;
nxJsonContents = JSON.stringify(nxJson);
} catch {}
return [
{
hash: this.hashing.hashArray([nxJsonContents]),
file: 'nx.json',
},
];
}
} }
class ProjectHasher { class TaskHasher {
private sourceHashes: { [projectName: string]: Promise<string> } = {}; private DEFAULT_FILESET_CONFIG = [
private workspaceJson: ProjectsConfigurations; {
private nxJson: NxJsonConfiguration; projects: 'self',
fileset: 'default',
},
{
projects: 'dependencies',
fileset: 'default',
},
];
private filesetHashes: {
[taskId: string]: Promise<{ taskId: string; value: string }>;
} = {};
private tsConfigJson: TsconfigJsonConfiguration; private tsConfigJson: TsconfigJsonConfiguration;
private nxJson: NxJsonConfiguration;
constructor( constructor(
private readonly projectGraph: ProjectGraph, private readonly projectGraph: ProjectGraph,
private readonly hashing: HashingImpl, private readonly hashing: HashingImpl,
private readonly options: { selectivelyHashTsConfig: boolean } private readonly options: { selectivelyHashTsConfig: boolean }
) { ) {
this.workspaceJson = this.readWorkspaceConfigFile(workspaceFileName());
this.nxJson = this.readNxJsonConfigFile('nx.json');
this.tsConfigJson = this.readTsConfig(); this.tsConfigJson = this.readTsConfig();
this.nxJson = readNxJson();
} }
async hashProject( async hashTask(task: Task, visited: string[]): Promise<TaskGraphResult> {
projectName: string,
visited: string[],
filter: 'all-files' | 'exclude-tests-of-all' | 'exclude-tests-of-deps'
): Promise<ProjectHashResult> {
return Promise.resolve().then(async () => { return Promise.resolve().then(async () => {
const deps = this.projectGraph.dependencies[projectName] ?? []; const projectNode = this.projectGraph.nodes[task.target.project];
const depHashes = ( if (!projectNode) {
await Promise.all( return this.hashExternalDependency(task);
deps.map(async (d) => { }
if (visited.indexOf(d.target) > -1) { const projectGraphDeps =
return null; this.projectGraph.dependencies[task.target.project] ?? [];
} else {
visited.push(d.target); const filesetConfigs = this.filesetConfigs(task, projectNode);
return await this.hashProject(d.target, visited, filter); const self = await this.hashSelfFilesets(filesetConfigs, projectNode);
} const deps = await this.hashDepsTasks(
}) filesetConfigs,
) projectGraphDeps,
).filter((r) => !!r); visited
const filterForProject =
filter === 'all-files'
? 'all-files'
: filter === 'exclude-tests-of-deps' && visited[0] === projectName
? 'all-files'
: 'exclude-tests';
const projectHash = await this.hashProjectNodeSource(
projectName,
filterForProject
);
const nodes = depHashes.reduce(
(m, c) => {
return { ...m, ...c.nodes };
},
{ [projectName]: projectHash }
); );
const command = this.hashCommand(task);
const nodes = deps.reduce((m, c) => {
return { ...m, ...c.nodes };
}, {});
self.forEach((r) => (nodes[r.taskId] = r.value));
const value = this.hashing.hashArray([ const value = this.hashing.hashArray([
...depHashes.map((d) => d.value), command,
projectHash, ...self.map((d) => d.value),
...deps.map((d) => d.value),
]); ]);
return { value, nodes };
return { value, command, nodes };
}); });
} }
async hashProjectNodeSource( private async hashDepsTasks(
projectName: string, config: FilesetDependencyConfig[],
filter: 'all-files' | 'exclude-tests' projectGraphDeps: ProjectGraphDependency[],
visited: string[]
) { ) {
const mapKey = `${projectName}-${filter}`; return (
if (!this.sourceHashes[mapKey]) { await Promise.all(
this.sourceHashes[mapKey] = new Promise(async (res) => { config
.filter((fileset) => fileset.projects === 'dependencies')
.map(async (fileset) => {
return await Promise.all(
projectGraphDeps.map(async (d) => {
if (visited.indexOf(d.target) > -1) {
return null;
} else {
visited.push(d.target);
return await this.hashTask(
{
id: `${d.target}:$fileset:${fileset.fileset}`,
target: {
project: d.target,
target: '$fileset',
configuration: fileset.fileset,
},
overrides: {},
},
visited
);
}
})
);
})
)
)
.flat()
.filter((r) => !!r);
}
private async hashSelfFilesets(
config: FilesetDependencyConfig[],
projectNode: ProjectGraphProjectNode<any>
) {
return await Promise.all(
config
.filter((fileset) => fileset.projects === 'self')
.map((fileset) =>
this.hashFilesetSource(projectNode.name, fileset.fileset)
)
);
}
private filesetConfigs(
task: Task,
projectNode: ProjectGraphProjectNode<any>
): FilesetDependencyConfig[] {
if (task.target.target === '$fileset') {
return [
{
fileset: task.target.configuration,
projects: 'self',
},
{
fileset: task.target.configuration,
projects: 'dependencies',
},
];
} else {
const targetData = projectNode.data.targets[task.target.target];
const targetDefaults = this.nxJson.targetDefaults[task.target.target];
// task from TaskGraph can be added here
return expandFilesetConfigSyntaxSugar(
targetData.dependsOnFilesets ||
targetDefaults?.dependsOnFilesets ||
this.DEFAULT_FILESET_CONFIG
);
}
}
private hashExternalDependency(task: Task) {
const n = this.projectGraph.externalNodes[task.target.project];
const version = n?.data?.version;
let hash: string;
if (version) {
hash = this.hashing.hashArray([version]);
} else {
// unknown dependency
// this may occur if a file has a dependency to a npm package
// which is not directly registestered in package.json
// but only indirectly through dependencies of registered
// npm packages
// when it is at a later stage registered in package.json
// the cache project graph will not know this module but
// the new project graph will know it
// The actual checksum added here is of no importance as
// the version is unknown and may only change when some
// other change occurs in package.json and/or package-lock.json
hash = `__${task.target.project}__`;
}
return {
value: hash,
command: '',
nodes: {
[task.target.project]: version || hash,
},
};
}
private hashCommand(task: Task) {
const overrides = { ...task.overrides };
delete overrides['__overrides_unparsed__'];
const sortedOverrides = {};
for (let k of Object.keys(overrides).sort()) {
sortedOverrides[k] = overrides[k];
}
return this.hashing.hashArray([
task.target.project ?? '',
task.target.target ?? '',
task.target.configuration ?? '',
JSON.stringify(sortedOverrides),
]);
}
private async hashFilesetSource(
projectName: string,
filesetName: string
): Promise<{ taskId: string; value: string }> {
const mapKey = `${projectName}:$fileset:${filesetName}`;
if (!this.filesetHashes[mapKey]) {
this.filesetHashes[mapKey] = new Promise(async (res) => {
const p = this.projectGraph.nodes[projectName]; const p = this.projectGraph.nodes[projectName];
if (!p) { const filesetPatterns = this.selectFilesetPatterns(p, filesetName);
const n = this.projectGraph.externalNodes[projectName]; const filteredFiles = this.filterFiles(p.data.files, filesetPatterns);
const version = n?.data?.version;
let hash: string;
if (version) {
hash = this.hashing.hashArray([version]);
} else {
// unknown dependency
// this may occur if a file has a dependency to a npm package
// which is not directly registestered in package.json
// but only indirectly through dependencies of registered
// npm packages
// when it is at a later stage registered in package.json
// the cache project graph will not know this module but
// the new project graph will know it
// The actual checksum added here is of no importance as
// the version is unknown and may only change when some
// other change occurs in package.json and/or package-lock.json
hash = `__${projectName}__`;
}
res(hash);
return;
}
const filteredFiles =
filter === 'all-files'
? p.data.files
: p.data.files.filter((f) => !this.isSpec(f.file));
const fileNames = filteredFiles.map((f) => f.file); const fileNames = filteredFiles.map((f) => f.file);
const values = filteredFiles.map((f) => f.hash); const values = filteredFiles.map((f) => f.hash);
const workspaceJson = JSON.stringify(
this.workspaceJson.projects[projectName] ?? ''
);
let tsConfig: string; let tsConfig: string;
tsConfig = this.hashTsConfig(p);
if (this.options.selectivelyHashTsConfig) { res({
tsConfig = this.removeOtherProjectsPathRecords(projectName); taskId: mapKey,
} else { value: this.hashing.hashArray([
tsConfig = JSON.stringify(this.tsConfigJson);
}
res(
this.hashing.hashArray([
...fileNames, ...fileNames,
...values, ...values,
workspaceJson, JSON.stringify({ ...p.data, files: undefined }),
tsConfig, tsConfig,
]) ]),
); });
}); });
} }
return this.sourceHashes[mapKey]; return this.filesetHashes[mapKey];
} }
private isSpec(file: string) { private selectFilesetPatterns(
return ( p: ProjectGraphProjectNode,
file.endsWith('.spec.tsx') || filesetName: string
file.endsWith('.test.tsx') || ) {
file.endsWith('-test.tsx') || if (filesetName == undefined) {
file.endsWith('-spec.tsx') || filesetName = 'default';
file.endsWith('.spec.ts') || }
file.endsWith('.test.ts') || const projectFilesets = p.data.filesets
file.endsWith('-test.ts') || ? p.data.filesets[filesetName]
file.endsWith('-spec.ts') || : null;
file.endsWith('.spec.js') || const defaultFilesets = this.nxJson.filesets
file.endsWith('.test.js') || ? this.nxJson.filesets[filesetName]
file.endsWith('-test.js') || : null;
file.endsWith('-spec.js') if (projectFilesets) return projectFilesets;
if (defaultFilesets) return defaultFilesets;
return null;
}
private filterFiles(files: FileData[], patterns: string[] | null) {
if (patterns === null) return files;
return files.filter(
(f) => !!patterns.find((pattern) => minimatch(f.file, pattern))
); );
} }
private removeOtherProjectsPathRecords(projectName: string) { private hashTsConfig(p: ProjectGraphProjectNode) {
const { paths, ...compilerOptions } = this.tsConfigJson.compilerOptions; if (this.options.selectivelyHashTsConfig) {
return this.removeOtherProjectsPathRecords(p);
} else {
return JSON.stringify(this.tsConfigJson);
}
}
const rootPath = this.workspaceJson.projects[projectName].root.split('/'); private removeOtherProjectsPathRecords(p: ProjectGraphProjectNode) {
const { paths, ...compilerOptions } = this.tsConfigJson.compilerOptions;
const rootPath = p.data.root.split('/');
rootPath.shift(); rootPath.shift();
const pathAlias = `@${this.nxJson.npmScope}/${rootPath.join('/')}`; const pathAlias = `@${this.nxJson.npmScope}/${rootPath.join('/')}`;
@ -484,24 +541,20 @@ class ProjectHasher {
}; };
} }
} }
}
private readWorkspaceConfigFile(path: string): ProjectsConfigurations {
try { function expandFilesetConfigSyntaxSugar(
const res = readJsonFile(path); deps: (FilesetDependencyConfig | string)[]
res.projects ??= {}; ): FilesetDependencyConfig[] {
return resolveNewFormatWithInlineProjects(res); return deps.map((d) => {
} catch { if (typeof d === 'string') {
return { projects: {}, version: 2 }; if (d.startsWith('^')) {
} return { projects: 'dependencies', fileset: d.substring(1) };
} } else {
return { projects: 'self', fileset: d };
private readNxJsonConfigFile(path: string): NxJsonConfiguration { }
try { } else {
const res = readJsonFile(path); return d;
res.projects ??= {}; }
return res; });
} catch {
return {};
}
}
} }