fix(core): nx migrate should work in encapsulated Nx repos (#15338)

e2e/nx-misc/src/encapsulated.test.ts

@@ -8,6 +8,8 @@ import {
   cleanupProject,
   getPublishedVersion,
   uniq,
+  readJson,
+  readFile,
 } from '@nrwl/e2e/utils';
 import { bold } from 'chalk';
 
@@ -104,6 +106,161 @@ describe('encapsulated nx', () => {
     ).not.toThrow();
     expect(() => checkFilesExist());
   });
+
+  it('should work with migrate', () => {
+    updateFile(
+      `.nx/installation/node_modules/migrate-parent-package/package.json`,
+      JSON.stringify({
+        version: '1.0.0',
+        name: 'migrate-parent-package',
+        'nx-migrations': './migrations.json',
+      })
+    );
+
+    updateFile(
+      `.nx/installation/node_modules/migrate-parent-package/migrations.json`,
+      JSON.stringify({
+        schematics: {
+          run11: {
+            version: '1.1.0',
+            description: '1.1.0',
+            factory: './run11',
+          },
+          run20: {
+            version: '2.0.0',
+            description: '2.0.0',
+            implementation: './run20',
+          },
+        },
+      })
+    );
+
+    updateFile(
+      `.nx/installation/node_modules/migrate-parent-package/run11.js`,
+      `
+        exports.default = function default_1() {
+          return function(host) {
+            host.create('file-11', 'content11')
+          }
+        }
+        `
+    );
+
+    updateFile(
+      `.nx/installation/node_modules/migrate-parent-package/run20.js`,
+      `
+        exports.default = function (host) {
+           host.write('file-20', 'content20')
+        }
+        `
+    );
+
+    updateFile(
+      `.nx/installation/node_modules/migrate-child-package/package.json`,
+      JSON.stringify({
+        name: 'migrate-child-package',
+        version: '1.0.0',
+      })
+    );
+
+    /**
+     * Patches migration fetcher to load in migrations that we are using to test.
+     */
+    updateFile(
+      '.nx/installation/node_modules/nx/src/command-line/migrate.js',
+      (content) => {
+        const start = content.indexOf('// testing-fetch-start');
+        const end = content.indexOf('// testing-fetch-end');
+
+        const before = content.substring(0, start);
+        const after = content.substring(end);
+        const newFetch = `
+             function createFetcher(logger) {
+              return function fetch(packageName) {
+                if (packageName === 'migrate-parent-package') {
+                  return Promise.resolve({
+                    version: '2.0.0',
+                    generators: {
+                      'run11': {
+                        version: '1.1.0'
+                      },
+                      'run20': {
+                        version: '2.0.0',
+                        cli: 'nx'
+                      }
+                    },
+                    packageJsonUpdates: {
+                      'run-11': {version: '1.1.0', packages: {
+                        'migrate-child-package': {version: '9.0.0', alwaysAddToPackageJson: false},
+                      }},
+                    }
+                  });
+                } else {
+                  return Promise.resolve({version: '9.0.0'});
+                }
+              }
+            }
+            `;
+
+        return `${before}${newFetch}${after}`;
+      }
+    );
+
+    updateJson('nx.json', (j: NxJsonConfiguration) => {
+      j.installation = {
+        version: getPublishedVersion(),
+        plugins: {
+          'migrate-child-package': '1.0.0',
+        },
+      };
+      return j;
+    });
+    runEncapsulatedNx(
+      'migrate migrate-parent-package@2.0.0 --from="migrate-parent-package@1.0.0"',
+      {
+        env: {
+          ...process.env,
+          NX_MIGRATE_SKIP_INSTALL: 'true',
+          NX_MIGRATE_USE_LOCAL: 'true',
+          NX_WRAPPER_SKIP_INSTALL: 'true',
+        },
+      }
+    );
+
+    const nxJson: NxJsonConfiguration = readJson(`nx.json`);
+    expect(nxJson.installation.plugins['migrate-child-package']).toEqual(
+      '9.0.0'
+    );
+    // creates migrations.json
+    const migrationsJson = readJson(`migrations.json`);
+    expect(migrationsJson).toEqual({
+      migrations: [
+        {
+          package: 'migrate-parent-package',
+          version: '1.1.0',
+          name: 'run11',
+        },
+        {
+          package: 'migrate-parent-package',
+          version: '2.0.0',
+          name: 'run20',
+          cli: 'nx',
+        },
+      ],
+    });
+
+    // runs migrations
+    runEncapsulatedNx('migrate --run-migrations=migrations.json', {
+      env: {
+        ...process.env,
+        NX_MIGRATE_SKIP_INSTALL: 'true',
+        NX_MIGRATE_USE_LOCAL: 'true',
+        NX_WRAPPER_SKIP_INSTALL: 'true',
+      },
+    });
+    expect(readFile('file-11')).toEqual('content11');
+    expect(readFile('file-20')).toEqual('content20');
+  });
 });
 
 function assertNoRootPackages() {

e2e/utils/command-utils.ts

@@ -66,19 +66,20 @@ export function setMaxWorkers() {
 
 export function runCommand(
   command: string,
-  options?: Partial<ExecSyncOptions>
+  options?: Partial<ExecSyncOptions> & { failOnError?: boolean }
 ): string {
+  const { failOnError, ...childProcessOptions } = options ?? {};
   try {
     const r = execSync(command, {
       cwd: tmpProjPath(),
       stdio: ['pipe', 'pipe', 'pipe'],
       env: {
         ...getStrippedEnvironmentVariables(),
-        ...options?.env,
+        ...childProcessOptions?.env,
         FORCE_COLOR: 'false',
       },
       encoding: 'utf-8',
-      ...options,
+      ...childProcessOptions,
     }).toString();
     if (process.env.NX_VERBOSE_LOGGING) {
       console.log(r);
@@ -87,7 +88,7 @@ export function runCommand(
   } catch (e) {
     // this is intentional
     // npm ls fails if package is not found
-    if (e.stdout || e.stderr) {
+    if (!failOnError && (e.stdout || e.stderr)) {
       return e.stdout?.toString() + e.stderr?.toString();
     }
     throw e;

e2e/utils/create-project-utils.ts

@@ -17,7 +17,7 @@ import * as isCI from 'is-ci';
 
 import { angularCliVersion } from '@nrwl/workspace/src/utils/versions';
 import { dump } from '@zkochan/js-yaml';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 
 import { isVerbose } from './get-env-info';
 import { logError, logInfo } from './log-utils';
@@ -372,15 +372,15 @@ export function newLernaWorkspace({
 export function newEncapsulatedNxWorkspace({
   name = uniq('encapsulated'),
   pmc = getPackageManagerCommand(),
-} = {}): (command: string) => string {
+} = {}): (command: string, opts?: Partial<ExecSyncOptions>) => string {
   projName = name;
   ensureDirSync(tmpProjPath());
   runCommand(`${pmc.runUninstalledPackage} nx@latest init --encapsulated`);
-  return (command: string) => {
+  return (command: string, opts: Partial<ExecSyncOptions>) => {
     if (process.platform === 'win32') {
-      return runCommand(`./nx.bat ${command}`);
+      return runCommand(`./nx.bat ${command}`, { ...opts, failOnError: true });
     } else {
-      return runCommand(`./nx ${command}`);
+      return runCommand(`./nx ${command}`, { ...opts, failOnError: true });
     }
   };
 }

packages/angular/src/generators/ng-add/utilities/format-files-task.ts

@@ -1,6 +1,6 @@
 import type { Tree } from '@nrwl/devkit';
 import { getPackageManagerCommand } from '@nrwl/devkit';
-import { execSync } from 'child_process';
+import { runNxSync } from 'nx/src/utils/child-process';
 
 export function formatFilesTask(tree: Tree): void {
   if (
@@ -11,8 +11,7 @@ export function formatFilesTask(tree: Tree): void {
     return;
   }
 
-  const pmc = getPackageManagerCommand();
   try {
-    execSync(`${pmc.exec} nx format`, { cwd: tree.root, stdio: [0, 1, 2] });
+    runNxSync(`format`, { cwd: tree.root, stdio: [0, 1, 2] });
   } catch {}
 }

packages/nx/src/adapter/ngcli-adapter.ts

@@ -29,6 +29,7 @@ import {
 } from '../generators/utils/project-configuration';
 import { createProjectGraphAsync } from '../project-graph/project-graph';
 import { readJsonFile } from '../utils/fileutils';
+import { getNxRequirePaths } from '../utils/installation-directory';
 import { parseJson } from '../utils/json';
 import { NX_ERROR, NX_PREFIX } from '../utils/logger';
 import { readModulePackageJson } from '../utils/package-json';
@@ -643,9 +644,10 @@ function resolveMigrationsCollection(name: string): string {
   if (extname(name)) {
     collectionPath = require.resolve(name);
   } else {
-    const { path: packageJsonPath, packageJson } = readModulePackageJson(name, [
-      process.cwd(),
-    ]);
+    const { path: packageJsonPath, packageJson } = readModulePackageJson(
+      name,
+      getNxRequirePaths(process.cwd())
+    );
 
     let pkgJsonSchematics =
       packageJson['nx-migrations'] ?? packageJson['ng-update'];

packages/nx/src/command-line/connect.ts

@@ -10,6 +10,7 @@ import {
 import { existsSync } from 'fs';
 import { join } from 'path';
 import { workspaceRoot } from '../utils/workspace-root';
+import { runNxSync } from '../utils/child-process';
 
 export async function connectToNxCloudIfExplicitlyAsked(opts: {
   [k: string]: any;
@@ -24,13 +25,7 @@ export async function connectToNxCloudIfExplicitlyAsked(opts: {
     output.log({
       title: '--cloud requires the workspace to be connected to Nx Cloud.',
     });
-    const pmc = getPackageManagerCommand();
-    const nxCommand = existsSync(join(workspaceRoot, 'package.json'))
-      ? `${pmc.exec} nx`
-      : process.platform === 'win32'
-      ? './nx.bat'
-      : './nx';
-    execSync(`${nxCommand} connect-to-nx-cloud`, {
+    runNxSync(`connect-to-nx-cloud`, {
       stdio: [0, 1, 2],
     });
     output.success({
@@ -60,11 +55,6 @@ export async function connectToNxCloudCommand(
   const res = await connectToNxCloudPrompt(promptOverride);
   if (!res) return false;
   const pmc = getPackageManagerCommand();
-  const nxCommand = pmc
-    ? `${pmc.exec} nx`
-    : process.platform === 'win32'
-    ? './nx.bat'
-    : './nx';
   if (pmc) {
     execSync(`${pmc.addDev} @nrwl/nx-cloud@latest`);
   } else {
@@ -76,7 +66,7 @@ export async function connectToNxCloudCommand(
       ).toString();
     }
   }
-  execSync(`${nxCommand} g @nrwl/nx-cloud:init`, {
+  runNxSync(`g @nrwl/nx-cloud:init`, {
     stdio: [0, 1, 2],
   });
   return true;

packages/nx/src/command-line/migrate.ts

@@ -53,6 +53,9 @@ import { nxVersion } from '../utils/versions';
 import { existsSync } from 'fs';
 import { workspaceRoot } from '../utils/workspace-root';
 import { isCI } from '../utils/is-ci';
+import { getNxRequirePaths } from '../utils/installation-directory';
+import { readNxJson } from '../config/configuration';
+import { runNxSync } from '../utils/child-process';
 
 export interface ResolvedMigrationConfiguration extends MigrationsJson {
   packageGroup?: ArrayPackageGroup;
@@ -100,7 +103,8 @@ function normalizeSlashes(packageName: string): string {
 }
 
 export interface MigratorOptions {
-  packageJson: PackageJson;
+  packageJson?: PackageJson;
+  nxInstallation?: NxJsonConfiguration['installation'];
   getInstalledPackageVersion: (
     pkg: string,
     overrides?: Record<string, string>
@@ -116,7 +120,7 @@ export interface MigratorOptions {
 }
 
 export class Migrator {
-  private readonly packageJson: MigratorOptions['packageJson'];
+  private readonly packageJson?: MigratorOptions['packageJson'];
   private readonly getInstalledPackageVersion: MigratorOptions['getInstalledPackageVersion'];
   private readonly fetch: MigratorOptions['fetch'];
   private readonly installedPkgVersionOverrides: MigratorOptions['from'];
@@ -126,9 +130,11 @@ export class Migrator {
   private readonly packageUpdates: Record<string, PackageUpdate> = {};
   private readonly collectedVersions: Record<string, string> = {};
   private readonly promptAnswers: Record<string, boolean> = {};
+  private readonly nxInstallation: NxJsonConfiguration['installation'] | null;
 
   constructor(opts: MigratorOptions) {
     this.packageJson = opts.packageJson;
+    this.nxInstallation = opts.nxInstallation;
     this.getInstalledPackageVersion = opts.getInstalledPackageVersion;
     this.fetch = opts.fetch;
     this.installedPkgVersionOverrides = opts.from;
@@ -400,33 +406,34 @@ export class Migrator {
         continue;
       }
 
-      const { dependencies, devDependencies } = this.packageJson;
-      packageJsonUpdate.packages = Object.entries(packageJsonUpdate.packages)
-        .filter(
-          ([packageName, packageUpdate]) =>
-            (!packageUpdate.ifPackageInstalled ||
-              this.getPkgVersion(packageUpdate.ifPackageInstalled)) &&
-            (packageUpdate.alwaysAddToPackageJson ||
-              packageUpdate.addToPackageJson ||
-              !!dependencies?.[packageName] ||
-              !!devDependencies?.[packageName]) &&
-            (!this.collectedVersions[packageName] ||
-              this.gt(
-                packageUpdate.version,
-                this.collectedVersions[packageName]
-              ))
-        )
-        .reduce((acc, [packageName, packageUpdate]) => {
-          acc[packageName] = {
+      const dependencies: Record<string, string> = {
+        ...this.packageJson?.dependencies,
+        ...this.packageJson?.devDependencies,
+        ...this.nxInstallation?.plugins,
+        ...(this.nxInstallation && { nx: this.nxInstallation.version }),
+      };
+
+      const filtered: Record<string, PackageUpdate> = {};
+      for (const [packageName, packageUpdate] of Object.entries(
+        packageJsonUpdate.packages
+      )) {
+        if (
+          this.shouldApplyPackageUpdate(
+            packageUpdate,
+            packageName,
+            dependencies
+          )
+        ) {
+          filtered[packageName] = {
             version: packageUpdate.version,
             addToPackageJson: packageUpdate.alwaysAddToPackageJson
               ? 'dependencies'
               : packageUpdate.addToPackageJson || false,
           };
-          return acc;
-        }, {} as Record<string, PackageUpdate>);
-
-      if (Object.keys(packageJsonUpdate.packages).length) {
+        }
+      }
+      if (Object.keys(filtered).length) {
+        packageJsonUpdate.packages = filtered;
         filteredPackageJsonUpdates.push(packageJsonUpdate);
       }
     }
@@ -434,6 +441,22 @@ export class Migrator {
     return filteredPackageJsonUpdates;
   }
 
+  private shouldApplyPackageUpdate(
+    packageUpdate: PackageUpdate,
+    packageName: string,
+    dependencies: Record<string, string>
+  ) {
+    return (
+      (!packageUpdate.ifPackageInstalled ||
+        this.getPkgVersion(packageUpdate.ifPackageInstalled)) &&
+      (packageUpdate.alwaysAddToPackageJson ||
+        packageUpdate.addToPackageJson ||
+        !!dependencies?.[packageName]) &&
+      (!this.collectedVersions[packageName] ||
+        this.gt(packageUpdate.version, this.collectedVersions[packageName]))
+    );
+  }
+
   private addPackageUpdate(name: string, packageUpdate: PackageUpdate): void {
     if (
       !this.packageUpdates[name] ||
@@ -732,9 +755,10 @@ function createInstalledPackageVersionsResolver(
       }
 
       if (!cache[packageName]) {
-        const { packageJson, path } = readModulePackageJson(packageName, [
-          root,
-        ]);
+        const { packageJson, path } = readModulePackageJson(
+          packageName,
+          getNxRequirePaths()
+        );
         // old workspaces would have the temp installation of nx in the cache,
         // so the resolved package is not the one we need
         if (!path.startsWith(workspaceRoot)) {
@@ -970,7 +994,7 @@ function readPackageMigrationConfig(
 ): PackageMigrationConfig {
   const { path: packageJsonPath, packageJson: json } = readModulePackageJson(
     packageName,
-    [dir]
+    getNxRequirePaths(dir)
   );
 
   const config = readNxMigrateConfig(json);
@@ -1043,7 +1067,7 @@ function updatePackageJson(
   });
 }
 
-function updateInstallationDetails(
+async function updateInstallationDetails(
   root: string,
   updatedPackages: Record<string, PackageUpdate>
 ) {
@@ -1064,7 +1088,9 @@ function updateInstallationDetails(
     for (const dep in nxJson.installation.plugins) {
       const update = updatedPackages[dep];
       if (update) {
-        nxJson.installation.plugins[dep] = update.version;
+        nxJson.installation.plugins[dep] = valid(update.version)
+          ? update.version
+          : await resolvePackageVersionUsingRegistry(dep, update.version);
       }
     }
   }
@@ -1101,10 +1127,13 @@ async function generateMigrationsJsonAndUpdatePackageJson(
 ) {
   const pmc = getPackageManagerCommand();
   try {
-    let originalPackageJson = readJsonFile<PackageJson>(
-      join(root, 'package.json')
-    );
-    const from = readNxVersion(originalPackageJson);
+    const rootPkgJsonPath = join(root, 'package.json');
+    let originalPackageJson = existsSync(rootPkgJsonPath)
+      ? readJsonFile<PackageJson>(rootPkgJsonPath)
+      : null;
+    const originalNxInstallation = readNxJson().installation;
+    const from =
+      originalNxInstallation?.version ?? readNxVersion(originalPackageJson);
 
     try {
       if (
@@ -1135,6 +1164,7 @@ async function generateMigrationsJsonAndUpdatePackageJson(
 
     const migrator = new Migrator({
       packageJson: originalPackageJson,
+      nxInstallation: originalNxInstallation,
       getInstalledPackageVersion: createInstalledPackageVersionsResolver(root),
       fetch: createFetcher(),
       from: opts.from,
@@ -1149,7 +1179,7 @@ async function generateMigrationsJsonAndUpdatePackageJson(
     );
 
     updatePackageJson(root, packageUpdates);
-    updateInstallationDetails(root, packageUpdates);
+    await updateInstallationDetails(root, packageUpdates);
 
     if (migrations.length > 0) {
       createMigrationsFile(root, [
@@ -1352,8 +1382,7 @@ async function runMigrations(
   if (!__dirname.startsWith(workspaceRoot)) {
     // we are running from a temp installation with nx latest, switch to running
     // from local installation
-    const pmc = getPackageManagerCommand();
-    execSync(`${pmc.exec} nx migrate ${args.join(' ')}`, {
+    runNxSync(`migrate ${args.join(' ')}`, {
       stdio: ['inherit', 'inherit', 'inherit'],
       env: {
         ...process.env,
@@ -1410,11 +1439,17 @@ async function runMigrations(
 }
 
 function getStringifiedPackageJsonDeps(root: string): string {
-  const { dependencies, devDependencies } = readJsonFile<PackageJson>(
-    join(root, 'package.json')
-  );
+  try {
+    const { dependencies, devDependencies } = readJsonFile<PackageJson>(
+      join(root, 'package.json')
+    );
 
-  return JSON.stringify([dependencies, devDependencies]);
+    return JSON.stringify([dependencies, devDependencies]);
+  } catch {
+    // We don't really care if the .nx/installation property changes,
+    // whenever nxw is invoked it will handle the dep updates.
+    return '';
+  }
 }
 
 function commitChanges(commitMessage: string): string | null {

packages/nx/src/command-line/nx-commands.ts

@@ -8,6 +8,7 @@ import { workspaceRoot } from '../utils/workspace-root';
 import { getPackageManagerCommand } from '../utils/package-manager';
 import { writeJsonFile } from '../utils/fileutils';
 import { WatchArguments } from './watch';
+import { runNxSync } from '../utils/child-process';
 
 // Ensure that the output takes up the available width of the terminal.
 yargs.wrap(yargs.terminalWidth());
@@ -1090,8 +1091,7 @@ function withListOptions(yargs) {
 
 function runMigration() {
   const runLocalMigrate = () => {
-    const pmc = getPackageManagerCommand();
-    execSync(`${pmc.exec} nx _migrate ${process.argv.slice(3).join(' ')}`, {
+    runNxSync(`_migrate ${process.argv.slice(3).join(' ')}`, {
       stdio: ['inherit', 'inherit', 'inherit'],
     });
   };

packages/nx/src/command-line/view-logs.ts

@@ -2,6 +2,7 @@ import { getPackageManagerCommand } from '../utils/package-manager';
 import { execSync } from 'child_process';
 import { isNxCloudUsed } from '../utils/nx-cloud-utils';
 import { output } from '../utils/output';
+import { runNxSync } from '../utils/child-process';
 
 export async function viewLogs(): Promise<number> {
   const pmc = getPackageManagerCommand();
@@ -48,10 +49,9 @@ export async function viewLogs(): Promise<number> {
       output.log({
         title: 'Connecting to Nx Cloud',
       });
-      execSync(
-        `${pmc.exec} nx g @nrwl/nx-cloud:init --installation-source=view-logs`,
-        { stdio: 'ignore' }
-      );
+      runNxSync(`g @nrwl/nx-cloud:init --installation-source=view-logs`, {
+        stdio: 'ignore',
+      });
     } catch (e) {
       output.log({
         title: 'Failed to connect to Nx Cloud',

packages/nx/src/nx-init/encapsulated/nxw.ts

@@ -99,8 +99,10 @@ function ensureUpToDateInstallation() {
   }
 }
 
-if (require.main === module && !process.env.NX_WRAPPER_SET) {
+if (require.main === module || !process.env.NX_WRAPPER_SET) {
+  if (!process.env.NX_WRAPPER_SKIP_INSTALL) {
+    ensureUpToDateInstallation();
+  }
   process.env.NX_WRAPPER_SET = 'true';
-  ensureUpToDateInstallation();
   require('./installation/node_modules/nx/bin/nx');
 }

packages/nx/src/nx-init/utils.ts

@@ -6,6 +6,7 @@ import {
   getPackageManagerCommand,
   PackageManagerCommands,
 } from '../utils/package-manager';
+import { runNxSync } from '../utils/child-process';
 
 export function askAboutNxCloud() {
   return enquirer
@@ -121,12 +122,8 @@ export function runInstall(
 }
 
 export function initCloud(repoRoot: string) {
-  const pmc = getPackageManagerCommand();
-  execSync(
-    `${pmc.exec} nx g @nrwl/nx-cloud:init --installationSource=add-nx-to-monorepo`,
-    {
-      stdio: [0, 1, 2],
-      cwd: repoRoot,
-    }
-  );
+  runNxSync(`g @nrwl/nx-cloud:init --installationSource=add-nx-to-monorepo`, {
+    stdio: [0, 1, 2],
+    cwd: repoRoot,
+  });
 }

packages/nx/src/utils/child-process.ts

@@ -0,0 +1,28 @@
+import { execSync, ExecSyncOptions } from 'child_process';
+import { existsSync } from 'fs';
+import { join, relative } from 'path';
+import { getPackageManagerCommand } from './package-manager';
+import { workspaceRoot, workspaceRootInner } from './workspace-root';
+
+export function runNxSync(
+  cmd: string,
+  options?: ExecSyncOptions & { cwd?: string }
+) {
+  let baseCmd: string;
+  if (existsSync(join(workspaceRoot, 'package.json'))) {
+    baseCmd = `${getPackageManagerCommand().exec} nx`;
+  } else {
+    options ??= {};
+    options.cwd ??= process.cwd();
+    const offsetFromRoot = relative(
+      options.cwd,
+      workspaceRootInner(options.cwd, null)
+    );
+    if (process.platform === 'win32') {
+      baseCmd = '.\\' + join(`${offsetFromRoot}`, 'nx.bat');
+    } else {
+      baseCmd = './' + join(`${offsetFromRoot}`, 'nx');
+    }
+  }
+  execSync(`${baseCmd} ${cmd}`, options);
+}