fix(core): filter branch in preparation for nx import (#27652)

This PR fixes an issue with `nx import` where multiple directories
cannot be imported from the same repo. We now use `git filter-repo` (if
installed) or `git filter-branch` (fallback) to prepare the repo for
import so anything not in the subdirectory will be ignored (i.e. cleaner
history).

Note: `filter-repo` is much faster but requires the user to install it
first via their package manager (e.g. `brew install git-filter-repo` or
`apt install git-filter-repo`). We fallback to `git filter-branch` since
it comes with git, but it is slower, so the process may take 10+ minutes
on really large monorepos (e.g. next.js).

Also:
- Use `await` before returning a promise to Node can maintain correct
stacktrace
- Remove logic for `if (relativeSourceDir === '')` since using
`filter-branch` moves all the files to the root (`.`)
- Default destination project location to be the same as source (e.g.
importing `packages/a` will go to `packages/a` unless user types in
something else)
- Add `--depth` option if users don't want to clone with full history
(default is full history)
- Fix issues with special characters causing `git ls-files` + `git mv`
to since `mv` doesn't work with escaped names

<!-- If this is a particularly complex change or feature addition, you
can request a dedicated Nx release for this pull request branch. Mention
someone from the Nx team or the `@nrwl/nx-pipelines-reviewers` and they
will confirm if the PR warrants its own release for testing purposes,
and generate it for you if appropriate. -->

## Current Behavior
<!-- This is the behavior we have today -->

## Expected Behavior
<!-- This is the behavior we should expect with the changes in this PR
-->

## Related Issue(s)
<!-- Please link the issue being fixed so it gets closed when this is
merged. -->

Fixes #

e2e/nx/src/import.test.ts

@@ -8,7 +8,7 @@ import {
   updateFile,
   e2eCwd,
 } from '@nx/e2e/utils';
-import { mkdirSync, rmdirSync } from 'fs';
+import { writeFileSync, mkdirSync, rmdirSync } from 'fs';
 import { execSync } from 'node:child_process';
 import { join } from 'path';
 
@@ -86,4 +86,54 @@ describe('Nx Import', () => {
     );
     runCLI(`vite:build created-vite-app`);
   });
+
+  it('should be able to import two directories from same repo', () => {
+    // Setup repo with two packages: a and b
+    const repoPath = join(tempImportE2ERoot, 'repo');
+    mkdirSync(repoPath, { recursive: true });
+    writeFileSync(join(repoPath, 'README.md'), `# Repo`);
+    execSync(`git init`, {
+      cwd: repoPath,
+    });
+    execSync(`git add .`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -am "initial commit"`, {
+      cwd: repoPath,
+    });
+    execSync(`git checkout -b main`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/a'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/a/README.md'), `# A`);
+    execSync(`git add packages/a`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package a"`, {
+      cwd: repoPath,
+    });
+    mkdirSync(join(repoPath, 'packages/b'), { recursive: true });
+    writeFileSync(join(repoPath, 'packages/b/README.md'), `# B`);
+    execSync(`git add packages/b`, {
+      cwd: repoPath,
+    });
+    execSync(`git commit -m "add package b"`, {
+      cwd: repoPath,
+    });
+
+    runCLI(
+      `import ${repoPath} packages/a --ref main --source packages/a --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+    runCLI(
+      `import ${repoPath} packages/b --ref main --source packages/b --no-interactive`,
+      {
+        verbose: true,
+      }
+    );
+
+    checkFilesExist('packages/a/README.md', 'packages/b/README.md');
+  });
 });

packages/nx/src/command-line/import/command-object.ts

@@ -28,6 +28,11 @@ export const yargsImportCommand: CommandModule = {
             type: 'string',
             description: 'The branch from the source repository to import',
           })
+          .option('depth', {
+            type: 'number',
+            description:
+              'The depth to clone the source repository (limit this for faster git clone)',
+          })
           .option('interactive', {
             type: 'boolean',
             description: 'Interactive mode',

packages/nx/src/command-line/import/import.ts

@@ -1,4 +1,8 @@
-import { join, relative, resolve } from 'path';
+import { dirname, join, relative, resolve } from 'path';
+import { minimatch } from 'minimatch';
+import { existsSync, promises as fsp } from 'node:fs';
+import * as chalk from 'chalk';
+import { load as yamlLoad } from '@zkochan/js-yaml';
 import { cloneFromUpstream, GitRepository } from '../../utils/git-utils';
 import { stat, mkdir, rm } from 'node:fs/promises';
 import { tmpdir } from 'tmp';
@@ -11,6 +15,9 @@ import { workspaceRoot } from '../../utils/workspace-root';
 import {
   detectPackageManager,
   getPackageManagerCommand,
+  isWorkspacesEnabled,
+  PackageManager,
+  PackageManagerCommands,
 } from '../../utils/package-manager';
 import { resetWorkspaceContext } from '../../utils/workspace-context';
 import { runInstall } from '../init/implementation/utils';
@@ -21,6 +28,7 @@ import {
   getPackagesInPackageManagerWorkspace,
   needsInstall,
 } from './utils/needs-install';
+import { readPackageJson } from '../../project-graph/file-utils';
 
 const importRemoteName = '__tmp_nx_import__';
 
@@ -41,6 +49,10 @@ export interface ImportOptions {
    * The directory in the destination repo to import into
    */
   destination: string;
+  /**
+   * The depth to clone the source repository (limit this for faster clone times)
+   */
+  depth: number;
 
   verbose: boolean;
   interactive: boolean;
@@ -90,7 +102,7 @@ export async function importHandler(options: ImportOptions) {
 
   const sourceRepoPath = join(tempImportDirectory, 'repo');
   const spinner = createSpinner(
-    `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath}`
+    `Cloning ${sourceRemoteUrl} into a temporary directory: ${sourceRepoPath} (Use --depth to limit commit history and speed up clone times)`
   ).start();
   try {
     await rm(tempImportDirectory, { recursive: true });
@@ -101,6 +113,7 @@ export async function importHandler(options: ImportOptions) {
   try {
     sourceGitClient = await cloneFromUpstream(sourceRemoteUrl, sourceRepoPath, {
       originName: importRemoteName,
+      depth: options.depth,
     });
   } catch (e) {
     spinner.fail(`Failed to clone ${sourceRemoteUrl} into ${sourceRepoPath}`);
@@ -110,6 +123,9 @@ export async function importHandler(options: ImportOptions) {
   }
   spinner.succeed(`Cloned into ${sourceRepoPath}`);
 
+  // Detecting the package manager before preparing the source repo for import.
+  const sourcePackageManager = detectPackageManager(sourceGitClient.root);
+
   if (!ref) {
     const branchChoices = await sourceGitClient.listBranches();
     ref = (
@@ -149,6 +165,7 @@ export async function importHandler(options: ImportOptions) {
           name: 'destination',
           message: 'Where in this workspace should the code be imported into?',
           required: true,
+          initial: source ? source : undefined,
         },
       ])
     ).destination;
@@ -157,6 +174,23 @@ export async function importHandler(options: ImportOptions) {
   const absSource = join(sourceRepoPath, source);
   const absDestination = join(process.cwd(), destination);
 
+  const destinationGitClient = new GitRepository(process.cwd());
+  await assertDestinationEmpty(destinationGitClient, absDestination);
+
+  const tempImportBranch = getTempImportBranch(ref);
+  await sourceGitClient.addFetchRemote(importRemoteName, ref);
+  await sourceGitClient.fetch(importRemoteName, ref);
+  spinner.succeed(`Fetched ${ref} from ${sourceRemoteUrl}`);
+  spinner.start(
+    `Checking out a temporary branch, ${tempImportBranch} based on ${ref}`
+  );
+  await sourceGitClient.checkout(tempImportBranch, {
+    new: true,
+    base: `${importRemoteName}/${ref}`,
+  });
+
+  spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`);
+
   try {
     await stat(absSource);
   } catch (e) {
@@ -165,11 +199,6 @@ export async function importHandler(options: ImportOptions) {
     );
   }
 
-  const destinationGitClient = new GitRepository(process.cwd());
-  await assertDestinationEmpty(destinationGitClient, absDestination);
-
-  const tempImportBranch = getTempImportBranch(ref);
-
   const packageManager = detectPackageManager(workspaceRoot);
 
   const originalPackageWorkspaces = await getPackagesInPackageManagerWorkspace(
@@ -186,8 +215,7 @@ export async function importHandler(options: ImportOptions) {
     source,
     relativeDestination,
     tempImportBranch,
-    sourceRemoteUrl,
-    importRemoteName
+    sourceRemoteUrl
   );
 
   await createTemporaryRemote(
@@ -220,23 +248,75 @@ export async function importHandler(options: ImportOptions) {
     options.interactive
   );
 
-  if (plugins.length > 0) {
-    output.log({ title: 'Installing Plugins' });
-    installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts);
-
-    await destinationGitClient.amendCommit();
-  } else if (await needsInstall(packageManager, originalPackageWorkspaces)) {
-    output.log({
-      title: 'Installing dependencies for imported code',
+  if (packageManager !== sourcePackageManager) {
+    output.warn({
+      title: `Mismatched package managers`,
+      bodyLines: [
+        `The source repository is using a different package manager (${sourcePackageManager}) than this workspace (${packageManager}).`,
+        `This could lead to install issues due to discrepancies in "package.json" features.`,
+      ],
     });
+  }
 
-    runInstall(workspaceRoot, getPackageManagerCommand(packageManager));
+  // If install fails, we should continue since the errors could be resolved later.
+  let installFailed = false;
+  if (plugins.length > 0) {
+    try {
+      output.log({ title: 'Installing Plugins' });
+      installPlugins(workspaceRoot, plugins, pmc, updatePackageScripts);
 
-    await destinationGitClient.amendCommit();
+      await destinationGitClient.amendCommit();
+    } catch (e) {
+      installFailed = true;
+      output.error({
+        title: `Install failed: ${e.message || 'Unknown error'}`,
+        bodyLines: [e.stack],
+      });
+    }
+  } else if (await needsInstall(packageManager, originalPackageWorkspaces)) {
+    try {
+      output.log({
+        title: 'Installing dependencies for imported code',
+      });
+
+      runInstall(workspaceRoot, getPackageManagerCommand(packageManager));
+
+      await destinationGitClient.amendCommit();
+    } catch (e) {
+      installFailed = true;
+      output.error({
+        title: `Install failed: ${e.message || 'Unknown error'}`,
+        bodyLines: [e.stack],
+      });
+    }
   }
 
   console.log(await destinationGitClient.showStat());
 
+  if (installFailed) {
+    const pmc = getPackageManagerCommand(packageManager);
+    output.warn({
+      title: `The import was successful, but the install failed`,
+      bodyLines: [
+        `You may need to run "${pmc.install}" manually to resolve the issue. The error is logged above.`,
+      ],
+    });
+  }
+
+  await warnOnMissingWorkspacesEntry(packageManager, pmc, relativeDestination);
+
+  // When only a subdirectory is imported, there might be devDependencies in the root package.json file
+  // that needs to be ported over as well.
+  if (ref) {
+    output.log({
+      title: `Check root dependencies`,
+      bodyLines: [
+        `"dependencies" and "devDependencies" are not imported from the source repository (${sourceRemoteUrl}).`,
+        `You may need to add some of those dependencies to this workspace in order to run tasks successfully.`,
+      ],
+    });
+  }
+
   output.log({
     title: `Merging these changes into ${getBaseRef(nxJson)}`,
     bodyLines: [
@@ -274,3 +354,77 @@ async function createTemporaryRemote(
   await destinationGitClient.addGitRemote(remoteName, sourceRemoteUrl);
   await destinationGitClient.fetch(remoteName);
 }
+
+// If the user imports a project that isn't in NPM/Yarn/PNPM workspaces, then its dependencies
+// will not be installed. We should warn users and provide instructions on how to fix this.
+async function warnOnMissingWorkspacesEntry(
+  pm: PackageManager,
+  pmc: PackageManagerCommands,
+  pkgPath: string
+) {
+  if (!isWorkspacesEnabled(pm, workspaceRoot)) {
+    output.warn({
+      title: `Missing workspaces in package.json`,
+      bodyLines:
+        pm === 'npm'
+          ? [
+              `We recommend enabling NPM workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://docs.npmjs.com/cli/using-npm/workspaces`,
+            ]
+          : pm === 'yarn'
+          ? [
+              `We recommend enabling Yarn workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://yarnpkg.com/features/workspaces`,
+            ]
+          : pm === 'bun'
+          ? [
+              `We recommend enabling Bun workspaces to install dependencies for the imported project.`,
+              `Add \`"workspaces": ["${pkgPath}"]\` to package.json and run "${pmc.install}".`,
+              `See: https://bun.sh/docs/install/workspaces`,
+            ]
+          : [
+              `We recommend enabling PNPM workspaces to install dependencies for the imported project.`,
+              `Add the following entry to to pnpm-workspace.yaml and run "${pmc.install}":`,
+              chalk.bold(`packages:\n  - '${pkgPath}'`),
+              `See: https://pnpm.io/workspaces`,
+            ],
+    });
+  } else {
+    // Check if the new package is included in existing workspaces entries. If not, warn the user.
+    let workspaces: string[] | null = null;
+
+    if (pm === 'npm' || pm === 'yarn' || pm === 'bun') {
+      const packageJson = readPackageJson();
+      workspaces = packageJson.workspaces;
+    } else if (pm === 'pnpm') {
+      const yamlPath = join(workspaceRoot, 'pnpm-workspace.yaml');
+      if (existsSync(yamlPath)) {
+        const yamlContent = await fsp.readFile(yamlPath, 'utf-8');
+        const yaml = yamlLoad(yamlContent);
+        workspaces = yaml.packages;
+      }
+    }
+
+    if (workspaces) {
+      const isPkgIncluded = workspaces.some((w) => minimatch(pkgPath, w));
+      if (!isPkgIncluded) {
+        const pkgsDir = dirname(pkgPath);
+        output.warn({
+          title: `Project missing in workspaces`,
+          bodyLines:
+            pm === 'npm' || pm === 'yarn' || pm === 'bun'
+              ? [
+                  `The imported project (${pkgPath}) is missing the "workspaces" field in package.json.`,
+                  `Add "${pkgsDir}/*" to workspaces run "${pmc.install}".`,
+                ]
+              : [
+                  `The imported project (${pkgPath}) is missing the "packages" field in pnpm-workspaces.yaml.`,
+                  `Add "${pkgsDir}/*" to packages run "${pmc.install}".`,
+                ],
+        });
+      }
+    }
+  }
+}

packages/nx/src/command-line/import/utils/prepare-source-repo.ts

@@ -1,6 +1,6 @@
 import * as createSpinner from 'ora';
-import { basename, dirname, join, relative } from 'path';
-import { copyFile, mkdir, rm } from 'node:fs/promises';
+import { dirname, join, relative } from 'path';
+import { mkdir, rm } from 'node:fs/promises';
 import { GitRepository } from '../../../utils/git-utils';
 
 export async function prepareSourceRepo(
@@ -9,124 +9,66 @@ export async function prepareSourceRepo(
   source: string,
   relativeDestination: string,
   tempImportBranch: string,
-  sourceRemoteUrl: string,
-  originName: string
+  sourceRemoteUrl: string
 ) {
   const spinner = createSpinner().start(
     `Fetching ${ref} from ${sourceRemoteUrl}`
   );
-  await gitClient.addFetchRemote(originName, ref);
-  await gitClient.fetch(originName, ref);
-  spinner.succeed(`Fetched ${ref} from ${sourceRemoteUrl}`);
-  spinner.start(
-    `Checking out a temporary branch, ${tempImportBranch} based on ${ref}`
-  );
-  await gitClient.checkout(tempImportBranch, {
-    new: true,
-    base: `${originName}/${ref}`,
-  });
-  spinner.succeed(`Created a ${tempImportBranch} branch based on ${ref}`);
   const relativeSourceDir = relative(
     gitClient.root,
     join(gitClient.root, source)
   );
 
+  if (relativeSourceDir !== '') {
+    if (await gitClient.hasFilterRepoInstalled()) {
+      spinner.start(
+        `Filtering git history to only include files in ${relativeSourceDir}`
+      );
+      await gitClient.filterRepo(relativeSourceDir);
+    } else {
+      spinner.start(
+        `Filtering git history to only include files in ${relativeSourceDir} (this might take a few minutes -- install git-filter-repo for faster performance)`
+      );
+      await gitClient.filterBranch(relativeSourceDir, tempImportBranch);
+    }
+    spinner.succeed(
+      `Filtered git history to only include files in ${relativeSourceDir}`
+    );
+  }
+
   const destinationInSource = join(gitClient.root, relativeDestination);
   spinner.start(`Moving files and git history to ${destinationInSource}`);
-  if (relativeSourceDir === '') {
-    const files = await gitClient.getGitFiles('.');
+
+  // The result of filter-branch will contain only the files in the subdirectory at its root.
+  const files = await gitClient.getGitFiles('.');
+  try {
+    await rm(destinationInSource, {
+      recursive: true,
+    });
+  } catch {}
+  await mkdir(destinationInSource, { recursive: true });
+  for (const file of files) {
+    spinner.start(
+      `Moving files and git history to ${destinationInSource}: ${file}`
+    );
+
+    const newPath = join(destinationInSource, file);
+
+    await mkdir(dirname(newPath), { recursive: true });
     try {
-      await rm(destinationInSource, {
-        recursive: true,
-      });
-    } catch {}
-    await mkdir(destinationInSource, { recursive: true });
-    const gitignores = new Set<string>();
-    for (const file of files) {
-      if (basename(file) === '.gitignore') {
-        gitignores.add(file);
-        continue;
-      }
-      spinner.start(
-        `Moving files and git history to ${destinationInSource}: ${file}`
-      );
-
-      const newPath = join(destinationInSource, file);
-
-      await mkdir(dirname(newPath), { recursive: true });
-      try {
-        await gitClient.move(file, newPath);
-      } catch {
-        await wait(100);
-        await gitClient.move(file, newPath);
-      }
-    }
-
-    await gitClient.commit(
-      `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
-    );
-
-    for (const gitignore of gitignores) {
-      await gitClient.move(gitignore, join(destinationInSource, gitignore));
-    }
-    await gitClient.amendCommit();
-    for (const gitignore of gitignores) {
-      await copyFile(
-        join(destinationInSource, gitignore),
-        join(gitClient.root, gitignore)
-      );
-    }
-  } else {
-    let needsSquash = false;
-    const needsMove = destinationInSource !== join(gitClient.root, source);
-    if (needsMove) {
-      try {
-        await rm(destinationInSource, {
-          recursive: true,
-        });
-        await gitClient.commit(
-          `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
-        );
-        needsSquash = true;
-      } catch {}
-
-      await mkdir(destinationInSource, { recursive: true });
-    }
-
-    const files = await gitClient.getGitFiles('.');
-    for (const file of files) {
-      if (file === '.gitignore') {
-        continue;
-      }
-      spinner.start(
-        `Moving files and git history to ${destinationInSource}: ${file}`
-      );
-
-      if (!relative(source, file).startsWith('..')) {
-        if (needsMove) {
-          const newPath = join(destinationInSource, file);
-
-          await mkdir(dirname(newPath), { recursive: true });
-          try {
-            await gitClient.move(file, newPath);
-          } catch {
-            await wait(100);
-            await gitClient.move(file, newPath);
-          }
-        }
-      } else {
-        await rm(join(gitClient.root, file), {
-          recursive: true,
-        });
-      }
-    }
-    await gitClient.commit(
-      `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
-    );
-    if (needsSquash) {
-      await gitClient.squashLastTwoCommits();
+      await gitClient.move(file, newPath);
+    } catch {
+      await wait(100);
+      await gitClient.move(file, newPath);
     }
   }
+
+  await gitClient.commit(
+    `chore(repo): move ${source} to ${relativeDestination} to prepare to be imported`
+  );
+
+  await gitClient.amendCommit();
+
   spinner.succeed(
     `${sourceRemoteUrl} has been prepared to be imported into this workspace on a temporary branch: ${tempImportBranch} in ${gitClient.root}`
   );

packages/nx/src/utils/__snapshots__/git-utils.spec.ts.snap

@@ -1,37 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`git utils tests updateRebaseFile should squash the last 2 commits 1`] = `
-"pick 6a642190 chore(repo): hi
-pick 022528d9 chore(repo): prepare for import
-fixup 84ef7741 feat(repo): complete import of git@github.com:FrozenPandaz/created-vite-app.git
-
-# Rebase 3441f39e..84ef7741 onto 3441f39e (3 commands)
-#
-# Commands:
-# p, pick <commit> = use commit
-# r, reword <commit> = use commit, but edit the commit message
-# e, edit <commit> = use commit, but stop for amending
-# s, squash <commit> = use commit, but meld into previous commit
-# f, fixup [-C | -c] <commit> = like "squash" but keep only the previous
-#                    commit's log message, unless -C is used, in which case
-#                    keep only this commit's message; -c is same as -C but
-#                    opens the editor
-# x, exec <command> = run command (the rest of the line) using shell
-# b, break = stop here (continue rebase later with 'git rebase --continue')
-# d, drop <commit> = remove commit
-# l, label <label> = label current HEAD with a name
-# t, reset <label> = reset HEAD to a label
-# m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
-#         create a merge commit using the original merge commit's
-#         message (or the oneline, if no original merge commit was
-#         specified); use -c <commit> to reword the commit message
-# u, update-ref <ref> = track a placeholder for the <ref> to be updated
-#                       to this position in the new commits. The <ref> is
-#                       updated at the end of the rebase
-#
-# These lines can be re-ordered; they are executed from top to bottom.
-#
-# If you remove a line here THAT COMMIT WILL BE LOST.
-#
-# However, if you remove everything, the rebase will be aborted."
-`;

packages/nx/src/utils/git-utils.spec.ts

@@ -1,7 +1,6 @@
 import {
   extractUserAndRepoFromGitHubUrl,
   getGithubSlugOrNull,
-  updateRebaseFile,
 } from './git-utils';
 import { execSync } from 'child_process';
 
@@ -221,48 +220,4 @@ describe('git utils tests', () => {
       });
     });
   });
-
-  describe('updateRebaseFile', () => {
-    let rebaseFileContents;
-
-    beforeEach(() => {
-      rebaseFileContents = `pick 6a642190 chore(repo): hi
-pick 022528d9 chore(repo): prepare for import
-pick 84ef7741 feat(repo): complete import of git@github.com:FrozenPandaz/created-vite-app.git
-
-# Rebase 3441f39e..84ef7741 onto 3441f39e (3 commands)
-#
-# Commands:
-# p, pick <commit> = use commit
-# r, reword <commit> = use commit, but edit the commit message
-# e, edit <commit> = use commit, but stop for amending
-# s, squash <commit> = use commit, but meld into previous commit
-# f, fixup [-C | -c] <commit> = like "squash" but keep only the previous
-#                    commit's log message, unless -C is used, in which case
-#                    keep only this commit's message; -c is same as -C but
-#                    opens the editor
-# x, exec <command> = run command (the rest of the line) using shell
-# b, break = stop here (continue rebase later with 'git rebase --continue')
-# d, drop <commit> = remove commit
-# l, label <label> = label current HEAD with a name
-# t, reset <label> = reset HEAD to a label
-# m, merge [-C <commit> | -c <commit>] <label> [# <oneline>]
-#         create a merge commit using the original merge commit's
-#         message (or the oneline, if no original merge commit was
-#         specified); use -c <commit> to reword the commit message
-# u, update-ref <ref> = track a placeholder for the <ref> to be updated
-#                       to this position in the new commits. The <ref> is
-#                       updated at the end of the rebase
-#
-# These lines can be re-ordered; they are executed from top to bottom.
-#
-# If you remove a line here THAT COMMIT WILL BE LOST.
-#
-# However, if you remove everything, the rebase will be aborted.`;
-    });
-
-    it('should squash the last 2 commits', () => {
-      expect(updateRebaseFile(rebaseFileContents)).toMatchSnapshot();
-    });
-  });
 });

packages/nx/src/utils/git-utils.ts

@@ -1,8 +1,6 @@
-import { exec, ExecOptions, execSync, ExecSyncOptions } from 'child_process';
+import { exec, ExecOptions, execSync } from 'child_process';
+import { dirname, posix, sep } from 'path';
 import { logger } from '../devkit-exports';
-import { dirname, join } from 'path';
-
-const SQUASH_EDITOR = join(__dirname, 'squash.js');
 
 function execAsync(command: string, execOptions: ExecOptions) {
   return new Promise<string>((res, rej) => {
@@ -18,12 +16,17 @@ function execAsync(command: string, execOptions: ExecOptions) {
 export async function cloneFromUpstream(
   url: string,
   destination: string,
-  { originName } = { originName: 'origin' }
+  { originName, depth }: { originName: string; depth?: number } = {
+    originName: 'origin',
+  }
 ) {
   await execAsync(
-    `git clone ${url} ${destination} --depth 1 --origin ${originName}`,
+    `git clone ${url} ${destination} ${
+      depth ? `--depth ${depth}` : ''
+    } --origin ${originName}`,
     {
       cwd: dirname(destination),
+      maxBuffer: 10 * 1024 * 1024,
     }
   );
 
@@ -42,18 +45,12 @@ export class GitRepository {
       .trim();
   }
 
-  addFetchRemote(remoteName: string, branch: string) {
-    return this.execAsync(
+  async addFetchRemote(remoteName: string, branch: string) {
+    return await this.execAsync(
       `git config --add remote.${remoteName}.fetch "+refs/heads/${branch}:refs/remotes/${remoteName}/${branch}"`
     );
   }
 
-  private execAsync(command: string) {
-    return execAsync(command, {
-      cwd: this.root,
-    });
-  }
-
   async showStat() {
     return await this.execAsync(`git show --stat`);
   }
@@ -71,30 +68,26 @@ export class GitRepository {
   }
 
   async getGitFiles(path: string) {
-    return (await this.execAsync(`git ls-files ${path}`))
+    // Use -z to return file names exactly as they are stored in git, separated by NULL (\x00) character.
+    // This avoids problems with special characters in file names.
+    return (await this.execAsync(`git ls-files -z ${path}`))
       .trim()
-      .split('\n')
+      .split('\x00')
       .map((s) => s.trim())
       .filter(Boolean);
   }
 
   async reset(ref: string) {
-    return this.execAsync(`git reset ${ref} --hard`);
-  }
-
-  async squashLastTwoCommits() {
-    return this.execAsync(
-      `git -c core.editor="node ${SQUASH_EDITOR}" rebase --interactive --no-autosquash HEAD~2`
-    );
+    return await this.execAsync(`git reset ${ref} --hard`);
   }
 
   async mergeUnrelatedHistories(ref: string, message: string) {
-    return this.execAsync(
+    return await this.execAsync(
       `git merge ${ref} -X ours --allow-unrelated-histories -m "${message}"`
     );
   }
   async fetch(remote: string, ref?: string) {
-    return this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`);
+    return await this.execAsync(`git fetch ${remote}${ref ? ` ${ref}` : ''}`);
   }
 
   async checkout(
@@ -104,7 +97,7 @@ export class GitRepository {
       base: string;
     }
   ) {
-    return this.execAsync(
+    return await this.execAsync(
       `git checkout ${opts.new ? '-b ' : ' '}${branch}${
         opts.base ? ' ' + opts.base : ''
       }`
@@ -112,50 +105,77 @@ export class GitRepository {
   }
 
   async move(path: string, destination: string) {
-    return this.execAsync(`git mv "${path}" "${destination}"`);
+    return await this.execAsync(
+      `git mv ${this.quotePath(path)} ${this.quotePath(destination)}`
+    );
   }
 
   async push(ref: string, remoteName: string) {
-    return this.execAsync(`git push -u -f ${remoteName} ${ref}`);
+    return await this.execAsync(`git push -u -f ${remoteName} ${ref}`);
   }
 
   async commit(message: string) {
-    return this.execAsync(`git commit -am "${message}"`);
+    return await this.execAsync(`git commit -am "${message}"`);
   }
   async amendCommit() {
-    return this.execAsync(`git commit --amend -a --no-edit`);
+    return await this.execAsync(`git commit --amend -a --no-edit`);
   }
 
-  deleteGitRemote(name: string) {
-    return this.execAsync(`git remote rm ${name}`);
+  async deleteGitRemote(name: string) {
+    return await this.execAsync(`git remote rm ${name}`);
   }
 
-  deleteBranch(branch: string) {
-    return this.execAsync(`git branch -D ${branch}`);
+  async addGitRemote(name: string, url: string) {
+    return await this.execAsync(`git remote add ${name} ${url}`);
   }
 
-  addGitRemote(name: string, url: string) {
-    return this.execAsync(`git remote add ${name} ${url}`);
+  async hasFilterRepoInstalled() {
+    try {
+      await this.execAsync(`git filter-repo --help`);
+      return true;
+    } catch {
+      return false;
+    }
   }
-}
 
-/**
- * This is used by the squash editor script to update the rebase file.
- */
-export function updateRebaseFile(contents: string): string {
-  const lines = contents.split('\n');
-  const lastCommitIndex = lines.findIndex((line) => line === '') - 1;
+  // git-filter-repo is much faster than filter-branch, but needs to be installed by user
+  // Use `hasFilterRepoInstalled` to check if it's installed
+  async filterRepo(subdirectory: string) {
+    // filter-repo requires POSIX path to work
+    const posixPath = subdirectory.split(sep).join(posix.sep);
+    return await this.execAsync(
+      `git filter-repo -f --subdirectory-filter ${this.quotePath(posixPath)}`
+    );
+  }
 
-  lines[lastCommitIndex] = lines[lastCommitIndex].replace('pick', 'fixup');
-  return lines.join('\n');
-}
+  async filterBranch(subdirectory: string, branchName: string) {
+    // filter-repo requires POSIX path to work
+    const posixPath = subdirectory.split(sep).join(posix.sep);
+    // We need non-ASCII file names to not be quoted, or else filter-branch will exclude them.
+    await this.execAsync(`git config core.quotepath false`);
+    return await this.execAsync(
+      `git filter-branch --subdirectory-filter ${this.quotePath(
+        posixPath
+      )} -- ${branchName}`
+    );
+  }
 
-export function fetchGitRemote(
-  name: string,
-  branch: string,
-  execOptions: ExecSyncOptions
-) {
-  return execSync(`git fetch ${name} ${branch} --depth 1`, execOptions);
+  private execAsync(command: string) {
+    return execAsync(command, {
+      cwd: this.root,
+      maxBuffer: 10 * 1024 * 1024,
+    });
+  }
+
+  private quotePath(path: string) {
+    return process.platform === 'win32'
+      ? // Windows/CMD only understands double-quotes, single-quotes are treated as part of the file name
+        // Bash and other shells will substitute `$` in file names with a variable value.
+        `"${path}"`
+      : // e.g. `git mv "$$file.txt" "libs/a/$$file.txt"` will not work since `$$` is swapped with the PID of the last process.
+        // Using single-quotes prevents this substitution.
+        `'${path}'`;
+  }
 }
 
 /**

packages/nx/src/utils/squash.ts

@@ -1,14 +0,0 @@
-import { readFileSync, writeFileSync } from 'fs';
-import { updateRebaseFile } from './git-utils';
-
-// This script is used as an editor for git rebase -i
-
-// This is the file which git creates. When this script exits, the updates should be written to this file.
-const filePath = process.argv[2];
-
-// Change the second commit from pick to fixup
-const contents = readFileSync(filePath).toString();
-const newContents = updateRebaseFile(contents);
-
-// Write the updated contents back to the file
-writeFileSync(filePath, newContents);