fix(core): improve yarn tarball detection (#18335)

packages/nx/src/plugins/js/lock-file/yarn-parser.spec.ts

@@ -616,6 +616,191 @@ describe('yarn LockFile utility', () => {
     });
   });
 
+  describe('auxiliary tagged version ranges', () => {
+    beforeEach(() => {
+      const fileSys = {
+        'node_modules/@nrwl/nx-cloud/package.json': '{"version": "16.1.1"}',
+        'node_modules/nx-cloud/package.json': '{"version": "16.1.1"}',
+        'node_modules/postgres/package.json': '{"version": "3.2.4"}',
+      };
+      vol.fromJSON(fileSys, '/root');
+    });
+
+    it('should parse tagged version range in combination with semver', () => {
+      const lockFile = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+"@nrwl/nx-cloud@16.1.1":
+  version "16.1.1"
+  resolved "http://localhost:4873/@nrwl%2fnx-cloud/-/nx-cloud-16.1.1.tgz#9e1ed2acff11ab0ae8a4c0c045d412034ac26be7"
+  integrity sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==
+  dependencies:
+    nx-cloud "16.1.1"
+
+nx-cloud@16.1.1, nx-cloud@latest:
+  version "16.1.1"
+  resolved "http://localhost:4873/nx-cloud/-/nx-cloud-16.1.1.tgz#103ae0f13f5eb05d6ddd6d9bfcafc56cf295a59a"
+  integrity sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==
+  dependencies:
+    "@nrwl/nx-cloud" "16.1.1"
+
+postgres@charsleysa/postgres#fix-errors-compiled:
+  version "3.2.4"
+  resolved "https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb"
+  `;
+      const packageJson: PackageJson = {
+        name: '@my-ns/example',
+        version: '0.0.1',
+        type: 'commonjs',
+        dependencies: {
+          'nx-cloud': 'latest',
+          postgres: 'charsleysa/postgres#fix-errors-compiled',
+        },
+      };
+
+      const builder = new ProjectGraphBuilder();
+      parseYarnLockfile(lockFile, packageJson, builder);
+      const graph = builder.getUpdatedProjectGraph();
+      expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
+        {
+          "data": {
+            "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
+            "packageName": "@nrwl/nx-cloud",
+            "version": "16.1.1",
+          },
+          "name": "npm:@nrwl/nx-cloud",
+          "type": "npm",
+        }
+      `);
+      expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
+        {
+          "data": {
+            "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
+            "packageName": "nx-cloud",
+            "version": "16.1.1",
+          },
+          "name": "npm:nx-cloud",
+          "type": "npm",
+        }
+      `);
+      expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
+        {
+          "data": {
+            "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+            "packageName": "postgres",
+            "version": "https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+          },
+          "name": "npm:postgres",
+          "type": "npm",
+        }
+      `);
+    });
+
+    it('should parse tagged version range in combination with semver in reverse order', () => {
+      const lockFile = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+"@nrwl/nx-cloud@16.1.1":
+  version "16.1.1"
+  resolved "http://localhost:4873/@nrwl%2fnx-cloud/-/nx-cloud-16.1.1.tgz#9e1ed2acff11ab0ae8a4c0c045d412034ac26be7"
+  integrity sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==
+  dependencies:
+    nx-cloud "16.1.1"
+
+nx-cloud@latest, nx-cloud@16.1.1:
+  version "16.1.1"
+  resolved "http://localhost:4873/nx-cloud/-/nx-cloud-16.1.1.tgz#103ae0f13f5eb05d6ddd6d9bfcafc56cf295a59a"
+  integrity sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==
+  dependencies:
+    "@nrwl/nx-cloud" "16.1.1"
+
+postgres@charsleysa/postgres#fix-errors-compiled:
+  version "3.2.4"
+  resolved "https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb"
+    `;
+      const packageJson: PackageJson = {
+        name: '@my-ns/example',
+        version: '0.0.1',
+        type: 'commonjs',
+        dependencies: {
+          'nx-cloud': 'latest',
+          postgres: 'charsleysa/postgres#fix-errors-compiled',
+        },
+      };
+
+      const builder = new ProjectGraphBuilder();
+      parseYarnLockfile(lockFile, packageJson, builder);
+      const graph = builder.getUpdatedProjectGraph();
+      expect(graph.externalNodes['npm:@nrwl/nx-cloud']).toMatchInlineSnapshot(`
+          {
+            "data": {
+              "hash": "sha512-iJIPP46+saFZK748FKU4u4YZH+Sv3ZvZPbMwGVMhwqhOYcrlO5aSa0lpilyoN8WuhooKNqcCfiqshx6V577fTg==",
+              "packageName": "@nrwl/nx-cloud",
+              "version": "16.1.1",
+            },
+            "name": "npm:@nrwl/nx-cloud",
+            "type": "npm",
+          }
+        `);
+      expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
+          {
+            "data": {
+              "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
+              "packageName": "nx-cloud",
+              "version": "16.1.1",
+            },
+            "name": "npm:nx-cloud",
+            "type": "npm",
+          }
+        `);
+      expect(graph.externalNodes['npm:postgres']).toMatchInlineSnapshot(`
+          {
+            "data": {
+              "hash": "postgres|https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+              "packageName": "postgres",
+              "version": "https://codeload.github.com/charsleysa/postgres/tar.gz/3b1a01b2da3e2fafb1a79006f838eff11a8de3cb",
+            },
+            "name": "npm:postgres",
+            "type": "npm",
+          }
+        `);
+    });
+
+    it('should parse tagged version range', () => {
+      const lockFile = `# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+nx-cloud@latest:
+  version "16.1.1"
+  resolved "http://localhost:4873/nx-cloud/-/nx-cloud-16.1.1.tgz#103ae0f13f5eb05d6ddd6d9bfcafc56cf295a59a"
+  integrity sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==
+    `;
+      const packageJson: PackageJson = {
+        name: '@my-ns/example',
+        version: '0.0.1',
+        type: 'commonjs',
+        dependencies: {
+          'nx-cloud': 'latest',
+        },
+      };
+
+      const builder = new ProjectGraphBuilder();
+      parseYarnLockfile(lockFile, packageJson, builder);
+      const graph = builder.getUpdatedProjectGraph();
+      expect(graph.externalNodes['npm:nx-cloud']).toMatchInlineSnapshot(`
+          {
+            "data": {
+              "hash": "sha512-Rq7ynvkYzAJ67N3pDqU6cMqwvWP7WXJGP4EFjLxgUrRHNCccqDPggeAqePodfk3nZEUrZB8F5QBKZuuw1DR3oA==",
+              "packageName": "nx-cloud",
+              "version": "16.1.1",
+            },
+            "name": "npm:nx-cloud",
+            "type": "npm",
+          }
+        `);
+    });
+  });
+
   describe('auxiliary packages PnP', () => {
     it('should parse yarn berry pnp', () => {
       const berryLockFile = require(joinPathFragments(

packages/nx/src/plugins/js/lock-file/yarn-parser.ts

@@ -14,10 +14,6 @@ import { sortObjectByKeys } from '../../../utils/object-sort';
  * - Classic has resolved and integrity
  * - Berry has resolution, checksum, languageName and linkType
  */
-type YarnLockFile = {
-  __metadata?: {};
-} & Record<string, YarnDependency>;
-
 type YarnDependency = {
   version: string;
   dependencies?: Record<string, string>;
@@ -41,12 +37,17 @@ export function parseYarnLockfile(
   builder: ProjectGraphBuilder
 ) {
   const { parseSyml } = require('@yarnpkg/parsers');
-  const data = parseSyml(lockFileContent);
+  const { __metadata, ...dependencies } = parseSyml(lockFileContent);
+  const isBerry = !!__metadata;
 
   // we use key => node map to avoid duplicate work when parsing keys
   const keyMap = new Map<string, ProjectGraphExternalNode>();
-  addNodes(data, packageJson, builder, keyMap);
+
-  addDependencies(data, builder, keyMap);
+  // yarn classic splits keys when parsing so we need to stich them back together
+  const groupedDependencies = groupDependencies(dependencies, isBerry);
+
+  addNodes(groupedDependencies, packageJson, builder, keyMap, isBerry);
+  addDependencies(groupedDependencies, builder, keyMap);
 }
 
 function getPackageNames(keys: string): string[] {
@@ -59,12 +60,12 @@ function getPackageNames(keys: string): string[] {
 }
 
 function addNodes(
-  { __metadata, ...dependencies }: YarnLockFile,
+  dependencies: Record<string, YarnDependency>,
   packageJson: NormalizedPackageJson,
   builder: ProjectGraphBuilder,
-  keyMap: Map<string, ProjectGraphExternalNode>
+  keyMap: Map<string, ProjectGraphExternalNode>,
+  isBerry: boolean
 ) {
-  const isBerry = !!__metadata;
   const nodes: Map<string, Map<string, ProjectGraphExternalNode>> = new Map();
   const combinedDeps = {
     ...packageJson.dependencies,
@@ -200,20 +201,33 @@ function findVersion(
     return snapshot.resolution.slice(packageName.length + 1);
   }
 
-  if (!isBerry && snapshot.resolved && !isValidVersionRange(versionRange)) {
+  if (!isBerry && isTarballPackage(versionRange, snapshot)) {
     return snapshot.resolved;
   }
   // otherwise it's a standard version
   return snapshot.version;
 }
 
-// check if value can be parsed as a semver range
+// check if snapshot represents tarball package
-function isValidVersionRange(versionRange: string): boolean {
+function isTarballPackage(
+  versionRange: string,
+  snapshot: YarnDependency
+): boolean {
+  // if resolved is missing it's internal link
+  if (!snapshot.resolved) {
+    return false;
+  }
+  // tarballs have no integrity
+  if (snapshot.integrity) {
+    return false;
+  }
   try {
     new Range(versionRange);
-    return true;
+    // range is a valid semver
-  } catch {
     return false;
+  } catch {
+    // range is not a valid semver, it can be an npm tag or url part of a tarball
+    return snapshot.version && !snapshot.resolved.includes(snapshot.version);
   }
 }
 
@@ -225,7 +239,7 @@ function getHoistedVersion(packageName: string): string {
 }
 
 function addDependencies(
-  { __metadata, ...dependencies }: YarnLockFile,
+  dependencies: Record<string, YarnDependency>,
   builder: ProjectGraphBuilder,
   keyMap: Map<string, ProjectGraphExternalNode>
 ) {
@@ -288,8 +302,12 @@ export function stringifyYarnLockfile(
 }
 
 function groupDependencies(
-  dependencies: Record<string, YarnDependency>
+  dependencies: Record<string, YarnDependency>,
+  isBerry: boolean
 ): Record<string, YarnDependency> {
+  if (isBerry) {
+    return dependencies;
+  }
   let groupedDependencies: Record<string, YarnDependency>;
   const resolutionMap = new Map<string, YarnDependency>();
   const snapshotMap = new Map<YarnDependency, Set<string>>();
@@ -342,13 +360,8 @@ function mapSnapshots(
     ...packageJson.peerDependencies,
   };
 
-  let groupedDependencies: Record<string, YarnDependency>;
-  if (isBerry) {
-    groupedDependencies = dependencies;
-  } else {
   // yarn classic splits keys when parsing so we need to stich them back together
-    groupedDependencies = groupDependencies(dependencies);
+  const groupedDependencies = groupDependencies(dependencies, isBerry);
-  }
 
   // collect snapshots and their matching keys
   Object.values(nodes).forEach((node) => {