fix: hash check when downloading Yarn Berry from npm (#439)

Co-authored-by: Antoine du Hamel <duhamelantoine1995@gmail.com>
Co-authored-by: Maël Nison <nison.mael@gmail.com>

Author: 3 people

config.json

@@ -149,7 +149,8 @@
           },
           "npmRegistry": {
             "type": "npm",
-            "package": "@yarnpkg/cli-dist"
+            "package": "@yarnpkg/cli-dist",
+            "bin": "bin/yarn.js"
           },
           "commands": {
             "use": [

sources/corepackUtils.ts

@@ -15,7 +15,7 @@ import * as httpUtils                                          from './httpUtils
 import * as nodeUtils                                          from './nodeUtils';
 import * as npmRegistryUtils                                   from './npmRegistryUtils';
 import {RegistrySpec, Descriptor, Locator, PackageManagerSpec} from './types';
-import {BinList, BinSpec, InstallSpec}                         from './types';
+import {BinList, BinSpec, InstallSpec, DownloadSpec}           from './types';
 
 export function getRegistryFromPackageManagerSpec(spec: PackageManagerSpec) {
   return process.env.COREPACK_NPM_REGISTRY
@@ -132,6 +132,66 @@ function isValidBinSpec(x: unknown): x is BinSpec {
   return typeof x === `object` && x !== null && !Array.isArray(x) && Object.keys(x).length > 0;
 }
 
+async function download(installTarget: string, url: string, algo: string, binPath: string | null = null): Promise<DownloadSpec> {
+  // Creating a temporary folder inside the install folder means that we
+  // are sure it'll be in the same drive as the destination, so we can
+  // just move it there atomically once we are done
+
+  const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
+  debugUtils.log(`Downloading to ${tmpFolder}`);
+
+  const stream = await httpUtils.fetchUrlStream(url);
+
+  const parsedUrl = new URL(url);
+  const ext = path.posix.extname(parsedUrl.pathname);
+
+  let outputFile: string | null = null;
+  let sendTo: any;
+
+  if (ext === `.tgz`) {
+    const {default: tar} = await import(`tar`);
+    sendTo = tar.x({
+      strip: 1,
+      cwd: tmpFolder,
+      filter: binPath ? path => {
+        const pos = path.indexOf(`/`);
+        return pos !== -1 && path.slice(pos + 1) === binPath;
+      } : undefined,
+    });
+  } else if (ext === `.js`) {
+    outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
+    sendTo = fs.createWriteStream(outputFile);
+  }
+  stream.pipe(sendTo);
+
+  let hash = !binPath ? stream.pipe(createHash(algo)) : null;
+  await once(sendTo, `finish`);
+
+  if (binPath) {
+    const downloadedBin = path.join(tmpFolder, binPath);
+    outputFile = path.join(tmpFolder, path.basename(downloadedBin));
+    try {
+      await renameSafe(downloadedBin, outputFile);
+    } catch (err) {
+      if ((err as nodeUtils.NodeError)?.code === `ENOENT`)
+        throw new Error(`Cannot locate '${binPath}' in downloaded tarball`, {cause: err});
+
+      throw err;
+    }
+
+    // Calculate the hash of the bin file
+    const fileStream = fs.createReadStream(outputFile);
+    hash = fileStream.pipe(createHash(algo));
+    await once(fileStream, `close`);
+  }
+
+  return {
+    tmpFolder,
+    outputFile,
+    hash: hash!.digest(`hex`),
+  };
+}
+
 export async function installVersion(installTarget: string, locator: Locator, {spec}: {spec: PackageManagerSpec}): Promise<InstallSpec> {
   const locatorIsASupportedPackageManager = isSupportedPackageManagerLocator(locator);
   const locatorReference = locatorIsASupportedPackageManager ? semver.parse(locator.reference)! : parseURLReference(locator);
@@ -159,12 +219,16 @@ export async function installVersion(installTarget: string, locator: Locator, {s
   }
 
   let url: string;
+  let binPath: string | null = null;
   if (locatorIsASupportedPackageManager) {
     url = spec.url.replace(`{}`, version);
     if (process.env.COREPACK_NPM_REGISTRY) {
       const registry = getRegistryFromPackageManagerSpec(spec);
       if (registry.type === `npm`) {
         url = await npmRegistryUtils.fetchTarballUrl(registry.package, version);
+        if (registry.bin) {
+          binPath = registry.bin;
+        }
       } else {
         url = url.replace(
           npmRegistryUtils.DEFAULT_NPM_REGISTRY_URL,
@@ -182,33 +246,9 @@ export async function installVersion(installTarget: string, locator: Locator, {s
     }
   }
 
-  // Creating a temporary folder inside the install folder means that we
-  // are sure it'll be in the same drive as the destination, so we can
-  // just move it there atomically once we are done
-
-  const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
-  debugUtils.log(`Installing ${locator.name}@${version} from ${url} to ${tmpFolder}`);
-  const stream = await httpUtils.fetchUrlStream(url);
-
-  const parsedUrl = new URL(url);
-  const ext = path.posix.extname(parsedUrl.pathname);
-
-  let outputFile: string | null = null;
-
-  let sendTo: any;
-  if (ext === `.tgz`) {
-    const {default: tar} = await import(`tar`);
-    sendTo = tar.x({strip: 1, cwd: tmpFolder});
-  } else if (ext === `.js`) {
-    outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
-    sendTo = fs.createWriteStream(outputFile);
-  }
-
-  stream.pipe(sendTo);
-
+  debugUtils.log(`Installing ${locator.name}@${version} from ${url}`);
   const algo = build[0] ?? `sha256`;
-  const hash = stream.pipe(createHash(algo));
-  await once(sendTo, `finish`);
+  const {tmpFolder, outputFile, hash: actualHash} = await download(installTarget, url, algo, binPath);
 
   let bin: BinSpec | BinList;
   const isSingleFile = outputFile !== null;
@@ -240,7 +280,6 @@ export async function installVersion(installTarget: string, locator: Locator, {s
     }
   }
 
-  const actualHash = hash.digest(`hex`);
   if (build[1] && actualHash !== build[1])
     throw new Error(`Mismatch hashes. Expected ${build[1]}, got ${actualHash}`);
 
@@ -305,6 +344,14 @@ export async function installVersion(installTarget: string, locator: Locator, {s
   };
 }
 
+async function renameSafe(oldPath: fs.PathLike, newPath: fs.PathLike) {
+  if (process.platform === `win32`) {
+    await renameUnderWindows(oldPath, newPath);
+  } else {
+    await fs.promises.rename(oldPath, newPath);
+  }
+}
+
 async function renameUnderWindows(oldPath: fs.PathLike, newPath: fs.PathLike) {
   // Windows malicious file analysis blocks files currently under analysis, so we need to wait for file release
   const retries = 5;

sources/types.ts

@@ -25,6 +25,7 @@ export function isSupportedPackageManager(value: string): value is SupportedPack
 export interface NpmRegistrySpec {
   type: `npm`;
   package: string;
+  bin?: string;
 }
 
 export interface UrlRegistrySpec {
@@ -59,6 +60,12 @@ export interface InstallSpec {
   hash: string;
 }
 
+export interface DownloadSpec {
+  tmpFolder: string;
+  outputFile: string | null;
+  hash: string;
+}
+
 /**
  * The data structure found in config.json
  */

tests/main.test.ts

@@ -787,19 +787,19 @@ it(`should download yarn berry from custom registry`, async () => {
     process.env.COREPACK_ENABLE_DOWNLOAD_PROMPT = `1`;
 
     await xfs.writeJsonPromise(ppath.join(cwd, `package.json` as Filename), {
-      packageManager: `yarn@3.0.0`,
+      packageManager: `yarn@3.0.0-rc.2+sha224.f83f6d1cbfac10ba6b516a62ccd2a72ccd857aa6c514d1cd7185ec60`,
     });
 
     await expect(runCli(cwd, [`yarn`, `--version`])).resolves.toMatchObject({
       exitCode: 0,
-      stdout: `3.0.0\n`,
-      stderr: `! Corepack is about to download https://registry.npmmirror.com/@yarnpkg/cli-dist/-/cli-dist-3.0.0.tgz\n`,
+      stdout: `3.0.0-rc.2\n`,
+      stderr: `! Corepack is about to download https://registry.npmmirror.com/@yarnpkg/cli-dist/-/cli-dist-3.0.0-rc.2.tgz\n`,
     });
 
     // Should keep working with cache
     await expect(runCli(cwd, [`yarn`, `--version`])).resolves.toMatchObject({
       exitCode: 0,
-      stdout: `3.0.0\n`,
+      stdout: `3.0.0-rc.2\n`,
       stderr: ``,
     });
   });