Merge branch 'main' into bump-node-version-14.21.3

Author: joshuarrrr

CHANGELOG.md

@@ -62,6 +62,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - [Data] Update `createAggConfig` so that newly created configs can be added to beginning of `aggConfig` array ([#3160](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3160))
 - Add disablePrototypePoisoningProtection configuration to prevent JS client from erroring when cluster utilizes JS reserved words ([#2992](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/2992))
 - [Multiple DataSource] Add support for SigV4 authentication ([#3058](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3058))
+- Make build scripts find and use the latest version of Node.js that satisfies `engines.node` ([#3467](https://github.com/opensearch-project/OpenSearch-Dashboards/issues/3467))
 
 ### 🐛 Bug Fixes
 
@@ -110,6 +111,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Add recording of functional test artifacts if they fail ([#3190](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3190))
 - Improve yarn's performance in workflows by caching yarn's cache folder ([#3194](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3194))
 - Fix detection of Chrome's version on Darwin during CI ([#3296](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3296))
+- Upgrade yarn version to be compatible with @openearch-project/opensearch ([#3443](https://github.com/opensearch-project/OpenSearch-Dashboards/pull/3443))
 
 ### 📝 Documentation
 

package.json

@@ -467,6 +467,6 @@
   },
   "engines": {
     "node": "^14.20.1",
-    "yarn": "^1.21.1"
+    "yarn": "^1.22.10"
   }
 }

src/dev/build/lib/config.test.ts

@@ -79,10 +79,10 @@ describe('#getOpenSearchDashboardsPkg()', () => {
   });
 });
 
-describe('#getNodeVersion()', () => {
-  it('returns the node version from the OpenSearch Dashboards package.json', async () => {
+describe('#getNodeRange()', () => {
+  it('returns the node version range from the OpenSearch Dashboards package.json', async () => {
     const config = await setup();
-    expect(config.getNodeVersion()).toEqual(pkg.engines.node);
+    expect(config.getNodeRange()).toEqual(pkg.engines.node);
   });
 });
 

src/dev/build/lib/config.ts

@@ -86,7 +86,7 @@ export class Config {
     private readonly targetAllPlatforms: boolean,
     private readonly targetPlatforms: TargetPlatforms,
     private readonly pkg: Package,
-    private readonly nodeVersion: string,
+    private readonly nodeRange: string,
     private readonly repoRoot: string,
     private readonly versionInfo: VersionInfo,
     public readonly isRelease: boolean
@@ -102,8 +102,8 @@ export class Config {
   /**
    * Get the node version required by OpenSearch Dashboards
    */
-  getNodeVersion() {
-    return this.nodeVersion;
+  getNodeRange() {
+    return this.nodeRange;
   }
 
   /**

src/dev/build/tasks/create_archives_sources_task.ts

@@ -50,7 +50,7 @@ export const CreateArchivesSources: Task = {
 
         // copy node.js install
         await scanCopy({
-          source: getNodeDownloadInfo(config, platform).extractDir,
+          source: (await getNodeDownloadInfo(config, platform)).extractDir,
           destination: build.resolvePathForPlatform(platform, 'node'),
         });
 

src/dev/build/tasks/nodejs/download_node_builds_task.ts

@@ -30,16 +30,17 @@
 
 import { download, GlobalTask } from '../../lib';
 import { getNodeShasums } from './node_shasums';
-import { getNodeDownloadInfo } from './node_download_info';
+import { getLatestNodeVersion, getNodeDownloadInfo } from './node_download_info';
 
 export const DownloadNodeBuilds: GlobalTask = {
   global: true,
   description: 'Downloading node.js builds for all platforms',
   async run(config, log) {
-    const shasums = await getNodeShasums(log, config.getNodeVersion());
+    const latestNodeVersion = await getLatestNodeVersion(config);
+    const shasums = await getNodeShasums(log, latestNodeVersion);
     await Promise.all(
       config.getTargetPlatforms().map(async (platform) => {
-        const { url, downloadPath, downloadName } = getNodeDownloadInfo(config, platform);
+        const { url, downloadPath, downloadName } = await getNodeDownloadInfo(config, platform);
         await download({
           log,
           url,

src/dev/build/tasks/nodejs/extract_node_builds_task.test.ts

@@ -28,10 +28,6 @@
  * under the License.
  */
 
-import { readFileSync } from 'fs';
-import Path from 'path';
-
-import { REPO_ROOT } from '@osd/utils';
 import {
   ToolingLog,
   ToolingLogCollectingWriter,
@@ -41,6 +37,7 @@ import {
 
 import { Config } from '../../lib';
 import { ExtractNodeBuilds } from './extract_node_builds_task';
+import { getLatestNodeVersion } from './node_download_info';
 
 jest.mock('../../lib/fs');
 jest.mock('../../lib/get_build_number');
@@ -53,14 +50,6 @@ log.setWriters([testWriter]);
 
 expect.addSnapshotSerializer(createAbsolutePathSerializer());
 
-const nodeVersion = readFileSync(Path.resolve(REPO_ROOT, '.node-version'), 'utf8').trim();
-expect.addSnapshotSerializer(
-  createRecursiveSerializer(
-    (s) => typeof s === 'string' && s.includes(nodeVersion),
-    (s) => s.split(nodeVersion).join('<node version>')
-  )
-);
-
 async function setup() {
   const config = await Config.create({
     isRelease: true,
@@ -73,6 +62,16 @@ async function setup() {
     },
   });
 
+  const realNodeVersion = await getLatestNodeVersion(config);
+  if (realNodeVersion) {
+    expect.addSnapshotSerializer(
+      createRecursiveSerializer(
+        (s) => typeof s === 'string' && s.includes(realNodeVersion),
+        (s) => s.split(realNodeVersion).join('<node version>')
+      )
+    );
+  }
+
   return { config };
 }
 

src/dev/build/tasks/nodejs/extract_node_builds_task.ts

@@ -37,7 +37,7 @@ export const ExtractNodeBuilds: GlobalTask = {
   async run(config) {
     await Promise.all(
       config.getTargetPlatforms().map(async (platform) => {
-        const { downloadPath, extractDir } = getNodeDownloadInfo(config, platform);
+        const { downloadPath, extractDir } = await getNodeDownloadInfo(config, platform);
         if (platform.isWindows()) {
           await unzip(downloadPath, extractDir, { strip: 1 });
         } else {

src/dev/build/tasks/nodejs/node_download_info.ts

@@ -29,11 +29,15 @@
  */
 
 import { basename } from 'path';
+import fetch from 'node-fetch';
+import semver from 'semver';
 
 import { Config, Platform } from '../../lib';
 
-export function getNodeDownloadInfo(config: Config, platform: Platform) {
-  const version = config.getNodeVersion();
+const NODE_RANGE_CACHE: { [key: string]: string } = {};
+
+export async function getNodeDownloadInfo(config: Config, platform: Platform) {
+  const version = await getLatestNodeVersion(config);
   const arch = platform.getNodeArch();
 
   const downloadName = platform.isWindows()
@@ -52,3 +56,23 @@ export function getNodeDownloadInfo(config: Config, platform: Platform) {
     version,
   };
 }
+
+export async function getLatestNodeVersion(config: Config) {
+  const range = config.getNodeRange();
+  // Check cache and return if known
+  if (NODE_RANGE_CACHE[range]) return NODE_RANGE_CACHE[range];
+
+  const releaseDoc = await fetch('https://nodejs.org/dist/index.json');
+  const releaseList: [{ version: string }] = await releaseDoc.json();
+  const releases = releaseList.map(({ version }) => version.replace(/^v/, ''));
+  const maxVersion = semver.maxSatisfying(releases, range);
+
+  if (!maxVersion) {
+    throw new Error(`Cannot find a version of Node.js that satisfies ${range}.`);
+  }
+
+  // Cache it
+  NODE_RANGE_CACHE[range] = maxVersion;
+
+  return maxVersion;
+}

src/dev/build/tasks/nodejs/verify_existing_node_builds_task.test.ts

@@ -28,15 +28,10 @@
  * under the License.
  */
 
-import Path from 'path';
-import Fs from 'fs';
-
-import { REPO_ROOT } from '@osd/utils';
 import {
   ToolingLog,
   ToolingLogCollectingWriter,
   createAnyInstanceSerializer,
-  createRecursiveSerializer,
 } from '@osd/dev-utils';
 
 import { Config, Platform } from '../../lib';
@@ -48,7 +43,7 @@ jest.mock('../../lib/fs');
 jest.mock('../../lib/get_build_number');
 
 const { getNodeShasums } = jest.requireMock('./node_shasums');
-const { getNodeDownloadInfo } = jest.requireMock('./node_download_info');
+const { getNodeDownloadInfo, getLatestNodeVersion } = jest.requireMock('./node_download_info');
 const { getFileHash } = jest.requireMock('../../lib/fs');
 
 const log = new ToolingLog();
@@ -58,14 +53,6 @@ log.setWriters([testWriter]);
 expect.addSnapshotSerializer(createAnyInstanceSerializer(Config));
 expect.addSnapshotSerializer(createAnyInstanceSerializer(ToolingLog));
 
-const nodeVersion = Fs.readFileSync(Path.resolve(REPO_ROOT, '.node-version'), 'utf8').trim();
-expect.addSnapshotSerializer(
-  createRecursiveSerializer(
-    (s) => typeof s === 'string' && s.includes(nodeVersion),
-    (s) => s.split(nodeVersion).join('<node version>')
-  )
-);
-
 async function setup(actualShaSums?: Record<string, string>) {
   const config = await Config.create({
     isRelease: true,
@@ -74,6 +61,7 @@ async function setup(actualShaSums?: Record<string, string>) {
       linux: false,
       linuxArm: false,
       darwin: false,
+      windows: false,
     },
   });
 
@@ -89,9 +77,12 @@ async function setup(actualShaSums?: Record<string, string>) {
     return {
       downloadPath: `${platform.getName()}:${platform.getNodeArch()}:downloadPath`,
       downloadName: `${platform.getName()}:${platform.getNodeArch()}:downloadName`,
+      version: '<node version>',
     };
   });
 
+  getLatestNodeVersion.mockReturnValue('<node version>');
+
   getFileHash.mockImplementation((downloadPath: string) => {
     if (actualShaSums?.[downloadPath]) {
       return actualShaSums[downloadPath];
@@ -176,27 +167,31 @@ it('checks shasums for each downloaded node build', async () => {
           "value": Object {
             "downloadName": "linux:linux-x64:downloadName",
             "downloadPath": "linux:linux-x64:downloadPath",
+            "version": "<node version>",
           },
         },
         Object {
           "type": "return",
           "value": Object {
             "downloadName": "linux:linux-arm64:downloadName",
             "downloadPath": "linux:linux-arm64:downloadPath",
+            "version": "<node version>",
           },
         },
         Object {
           "type": "return",
           "value": Object {
             "downloadName": "darwin:darwin-x64:downloadName",
             "downloadPath": "darwin:darwin-x64:downloadPath",
+            "version": "<node version>",
           },
         },
         Object {
           "type": "return",
           "value": Object {
             "downloadName": "win32:win32-x64:downloadName",
             "downloadPath": "win32:win32-x64:downloadPath",
+            "version": "<node version>",
           },
         },
       ],

src/dev/build/tasks/nodejs/verify_existing_node_builds_task.ts

@@ -29,18 +29,19 @@
  */
 
 import { getFileHash, GlobalTask } from '../../lib';
-import { getNodeDownloadInfo } from './node_download_info';
+import { getNodeDownloadInfo, getLatestNodeVersion } from './node_download_info';
 import { getNodeShasums } from './node_shasums';
 
 export const VerifyExistingNodeBuilds: GlobalTask = {
   global: true,
   description: 'Verifying previously downloaded node.js build for all platforms',
   async run(config, log) {
-    const shasums = await getNodeShasums(log, config.getNodeVersion());
+    const latestNodeVersion = await getLatestNodeVersion(config);
+    const shasums = await getNodeShasums(log, latestNodeVersion);
 
     await Promise.all(
       config.getTargetPlatforms().map(async (platform) => {
-        const { downloadPath, downloadName } = getNodeDownloadInfo(config, platform);
+        const { downloadPath, downloadName } = await getNodeDownloadInfo(config, platform);
 
         const sha256 = await getFileHash(downloadPath, 'sha256');
         if (sha256 !== shasums[downloadName]) {

src/dev/build/tasks/notice_file_task.ts

@@ -57,7 +57,7 @@ export const CreateNoticeFile: Task = {
 
     log.info('Generating build notice');
 
-    const { extractDir: nodeDir, version: nodeVersion } = getNodeDownloadInfo(
+    const { extractDir: nodeDir, version: nodeVersion } = await getNodeDownloadInfo(
       config,
       config.hasSpecifiedPlatform()
         ? config.getPlatform(

src/dev/build/tasks/verify_env_task.ts

@@ -28,17 +28,20 @@
  * under the License.
  */
 
+import semver from 'semver';
 import { GlobalTask } from '../lib';
 
 export const VerifyEnv: GlobalTask = {
   global: true,
   description: 'Verifying environment meets requirements',
 
   async run(config, log) {
-    const version = `v${config.getNodeVersion()}`;
+    const range = config.getNodeRange();
 
-    if (version !== process.version) {
-      throw new Error(`Invalid nodejs version, please use ${version}`);
+    if (!semver.satisfies(process.version, range)) {
+      throw new Error(
+        `Invalid Node.js version (${process.version}); please use a version that satisfies ${range}.`
+      );
     }
 
     log.success('Node.js version verified');