[Tests] updated artifact script

Updating artifact to enable running integration tests and functional
tests to pull an artifact from the current hosted distributions.

At the time of this commit, there is not manifest hosted but there are
static links which can have an unknown amount of RC versions if the
GA snapshot does not exist for that version. But the assumption
is that it will not be too high.

Deprecating the previous implementation but we can remove that in a
future iteration. Wanted to leave that available incase others use
that for custom manifests.

Enable tests that depended on snapshots as well.

Issues resolved:
opensearch-project#242
opensearch-project#19

Signed-off-by: Kawika Avilla <kavilla414@gmail.com>

Author: kavilla

packages/osd-opensearch-archiver/src/lib/docs/__tests__/generate_doc_records_stream.ts

@@ -104,7 +104,7 @@ describe('opensearchArchiver: createGenerateDocRecordsStream()', () => {
         await delay(200);
         return {
           _scroll_id: 'index1ScrollId',
-          hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_foo' }] },
+          hits: { total: 2, hits: [{ _id: 1, _index: '.opensearch_dashboards_1' }] },
         };
       },
       async (name, params) => {

packages/osd-opensearch/src/artifact.js

@@ -29,7 +29,6 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
-
 const fetch = require('node-fetch');
 const AbortController = require('abort-controller');
 const fs = require('fs');
@@ -40,8 +39,11 @@ const { createHash } = require('crypto');
 const path = require('path');
 
 const asyncPipeline = promisify(pipeline);
-const DAILY_SNAPSHOTS_BASE_URL = '';
+const DAILY_SNAPSHOTS_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
+// TODO: [RENAMEME] currently do not have an existing replacement
 const PERMANENT_SNAPSHOTS_BASE_URL = '';
+// Since we do not have a manifest URL, limiting how many RC checks to 5 should be more than enough
+const MAX_RC_CHECK = 5;
 
 const { cache } = require('./utils');
 const { resolveCustomSnapshotUrl } = require('./custom_snapshots');
@@ -96,6 +98,21 @@ async function fetchSnapshotManifest(url, log) {
   return { abc, resp, json };
 }
 
+async function verifySnapshotUrl(url, log) {
+  log.info('Verifying snapshot URL at %s', chalk.bold(url));
+
+  const abc = new AbortController();
+  const resp = await retry(
+    log,
+    async () => await fetch(url, { signal: abc.signal }, { method: 'HEAD' })
+  );
+
+  return { abc, resp };
+}
+
+/**
+ * @deprecated This method should not be used, uses logic for resources we do not have access to.
+ */
 async function getArtifactSpecForSnapshot(urlVersion, license, log) {
   const desiredVersion = urlVersion.replace('-SNAPSHOT', '');
   const desiredLicense = license === 'oss' ? 'oss' : 'default';
@@ -153,6 +170,76 @@ async function getArtifactSpecForSnapshot(urlVersion, license, log) {
   };
 }
 
+async function getArtifactSpecForSnapshotFromUrl(urlVersion, log) {
+  const desiredVersion = urlVersion.replace('-SNAPSHOT', '');
+
+  if (process.env.OPENSEARCH_SNAPSHOT_MANIFEST) {
+    return await getArtifactSpecForSnapshot(urlVersion, 'oss', log);
+  }
+
+  // [RENAMEME] Need replacement for other platforms.
+  const platform = process.platform === 'win32' ? 'windows' : process.platform;
+  const arch = process.arch === 'arm64' ? 'arm64' : 'x64';
+  if (platform !== 'linux' || arch !== 'x64') {
+    throw createCliError(`Snapshots are only available for Linux x64`);
+  }
+
+  const latestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}`;
+  const latestFile = `opensearch-${desiredVersion}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+  const completeLatestUrl = `${latestUrl}/${latestFile}`;
+
+  let { abc, resp } = await verifySnapshotUrl(completeLatestUrl, log);
+
+  if (resp.ok) {
+    return {
+      url: completeLatestUrl,
+      checksumUrl: completeLatestUrl + '.sha512',
+      checksumType: 'sha512',
+      filename: latestFile,
+    };
+  }
+
+  log.info(
+    'Daily general-availability snapshot URL not found for current version, falling back to release-candidate snapshot URL.'
+  );
+
+  let completeUrl = null;
+  let snapshotFile = null;
+
+  // This checks and uses an RC if a RC exists at a higher increment than RC1 or it tries to use RC1
+  // This is in replacement of having a manifest URL, so the exact RC number is unknown but expect it not to be a large number
+  let rcCheck = MAX_RC_CHECK;
+  do {
+    const secondaryLatestUrl = `${DAILY_SNAPSHOTS_BASE_URL}/${desiredVersion}-rc${rcCheck}`;
+    const secondaryLatestFile = `opensearch-${desiredVersion}-rc${rcCheck}-SNAPSHOT-${platform}-${arch}-latest.tar.gz`;
+    const completeSecondaryLatestUrl = `${secondaryLatestUrl}/${secondaryLatestFile}`;
+    ({ abc, resp } = await verifySnapshotUrl(completeSecondaryLatestUrl, log));
+
+    if (resp.ok) {
+      completeUrl = completeSecondaryLatestUrl;
+      snapshotFile = secondaryLatestFile;
+      break;
+    }
+  } while (rcCheck-- >= 1);
+
+  if (resp.status === 404 || !completeUrl || !snapshotFile) {
+    abc.abort();
+    throw createCliError(`Snapshots for ${desiredVersion} are not available`);
+  }
+
+  if (!resp.ok) {
+    abc.abort();
+    throw new Error(`Unable to read snapshot url: ${resp.statusText}`);
+  }
+
+  return {
+    url: completeUrl,
+    checksumUrl: completeUrl + '.sha512',
+    checksumType: 'sha512',
+    filename: snapshotFile,
+  };
+}
+
 exports.Artifact = class Artifact {
   /**
    * Fetch an Artifact from the Artifact API for a license level and version
@@ -168,7 +255,7 @@ exports.Artifact = class Artifact {
       return new Artifact(customSnapshotArtifactSpec, log);
     }
 
-    const artifactSpec = await getArtifactSpecForSnapshot(urlVersion, license, log);
+    const artifactSpec = await getArtifactSpecForSnapshotFromUrl(urlVersion, log);
     return new Artifact(artifactSpec, log);
   }
 

packages/osd-opensearch/src/artifact.test.js

@@ -40,26 +40,33 @@ import { Artifact } from './artifact';
 const log = new ToolingLog();
 let MOCKS;
 
+const DAILY_SNAPSHOT_BASE_URL = 'https://artifacts.opensearch.org/snapshots/core/opensearch';
+const PERMANENT_SNAPSHOT_BASE_URL = '';
+
+const ORIGINAL_PLATFROM = process.platform;
+const ORIGINAL_ARCHITECTURE = process.arch;
 const PLATFORM = process.platform === 'win32' ? 'windows' : process.platform;
 const ARCHITECTURE = process.arch === 'arm64' ? 'arm64' : 'x64';
 const MOCK_VERSION = 'test-version';
-const MOCK_URL = 'http://127.0.0.1:12345';
-const MOCK_FILENAME = 'test-filename';
+const MOCK_RC_VERSION = `test-version-rc4`;
+const MOCK_FILENAME = 'opensearch-test-version-SNAPSHOT-linux-x64-latest.tar.gz';
+const MOCK_RC_FILENAME = `opensearch-test-version-rc4-SNAPSHOT-linux-x64-latest.tar.gz`;
+const MOCK_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/${MOCK_FILENAME}`;
+const MOCK_RC_URL = `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_RC_VERSION}/${MOCK_RC_FILENAME}`;
 
-const DAILY_SNAPSHOT_BASE_URL = '';
-const PERMANENT_SNAPSHOT_BASE_URL = '';
+const itif = process.platfom !== 'linux' || process.arch !== 'x64' ? it : it.skip;
 
 const createArchive = (params = {}) => {
-  const license = params.license || 'default';
   const architecture = params.architecture || ARCHITECTURE;
+  const useRCVersion = params.useRCVersion || false;
 
   return {
-    license: 'default',
+    license: 'oss',
     architecture,
-    version: MOCK_VERSION,
-    url: MOCK_URL + `/${license}`,
+    version: !useRCVersion ? MOCK_VERSION : MOCK_RC_VERSION,
+    url: !useRCVersion ? MOCK_URL : MOCK_RC_URL,
     platform: PLATFORM,
-    filename: MOCK_FILENAME + `-${architecture}.${license}`,
+    filename: !useRCVersion ? MOCK_FILENAME : MOCK_RC_FILENAME,
     ...params,
   };
 };
@@ -92,99 +99,87 @@ beforeEach(() => {
   jest.resetAllMocks();
 
   MOCKS = {
-    valid: {
-      archives: [createArchive({ license: 'oss' }), createArchive({ license: 'default' })],
+    GA: {
+      archives: [createArchive({ useRCVersion: false })],
+    },
+    RC: {
+      archives: [createArchive({ useRCVersion: true })],
     },
     multipleArch: {
       archives: [
-        createArchive({ architecture: 'fake_arch', license: 'oss' }),
-        createArchive({ architecture: ARCHITECTURE, license: 'oss' }),
+        createArchive({ architecture: 'fake_arch', useRCVersion: false }),
+        createArchive({ architecture: ARCHITECTURE, useRCVersion: false }),
       ],
     },
   };
 });
 
-const artifactTest = (requestedLicense, expectedLicense, fetchTimesCalled = 1) => {
+const artifactTest = (fetchTimesCalled = 1) => {
   return async () => {
-    const artifact = await Artifact.getSnapshot(requestedLicense, MOCK_VERSION, log);
+    const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
+    const expectedUrl = fetchTimesCalled === 1 ? MOCK_URL : MOCK_RC_URL;
+    const expectedFilename = fetchTimesCalled === 1 ? MOCK_FILENAME : MOCK_RC_FILENAME;
     expect(fetch).toHaveBeenCalledTimes(fetchTimesCalled);
-    expect(fetch.mock.calls[0][0]).toEqual(
-      `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest-verified.json`
-    );
-    if (fetchTimesCalled === 2) {
-      expect(fetch.mock.calls[1][0]).toEqual(
-        `${PERMANENT_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest.json`
-      );
+    expect(fetch.mock.calls[0][0]).toEqual(MOCK_URL);
+    if (fetchTimesCalled !== 1) {
+      expect(fetch.mock.calls[fetchTimesCalled - 1][0]).toEqual(MOCK_RC_URL);
     }
-    expect(artifact.getUrl()).toEqual(MOCK_URL + `/${expectedLicense}`);
-    expect(artifact.getChecksumUrl()).toEqual(MOCK_URL + `/${expectedLicense}.sha512`);
+    expect(artifact.getUrl()).toEqual(expectedUrl);
+    expect(artifact.getChecksumUrl()).toEqual(expectedUrl + '.sha512');
     expect(artifact.getChecksumType()).toEqual('sha512');
-    expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.${expectedLicense}`);
+    expect(artifact.getFilename()).toEqual(expectedFilename);
   };
 };
 
 describe('Artifact', () => {
   describe('getSnapshot()', () => {
-    describe('with default snapshot', () => {
-      beforeEach(() => {
-        mockFetch(MOCKS.valid);
-      });
-
-      it('should return artifact metadata for a daily oss artifact', artifactTest('oss', 'oss'));
+    itif('should return artifact metadata for a daily GA artifact', () => {
+      mockFetch(MOCKS.GA);
+      artifactTest();
+    });
 
-      it(
-        'should return artifact metadata for a daily default artifact',
-        artifactTest('default', 'default')
-      );
+    itif('should return artifact metadata for a RC artifact', () => {
+      fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
+      fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
+      mockFetch(MOCKS.RC);
+      artifactTest(3);
+    });
 
-      it(
-        'should default to default license with anything other than "oss"',
-        artifactTest('INVALID_LICENSE', 'default')
+    itif('should throw when an artifact cannot be found for the specified parameters', async () => {
+      fetch.mockReturnValue(Promise.resolve(new Response('', { status: 404 })));
+      await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
+        'Snapshots for INVALID_VERSION are not available'
       );
-
-      it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
-        await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
-          "couldn't find an artifact"
-        );
-      });
     });
 
-    describe('with missing default snapshot', () => {
-      beforeEach(() => {
-        fetch.mockReturnValueOnce(Promise.resolve(new Response('', { status: 404 })));
-        mockFetch(MOCKS.valid);
+    describe('with snapshots for multiple architectures', () => {
+      afterAll(() => {
+        Object.defineProperties(process, {
+          platform: {
+            value: ORIGINAL_PLATFROM,
+          },
+          arch: {
+            value: ORIGINAL_ARCHITECTURE,
+          },
+        });
       });
 
-      it(
-        'should return artifact metadata for a permanent oss artifact',
-        artifactTest('oss', 'oss', 2)
-      );
-
-      it(
-        'should return artifact metadata for a permanent default artifact',
-        artifactTest('default', 'default', 2)
-      );
-
-      it(
-        'should default to default license with anything other than "oss"',
-        artifactTest('INVALID_LICENSE', 'default', 2)
-      );
-
-      it('should throw when an artifact cannot be found in the manifest for the specified parameters', async () => {
-        await expect(Artifact.getSnapshot('default', 'INVALID_VERSION', log)).rejects.toThrow(
-          "couldn't find an artifact"
+      it('should throw when on a non-Linux platform', async () => {
+        Object.defineProperty(process, 'platform', {
+          value: 'win32',
+        });
+        await expect(Artifact.getSnapshot('default', 'INVALID_PLATFORM', log)).rejects.toThrow(
+          'Snapshots are only available for Linux x64'
         );
       });
-    });
-
-    describe('with snapshots for multiple architectures', () => {
-      beforeEach(() => {
-        mockFetch(MOCKS.multipleArch);
-      });
 
-      it('should return artifact metadata for the correct architecture', async () => {
-        const artifact = await Artifact.getSnapshot('oss', MOCK_VERSION, log);
-        expect(artifact.getFilename()).toEqual(MOCK_FILENAME + `-${ARCHITECTURE}.oss`);
+      it('should throw when on a non-x64 arch', async () => {
+        Object.defineProperty(process, 'arch', {
+          value: 'arm64',
+        });
+        await expect(Artifact.getSnapshot('default', 'INVALID_ARCH', log)).rejects.toThrow(
+          'Snapshots are only available for Linux x64'
+        );
       });
     });
 
@@ -193,7 +188,7 @@ describe('Artifact', () => {
 
       beforeEach(() => {
         process.env.OPENSEARCH_SNAPSHOT_MANIFEST = CUSTOM_URL;
-        mockFetch(MOCKS.valid);
+        mockFetch(MOCKS.GA);
       });
 
       it('should use the custom URL when looking for a snapshot', async () => {
@@ -205,23 +200,5 @@ describe('Artifact', () => {
         delete process.env.OPENSEARCH_SNAPSHOT_MANIFEST;
       });
     });
-
-    describe('with latest unverified snapshot', () => {
-      beforeEach(() => {
-        process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED = 1;
-        mockFetch(MOCKS.valid);
-      });
-
-      it('should use the daily unverified URL when looking for a snapshot', async () => {
-        await Artifact.getSnapshot('oss', MOCK_VERSION, log);
-        expect(fetch.mock.calls[0][0]).toEqual(
-          `${DAILY_SNAPSHOT_BASE_URL}/${MOCK_VERSION}/manifest-latest.json`
-        );
-      });
-
-      afterEach(() => {
-        delete process.env.OSD_OPENSEARCH_SNAPSHOT_USE_UNVERIFIED;
-      });
-    });
   });
 });

src/core/server/ui_settings/create_or_upgrade_saved_config/integration_tests/create_or_upgrade.test.ts

@@ -95,8 +95,7 @@ describe('createOrUpgradeSavedConfig()', () => {
     await osd.stop();
   }, 30000);
 
-  // TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL
-  xit('upgrades the previous version on each increment', async function () {
+  it('upgrades the previous version on each increment', async function () {
     // ------------------------------------
     // upgrade to 5.4.0
     await createOrUpgradeSavedConfig({

src/core/server/ui_settings/integration_tests/index.test.ts

@@ -36,8 +36,7 @@ import { docExistsSuite } from './doc_exists';
 import { docMissingSuite } from './doc_missing';
 import { docMissingAndIndexReadOnlySuite } from './doc_missing_and_index_read_only';
 
-// TODO: [RENAMEME] Test can be enabled once there is a valid snapshot URL
-xdescribe('uiSettings/routes', function () {
+describe('uiSettings/routes', function () {
   jest.setTimeout(10000);
 
   beforeAll(startServers);